@Component
public final class HandlerFactory {
private static final AdaptorLogger logger =
new AdaptorLogger(LoggerFactory.getLogger(HandlerFactory.class));
@Autowired private ApplicationContext context;
private Properties appProperties;
@Autowired private StringHelper stringHelper;
public HandlerFactory() throws AdaptorConfigurationException {
String envProperties = System.getProperty("env.properties");
logger.info("constructing HandlerFactory", "envProperties=\"{}\"", envProperties);
try (InputStream is = this.getClass().getClassLoader().getResourceAsStream(envProperties)) {
appProperties = new Properties();
appProperties.load(is);
logger.info("constructing HandlerFactory", "properties=\"{}\"", appProperties.toString());
} catch (IOException e) {
logger.alert(
ALERT_TYPE.ADAPTOR_FAILED_TO_START,
ALERT_CAUSE.INVALID_ADAPTOR_CONFIGURATION,
ALERT_SEVERITY.BLOCKER,
e.toString());
throw new AdaptorConfigurationException(e);
}
}
public Handler getHandler(final HandlerConfig handlerConfig)
throws AdaptorConfigurationException {
try {
final Class<? extends Handler> handlerClass =
Class.forName(handlerConfig.getHandlerClass()).asSubclass(Handler.class);
Handler handler = context.getBean(handlerClass);
Map<String, Object> handlerProperties = handlerConfig.getHandlerProperties();
for (Entry<String, Object> handlerProperty : handlerProperties.entrySet()) {
logger.info(
"building handler",
"handler_property_name=\"{}\" value=\"{}\" isString=\"{}\"",
handlerProperty.getKey(),
handlerProperty.getValue(),
(handlerProperty.getValue() instanceof String));
if (handlerProperty.getValue() instanceof String) {
String propValue = handlerProperty.getValue().toString();
String newValue = stringHelper.redeemToken(propValue, appProperties);
if (!propValue.equals(newValue)) {
handlerProperty.setValue(newValue);
logger.info(
"building handler",
"handler_property_name=\"{}\" old_value=\"{}\" new_value=\"{}\"",
handlerProperty.getKey(),
propValue,
newValue);
}
}
}
handler.setPropertyMap(handlerConfig.getHandlerProperties());
handler.setName(handlerConfig.getName());
handler.build();
logger.debug(
"building handler",
"handler_name=\"{}\" handler_properties=\"{}\"",
handler.getName(),
handlerConfig.getHandlerProperties());
return handler;
} catch (ClassNotFoundException e) {
logger.alert(
ALERT_TYPE.ADAPTOR_FAILED_TO_START,
ALERT_CAUSE.INVALID_ADAPTOR_CONFIGURATION,
ALERT_SEVERITY.BLOCKER,
e.toString());
throw new AdaptorConfigurationException(e);
}
}
}
/**
* Memory based DataChannel stores data in a collection backed by an ArrayList. This implementation
* supports replicating channel feature by maintaining a consumed-event-index for each consumer.
* Once an event at an index has been consumed by all the consumers, the event is freed from the
* list and space is made available.
*
* @author Neeraj Jain TODO define data structure MultiplexingQueue MultipleConsumerQueue
*/
@Component
@Scope("prototype")
public class MemoryChannel extends AbstractChannel {
private String channelId;
private static final AdaptorLogger logger =
new AdaptorLogger(LoggerFactory.getLogger(MemoryChannel.class));
private Set<String> consumerNames = new HashSet<>();
/*
* DataChannel maintains the data in an array list, need random access to
* the elements.
*/
private List<Event> eventList = Collections.synchronizedList(new ArrayList<Event>());
// private List<Event> eventList = new ArrayList<Event>();
/*
* Maintains a mapping between the consumer and the last index it fetched.
* The take request will return the data from the next index.
*/
private Map<String, Integer> consumerToTakenIndexMap = new HashMap<>();
/*
* Maintains a mapping between the index in the eventList and how many times
* it has been consumed. Once the element has been consumed by all the
* consumers, the take method should remove it from the list.
*/
private Map<Integer, Integer> indexToTakenCountMap = new HashMap<>();
/*
* Number of elements that have been removed from the eventList. Need this
* to compute the next index.
*/
private int removedCount;
// private String counterNameTake;
// private String counterNamePut;
private boolean printStats;
private long printStatsDurationInSeconds;
private long putCount;
private long takeCount;
private long channelSizeInBytes = 0l;
private long maxSizeInBytes = 0l;
private boolean channelStopped = false;
private long channelCapacity;
@Override
public void build() {
channelId = UUID.randomUUID().toString();
printStats =
PropertyHelper.getBooleanProperty(getProperties(), ChannelConfigConstants.PRINT_STATS);
channelCapacity =
PropertyHelper.getLongProperty(
getProperties(),
ChannelConfigConstants.CHANNEL_CAPACITY,
MemoryChannelConstants.DEFAULT_CHANNEL_CAPACITY);
printStatsDurationInSeconds =
PropertyHelper.getLongProperty(
getProperties(),
ChannelConfigConstants.PRINT_STATS_DURATION_IN_SECONDS,
MemoryChannelConstants.DEFAULT_PRINT_STATS_DURATION_IN_SECONDS);
logger.debug(
"building memory channel-notsync",
"channel_name=\"{}\" channel_id=\"{}\" printStats=\"{}\" printStatsDurationInSeconds=\"{}\" channelCapacity=\"{}\"",
getName(),
channelId,
printStats,
printStatsDurationInSeconds,
channelCapacity);
printStatsDurationInSeconds = printStatsDurationInSeconds * 1000;
}
/**
* Add a new item to the storage, after ensuring that it can hold more. Ensure that the storage
* has enough capacity to hold more items.
*/
@Override
public void put(Event arg0) {
/*
* Put the data in the eventList and notifyAll.
*
*/
logger.debug(
"putting event on memory channel",
"channel_name=\"{}\" channelSizeInBytes=\"{}\"",
getName(),
channelSizeInBytes);
synchronized (this) {
while ((channelSizeInBytes + arg0.getBody().length) > channelCapacity) {
try {
logger.debug(
"waitingto put event on memory channel",
"channel_name=\"{}\" channelCapacity=\"{}\"",
getName(),
channelCapacity);
wait(1000);
} catch (InterruptedException e) {
// nothing to do here
}
}
channelSizeInBytes = channelSizeInBytes + arg0.getBody().length;
if (channelSizeInBytes >= maxSizeInBytes) {
maxSizeInBytes = channelSizeInBytes;
}
eventList.add(arg0);
notifyAll();
}
putCount++;
}
/**
* This method, if used with default bigdime implementation, will throw an
* UnsupportedOperationException since bigdime expects the name of the consumer in the request.
* Consider using {@link MemoryChannel#take(String)} method instead.
*
* @throws ChannelException if the there is no data available on Channel
*/
@Override
public synchronized Event take() {
if ((consumerNames == null) || consumerNames.isEmpty()) {
return take("default");
} else {
throw new UnsupportedOperationException(
"this channel has registered consumers, invoking take method without parameters is not supported.");
}
}
/**
* This method, if used with default bigdime implementation, will throw an
* UnsupportedOperationException since bigdime expects the name of the consumer in the request.
* Consider using {@link MemoryChannel#take(String, int))} method instead.
*/
@Override
public synchronized List<Event> take(int size) {
if ((consumerNames == null) || consumerNames.isEmpty()) {
return take("default", size);
} else {
throw new UnsupportedOperationException(
"this channel has registered consumers, invoking take method without parameters is not supported.");
}
}
@Override
public synchronized boolean registerConsumer(String consumerName) {
logger.info(
"registering consumers",
"_message=\"before registering\" channel_name=\"{}\" consumers=\"{}\"",
getName(),
consumerNames);
boolean registered = consumerNames.add(consumerName);
logger.info(
"registering consumers",
"_message=\"after registering\" channel_name=\"{}\" consumers=\"{}\"",
getName(),
consumerNames);
return registered;
}
/*
* @formatter:off
* 1. Who is consuming? The channel needs to maintain a set of consumerNames.
* 2. For each consumer, maintain the takenIndex. consumer->takenIndex map.
* 3. For each index, maintain the fetch count. index->takenCount map.
* 4. For any take request, get the takenIndex for the consumer.
* if takenIndex is null, event with index=0 should be returned.
* if the takenIndex is=0, event with index=1 should be returned.
* if the takenIndex is=n, event with index=n+1 should be returned.
* 5. For each take request, update the indexToTakenCountMap map.
* if the takenCount for any index is equals to channelConsumerCount, remove the value from the eventList.
* when the value is removed from the eventList, set/increment removedCount.
* c1 and c2 on a eventList with 0 1 2 3 4 5
* consumerToTakenIndexMap = {} indexToTakenCountMap = {}
*
* c1.take()
* takenIndex=null=>0;
* removedCount = 0;
* newTakenIndex=0; get i0
* consumerToTakenIndexMap = {c1,0}
* indexToTakenCountMap = {0,1}
* removedCount = 0;
* data = i0 i1 i2 i3 i4 i5
*
* c1.take()
* takenIndex=0=>1;
* removedCount = 0;
* newTakenIndex=1; get i1
* consumerToTakenIndexMap = {c1,1}
* indexToTakenCountMap = {{0,1}, {1,1}}
* data = i0 i1 i2 i3 i4 i5
*
* c2.take()
* takenIndex=null=>0;
* removedCount = 0;
* newTakenIndex=0; get i0
* consumerToTakenIndexMap = {{c1,1}, {c2,0}}
* indexToTakenCountMap = {{0,2}, {1,1}} => {{1,1}}
* removedCount = 1;
* data = i1 i2 i3 i4 i5
*
* c1.take()
* takenIndex=1=>2;
* removedCount = 1;
* newTakenIndex=1; get i2
* consumerToTakenIndexMap = {{c1,2}, {c2,0}}
* indexToTakenCountMap = {{1,1}, {2,1}}
* removedCount = 1;
* data = i1 i2 i3 i4 i5
*
* c2.take()
* takenIndex=0=>1;
* removedCount = 1;
* newTakenIndex=0; get i1
* consumerToTakenIndexMap = {{c1,2}, {c2,1}}
* indexToTakenCountMap = {{1,2}, {2,1}} => {{2,1}}
* removedCount = 2;
* data = i2 i3 i4 i5
*
* c1.take()
* takenIndex=2=>3;
* removedCount = 2;
* newTakenIndex=1; get i3
* consumerToTakenIndexMap = {{c1,3}, {c2,0}}
* indexToTakenCountMap = {{2,1}, {3,1}}
* removedCount = 2;
* data = i2 i3 i4 i5
* @formatter:on
*/
@Override
public synchronized Event take(String consumerName) {
List<Event> events = take(consumerName, 1);
return events.get(0);
}
@Override
public synchronized List<Event> take(final String consumerName, final int size) {
// start: update the takenIndex for this consumer
Integer takenIndex = consumerToTakenIndexMap.get(consumerName);
if (takenIndex == null) {
takenIndex = 0;
} else {
takenIndex++;
}
int newTakenStartIndex = takenIndex - removedCount;
int availableEventsCount = eventList.size() - newTakenStartIndex;
if (availableEventsCount == 0) {
throw new ChannelException("No data found on channel");
}
int fetchSize = size;
if (availableEventsCount < size) {
fetchSize = availableEventsCount;
}
int newTakenEndIndex = newTakenStartIndex + fetchSize;
List<Event> takenEvent =
new ArrayList<>(eventList.subList(newTakenStartIndex, newTakenEndIndex));
consumerToTakenIndexMap.put(consumerName, (takenIndex + fetchSize) - 1);
// start:update the taken count for this index
for (int i = 0; i < fetchSize; i++) {
Integer takenCount = indexToTakenCountMap.get(takenIndex + i);
if (takenCount == null) {
takenCount = 0;
}
takenCount++;
if (takenCount == consumerNames.size()) { // newTakenIndex must
// always be zero
Event removedEvent = eventList.remove(0);
channelSizeInBytes = channelSizeInBytes - removedEvent.getBody().length;
indexToTakenCountMap.remove(takenIndex + i);
removedCount++;
} else {
indexToTakenCountMap.put(takenIndex + i, takenCount);
}
}
takeCount++;
notifyAll();
return takenEvent;
}
public long getPutCount() {
return putCount;
}
public long getTakeCount() {
return takeCount;
}
public void printStats() {
logger.info(
"print MemoryChannelStats",
"channel_id=\"{}\" channel_name=\"{}\" channelCapacity=\"{}\" total_puts=\"{}\" total_takes=\"{}\" current_list_size=\"{}\" size_in_bytes=\"{}\" max_size_in_bytes=\"{}\" current_consumer_count=\"{}\" consumer_names=\"{}\" this=\"{}\"",
getChannelId(),
MemoryChannel.this.getName(),
channelCapacity,
putCount,
takeCount,
eventList.size(),
channelSizeInBytes,
maxSizeInBytes,
consumerNames.size(),
consumerNames,
this.getClass());
}
@Override
public void start() {
logger.info(
"starting MemoryChannelStats",
"starting MemoryChannel, channel_name=\"{}\" channel_id=\"{}\"",
getName(),
getChannelId());
if (printStats) {
logger.info("starting MemoryChannelStats", "starting MemoryChannel");
// printStatsDurationInSeconds = printStatsDurationInSeconds * 1000;
startStatsThread();
}
logger.info(
"starting MemoryChannelStats",
"started MemoryChannel, channel_name=\"{}\" channel_id=\"{}\"",
getName(),
getChannelId());
}
@Override
public void stop() {
channelStopped = true;
}
/**
* UUID for channel, set during build method. Only used for internal purposes.
*
* @return
*/
public String getChannelId() {
return channelId;
}
private void startStatsThread() {
new Thread() {
@Override
public void run() {
while (!channelStopped) {
try {
logger.info(
"heathcheck thread for MemoryChannel",
"printing stats, printStatsDuration=\"{}\"",
printStatsDurationInSeconds);
printStats();
sleep(printStatsDurationInSeconds);
} catch (Exception e) {
logger.warn(
"heathcheck thread for MemoryChannel",
"health check thread received an exception, will duck it. printStatsDurationInSeconds=\"{}\"",
printStatsDurationInSeconds,
e);
}
}
}
}.start();
logger.info(
"started heathcheck thread for MemoryChannel",
"channel_id=\"{}\" thread_name=\"{}\" channel_name=\"{}\" ",
getChannelId(),
getName(),
MemoryChannel.this.getName());
}
}