protected void applyOperatorsAndSend(List<String> tuple) {
long timestamp = 0;
if ((MyUtilities.isCustomTimestampMode(getConf())
&& getHierarchyPosition() == StormComponent.NEXT_TO_LAST_COMPONENT)
|| MyUtilities.isWindowTimestampMode(getConf())) timestamp = System.currentTimeMillis();
tuple = _operatorChain.process(tuple, timestamp);
if (tuple == null) return;
_numSentTuples++;
_pendingTuples++;
printTuple(tuple);
if (MyUtilities.isSending(getHierarchyPosition(), _aggBatchOutputMillis)) {
int tuplekey = Integer.parseInt(tuple.get(_keyIndex));
if (_frequentSet.contains(tuplekey))
tupleSend(
SynchronizedStormDataSourceInterface.SHUFFLE_GROUPING_STREAMID, tuple, null, timestamp);
else tupleSend(tuple, null, timestamp);
}
if (MyUtilities.isPrintLatency(getHierarchyPosition(), getConf())) {
printTupleLatency(_numSentTuples - 1, timestamp);
}
}
public StormOperator(
ArrayList<Component> parentEmitters,
ComponentProperties cp,
List<String> allCompNames,
int hierarchyPosition,
TopologyBuilder builder,
TopologyKiller killer,
Config conf) {
super(cp, allCompNames, hierarchyPosition, conf);
_aggBatchOutputMillis = cp.getBatchOutputMillis();
final int parallelism = SystemParameters.getInt(conf, getID() + "_PAR");
// if(parallelism > 1 && distinct != null){
// throw new RuntimeException(_componentName +
// ": Distinct operator cannot be specified for multiThreaded bolts!");
// }
_operatorChain = cp.getChainOperator();
InputDeclarer currentBolt = builder.setBolt(getID(), this, parallelism);
_fullHashList = cp.getFullHashList();
for (StormEmitter parentEmitter : parentEmitters) {
// This is specific only to the Synchronized operator
// add the shuffling stream grouping
if (parentEmitter instanceof SignaledDataSourceComponentInterface) {
final String[] emitterIDs = parentEmitter.getEmitterIDs();
for (final String emitterID : emitterIDs)
currentBolt.shuffleGrouping(
emitterID, SynchronizedStormDataSourceInterface.SHUFFLE_GROUPING_STREAMID);
}
if (MyUtilities.isManualBatchingMode(getConf()))
currentBolt =
MyUtilities.attachEmitterBatch(conf, _fullHashList, currentBolt, parentEmitter);
else
currentBolt =
MyUtilities.attachEmitterHash(conf, _fullHashList, currentBolt, parentEmitter);
}
if (getHierarchyPosition() == FINAL_COMPONENT && (!MyUtilities.isAckEveryTuple(conf)))
killer.registerComponent(this, parallelism);
if (cp.getPrintOut() && _operatorChain.isBlocking())
currentBolt.allGrouping(killer.getID(), SystemParameters.DUMP_RESULTS_STREAM);
}
/*
* whatever is inside this method is done only once
*/
private void eofFinalization() {
printContent();
if (!MyUtilities.isAckEveryTuple(getConf()))
if (getHierarchyPosition() == FINAL_COMPONENT) {
if (!_hasSentEOF) {
_hasSentEOF = true; // to ensure we will not send multiple
// EOF per single spout
getCollector().emit(SystemParameters.EOF_STREAM, new Values(SystemParameters.EOF));
}
} else if (!_hasSentLastAck) {
LOG.info(getID() + ":Has sent last_ack, tuples sent:" + _numSentTuples);
_hasSentLastAck = true;
final List<String> lastTuple =
new ArrayList<String>(Arrays.asList(SystemParameters.LAST_ACK));
tupleSend(lastTuple, null, 0);
}
if (_operatorChain != null) {
_operatorChain.finalizeProcessing();
}
}
protected void applyOperatorsAndSend(
Tuple stormTupleRcv, List<String> tuple, boolean isLastInBatch) {
long timestamp = 0;
if (MyUtilities.isCustomTimestampMode(getConf())
|| MyUtilities.isWindowTimestampMode(getConf()))
timestamp = stormTupleRcv.getLongByField(StormComponent.TIMESTAMP);
if (MyUtilities.isAggBatchOutputMode(_aggBatchOutputMillis))
try {
_semAgg.acquire();
} catch (final InterruptedException ex) {
}
tuple = _operatorChain.process(tuple, timestamp);
if (MyUtilities.isAggBatchOutputMode(_aggBatchOutputMillis)) _semAgg.release();
if (tuple == null) {
getCollector().ack(stormTupleRcv);
return;
}
_numSentTuples++;
printTuple(tuple);
if (MyUtilities.isSending(getHierarchyPosition(), _aggBatchOutputMillis)
|| MyUtilities.isWindowTimestampMode(getConf())) {
tupleSend(tuple, stormTupleRcv, timestamp);
}
if (MyUtilities.isPrintLatency(getHierarchyPosition(), getConf())) {
if (MyUtilities.isManualBatchingMode(getConf())) {
if (isLastInBatch) {
timestamp = stormTupleRcv.getLongByField(StormComponent.TIMESTAMP); // getLong(2);
printTupleLatency(_numSentTuples - 1, timestamp);
}
} else {
timestamp = stormTupleRcv.getLongByField(StormComponent.TIMESTAMP); // getLong(3);
printTupleLatency(_numSentTuples - 1, timestamp);
}
}
}
@Override
public void aggBatchSend() {
if (MyUtilities.isAggBatchOutputMode(_aggBatchOutputMillis))
if (_operatorChain != null) {
final Operator lastOperator = _operatorChain.getLastOperator();
if (lastOperator instanceof AggregateOperator) {
try {
_semAgg.acquire();
} catch (final InterruptedException ex) {
}
// sending
final AggregateOperator agg = (AggregateOperator) lastOperator;
final List<String> tuples = agg.getContent();
for (final String tuple : tuples)
tupleSend(MyUtilities.stringToTuple(tuple, getConf()), null, 0);
// clearing
agg.clearStorage();
_semAgg.release();
}
}
}
