public FeedRuntimeInputHandler(
IHyracksTaskContext ctx,
FeedConnectionId connectionId,
FeedRuntimeId runtimeId,
IFrameWriter coreOperator,
FeedPolicyAccessor fpa,
boolean bufferingEnabled,
FrameTupleAccessor fta,
RecordDescriptor recordDesc,
IFeedManager feedManager,
int nPartitions)
throws IOException {
this.connectionId = connectionId;
this.runtimeId = runtimeId;
this.coreOperator = coreOperator;
this.bufferingEnabled = bufferingEnabled;
this.feedPolicyAccessor = fpa;
this.spiller = new FeedFrameSpiller(ctx, connectionId, runtimeId, fpa);
this.discarder = new FeedFrameDiscarder(ctx, connectionId, runtimeId, fpa, this);
this.exceptionHandler =
new FeedExceptionHandler(ctx, fta, recordDesc, feedManager, connectionId);
this.mode = Mode.PROCESS;
this.lastMode = Mode.PROCESS;
this.finished = false;
this.fpa = fpa;
this.feedManager = feedManager;
this.pool =
(DataBucketPool)
feedManager.getFeedMemoryManager().getMemoryComponent(IFeedMemoryComponent.Type.POOL);
this.frameCollection =
(FrameCollection)
feedManager
.getFeedMemoryManager()
.getMemoryComponent(IFeedMemoryComponent.Type.COLLECTION);
this.frameEventCallback = new FrameEventCallback(fpa, this, coreOperator);
this.mBuffer =
MonitoredBuffer.getMonitoredBuffer(
ctx,
this,
coreOperator,
fta,
recordDesc,
feedManager.getFeedMetricCollector(),
connectionId,
runtimeId,
exceptionHandler,
frameEventCallback,
nPartitions,
fpa);
this.mBuffer.start();
this.throttlingEnabled = false;
}
private void bufferDataUntilRecovery(ByteBuffer frame) throws Exception {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Bufferring data until recovery is complete " + this.runtimeId);
}
if (frameCollection == null) {
this.frameCollection =
(FrameCollection)
feedManager
.getFeedMemoryManager()
.getMemoryComponent(IFeedMemoryComponent.Type.COLLECTION);
}
if (frameCollection == null) {
discarder.processMessage(frame);
if (LOGGER.isLoggable(Level.WARNING)) {
LOGGER.warning("Running low on memory! DISCARDING FRAME ");
}
} else {
boolean success = frameCollection.addFrame(frame);
if (!success) {
if (fpa.spillToDiskOnCongestion()) {
if (frame != null) {
spiller.processMessage(frame);
} // TODO handle the else case
} else {
discarder.processMessage(frame);
}
}
}
}
public void close() {
if (mBuffer != null) {
boolean disableMonitoring = !this.mode.equals(Mode.STALL);
if (frameCollection != null) {
feedManager.getFeedMemoryManager().releaseMemoryComponent(frameCollection);
}
if (pool != null) {
feedManager.getFeedMemoryManager().releaseMemoryComponent(pool);
}
mBuffer.close(false, disableMonitoring);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info(
"Closed input side handler for "
+ this.runtimeId
+ " disabled monitoring "
+ disableMonitoring
+ " Mode for runtime "
+ this.mode);
}
}
}
public void reportUnresolvableCongestion() throws HyracksDataException {
if (this.runtimeId.getFeedRuntimeType().equals(FeedRuntimeType.COMPUTE)) {
FeedCongestionMessage congestionReport =
new FeedCongestionMessage(
connectionId, runtimeId, mBuffer.getInflowRate(), mBuffer.getOutflowRate(), mode);
feedManager.getFeedMessageService().sendMessage(congestionReport);
if (LOGGER.isLoggable(Level.WARNING)) {
LOGGER.warning("Congestion reported " + this.connectionId + " " + this.runtimeId);
}
} else {
if (LOGGER.isLoggable(Level.WARNING)) {
LOGGER.warning("Unresolvable congestion at " + this.connectionId + " " + this.runtimeId);
}
}
}
public void setThrottlingEnabled(boolean throttlingEnabled) {
if (this.throttlingEnabled != throttlingEnabled) {
this.throttlingEnabled = throttlingEnabled;
IFeedMessage throttlingEnabledMesg =
new ThrottlingEnabledFeedMessage(connectionId, runtimeId);
feedManager.getFeedMessageService().sendMessage(throttlingEnabledMesg);
if (LOGGER.isLoggable(Level.WARNING)) {
LOGGER.warning(
"Throttling "
+ throttlingEnabled
+ " for "
+ this.connectionId
+ "["
+ runtimeId
+ "]");
}
}
}
private void processBufferredBacklog() throws HyracksDataException {
try {
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Processing backlog " + this.runtimeId);
}
if (frameCollection != null) {
Iterator<ByteBuffer> backlog = frameCollection.getFrameCollectionIterator();
while (backlog.hasNext()) {
process(backlog.next());
nProcessed++;
}
DataBucket bucket = pool.getDataBucket();
bucket.setContentType(ContentType.EOSD);
bucket.setDesiredReadCount(1);
mBuffer.sendMessage(bucket);
feedManager.getFeedMemoryManager().releaseMemoryComponent(frameCollection);
frameCollection = null;
}
} catch (Exception e) {
e.printStackTrace();
throw new HyracksDataException(e);
}
}