private void processSpilledBacklog() throws HyracksDataException { try { Iterator<ByteBuffer> backlog = spiller.replayData(); while (backlog.hasNext()) { process(backlog.next()); nProcessed++; } DataBucket bucket = pool.getDataBucket(); bucket.setContentType(ContentType.EOSD); bucket.setDesiredReadCount(1); mBuffer.sendMessage(bucket); spiller.reset(); } catch (Exception e) { e.printStackTrace(); throw new HyracksDataException(e); } }
private void processBufferredBacklog() throws HyracksDataException { try { if (LOGGER.isLoggable(Level.INFO)) { LOGGER.info("Processing backlog " + this.runtimeId); } if (frameCollection != null) { Iterator<ByteBuffer> backlog = frameCollection.getFrameCollectionIterator(); while (backlog.hasNext()) { process(backlog.next()); nProcessed++; } DataBucket bucket = pool.getDataBucket(); bucket.setContentType(ContentType.EOSD); bucket.setDesiredReadCount(1); mBuffer.sendMessage(bucket); feedManager.getFeedMemoryManager().releaseMemoryComponent(frameCollection); frameCollection = null; } } catch (Exception e) { e.printStackTrace(); throw new HyracksDataException(e); } }
protected void process(ByteBuffer frame) throws HyracksDataException { boolean frameProcessed = false; while (!frameProcessed) { try { if (!bufferingEnabled) { coreOperator.nextFrame(frame); // synchronous mBuffer.sendReport(frame); } else { DataBucket bucket = pool.getDataBucket(); if (bucket != null) { if (frame != null) { bucket.reset(frame); // created a copy here bucket.setContentType(ContentType.DATA); } else { bucket.setContentType(ContentType.EOD); } bucket.setDesiredReadCount(1); mBuffer.sendMessage(bucket); mBuffer.sendReport(frame); nProcessed++; } else { if (fpa.spillToDiskOnCongestion()) { if (frame != null) { boolean spilled = spiller.processMessage(frame); if (spilled) { setMode(Mode.SPILL); } else { reportUnresolvableCongestion(); } } } else if (fpa.discardOnCongestion()) { boolean discarded = discarder.processMessage(frame); if (!discarded) { reportUnresolvableCongestion(); } } else if (fpa.throttlingEnabled()) { setThrottlingEnabled(true); } else { reportUnresolvableCongestion(); } } } frameProcessed = true; } catch (Exception e) { if (feedPolicyAccessor.continueOnSoftFailure()) { frame = exceptionHandler.handleException(e, frame); if (frame == null) { frameProcessed = true; if (LOGGER.isLoggable(Level.WARNING)) { LOGGER.warning( "Encountered exception! " + e.getMessage() + "Insufficient information, Cannot extract failing tuple"); } } } else { if (LOGGER.isLoggable(Level.WARNING)) { LOGGER.warning( "Ingestion policy does not require recovering from tuple. Feed would terminate"); } mBuffer.close(false); throw new HyracksDataException(e); } } } }