private void processBatchedItems() throws ProcessingException { final int effectiveBatchSize = determineBatchSize(); List<E> batch = getItemsFromQueue(effectiveBatchSize); final int retryAttempts = config.getRetryAttempts(); int executionsLeft = retryAttempts + 1; while (executionsLeft-- > 0) { try { processor.process(batch); break; } catch (final RuntimeException e) { LOGGER.warn( "processBatchedItems caught error while processing batch of " + batch.size() + " error " + e); if (executionsLeft <= 0) { for (E item : batch) { try { processor.throwAway(item, e); } catch (final Throwable th) { LOGGER.warn( "processBatchedItems caught error while throwing away an item: " + item + " error " + th); } } } else { LOGGER.warn( getThreadName() + " : processBatchedItems() : exception during processing, retrying in " + retryAttempts + " milliseconds, " + executionsLeft + " retries left : " + e); try { Thread.sleep(config.getRetryAttemptDelay()); } catch (InterruptedException e1) { Thread.currentThread().interrupt(); throw e; } } } } removeFromQueue(effectiveBatchSize); }
private void doProcessItems() throws ProcessingException { // process the quarantined items and remove them as they're processed // don't process work if this node's operations have been disabled if (!cluster.areOperationsEnabled()) { return; } else { if (config.isBatchingEnabled() && config.getBatchSize() > 0) { processBatchedItems(); } else { processListSnapshot(); } if (toolkitList.isEmpty() && stopState == STOP_STATE.STOP_REQUESTED) { signalStop(); } } }
// Do not take any clustered write lock in this path. public void add(final E item) { if (null == item) return; int maxQueueSize = config.getMaxQueueSize(); bucketWriteLock.lock(); boolean interrupted = false; try { if (maxQueueSize != UNLIMITED_QUEUE_SIZE) { while (!isCancelled() && toolkitList.size() >= maxQueueSize) { try { bucketNotFull.await(); } catch (final InterruptedException e) { interrupted = true; } } } boolean signalNotEmpty = toolkitList.isEmpty(); toolkitList.unlockedAdd(item); if (signalNotEmpty) { bucketNotEmpty.signalAll(); } } finally { bucketWriteLock.unlock(); if (interrupted) { Thread.currentThread().interrupt(); } } }
private int determineBatchSize() { int batchSize = config.getBatchSize(); int listSize = toolkitList.size(); if (listSize < batchSize) { batchSize = listSize; } return batchSize; }
@Override public synchronized AsyncCoordinator getOrCreateAsyncCoordinator( final Ehcache cache, final AsyncConfig config) { final String fullAsyncName = toolkitInstanceFactory.getFullAsyncName(cache); final ToolkitCache<String, AsyncConfig> configMap = toolkitInstanceFactory.getOrCreateAsyncConfigMap(); AsyncConfig oldConfig = configMap.putIfAbsent(fullAsyncName, config); if (oldConfig != null && !oldConfig.equals(config)) { throw new IllegalArgumentException( "can not get AsyncCoordinator " + fullAsyncName + " for same name but different configs.\nExisting config\n" + oldConfig + "\nNew Config\n" + config); } AsyncCoordinatorImpl async = localMap.get(fullAsyncName); if (async != null) { if (oldConfig == null) { throw new IllegalArgumentException( "AsyncCoordinator " + fullAsyncName + " created for this node but entry not present in configMap"); } } else { Callback stopCallable = new Callback() { @Override public void callback() { synchronized (AsyncCoordinatorFactoryImpl.this) { localMap.remove(fullAsyncName); } } }; async = new AsyncCoordinatorImpl(fullAsyncName, config, toolkitInstanceFactory, stopCallable); localMap.put(fullAsyncName, async); } return async; }
private void processSingleItem() throws ProcessingException { // process the next item final E item = getItemsFromQueue(1).get(0); final int retryAttempts = config.getRetryAttempts(); int executionsLeft = retryAttempts + 1; while (executionsLeft-- > 0) { try { processor.process(item); break; } catch (final RuntimeException e) { if (executionsLeft <= 0) { try { processor.throwAway(item, e); } catch (final Throwable th) { LOGGER.warn( "processSingleItem caught error while throwing away an item: " + item + " " + th); } } else { LOGGER.warn( getThreadName() + " : processSingleItem() : exception during processing, retrying in " + retryAttempts + " milliseconds, " + executionsLeft + " retries left : " + e.getMessage()); try { Thread.sleep(config.getRetryAttemptDelay()); } catch (InterruptedException e1) { Thread.currentThread().interrupt(); throw e; } } } } removeFromQueue(1); }
public ProcessingBucket( String bucketName, AsyncConfig config, ToolkitListInternal<E> toolkitList, ClusterInfo cluster, ItemProcessor<E> processor, boolean workingOnDeadBucket) { this.bucketName = bucketName; this.config = config; this.cluster = cluster; this.processor = processor; this.toolkitList = toolkitList; this.baselineTimestampMillis = System.currentTimeMillis(); ReentrantReadWriteLock bucketLock = new ReentrantReadWriteLock(); this.bucketReadLock = bucketLock.readLock(); this.bucketWriteLock = bucketLock.writeLock(); this.bucketNotEmpty = bucketWriteLock.newCondition(); this.bucketNotFull = bucketWriteLock.newCondition(); this.stoppedButBucketNotEmpty = bucketWriteLock.newCondition(); this.workDelay = new AtomicLong(config.getWorkDelay()); this.workingOnDeadBucket = workingOnDeadBucket; this.processingWorkerRunnable = new ProcessingWorker(threadNamePrefix + bucketName); this.destroyAfterStop = true; }
/** * This method process items from bucket. Execution of this method does not guarantee that items * from a non empty bucket will be processed. */ private void processItems() throws ProcessingException { final int workSize; bucketWriteLock.lock(); try { // set some state related to this processing run lastProcessingTimeMillis = baselinedCurrentTimeMillis(); workSize = toolkitList.size(); // if there's no work that needs to be done, stop the processing if (0 == workSize) { LOGGER.warn(getThreadName() + " : processItems : nothing to process"); return; } // filter might remove items from list, so this should be with-in writeLock filterQuarantined(); } finally { bucketWriteLock.unlock(); } // if the batching is enabled and work size is smaller than batch size, don't process anything // as long as the // max allowed fall behind delay hasn't expired final int batchSize = config.getBatchSize(); if (config.isBatchingEnabled() && batchSize > 0) { // wait for another round if the batch size hasn't been filled up yet and the max write delay // hasn't expired yet if (workSize < batchSize && config.getMaxAllowedFallBehind() > lastProcessingTimeMillis - lastWorkDoneMillis) { bucketReadLock.lock(); try { if (stopState == STOP_STATE.NORMAL) { LOGGER.warn( getThreadName() + " : processItems : only " + workSize + " work items available, waiting for " + batchSize + " items to fill up a batch"); return; } } finally { bucketReadLock.unlock(); } } // enforce the rate limit and wait for another round if too much would be processed compared // to // the last time when a batch was executed final int rateLimit = config.getRateLimit(); if (rateLimit > 0) { final long secondsSinceLastWorkDone; final int effectiveBatchSize; bucketReadLock.lock(); try { if (stopState == STOP_STATE.NORMAL) { secondsSinceLastWorkDone = (baselinedCurrentTimeMillis() - lastWorkDoneMillis) / 1000; effectiveBatchSize = determineBatchSize(); long maxBatchSizeSinceLastWorkDone = rateLimit * secondsSinceLastWorkDone; if (effectiveBatchSize > maxBatchSizeSinceLastWorkDone) { LOGGER.warn( getThreadName() + " : processItems() : last work was done " + secondsSinceLastWorkDone + " seconds ago, processing " + effectiveBatchSize + " batch items would exceed the rate limit of " + rateLimit + ", waiting for a while."); return; } } } finally { bucketReadLock.unlock(); } } } bucketWriteLock.lock(); try { lastWorkDoneMillis = baselinedCurrentTimeMillis(); } finally { bucketWriteLock.unlock(); } doProcessItems(); }