@Override
public void reclaim() {
// Simply put all entries into consumingEntries and clear those up from the entry cache as
// well.
for (SimpleQueueEntry entry : entries) {
consumingEntries.put(entry.getRowKey(), entry);
entryCache.remove(entry.getRowKey());
}
}
@Override
public DequeueResult<byte[]> dequeue(int maxBatchSize) throws IOException {
Preconditions.checkArgument(maxBatchSize > 0, "Batch size must be > 0.");
// pre-compute the "claimed" state content in case of FIFO.
byte[] claimedStateValue = null;
if (consumerConfig.getDequeueStrategy() == DequeueStrategy.FIFO
&& consumerConfig.getGroupSize() > 1) {
claimedStateValue = encodeStateColumn(ConsumerEntryState.CLAIMED);
}
while (consumingEntries.size() < maxBatchSize && getEntries(consumingEntries, maxBatchSize)) {
// ANDREAS: this while loop should stop once getEntries/populateCache reaches the end of the
// queue. Currently, it
// will retry as long as it gets at least one entry in every round, even if that is an entry
// that must be ignored
// because it cannot be claimed.
// ANDREAS: It could be a problem that we always read to the end of the queue. This way one
// flowlet instance may
// always all entries, while others are idle.
// For FIFO, need to try claiming the entry if group size > 1
if (consumerConfig.getDequeueStrategy() == DequeueStrategy.FIFO
&& consumerConfig.getGroupSize() > 1) {
Iterator<Map.Entry<byte[], SimpleQueueEntry>> iterator =
consumingEntries.entrySet().iterator();
while (iterator.hasNext()) {
SimpleQueueEntry entry = iterator.next().getValue();
if (entry.getState() == null
|| QueueEntryRow.getStateInstanceId(entry.getState())
>= consumerConfig.getGroupSize()) {
// If not able to claim it, remove it, and move to next one.
if (!claimEntry(entry.getRowKey(), claimedStateValue)) {
iterator.remove();
}
}
}
}
}
// If nothing get dequeued, return the empty result.
if (consumingEntries.isEmpty()) {
return EMPTY_RESULT;
}
return new SimpleDequeueResult(consumingEntries.values());
}
@Override
public byte[] apply(SimpleQueueEntry input) {
return input.getData();
}
