String fillMessageMutation(MutationBatch mb, Message message) throws MessageQueueException {
// Get the execution time from the message or set to current time so it runs immediately
long curTimeMicros;
if (!message.hasTrigger()) {
curTimeMicros =
TimeUnit.MICROSECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS);
} else {
curTimeMicros =
TimeUnit.MICROSECONDS.convert(
message.getTrigger().getTriggerTime(), TimeUnit.MILLISECONDS);
}
curTimeMicros += (counter.incrementAndGet() % 1000);
// Update the message for the new token
message.setToken(TimeUUIDUtils.getMicrosTimeUUID(curTimeMicros));
// Set up the queue entry
MessageQueueEntry entry =
MessageQueueEntry.newMessageEntry(
message.getPriority(), message.getToken(), MessageQueueEntryState.Waiting);
// Convert the message object to JSON
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try {
mapper.writeValue(baos, message);
baos.flush();
} catch (Exception e) {
throw new MessageQueueException("Failed to serialize message data: " + message, e);
}
// Write the queue entry
String shardKey = getShardKey(message);
mb.withRow(queueColumnFamily, shardKey)
.putColumn(entry, new String(baos.toByteArray()), metadata.getRetentionTimeout());
// Write the lookup from queue key to queue entry
if (message.hasKey()) {
mb.withRow(keyIndexColumnFamily, getCompositeKey(getName(), message.getKey()))
.putEmptyColumn(
MessageMetadataEntry.newMessageId(getCompositeKey(shardKey, entry.getMessageId())),
metadata.getRetentionTimeout());
}
// Allow hook processing
for (MessageQueueHooks hook : hooks) {
hook.beforeSendMessage(message, mb);
}
// Update state and retun the token
stats.incSendMessageCount();
return getCompositeKey(shardKey, entry.getMessageId());
}
private void fillAckMutation(MessageContext context, MutationBatch mb) {
queue.stats.incAckMessageCount();
Message message = context.getMessage();
// Token refers to the timeout event. If 0 (i.e. no) timeout was specified
// then the token will not exist
if (message.getToken() != null) {
MessageQueueEntry entry = MessageQueueEntry.newBusyEntry(message);
// Remove timeout entry from the queue
mb.withRow(queue.queueColumnFamily, queue.getShardKey(message)).deleteColumn(entry);
// Remove entry lookup from the key, if one exists
if (message.hasKey()) {
mb.withRow(
queue.keyIndexColumnFamily,
queue.getCompositeKey(queue.getName(), message.getKey()))
.putEmptyColumn(
MessageMetadataEntry.newMessageId(
queue.getCompositeKey(queue.getShardKey(message), entry.getMessageId())),
queue.metadataDeleteTTL);
if (message.isKeepHistory()) {
MessageHistory history = context.getHistory();
if (history.getStatus() == MessageStatus.RUNNING) {
history.setStatus(MessageStatus.DONE);
}
history.setEndTime(
TimeUnit.MICROSECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS));
try {
mb.withRow(queue.historyColumnFamily, message.getKey())
.putColumn(
history.getToken(),
queue.serializeToString(context.getHistory()),
queue.metadata.getHistoryTtl()); // TTL
} catch (Exception e) {
LOG.warn("Error serializing message history for " + message.getKey(), e);
}
}
}
// Run hooks
for (MessageQueueHooks hook : queue.hooks) {
hook.beforeAckMessage(message, mb);
}
}
if (context.getNextMessage() != null) {
try {
queue.fillMessageMutation(mb, context.getNextMessage());
} catch (MessageQueueException e) {
LOG.warn("Error filling nextMessage for " + message.getKey(), e);
}
}
}
private List<MessageContext> readMessagesInternal(
String shardName,
int itemsToPop,
int lockColumnCount,
MessageQueueEntry lockColumn,
ColumnListMutation<MessageQueueEntry> rowMutation,
MutationBatch m,
long curTimeMicros)
throws BusyLockException, MessageQueueException {
try {
List<MessageContext> entries = Lists.newArrayList();
RangeEndpoint re =
ShardedDistributedMessageQueue.entrySerializer
.makeEndpoint((byte) MessageQueueEntryType.Message.ordinal(), Equality.EQUAL)
.append((byte) 0, Equality.EQUAL);
if (lockColumn != null) {
re.append(lockColumn.getTimestamp(), Equality.LESS_THAN_EQUALS);
} else {
re.append(TimeUUIDUtils.getMicrosTimeUUID(curTimeMicros), Equality.LESS_THAN_EQUALS);
}
ColumnList<MessageQueueEntry> result =
queue
.keyspace
.prepareQuery(queue.queueColumnFamily)
.setConsistencyLevel(queue.consistencyLevel)
.getKey(shardName)
.withColumnRange(
new RangeBuilder()
.setLimit(itemsToPop + (lockColumn == null ? 0 : (lockColumnCount + 1)))
.setEnd(re.toBytes())
.build())
.execute()
.getResult();
for (Column<MessageQueueEntry> column : result) {
if (itemsToPop == 0) {
break;
}
MessageQueueEntry entry = column.getName();
switch (entry.getType()) {
case Lock:
// TODO: Track number of locks read and make sure we don't exceed itemsToPop
// We have the lock
if (lockColumn != null && entry.getState() == MessageQueueEntryState.Acquired) {
if (!entry.getTimestamp().equals(lockColumn.getTimestamp())) {
throw new BusyLockException("Someone else snuck in");
}
}
break;
case Message:
{
try {
itemsToPop--;
// First, we always want to remove the old item
String messageId = queue.getCompositeKey(shardName, entry.getMessageId());
rowMutation.deleteColumn(entry);
// Next, parse the message metadata and add a timeout entry
final Message message = queue.extractMessageFromColumn(column);
// Update the message state
if (message != null) {
MessageContext context = new MessageContext();
context.setMessage(message);
// Message has a trigger so we need to figure out if it is an
// unfinished repeating trigger and re-add it.
if (message.hasTrigger()) {
// Read back all messageIds associated with this key and check to see if we have
// duplicates.
String groupRowKey = queue.getCompositeKey(queue.getName(), message.getKey());
try {
// Use consistency level
ColumnList<MessageMetadataEntry> columns =
queue
.keyspace
.prepareQuery(queue.keyIndexColumnFamily)
.getRow(groupRowKey)
.withColumnRange(
ShardedDistributedMessageQueue.metadataSerializer
.buildRange()
.greaterThanEquals(
(byte) MessageMetadataEntryType.MessageId.ordinal())
.lessThanEquals(
(byte) MessageMetadataEntryType.MessageId.ordinal())
.build())
.execute()
.getResult();
MessageMetadataEntry mostRecentMessageMetadata = null;
long mostRecentTriggerTime = 0;
for (Column<MessageMetadataEntry> currMessageEntry : columns) {
MessageQueueEntry pendingMessageEntry =
MessageQueueEntry.fromMetadata(currMessageEntry.getName());
if (currMessageEntry.getTtl() == 0) {
long currMessageTriggerTime =
pendingMessageEntry.getTimestamp(TimeUnit.MICROSECONDS);
// First message we found, so treat as the most recent
if (mostRecentMessageMetadata == null) {
mostRecentMessageMetadata = currMessageEntry.getName();
mostRecentTriggerTime = currMessageTriggerTime;
} else {
// This message's trigger time is after what we thought was the most
// recent.
// Discard the previous 'most' recent and accept this one instead
if (currMessageTriggerTime > mostRecentTriggerTime) {
LOG.warn(
"Need to discard : "
+ entry.getMessageId()
+ " => "
+ mostRecentMessageMetadata.getName());
m.withRow(
queue.keyIndexColumnFamily,
queue.getCompositeKey(queue.getName(), message.getKey()))
.putEmptyColumn(
mostRecentMessageMetadata, queue.metadataDeleteTTL);
mostRecentTriggerTime = currMessageTriggerTime;
mostRecentMessageMetadata = currMessageEntry.getName();
} else {
LOG.warn(
"Need to discard : "
+ entry.getMessageId()
+ " => "
+ currMessageEntry.getName());
m.withRow(
queue.keyIndexColumnFamily,
queue.getCompositeKey(queue.getName(), message.getKey()))
.putEmptyColumn(
currMessageEntry.getName(), queue.metadataDeleteTTL);
}
}
}
}
if (mostRecentMessageMetadata != null) {
if (!mostRecentMessageMetadata.getName().endsWith(entry.getMessageId())) {
throw new DuplicateMessageException("Duplicate trigger for " + messageId);
}
}
} catch (NotFoundException e) {
} catch (ConnectionException e) {
throw new MessageQueueException("Error fetching row " + groupRowKey, e);
}
// Update the trigger
final Message nextMessage;
Trigger trigger = message.getTrigger().nextTrigger();
if (trigger != null) {
nextMessage = message.clone();
nextMessage.setTrigger(trigger);
context.setNextMessage(nextMessage);
if (message.isAutoCommitTrigger()) {
queue.fillMessageMutation(m, nextMessage);
}
}
}
// Message has a key so we remove this item from the messages by key index.
// A timeout item will be added later
if (message.hasKey()) {
m.withRow(
queue.keyIndexColumnFamily,
queue.getCompositeKey(queue.getName(), message.getKey()))
.putEmptyColumn(
MessageMetadataEntry.newMessageId(messageId), queue.metadataDeleteTTL);
LOG.debug(
"Removing from key : "
+ queue.getCompositeKey(queue.getName(), message.getKey())
+ " : "
+ messageId);
if (message.isKeepHistory()) {
MessageHistory history = context.getHistory();
history.setToken(entry.getTimestamp());
history.setStartTime(curTimeMicros);
history.setTriggerTime(message.getTrigger().getTriggerTime());
history.setStatus(MessageStatus.RUNNING);
try {
m.withRow(queue.historyColumnFamily, message.getKey())
.putColumn(
entry.getTimestamp(),
queue.serializeToString(history),
queue.metadata.getHistoryTtl());
} catch (Exception e) {
LOG.warn("Error serializing history for key '" + message.getKey() + "'", e);
}
}
}
// Message has a timeout so we add a timeout event.
if (message.getTimeout() > 0) {
MessageQueueEntry timeoutEntry =
MessageQueueEntry.newMessageEntry(
(byte) 0,
TimeUUIDUtils.getMicrosTimeUUID(
curTimeMicros
+ TimeUnit.MICROSECONDS.convert(
message.getTimeout(), TimeUnit.SECONDS)
+ (queue.counter.incrementAndGet() % 1000)),
MessageQueueEntryState.Busy);
message.setToken(timeoutEntry.getTimestamp());
message.setRandom(timeoutEntry.getRandom());
m.withRow(queue.queueColumnFamily, queue.getShardKey(message))
.putColumn(
timeoutEntry,
column.getStringValue(),
queue.metadata.getRetentionTimeout());
MessageMetadataEntry messageIdEntry =
MessageMetadataEntry.newMessageId(
queue.getCompositeKey(
queue.getShardKey(message), timeoutEntry.getMessageId()));
// Add the timeout column to the key
if (message.hasKey()) {
m.withRow(
queue.keyIndexColumnFamily,
queue.getCompositeKey(queue.getName(), message.getKey()))
.putEmptyColumn(messageIdEntry, queue.metadata.getRetentionTimeout());
}
context.setAckMessageId(messageIdEntry.getName());
} else {
message.setToken(null);
}
// Update some stats
switch (entry.getState()) {
case Waiting:
queue.stats.incProcessCount();
break;
case Busy:
queue.stats.incReprocessCount();
break;
default:
LOG.warn("Unknown message state: " + entry.getState());
// TODO:
break;
}
entries.add(context);
} else {
queue.stats.incInvalidMessageCount();
// TODO: Add to poison queue
}
} catch (DuplicateMessageException e) {
// OK to ignore this error. All the proper columns will have been deleted in the
// batch.
}
break;
}
default:
{
// TODO: Error: Unknown type
break;
}
}
}
return entries;
} catch (BusyLockException e) {
queue.stats.incLockContentionCount();
throw e;
} catch (Exception e) {
throw new MessageQueueException("Error processing queue shard : " + shardName, e);
} finally {
try {
m.execute();
} catch (Exception e) {
throw new MessageQueueException("Error processing queue shard : " + shardName, e);
}
}
}