/**
* Fast check to see if a shard has messages to process
*
* @param shardName
* @throws MessageQueueException
*/
private boolean hasMessages(String shardName) throws MessageQueueException {
UUID currentTime = TimeUUIDUtils.getUniqueTimeUUIDinMicros();
try {
ColumnList<MessageQueueEntry> result =
keyspace
.prepareQuery(queueColumnFamily)
.setConsistencyLevel(consistencyLevel)
.getKey(shardName)
.withColumnRange(
new RangeBuilder()
.setLimit(1) // Read extra messages because of the lock column
.setStart(
entrySerializer
.makeEndpoint(
(byte) MessageQueueEntryType.Message.ordinal(), Equality.EQUAL)
.toBytes())
.setEnd(
entrySerializer
.makeEndpoint(
(byte) MessageQueueEntryType.Message.ordinal(), Equality.EQUAL)
.append((byte) 0, Equality.EQUAL)
.append(currentTime, Equality.LESS_THAN_EQUALS)
.toBytes())
.build())
.execute()
.getResult();
return !result.isEmpty();
} catch (ConnectionException e) {
throw new MessageQueueException("Error checking shard for messages. " + shardName, e);
}
}
private SampleEntity createSampleEntity(String id) {
Random prng = new Random();
SampleEntity entity = new SampleEntity();
entity.setId(id);
entity.setBooleanPrimitive(prng.nextBoolean());
entity.setBooleanObject(prng.nextBoolean());
entity.setBytePrimitive((byte) prng.nextInt(Byte.MAX_VALUE));
entity.setByteObject((byte) prng.nextInt(Byte.MAX_VALUE));
entity.setShortPrimitive((short) prng.nextInt(Short.MAX_VALUE));
entity.setShortObject((short) prng.nextInt(Short.MAX_VALUE));
entity.setIntPrimitive(prng.nextInt());
entity.setIntObject(prng.nextInt());
entity.setLongPrimitive(prng.nextLong());
entity.setLongObject(prng.nextLong());
entity.setFloatPrimitive(prng.nextFloat());
entity.setFloatObject(prng.nextFloat());
entity.setDoublePrimitive(prng.nextDouble());
entity.setDoubleObject(prng.nextDouble());
entity.setString(RandomStringUtils.randomAlphanumeric(16));
entity.setByteArray(RandomStringUtils.randomAlphanumeric(16).getBytes(Charsets.UTF_8));
entity.setDate(new Date());
entity.setUuid(TimeUUIDUtils.getUniqueTimeUUIDinMicros());
Foo foo = new Foo(prng.nextInt(), RandomStringUtils.randomAlphanumeric(4));
entity.setFoo(foo);
BarBar barbar = new BarBar();
barbar.i = prng.nextInt();
barbar.s = RandomStringUtils.randomAlphanumeric(4);
Bar bar = new Bar();
bar.i = prng.nextInt();
bar.s = RandomStringUtils.randomAlphanumeric(4);
bar.barbar = barbar;
entity.setBar(bar);
return entity;
}
String fillMessageMutation(MutationBatch mb, Message message) throws MessageQueueException {
// Get the execution time from the message or set to current time so it runs immediately
long curTimeMicros;
if (!message.hasTrigger()) {
curTimeMicros =
TimeUnit.MICROSECONDS.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS);
} else {
curTimeMicros =
TimeUnit.MICROSECONDS.convert(
message.getTrigger().getTriggerTime(), TimeUnit.MILLISECONDS);
}
curTimeMicros += (counter.incrementAndGet() % 1000);
// Update the message for the new token
message.setToken(TimeUUIDUtils.getMicrosTimeUUID(curTimeMicros));
// Set up the queue entry
MessageQueueEntry entry =
MessageQueueEntry.newMessageEntry(
message.getPriority(), message.getToken(), MessageQueueEntryState.Waiting);
// Convert the message object to JSON
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try {
mapper.writeValue(baos, message);
baos.flush();
} catch (Exception e) {
throw new MessageQueueException("Failed to serialize message data: " + message, e);
}
// Write the queue entry
String shardKey = getShardKey(message);
mb.withRow(queueColumnFamily, shardKey)
.putColumn(entry, new String(baos.toByteArray()), metadata.getRetentionTimeout());
// Write the lookup from queue key to queue entry
if (message.hasKey()) {
mb.withRow(keyIndexColumnFamily, getCompositeKey(getName(), message.getKey()))
.putEmptyColumn(
MessageMetadataEntry.newMessageId(getCompositeKey(shardKey, entry.getMessageId())),
metadata.getRetentionTimeout());
}
// Allow hook processing
for (MessageQueueHooks hook : hooks) {
hook.beforeSendMessage(message, mb);
}
// Update state and retun the token
stats.incSendMessageCount();
return getCompositeKey(shardKey, entry.getMessageId());
}
List<MessageContext> readMessagesFromShardUsingLockManager(String shardName, int itemToPop)
throws MessageQueueException, BusyLockException {
ShardLock lock = null;
try {
lock = queue.lockManager.acquireLock(shardName);
MutationBatch m =
queue.keyspace.prepareMutationBatch().setConsistencyLevel(queue.consistencyLevel);
ColumnListMutation<MessageQueueEntry> rowMutation =
m.withRow(queue.queueColumnFamily, shardName);
long curTimeMicros =
TimeUUIDUtils.getMicrosTimeFromUUID(TimeUUIDUtils.getUniqueTimeUUIDinMicros());
return readMessagesInternal(shardName, itemToPop, 0, null, rowMutation, m, curTimeMicros);
} catch (BusyLockException e) {
queue.stats.incLockContentionCount();
throw e;
} catch (Exception e) {
LOG.error("Error reading shard " + shardName, e);
throw new MessageQueueException("Error", e);
} finally {
queue.lockManager.releaseLock(lock);
}
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("Message[");
if (token != null) {
sb.append("token=" + token + " (" + TimeUUIDUtils.getMicrosTimeFromUUID(token) + ")");
}
if (random != null) sb.append(", random=" + random);
if (trigger != null) sb.append(", trigger=" + trigger);
if (parameters != null) sb.append(", parameters=" + parameters);
sb.append(", priority=" + priority);
sb.append(", timeout=" + timeout);
if (key != null) sb.append(", key=" + key);
if (hasUniqueKey) sb.append(", hasUniqueKey=" + hasUniqueKey);
if (taskClass != null) sb.append(", taskClass=" + taskClass);
if (isKeepHistory) sb.append(", isKeepHistory=" + isKeepHistory);
sb.append("]");
return sb.toString();
}
private List<MessageContext> readMessagesInternal(
String shardName,
int itemsToPop,
int lockColumnCount,
MessageQueueEntry lockColumn,
ColumnListMutation<MessageQueueEntry> rowMutation,
MutationBatch m,
long curTimeMicros)
throws BusyLockException, MessageQueueException {
try {
List<MessageContext> entries = Lists.newArrayList();
RangeEndpoint re =
ShardedDistributedMessageQueue.entrySerializer
.makeEndpoint((byte) MessageQueueEntryType.Message.ordinal(), Equality.EQUAL)
.append((byte) 0, Equality.EQUAL);
if (lockColumn != null) {
re.append(lockColumn.getTimestamp(), Equality.LESS_THAN_EQUALS);
} else {
re.append(TimeUUIDUtils.getMicrosTimeUUID(curTimeMicros), Equality.LESS_THAN_EQUALS);
}
ColumnList<MessageQueueEntry> result =
queue
.keyspace
.prepareQuery(queue.queueColumnFamily)
.setConsistencyLevel(queue.consistencyLevel)
.getKey(shardName)
.withColumnRange(
new RangeBuilder()
.setLimit(itemsToPop + (lockColumn == null ? 0 : (lockColumnCount + 1)))
.setEnd(re.toBytes())
.build())
.execute()
.getResult();
for (Column<MessageQueueEntry> column : result) {
if (itemsToPop == 0) {
break;
}
MessageQueueEntry entry = column.getName();
switch (entry.getType()) {
case Lock:
// TODO: Track number of locks read and make sure we don't exceed itemsToPop
// We have the lock
if (lockColumn != null && entry.getState() == MessageQueueEntryState.Acquired) {
if (!entry.getTimestamp().equals(lockColumn.getTimestamp())) {
throw new BusyLockException("Someone else snuck in");
}
}
break;
case Message:
{
try {
itemsToPop--;
// First, we always want to remove the old item
String messageId = queue.getCompositeKey(shardName, entry.getMessageId());
rowMutation.deleteColumn(entry);
// Next, parse the message metadata and add a timeout entry
final Message message = queue.extractMessageFromColumn(column);
// Update the message state
if (message != null) {
MessageContext context = new MessageContext();
context.setMessage(message);
// Message has a trigger so we need to figure out if it is an
// unfinished repeating trigger and re-add it.
if (message.hasTrigger()) {
// Read back all messageIds associated with this key and check to see if we have
// duplicates.
String groupRowKey = queue.getCompositeKey(queue.getName(), message.getKey());
try {
// Use consistency level
ColumnList<MessageMetadataEntry> columns =
queue
.keyspace
.prepareQuery(queue.keyIndexColumnFamily)
.getRow(groupRowKey)
.withColumnRange(
ShardedDistributedMessageQueue.metadataSerializer
.buildRange()
.greaterThanEquals(
(byte) MessageMetadataEntryType.MessageId.ordinal())
.lessThanEquals(
(byte) MessageMetadataEntryType.MessageId.ordinal())
.build())
.execute()
.getResult();
MessageMetadataEntry mostRecentMessageMetadata = null;
long mostRecentTriggerTime = 0;
for (Column<MessageMetadataEntry> currMessageEntry : columns) {
MessageQueueEntry pendingMessageEntry =
MessageQueueEntry.fromMetadata(currMessageEntry.getName());
if (currMessageEntry.getTtl() == 0) {
long currMessageTriggerTime =
pendingMessageEntry.getTimestamp(TimeUnit.MICROSECONDS);
// First message we found, so treat as the most recent
if (mostRecentMessageMetadata == null) {
mostRecentMessageMetadata = currMessageEntry.getName();
mostRecentTriggerTime = currMessageTriggerTime;
} else {
// This message's trigger time is after what we thought was the most
// recent.
// Discard the previous 'most' recent and accept this one instead
if (currMessageTriggerTime > mostRecentTriggerTime) {
LOG.warn(
"Need to discard : "
+ entry.getMessageId()
+ " => "
+ mostRecentMessageMetadata.getName());
m.withRow(
queue.keyIndexColumnFamily,
queue.getCompositeKey(queue.getName(), message.getKey()))
.putEmptyColumn(
mostRecentMessageMetadata, queue.metadataDeleteTTL);
mostRecentTriggerTime = currMessageTriggerTime;
mostRecentMessageMetadata = currMessageEntry.getName();
} else {
LOG.warn(
"Need to discard : "
+ entry.getMessageId()
+ " => "
+ currMessageEntry.getName());
m.withRow(
queue.keyIndexColumnFamily,
queue.getCompositeKey(queue.getName(), message.getKey()))
.putEmptyColumn(
currMessageEntry.getName(), queue.metadataDeleteTTL);
}
}
}
}
if (mostRecentMessageMetadata != null) {
if (!mostRecentMessageMetadata.getName().endsWith(entry.getMessageId())) {
throw new DuplicateMessageException("Duplicate trigger for " + messageId);
}
}
} catch (NotFoundException e) {
} catch (ConnectionException e) {
throw new MessageQueueException("Error fetching row " + groupRowKey, e);
}
// Update the trigger
final Message nextMessage;
Trigger trigger = message.getTrigger().nextTrigger();
if (trigger != null) {
nextMessage = message.clone();
nextMessage.setTrigger(trigger);
context.setNextMessage(nextMessage);
if (message.isAutoCommitTrigger()) {
queue.fillMessageMutation(m, nextMessage);
}
}
}
// Message has a key so we remove this item from the messages by key index.
// A timeout item will be added later
if (message.hasKey()) {
m.withRow(
queue.keyIndexColumnFamily,
queue.getCompositeKey(queue.getName(), message.getKey()))
.putEmptyColumn(
MessageMetadataEntry.newMessageId(messageId), queue.metadataDeleteTTL);
LOG.debug(
"Removing from key : "
+ queue.getCompositeKey(queue.getName(), message.getKey())
+ " : "
+ messageId);
if (message.isKeepHistory()) {
MessageHistory history = context.getHistory();
history.setToken(entry.getTimestamp());
history.setStartTime(curTimeMicros);
history.setTriggerTime(message.getTrigger().getTriggerTime());
history.setStatus(MessageStatus.RUNNING);
try {
m.withRow(queue.historyColumnFamily, message.getKey())
.putColumn(
entry.getTimestamp(),
queue.serializeToString(history),
queue.metadata.getHistoryTtl());
} catch (Exception e) {
LOG.warn("Error serializing history for key '" + message.getKey() + "'", e);
}
}
}
// Message has a timeout so we add a timeout event.
if (message.getTimeout() > 0) {
MessageQueueEntry timeoutEntry =
MessageQueueEntry.newMessageEntry(
(byte) 0,
TimeUUIDUtils.getMicrosTimeUUID(
curTimeMicros
+ TimeUnit.MICROSECONDS.convert(
message.getTimeout(), TimeUnit.SECONDS)
+ (queue.counter.incrementAndGet() % 1000)),
MessageQueueEntryState.Busy);
message.setToken(timeoutEntry.getTimestamp());
message.setRandom(timeoutEntry.getRandom());
m.withRow(queue.queueColumnFamily, queue.getShardKey(message))
.putColumn(
timeoutEntry,
column.getStringValue(),
queue.metadata.getRetentionTimeout());
MessageMetadataEntry messageIdEntry =
MessageMetadataEntry.newMessageId(
queue.getCompositeKey(
queue.getShardKey(message), timeoutEntry.getMessageId()));
// Add the timeout column to the key
if (message.hasKey()) {
m.withRow(
queue.keyIndexColumnFamily,
queue.getCompositeKey(queue.getName(), message.getKey()))
.putEmptyColumn(messageIdEntry, queue.metadata.getRetentionTimeout());
}
context.setAckMessageId(messageIdEntry.getName());
} else {
message.setToken(null);
}
// Update some stats
switch (entry.getState()) {
case Waiting:
queue.stats.incProcessCount();
break;
case Busy:
queue.stats.incReprocessCount();
break;
default:
LOG.warn("Unknown message state: " + entry.getState());
// TODO:
break;
}
entries.add(context);
} else {
queue.stats.incInvalidMessageCount();
// TODO: Add to poison queue
}
} catch (DuplicateMessageException e) {
// OK to ignore this error. All the proper columns will have been deleted in the
// batch.
}
break;
}
default:
{
// TODO: Error: Unknown type
break;
}
}
}
return entries;
} catch (BusyLockException e) {
queue.stats.incLockContentionCount();
throw e;
} catch (Exception e) {
throw new MessageQueueException("Error processing queue shard : " + shardName, e);
} finally {
try {
m.execute();
} catch (Exception e) {
throw new MessageQueueException("Error processing queue shard : " + shardName, e);
}
}
}
List<MessageContext> readMessagesFromShardUsingDefaultLock(String shardName, int itemsToPop)
throws MessageQueueException, BusyLockException {
MutationBatch m = null;
MessageQueueEntry lockColumn = null;
ColumnListMutation<MessageQueueEntry> rowMutation = null;
int lockColumnCount = 0;
// Try locking first
try {
// 1. Write the lock column
lockColumn = MessageQueueEntry.newLockEntry(MessageQueueEntryState.None);
long curTimeMicros = TimeUUIDUtils.getTimeFromUUID(lockColumn.getTimestamp());
m = queue.keyspace.prepareMutationBatch().setConsistencyLevel(queue.consistencyLevel);
m.withRow(queue.queueColumnFamily, shardName)
.putColumn(lockColumn, curTimeMicros + queue.lockTimeout, queue.lockTtl);
m.execute();
// 2. Read back lock columns and entries
ColumnList<MessageQueueEntry> result =
queue
.keyspace
.prepareQuery(queue.queueColumnFamily)
.setConsistencyLevel(queue.consistencyLevel)
.getKey(shardName)
.withColumnRange(
ShardedDistributedMessageQueue.entrySerializer
.buildRange()
.greaterThanEquals((byte) MessageQueueEntryType.Lock.ordinal())
.lessThanEquals((byte) MessageQueueEntryType.Lock.ordinal())
.build())
.execute()
.getResult();
m = queue.keyspace.prepareMutationBatch().setConsistencyLevel(queue.consistencyLevel);
rowMutation = m.withRow(queue.queueColumnFamily, shardName);
rowMutation.deleteColumn(lockColumn);
int lockCount = 0;
boolean lockAcquired = false;
lockColumnCount = result.size();
for (Column<MessageQueueEntry> column : result) {
MessageQueueEntry lock = column.getName();
if (lock.getType() == MessageQueueEntryType.Lock) {
lockColumnCount++;
// Stale lock so we can discard it
if (column.getLongValue() < curTimeMicros) {
queue.stats.incExpiredLockCount();
rowMutation.deleteColumn(lock);
} else if (lock.getState() == MessageQueueEntryState.Acquired) {
throw new BusyLockException("Not first lock");
} else {
lockCount++;
if (lockCount == 1 && lock.getTimestamp().equals(lockColumn.getTimestamp())) {
lockAcquired = true;
}
}
if (!lockAcquired) {
throw new BusyLockException("Not first lock");
}
// Write the acquired lock column
lockColumn =
MessageQueueEntry.newLockEntry(
lockColumn.getTimestamp(), MessageQueueEntryState.Acquired);
rowMutation.putColumn(lockColumn, curTimeMicros + queue.lockTimeout, queue.lockTtl);
}
}
} catch (BusyLockException e) {
queue.stats.incLockContentionCount();
throw e;
} catch (ConnectionException e) {
LOG.error("Error reading shard " + shardName, e);
throw new MessageQueueException("Error", e);
} finally {
try {
m.execute();
} catch (Exception e) {
throw new MessageQueueException("Error committing lock", e);
}
}
long curTimeMicros = TimeUUIDUtils.getMicrosTimeFromUUID(lockColumn.getTimestamp());
m = queue.keyspace.prepareMutationBatch().setConsistencyLevel(queue.consistencyLevel);
// First, release the lock column
rowMutation = m.withRow(queue.queueColumnFamily, shardName);
rowMutation.deleteColumn(lockColumn);
return readMessagesInternal(
shardName, itemsToPop, lockColumnCount, lockColumn, rowMutation, m, curTimeMicros);
}
@Test
public void testMultiRowUniqueness() {
DedicatedMultiRowUniquenessConstraint<UUID> constraint =
new DedicatedMultiRowUniquenessConstraint<UUID>(
keyspace, TimeUUIDUtils.getUniqueTimeUUIDinMicros())
.withConsistencyLevel(ConsistencyLevel.CL_ONE)
.withRow(CF_USER_UNIQUE_UUID, "user1")
.withRow(CF_EMAIL_UNIQUE_UUID, "*****@*****.**");
DedicatedMultiRowUniquenessConstraint<UUID> constraint2 =
new DedicatedMultiRowUniquenessConstraint<UUID>(
keyspace, TimeUUIDUtils.getUniqueTimeUUIDinMicros())
.withConsistencyLevel(ConsistencyLevel.CL_ONE)
.withRow(CF_USER_UNIQUE_UUID, "user1")
.withRow(CF_EMAIL_UNIQUE_UUID, "*****@*****.**");
try {
Column<UUID> c = constraint.getUniqueColumn();
Assert.fail();
} catch (Exception e) {
LOG.info(e.getMessage());
}
try {
constraint.acquire();
Column<UUID> c = constraint.getUniqueColumn();
LOG.info("Unique column is " + c.getName());
try {
constraint2.acquire();
Assert.fail("Should already be acquired");
} catch (NotUniqueException e) {
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
} finally {
try {
constraint2.release();
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
}
}
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
} finally {
try {
constraint.release();
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
}
}
try {
constraint2.acquire();
Column<UUID> c = constraint.getUniqueColumn();
LOG.info("Unique column is " + c.getName());
} catch (NotUniqueException e) {
Assert.fail("Should already be unique");
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
} finally {
try {
constraint2.release();
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
}
}
}
/**
* Get the micros time encoded in the token
*
* @return
*/
@JsonIgnore
public long getTokenTime() {
return TimeUUIDUtils.getMicrosTimeFromUUID(token);
}
