@Override
public void dropStorage() throws MessageQueueException {
try {
keyspace.dropColumnFamily(this.queueColumnFamily);
try {
Thread.sleep(SCHEMA_CHANGE_DELAY);
} catch (InterruptedException e) {
}
} catch (ConnectionException e) {
if (!e.getMessage().contains("already exist"))
throw new MessageQueueException(
"Failed to create column family for " + queueColumnFamily.getName(), e);
}
try {
keyspace.dropColumnFamily(this.keyIndexColumnFamily);
try {
Thread.sleep(SCHEMA_CHANGE_DELAY);
} catch (InterruptedException e) {
}
} catch (ConnectionException e) {
if (!e.getMessage().contains("already exist"))
throw new MessageQueueException(
"Failed to create column family for " + queueColumnFamily.getName(), e);
}
}
@Test
public void basicLifecycle() throws Exception {
final String id = "basicLifecycle";
EntityManager<SampleEntity, String> entityPersister =
new DefaultEntityManager.Builder<SampleEntity, String>()
.withEntityType(SampleEntity.class)
.withKeyspace(keyspace)
.withColumnFamily(CF_SAMPLE_ENTITY)
.build();
SampleEntity origEntity = createSampleEntity(id);
entityPersister.put(origEntity);
// use low-level astyanax API to confirm the write
{
ColumnList<String> cl =
keyspace.prepareQuery(CF_SAMPLE_ENTITY).getKey(id).execute().getResult();
// 19 simple columns
// 2 one-level-deep nested columns from Bar
// 2 two-level-deep nested columns from BarBar
Assert.assertEquals(23, cl.size());
// simple columns
Assert.assertEquals(origEntity.getString(), cl.getColumnByName("STRING").getStringValue());
Assert.assertArrayEquals(
origEntity.getByteArray(), cl.getColumnByName("BYTE_ARRAY").getByteArrayValue());
// nested fields
Assert.assertEquals(origEntity.getBar().i, cl.getColumnByName("BAR.i").getIntegerValue());
Assert.assertEquals(origEntity.getBar().s, cl.getColumnByName("BAR.s").getStringValue());
Assert.assertEquals(
origEntity.getBar().barbar.i, cl.getColumnByName("BAR.barbar.i").getIntegerValue());
Assert.assertEquals(
origEntity.getBar().barbar.s, cl.getColumnByName("BAR.barbar.s").getStringValue());
}
SampleEntity getEntity = entityPersister.get(id);
Assert.assertEquals(origEntity, getEntity);
entityPersister.delete(id);
// use low-level astyanax API to confirm the delete
{
ColumnList<String> cl =
keyspace.prepareQuery(CF_SAMPLE_ENTITY).getKey(id).execute().getResult();
Assert.assertEquals(0, cl.size());
}
}
/**
* @param _clusterName
* @param _server
* @param _keySpaceName
* @return KeySpace - the Cassandra KeySpace we are going to work in, and that we give back to the
* caller ( if everything goes well ).
* @throws java.lang.Exception
*/
public static final Keyspace doInit(
final String _clusterName, final String _server, final String _keySpaceName)
throws Exception {
AstyanaxContext<Keyspace> context =
new AstyanaxContext.Builder()
.forCluster(_clusterName)
.forKeyspace(_keySpaceName)
.withAstyanaxConfiguration(
new AstyanaxConfigurationImpl()
.setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
.setCqlVersion("3.0.0")
.setTargetCassandraVersion("2.0.4"))
.withConnectionPoolConfiguration(
new ConnectionPoolConfigurationImpl("ubicity file loading pool")
.setPort(9160)
.setMaxConnsPerHost(16)
.setSeeds(_server + ":9160")
.setMaxOperationsPerConnection(10000000)
.setConnectTimeout(10000))
.withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
.buildKeyspace(ThriftFamilyFactory.getInstance());
context.start();
Keyspace keySpace = context.getClient();
try {
keySpace.describeKeyspace();
logger.log(Level.INFO, "keyspace " + keySpace.getKeyspaceName() + " does exist");
} catch (BadRequestException bre) {
logger.log(
Level.INFO, "keyspace " + keySpace.getKeyspaceName() + " does NOT exist, creating it");
keySpace.createKeyspace(
ImmutableMap.<String, Object>builder()
.put(
"strategy_options",
ImmutableMap.<String, Object>builder().put("replication_factor", "1").build())
.put("strategy_class", "SimpleStrategy")
.build());
}
log = CassandraCrawlLogSchema.checkOrBuildMonitrixSchema(keySpace);
if (log == null) {
logger.log(Level.SEVERE, "could not create Monitrix ColumnFamily ( table ) ");
}
return keySpace;
}
@Override
public List<Message> peekMessagesByKey(String key) throws MessageQueueException {
String groupRowKey = getCompositeKey(getName(), key);
List<Message> messages = Lists.newArrayList();
try {
ColumnList<MessageMetadataEntry> columns =
keyspace
.prepareQuery(keyIndexColumnFamily)
.getRow(groupRowKey)
.withColumnRange(
metadataSerializer
.buildRange()
.greaterThanEquals((byte) MessageMetadataEntryType.MessageId.ordinal())
.lessThanEquals((byte) MessageMetadataEntryType.MessageId.ordinal())
.build())
.execute()
.getResult();
for (Column<MessageMetadataEntry> entry : columns) {
if (entry.getTtl() != 0) continue;
Message message = peekMessage(entry.getName().getName());
if (message != null) {
messages.add(peekMessage(entry.getName().getName()));
} else {
LOG.warn("No queue item for " + entry.getName());
}
}
} catch (NotFoundException e) {
} catch (ConnectionException e) {
throw new MessageQueueException("Error fetching row " + groupRowKey, e);
}
return messages;
}
/**
* Fast check to see if a shard has messages to process
*
* @param shardName
* @throws MessageQueueException
*/
private boolean hasMessages(String shardName) throws MessageQueueException {
UUID currentTime = TimeUUIDUtils.getUniqueTimeUUIDinMicros();
try {
ColumnList<MessageQueueEntry> result =
keyspace
.prepareQuery(queueColumnFamily)
.setConsistencyLevel(consistencyLevel)
.getKey(shardName)
.withColumnRange(
new RangeBuilder()
.setLimit(1) // Read extra messages because of the lock column
.setStart(
entrySerializer
.makeEndpoint(
(byte) MessageQueueEntryType.Message.ordinal(), Equality.EQUAL)
.toBytes())
.setEnd(
entrySerializer
.makeEndpoint(
(byte) MessageQueueEntryType.Message.ordinal(), Equality.EQUAL)
.append((byte) 0, Equality.EQUAL)
.append(currentTime, Equality.LESS_THAN_EQUALS)
.toBytes())
.build())
.execute()
.getResult();
return !result.isEmpty();
} catch (ConnectionException e) {
throw new MessageQueueException("Error checking shard for messages. " + shardName, e);
}
}
/**
* Return history for a single key for the specified time range
*
* <p>TODO: honor the time range :)
*/
@Override
public List<MessageHistory> getKeyHistory(String key, Long startTime, Long endTime, int count)
throws MessageQueueException {
List<MessageHistory> list = Lists.newArrayList();
ColumnList<UUID> columns;
try {
columns =
keyspace
.prepareQuery(historyColumnFamily)
.setConsistencyLevel(consistencyLevel)
.getRow(key)
.execute()
.getResult();
} catch (ConnectionException e) {
throw new MessageQueueException("Failed to load history for " + key, e);
}
for (Column<UUID> column : columns) {
try {
list.add(deserializeString(column.getStringValue(), MessageHistory.class));
} catch (Exception e) {
LOG.info("Error deserializing history entry", e);
}
}
return list;
}
/**
* Do the column update or delete for the given column and row key
*
* @param applicationScope We need to use this when getting the keyspace
* @param uniqueValue The unique value to write
* @param op The operation to write
*/
private MutationBatch doWrite(
ApplicationScope applicationScope, UniqueValue uniqueValue, RowOp op) {
final MutationBatch batch = keyspace.prepareMutationBatch();
final Id applicationId = applicationScope.getApplication();
final FieldKey fieldKey =
createUniqueValueKey(
applicationId, uniqueValue.getEntityId().getType(), uniqueValue.getField());
op.doLookup(batch.withRow(CF_UNIQUE_VALUES, ScopedRowKey.fromKey(applicationId, fieldKey)));
final EntityKey entityKey = createEntityUniqueLogKey(applicationId, uniqueValue.getEntityId());
op.doLog(
batch.withRow(CF_ENTITY_UNIQUE_VALUE_LOG, ScopedRowKey.fromKey(applicationId, entityKey)));
if (log.isDebugEnabled()) {
log.debug(
"Writing unique value version={} name={} value={} ",
new Object[] {
uniqueValue.getEntityVersion(),
uniqueValue.getField().getName(),
uniqueValue.getField().getValue()
});
}
return batch;
}
/**
* Read the data stored with the unique row. This data is normally a 'foreign' key to another
* column family.
*
* @return
* @throws Exception
*/
public ByteBuffer readData() throws Exception {
ColumnList<C> result =
keyspace
.prepareQuery(columnFamily)
.setConsistencyLevel(consistencyLevel)
.getKey(key)
.execute()
.getResult();
boolean hasColumn = false;
ByteBuffer data = null;
for (Column<C> column : result) {
if (column.getTtl() == 0) {
if (hasColumn) {
throw new IllegalStateException("Row has multiple uniquneness locks");
}
hasColumn = true;
data = column.getByteBufferValue();
}
}
if (!hasColumn) {
throw new NotFoundException(this.key.toString() + " has no uniquness lock");
}
return data;
}
@Override
public Message peekMessage(String messageId) throws MessageQueueException {
String[] parts = splitCompositeKey(messageId);
String shardKey = parts[0];
MessageQueueEntry entry = new MessageQueueEntry(parts[1]);
try {
Column<MessageQueueEntry> column =
keyspace
.prepareQuery(queueColumnFamily)
.setConsistencyLevel(consistencyLevel)
.getKey(shardKey)
.getColumn(entry)
.execute()
.getResult();
try {
ByteArrayInputStream bais = new ByteArrayInputStream(column.getByteArrayValue());
return mapper.readValue(bais, Message.class);
} catch (Exception e) {
LOG.warn("Error parsing message", e);
// Error parsing the message so we pass it on to the invalid message handler.
try {
return invalidMessageHandler.apply(column.getStringValue());
} catch (Exception e2) {
LOG.warn("Error handling invalid message message", e2);
throw new MessageQueueException("Error parsing message " + messageId);
}
}
} catch (NotFoundException e) {
return null;
} catch (ConnectionException e) {
throw new MessageQueueException("Error getting message " + messageId, e);
}
}
@Override
public Message peekMessageByKey(String key) throws MessageQueueException {
String groupRowKey = getCompositeKey(getName(), key);
try {
ColumnList<MessageMetadataEntry> columns =
keyspace
.prepareQuery(keyIndexColumnFamily)
.setConsistencyLevel(consistencyLevel)
.getRow(groupRowKey)
.withColumnRange(
metadataSerializer
.buildRange()
.greaterThanEquals((byte) MessageMetadataEntryType.MessageId.ordinal())
.lessThanEquals((byte) MessageMetadataEntryType.MessageId.ordinal())
.build())
.execute()
.getResult();
for (Column<MessageMetadataEntry> entry : columns) {
if (entry.getTtl() != 0) continue;
// Return the first one we get. Hmmm... maybe we want to do some validation checks here
return peekMessage(entry.getName().getName());
}
return null;
} catch (NotFoundException e) {
return null;
} catch (ConnectionException e) {
throw new MessageQueueException("Error fetching row " + groupRowKey, e);
}
}
@Override
public UniqueValueSet load(
final ApplicationScope appScope,
final ConsistencyLevel consistencyLevel,
final String type,
final Collection<Field> fields)
throws ConnectionException {
Preconditions.checkNotNull(fields, "fields are required");
Preconditions.checkArgument(fields.size() > 0, "More than 1 field must be specified");
final List<ScopedRowKey<FieldKey>> keys = new ArrayList<>(fields.size());
final Id applicationId = appScope.getApplication();
for (Field field : fields) {
final FieldKey key = createUniqueValueKey(applicationId, type, field);
final ScopedRowKey<FieldKey> rowKey = ScopedRowKey.fromKey(applicationId, key);
keys.add(rowKey);
}
final UniqueValueSetImpl uniqueValueSet = new UniqueValueSetImpl(fields.size());
Iterator<Row<ScopedRowKey<FieldKey>, EntityVersion>> results =
keyspace
.prepareQuery(CF_UNIQUE_VALUES)
.setConsistencyLevel(consistencyLevel)
.getKeySlice(keys)
.withColumnRange(new RangeBuilder().setLimit(1).build())
.execute()
.getResult()
.iterator();
while (results.hasNext()) {
final Row<ScopedRowKey<FieldKey>, EntityVersion> unique = results.next();
final Field field = parseRowKey(unique.getKey());
final Iterator<Column<EntityVersion>> columnList = unique.getColumns().iterator();
// sanity check, nothing to do, skip it
if (!columnList.hasNext()) {
continue;
}
final EntityVersion entityVersion = columnList.next().getName();
final UniqueValueImpl uniqueValue =
new UniqueValueImpl(field, entityVersion.getEntityId(), entityVersion.getEntityVersion());
uniqueValueSet.addValue(uniqueValue);
}
return uniqueValueSet;
}
@Test
public void testTtlOverride() throws Exception {
final String id = "testTtlAnnotation";
EntityManager<TtlEntity, String> entityPersister =
new DefaultEntityManager.Builder<TtlEntity, String>()
.withEntityType(TtlEntity.class)
.withKeyspace(keyspace)
.withColumnFamily(CF_SAMPLE_ENTITY)
.withTTL(5)
.build();
TtlEntity origEntity = createTtlEntity(id);
entityPersister.put(origEntity);
// use low-level astyanax API to confirm the write
{
ColumnList<String> cl =
keyspace.prepareQuery(CF_SAMPLE_ENTITY).getKey(id).execute().getResult();
// test column number
Assert.assertEquals(1, cl.size());
// test column value
Assert.assertEquals(origEntity.getColumn(), cl.getColumnByName("column").getStringValue());
// custom ttl
Assert.assertEquals(5, cl.getColumnByName("column").getTtl());
}
TtlEntity getEntity = entityPersister.get(id);
Assert.assertEquals(origEntity, getEntity);
// entity should still be alive after 3s since TTL is overriden to 5s
Thread.sleep(1000 * 3);
getEntity = entityPersister.get(id);
Assert.assertEquals(origEntity, getEntity);
// entity should expire after 3s since 6s have passed with 5s TTL
Thread.sleep(1000 * 3);
// use low-level astyanax API to confirm the TTL expiration
{
ColumnList<String> cl =
keyspace.prepareQuery(CF_SAMPLE_ENTITY).getKey(id).execute().getResult();
Assert.assertEquals(0, cl.size());
}
}
@Override
public boolean deleteMessageByKey(String key) throws MessageQueueException {
MutationBatch mb = keyspace.prepareMutationBatch().setConsistencyLevel(consistencyLevel);
String groupRowKey = getCompositeKey(getName(), key);
try {
ColumnList<MessageMetadataEntry> columns =
keyspace
.prepareQuery(keyIndexColumnFamily)
.setConsistencyLevel(consistencyLevel)
.getRow(groupRowKey)
.withColumnRange(
metadataSerializer
.buildRange()
.greaterThanEquals((byte) MessageMetadataEntryType.MessageId.ordinal())
.lessThanEquals((byte) MessageMetadataEntryType.MessageId.ordinal())
.build())
.execute()
.getResult();
for (Column<MessageMetadataEntry> entry : columns) {
String[] parts = splitCompositeKey(entry.getName().getName());
String shardKey = parts[0];
MessageQueueEntry queueEntry = new MessageQueueEntry(parts[1]);
mb.withRow(queueColumnFamily, shardKey).deleteColumn(queueEntry);
}
mb.withRow(keyIndexColumnFamily, groupRowKey).delete();
} catch (NotFoundException e) {
return false;
} catch (ConnectionException e) {
throw new MessageQueueException("Error fetching row " + groupRowKey, e);
}
try {
mb.execute();
} catch (ConnectionException e) {
throw new MessageQueueException("Error deleting queue item " + groupRowKey, e);
}
return true;
}
@Override
public void setConfigurationProperty(final String key, final String value)
throws StorageException {
try {
ensureColumnFamilyExists(SYSTEM_PROPERTIES_CF, "org.apache.cassandra.db.marshal.UTF8Type");
Keyspace ks = keyspaceContext.getEntity();
OperationResult<Void> result =
ks.prepareColumnMutation(PROPERTIES_CF, SYSTEM_PROPERTIES_KEY, key)
.setConsistencyLevel(ConsistencyLevel.CL_QUORUM)
.putValue(value, null)
.execute();
result.getResult();
} catch (ConnectionException e) {
throw new PermanentStorageException(e);
}
}
@Override
protected RowQuery<String, IndexColumnName> genQuery() {
RowQuery<String, IndexColumnName> query =
_keyspace
.prepareQuery(_field.getIndexCF())
.getKey(_indexKey.toString())
.withColumnRange(_field.buildPrefixRange(_prefix, pageCount));
return query;
}
private void doPut(Keyspace keyspace, CacheKey key, Integer dataOffset, Point d)
throws ConnectionException {
keyspace
.prepareQuery(columnFamily)
.withCql(CQL_STMT)
.asPreparedStatement()
.withByteBufferValue(key, cacheKeySerializer)
.withIntegerValue(dataOffset)
.withDoubleValue(d.getValue())
.execute();
}
@Override
public void mutateMany(Map<ByteBuffer, Mutation> mutations, TransactionHandle txh)
throws StorageException {
// null txh means a Transaction is calling this method
if (null != txh) {
// non-null txh -> make sure locks are valid
AstyanaxTransaction atxh = (AstyanaxTransaction) txh;
if (!atxh.isMutationStarted()) {
// This is the first mutate call in the transaction
atxh.mutationStarted();
// Verify all blind lock claims now
atxh.verifyAllLockClaims(); // throws GSE and unlocks everything on any lock failure
}
}
MutationBatch m =
keyspace
.prepareMutationBatch()
.setConsistencyLevel(writeLevel)
.withRetryPolicy(retryPolicy.duplicate());
final long delTS = TimestampProvider.getApproxNSSinceEpoch(false);
final long addTS = TimestampProvider.getApproxNSSinceEpoch(true);
for (Map.Entry<ByteBuffer, Mutation> ent : mutations.entrySet()) {
// The CLMs for additions and deletions are separated because
// Astyanax's operation timestamp cannot be set on a per-delete
// or per-addition basis.
ColumnListMutation<ByteBuffer> dels = m.withRow(cf, ent.getKey());
dels.setTimestamp(delTS);
ColumnListMutation<ByteBuffer> adds = m.withRow(cf, ent.getKey());
adds.setTimestamp(addTS);
Mutation titanMutation = ent.getValue();
if (titanMutation.hasDeletions()) {
for (ByteBuffer b : titanMutation.getDeletions()) {
dels.deleteColumn(b);
}
}
if (titanMutation.hasAdditions()) {
for (Entry e : titanMutation.getAdditions()) {
adds.putColumn(e.getColumn(), e.getValue(), null);
}
}
}
try {
m.execute();
} catch (ConnectionException e) {
throw new TemporaryStorageException(e);
}
}
/*
* (non-Javadoc)
*
* @see
* backtype.storm.contrib.cassandra.client.CassandraClient#writeTuple(backtype
* .storm.tuple.Tuple,
* backtype.storm.contrib.cassandra.bolt.mapper.TupleMapper)
*/
@Override
public void writeTuple(Tuple input, TupleMapper<T> tupleMapper) throws Exception {
String columnFamilyName = tupleMapper.mapToColumnFamily(input);
String rowKey = (String) tupleMapper.mapToRowKey(input);
MutationBatch mutation = keyspace.prepareMutationBatch();
ColumnFamily<String, T> columnFamily =
new ColumnFamily<String, T>(
columnFamilyName, StringSerializer.get(), this.getColumnNameSerializer(tupleMapper));
this.addTupleToMutation(input, columnFamily, rowKey, mutation, tupleMapper);
mutation.execute();
}
@Override
public void acquireAndApplyMutation(Function<MutationBatch, Boolean> callback)
throws NotUniqueException, Exception {
try {
// Phase 1: Write a unique column
MutationBatch m = keyspace.prepareMutationBatch().setConsistencyLevel(consistencyLevel);
if (data == null) {
m.withRow(columnFamily, key).putEmptyColumn(uniqueColumn, ttl);
} else {
m.withRow(columnFamily, key).putColumn(uniqueColumn, data, ttl);
}
m.execute();
// Phase 2: Read back all columns. There should be only 1
ColumnList<C> result =
keyspace
.prepareQuery(columnFamily)
.setConsistencyLevel(consistencyLevel)
.getKey(key)
.execute()
.getResult();
if (result.size() != 1) {
throw new NotUniqueException(key.toString());
}
// Phase 3: Persist the uniqueness with
m = keyspace.prepareMutationBatch().setConsistencyLevel(consistencyLevel);
if (callback != null) callback.apply(m);
if (data == null) {
m.withRow(columnFamily, key).putEmptyColumn(uniqueColumn, null);
} else {
m.withRow(columnFamily, key).putColumn(uniqueColumn, data, null);
}
m.execute();
} catch (Exception e) {
release();
throw e;
}
}
private static void createKeyspace() throws Exception {
keyspaceContext =
new AstyanaxContext.Builder()
.forCluster(TEST_CLUSTER_NAME)
.forKeyspace(TEST_KEYSPACE_NAME)
.withAstyanaxConfiguration(
new AstyanaxConfigurationImpl()
.setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
.setConnectionPoolType(ConnectionPoolType.TOKEN_AWARE))
.withConnectionPoolConfiguration(
new ConnectionPoolConfigurationImpl(TEST_CLUSTER_NAME + "_" + TEST_KEYSPACE_NAME)
.setSocketTimeout(30000)
.setMaxTimeoutWhenExhausted(2000)
.setMaxConnsPerHost(20)
.setInitConnsPerHost(10)
.setSeeds(SEEDS))
.withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
.buildKeyspace(ThriftFamilyFactory.getInstance());
keyspaceContext.start();
keyspace = keyspaceContext.getEntity();
try {
keyspace.dropKeyspace();
} catch (Exception e) {
e.printStackTrace();
}
keyspace.createKeyspace(
ImmutableMap.<String, Object>builder()
.put(
"strategy_options",
ImmutableMap.<String, Object>builder().put("replication_factor", "1").build())
.put("strategy_class", "SimpleStrategy")
.build());
keyspace.createColumnFamily(CF_SAMPLE_ENTITY, null);
keyspace.createColumnFamily(CF_SIMPLE_ENTITY, null);
}
@Override
public void clearStorage() throws StorageException {
try {
Cluster cluster = clusterContext.getEntity();
Keyspace ks = cluster.getKeyspace(keySpaceName);
// Not a big deal if Keyspace doesn't not exist (dropped manually by user or tests).
// This is called on per test setup basis to make sure that previous test cleaned
// everything up, so first invocation would always fail as Keyspace doesn't yet exist.
if (ks == null) return;
for (ColumnFamilyDefinition cf :
cluster.describeKeyspace(keySpaceName).getColumnFamilyList()) {
ks.truncateColumnFamily(new ColumnFamily<Object, Object>(cf.getName(), null, null));
}
} catch (ConnectionException e) {
throw new PermanentStorageException(e);
} finally {
close();
}
}
@Test
public void testAcquireAndMutate() throws Exception {
final String row = "testAcquireAndMutate";
final String dataColumn = "data";
final String value = "test";
ColumnPrefixUniquenessConstraint<String> unique =
new ColumnPrefixUniquenessConstraint<String>(keyspace, UNIQUE_CF, row)
.withConsistencyLevel(ConsistencyLevel.CL_ONE)
.withUniqueId("def");
try {
unique.acquireAndApplyMutation(
new Function<MutationBatch, Boolean>() {
@Override
public Boolean apply(MutationBatch m) {
m.withRow(UNIQUE_CF, row).putColumn(dataColumn, value, null);
return true;
}
});
String column = unique.readUniqueColumn();
Assert.assertNotNull(column);
} catch (Exception e) {
e.printStackTrace();
LOG.error("", e);
Assert.fail();
} finally {
}
ColumnList<String> columns = keyspace.prepareQuery(UNIQUE_CF).getKey(row).execute().getResult();
Assert.assertEquals(2, columns.size());
Assert.assertEquals(value, columns.getStringValue(dataColumn, null));
unique.release();
columns = keyspace.prepareQuery(UNIQUE_CF).getKey(row).execute().getResult();
Assert.assertEquals(1, columns.size());
Assert.assertEquals(value, columns.getStringValue(dataColumn, null));
}
@Override
public void map(
LongWritable key,
Text value,
OutputCollector<NullWritable, NullWritable> collector,
Reporter reporter)
throws IOException {
if (value.getLength() == 0) return;
byte[] raw = value.getBytes();
Map<String, Object> msg = mapper.readValue(raw, Map.class);
String rowId = createRowId(msg);
// System.out.println("rowId:" + rowId.toString());
if (rowId == null) {
// TODO ... Error Handler
return;
}
if (mb == null) {
mb = ks.prepareMutationBatch();
}
ColumnListMutation<String> c = mb.withRow(cf, rowId);
c.putColumn("raw", value.toString(), null);
if (storeAttirbute) {
for (String k : msg.keySet()) {
if (k.startsWith("__")) continue;
Object v = msg.get(k);
if (v == null) continue;
if (v.equals("")) continue;
c.putColumn(k.toLowerCase(), v.toString(), null);
}
}
try {
if (mb.getRowCount() > 300) {
OperationResult<Void> result = mb.execute();
mb = null;
}
} catch (ConnectionException e) {
e.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
mb = null;
}
}
@Test
public void doubleIdColumnAnnotation() throws Exception {
final String id = "doubleIdColumnAnnotation";
EntityManager<DoubleIdColumnEntity, String> entityPersister =
new DefaultEntityManager.Builder<DoubleIdColumnEntity, String>()
.withEntityType(DoubleIdColumnEntity.class)
.withKeyspace(keyspace)
.withColumnFamily(CF_SAMPLE_ENTITY)
.build();
DoubleIdColumnEntity origEntity = createDoubleIdColumnEntity(id);
entityPersister.put(origEntity);
// use low-level astyanax API to confirm the write
{
ColumnList<String> cl =
keyspace.prepareQuery(CF_SAMPLE_ENTITY).getKey(id).execute().getResult();
// test column number
Assert.assertEquals(3, cl.size());
// test column value
Assert.assertEquals(origEntity.getId(), cl.getColumnByName("id").getStringValue());
Assert.assertEquals(origEntity.getNum(), cl.getColumnByName("num").getIntegerValue());
Assert.assertEquals(origEntity.getStr(), cl.getColumnByName("str").getStringValue());
}
DoubleIdColumnEntity getEntity = entityPersister.get(id);
Assert.assertEquals(origEntity, getEntity);
entityPersister.delete(id);
// use low-level astyanax API to confirm the delete
{
ColumnList<String> cl =
keyspace.prepareQuery(CF_SAMPLE_ENTITY).getKey(id).execute().getResult();
Assert.assertEquals(0, cl.size());
}
}
public void insertToTimeSpanColumnFamily(int userId, int skuId, int priceValue) {
try {
OperationResult<CqlResult<Integer, String>> result =
keyspace
.prepareQuery(TIMESPAN_CF)
.withCql(INSERT_STATEMENT)
.asPreparedStatement()
.withIntegerValue(userId)
.withIntegerValue(skuId)
.withIntegerValue(priceValue)
.execute();
} catch (ConnectionException e) {
throw new RuntimeException("Failed to write to column", e);
}
}
@Override
public void clearMessages() throws MessageQueueException {
LOG.info("Clearing messages from '" + getName() + "'");
MutationBatch mb = keyspace.prepareMutationBatch().setConsistencyLevel(consistencyLevel);
for (MessageQueueShard partition : shardReaderPolicy.listShards()) {
mb.withRow(queueColumnFamily, partition.getName()).delete();
}
try {
mb.execute();
} catch (ConnectionException e) {
throw new MessageQueueException("Failed to clear messages from queue " + getName(), e);
}
}
/*
* (non-Javadoc)
*
* @see
* backtype.storm.contrib.cassandra.client.CassandraClient#writeTuples(java
* .util.List, backtype.storm.contrib.cassandra.bolt.mapper.TupleMapper)
*/
@Override
public void writeTuples(List<Tuple> inputs, TupleMapper<T> tupleMapper) throws Exception {
MutationBatch mutation = keyspace.prepareMutationBatch();
for (Tuple input : inputs) {
String columnFamilyName = tupleMapper.mapToColumnFamily(input);
String rowKey = (String) tupleMapper.mapToRowKey(input);
ColumnFamily<String, T> columnFamily =
new ColumnFamily<String, T>(
columnFamilyName, StringSerializer.get(), this.getColumnNameSerializer(tupleMapper));
this.addTupleToMutation(input, columnFamily, rowKey, mutation, tupleMapper);
}
try {
mutation.execute();
} catch (Exception e) {
LOG.error("Could not execute mutation.", e);
}
}
@Override
public void deleteMessage(String messageId) throws MessageQueueException {
String[] parts = splitCompositeKey(messageId);
String shardKey = parts[0];
MessageQueueEntry entry = new MessageQueueEntry(parts[1]);
try {
keyspace
.prepareColumnMutation(queueColumnFamily, shardKey, entry)
.setConsistencyLevel(consistencyLevel)
.deleteColumn()
.execute();
} catch (ConnectionException e) {
throw new MessageQueueException("Error deleting message " + messageId, e);
}
}
@Override
public void deleteMessages(Collection<String> messageIds) throws MessageQueueException {
MutationBatch mb = keyspace.prepareMutationBatch().setConsistencyLevel(consistencyLevel);
for (String messageId : messageIds) {
String[] parts = splitCompositeKey(messageId);
String shardKey = parts[0];
MessageQueueEntry entry = new MessageQueueEntry(parts[1]);
mb.withRow(queueColumnFamily, shardKey).deleteColumn(entry);
}
try {
mb.execute();
} catch (ConnectionException e) {
throw new MessageQueueException("Error deleting messages " + messageIds, e);
}
}
@Override
public ByteBuffer get(ByteBuffer key, ByteBuffer column, TransactionHandle txh)
throws StorageException {
try {
OperationResult<Column<ByteBuffer>> result =
keyspace
.prepareQuery(cf)
.setConsistencyLevel(readLevel)
.withRetryPolicy(retryPolicy.duplicate())
.getKey(key)
.getColumn(column)
.execute();
return result.getResult().getByteBufferValue();
} catch (NotFoundException e) {
return null;
} catch (ConnectionException e) {
throw new TemporaryStorageException(e);
}
}
