@Override
  public void dropStorage() throws MessageQueueException {
    try {
      keyspace.dropColumnFamily(this.queueColumnFamily);
      try {
        Thread.sleep(SCHEMA_CHANGE_DELAY);
      } catch (InterruptedException e) {
      }
    } catch (ConnectionException e) {
      if (!e.getMessage().contains("already exist"))
        throw new MessageQueueException(
            "Failed to create column family for " + queueColumnFamily.getName(), e);
    }

    try {
      keyspace.dropColumnFamily(this.keyIndexColumnFamily);
      try {
        Thread.sleep(SCHEMA_CHANGE_DELAY);
      } catch (InterruptedException e) {
      }
    } catch (ConnectionException e) {
      if (!e.getMessage().contains("already exist"))
        throw new MessageQueueException(
            "Failed to create column family for " + queueColumnFamily.getName(), e);
    }
  }
  @Test
  public void basicLifecycle() throws Exception {
    final String id = "basicLifecycle";
    EntityManager<SampleEntity, String> entityPersister =
        new DefaultEntityManager.Builder<SampleEntity, String>()
            .withEntityType(SampleEntity.class)
            .withKeyspace(keyspace)
            .withColumnFamily(CF_SAMPLE_ENTITY)
            .build();
    SampleEntity origEntity = createSampleEntity(id);

    entityPersister.put(origEntity);

    // use low-level astyanax API to confirm the write
    {
      ColumnList<String> cl =
          keyspace.prepareQuery(CF_SAMPLE_ENTITY).getKey(id).execute().getResult();

      // 19 simple columns
      // 2 one-level-deep nested columns from Bar
      // 2 two-level-deep nested columns from BarBar
      Assert.assertEquals(23, cl.size());

      // simple columns
      Assert.assertEquals(origEntity.getString(), cl.getColumnByName("STRING").getStringValue());
      Assert.assertArrayEquals(
          origEntity.getByteArray(), cl.getColumnByName("BYTE_ARRAY").getByteArrayValue());

      //  nested fields
      Assert.assertEquals(origEntity.getBar().i, cl.getColumnByName("BAR.i").getIntegerValue());
      Assert.assertEquals(origEntity.getBar().s, cl.getColumnByName("BAR.s").getStringValue());
      Assert.assertEquals(
          origEntity.getBar().barbar.i, cl.getColumnByName("BAR.barbar.i").getIntegerValue());
      Assert.assertEquals(
          origEntity.getBar().barbar.s, cl.getColumnByName("BAR.barbar.s").getStringValue());
    }

    SampleEntity getEntity = entityPersister.get(id);
    Assert.assertEquals(origEntity, getEntity);

    entityPersister.delete(id);

    // use low-level astyanax API to confirm the delete
    {
      ColumnList<String> cl =
          keyspace.prepareQuery(CF_SAMPLE_ENTITY).getKey(id).execute().getResult();
      Assert.assertEquals(0, cl.size());
    }
  }
  /**
   * @param _clusterName
   * @param _server
   * @param _keySpaceName
   * @return KeySpace - the Cassandra KeySpace we are going to work in, and that we give back to the
   *     caller ( if everything goes well ).
   * @throws java.lang.Exception
   */
  public static final Keyspace doInit(
      final String _clusterName, final String _server, final String _keySpaceName)
      throws Exception {
    AstyanaxContext<Keyspace> context =
        new AstyanaxContext.Builder()
            .forCluster(_clusterName)
            .forKeyspace(_keySpaceName)
            .withAstyanaxConfiguration(
                new AstyanaxConfigurationImpl()
                    .setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
                    .setCqlVersion("3.0.0")
                    .setTargetCassandraVersion("2.0.4"))
            .withConnectionPoolConfiguration(
                new ConnectionPoolConfigurationImpl("ubicity file loading pool")
                    .setPort(9160)
                    .setMaxConnsPerHost(16)
                    .setSeeds(_server + ":9160")
                    .setMaxOperationsPerConnection(10000000)
                    .setConnectTimeout(10000))
            .withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
            .buildKeyspace(ThriftFamilyFactory.getInstance());

    context.start();

    Keyspace keySpace = context.getClient();

    try {
      keySpace.describeKeyspace();
      logger.log(Level.INFO, "keyspace " + keySpace.getKeyspaceName() + " does exist");
    } catch (BadRequestException bre) {
      logger.log(
          Level.INFO, "keyspace " + keySpace.getKeyspaceName() + " does NOT exist, creating it");
      keySpace.createKeyspace(
          ImmutableMap.<String, Object>builder()
              .put(
                  "strategy_options",
                  ImmutableMap.<String, Object>builder().put("replication_factor", "1").build())
              .put("strategy_class", "SimpleStrategy")
              .build());
    }

    log = CassandraCrawlLogSchema.checkOrBuildMonitrixSchema(keySpace);

    if (log == null) {
      logger.log(Level.SEVERE, "could not create Monitrix ColumnFamily ( table ) ");
    }
    return keySpace;
  }
  @Override
  public List<Message> peekMessagesByKey(String key) throws MessageQueueException {
    String groupRowKey = getCompositeKey(getName(), key);
    List<Message> messages = Lists.newArrayList();
    try {
      ColumnList<MessageMetadataEntry> columns =
          keyspace
              .prepareQuery(keyIndexColumnFamily)
              .getRow(groupRowKey)
              .withColumnRange(
                  metadataSerializer
                      .buildRange()
                      .greaterThanEquals((byte) MessageMetadataEntryType.MessageId.ordinal())
                      .lessThanEquals((byte) MessageMetadataEntryType.MessageId.ordinal())
                      .build())
              .execute()
              .getResult();

      for (Column<MessageMetadataEntry> entry : columns) {
        if (entry.getTtl() != 0) continue;

        Message message = peekMessage(entry.getName().getName());
        if (message != null) {
          messages.add(peekMessage(entry.getName().getName()));
        } else {
          LOG.warn("No queue item for " + entry.getName());
        }
      }
    } catch (NotFoundException e) {
    } catch (ConnectionException e) {
      throw new MessageQueueException("Error fetching row " + groupRowKey, e);
    }
    return messages;
  }
  /**
   * Fast check to see if a shard has messages to process
   *
   * @param shardName
   * @throws MessageQueueException
   */
  private boolean hasMessages(String shardName) throws MessageQueueException {
    UUID currentTime = TimeUUIDUtils.getUniqueTimeUUIDinMicros();

    try {
      ColumnList<MessageQueueEntry> result =
          keyspace
              .prepareQuery(queueColumnFamily)
              .setConsistencyLevel(consistencyLevel)
              .getKey(shardName)
              .withColumnRange(
                  new RangeBuilder()
                      .setLimit(1) // Read extra messages because of the lock column
                      .setStart(
                          entrySerializer
                              .makeEndpoint(
                                  (byte) MessageQueueEntryType.Message.ordinal(), Equality.EQUAL)
                              .toBytes())
                      .setEnd(
                          entrySerializer
                              .makeEndpoint(
                                  (byte) MessageQueueEntryType.Message.ordinal(), Equality.EQUAL)
                              .append((byte) 0, Equality.EQUAL)
                              .append(currentTime, Equality.LESS_THAN_EQUALS)
                              .toBytes())
                      .build())
              .execute()
              .getResult();
      return !result.isEmpty();
    } catch (ConnectionException e) {
      throw new MessageQueueException("Error checking shard for messages. " + shardName, e);
    }
  }
  /**
   * Return history for a single key for the specified time range
   *
   * <p>TODO: honor the time range :)
   */
  @Override
  public List<MessageHistory> getKeyHistory(String key, Long startTime, Long endTime, int count)
      throws MessageQueueException {
    List<MessageHistory> list = Lists.newArrayList();
    ColumnList<UUID> columns;
    try {
      columns =
          keyspace
              .prepareQuery(historyColumnFamily)
              .setConsistencyLevel(consistencyLevel)
              .getRow(key)
              .execute()
              .getResult();
    } catch (ConnectionException e) {
      throw new MessageQueueException("Failed to load history for " + key, e);
    }

    for (Column<UUID> column : columns) {
      try {
        list.add(deserializeString(column.getStringValue(), MessageHistory.class));
      } catch (Exception e) {
        LOG.info("Error deserializing history entry", e);
      }
    }
    return list;
  }
  /**
   * Do the column update or delete for the given column and row key
   *
   * @param applicationScope We need to use this when getting the keyspace
   * @param uniqueValue The unique value to write
   * @param op The operation to write
   */
  private MutationBatch doWrite(
      ApplicationScope applicationScope, UniqueValue uniqueValue, RowOp op) {
    final MutationBatch batch = keyspace.prepareMutationBatch();

    final Id applicationId = applicationScope.getApplication();

    final FieldKey fieldKey =
        createUniqueValueKey(
            applicationId, uniqueValue.getEntityId().getType(), uniqueValue.getField());

    op.doLookup(batch.withRow(CF_UNIQUE_VALUES, ScopedRowKey.fromKey(applicationId, fieldKey)));

    final EntityKey entityKey = createEntityUniqueLogKey(applicationId, uniqueValue.getEntityId());

    op.doLog(
        batch.withRow(CF_ENTITY_UNIQUE_VALUE_LOG, ScopedRowKey.fromKey(applicationId, entityKey)));

    if (log.isDebugEnabled()) {
      log.debug(
          "Writing unique value version={} name={} value={} ",
          new Object[] {
            uniqueValue.getEntityVersion(),
            uniqueValue.getField().getName(),
            uniqueValue.getField().getValue()
          });
    }

    return batch;
  }
Ejemplo n.º 8
0
  /**
   * Read the data stored with the unique row. This data is normally a 'foreign' key to another
   * column family.
   *
   * @return
   * @throws Exception
   */
  public ByteBuffer readData() throws Exception {
    ColumnList<C> result =
        keyspace
            .prepareQuery(columnFamily)
            .setConsistencyLevel(consistencyLevel)
            .getKey(key)
            .execute()
            .getResult();

    boolean hasColumn = false;
    ByteBuffer data = null;
    for (Column<C> column : result) {
      if (column.getTtl() == 0) {
        if (hasColumn) {
          throw new IllegalStateException("Row has multiple uniquneness locks");
        }
        hasColumn = true;
        data = column.getByteBufferValue();
      }
    }

    if (!hasColumn) {
      throw new NotFoundException(this.key.toString() + " has no uniquness lock");
    }
    return data;
  }
  @Override
  public Message peekMessage(String messageId) throws MessageQueueException {
    String[] parts = splitCompositeKey(messageId);

    String shardKey = parts[0];
    MessageQueueEntry entry = new MessageQueueEntry(parts[1]);

    try {
      Column<MessageQueueEntry> column =
          keyspace
              .prepareQuery(queueColumnFamily)
              .setConsistencyLevel(consistencyLevel)
              .getKey(shardKey)
              .getColumn(entry)
              .execute()
              .getResult();
      try {
        ByteArrayInputStream bais = new ByteArrayInputStream(column.getByteArrayValue());
        return mapper.readValue(bais, Message.class);
      } catch (Exception e) {
        LOG.warn("Error parsing message", e);
        // Error parsing the message so we pass it on to the invalid message handler.
        try {
          return invalidMessageHandler.apply(column.getStringValue());
        } catch (Exception e2) {
          LOG.warn("Error handling invalid message message", e2);
          throw new MessageQueueException("Error parsing message " + messageId);
        }
      }
    } catch (NotFoundException e) {
      return null;
    } catch (ConnectionException e) {
      throw new MessageQueueException("Error getting message " + messageId, e);
    }
  }
  @Override
  public Message peekMessageByKey(String key) throws MessageQueueException {
    String groupRowKey = getCompositeKey(getName(), key);
    try {
      ColumnList<MessageMetadataEntry> columns =
          keyspace
              .prepareQuery(keyIndexColumnFamily)
              .setConsistencyLevel(consistencyLevel)
              .getRow(groupRowKey)
              .withColumnRange(
                  metadataSerializer
                      .buildRange()
                      .greaterThanEquals((byte) MessageMetadataEntryType.MessageId.ordinal())
                      .lessThanEquals((byte) MessageMetadataEntryType.MessageId.ordinal())
                      .build())
              .execute()
              .getResult();

      for (Column<MessageMetadataEntry> entry : columns) {
        if (entry.getTtl() != 0) continue;
        // Return the first one we get.  Hmmm... maybe we want to do some validation checks here
        return peekMessage(entry.getName().getName());
      }
      return null;
    } catch (NotFoundException e) {
      return null;
    } catch (ConnectionException e) {
      throw new MessageQueueException("Error fetching row " + groupRowKey, e);
    }
  }
  @Override
  public UniqueValueSet load(
      final ApplicationScope appScope,
      final ConsistencyLevel consistencyLevel,
      final String type,
      final Collection<Field> fields)
      throws ConnectionException {

    Preconditions.checkNotNull(fields, "fields are required");
    Preconditions.checkArgument(fields.size() > 0, "More than 1 field must be specified");

    final List<ScopedRowKey<FieldKey>> keys = new ArrayList<>(fields.size());

    final Id applicationId = appScope.getApplication();

    for (Field field : fields) {

      final FieldKey key = createUniqueValueKey(applicationId, type, field);

      final ScopedRowKey<FieldKey> rowKey = ScopedRowKey.fromKey(applicationId, key);

      keys.add(rowKey);
    }

    final UniqueValueSetImpl uniqueValueSet = new UniqueValueSetImpl(fields.size());

    Iterator<Row<ScopedRowKey<FieldKey>, EntityVersion>> results =
        keyspace
            .prepareQuery(CF_UNIQUE_VALUES)
            .setConsistencyLevel(consistencyLevel)
            .getKeySlice(keys)
            .withColumnRange(new RangeBuilder().setLimit(1).build())
            .execute()
            .getResult()
            .iterator();

    while (results.hasNext()) {

      final Row<ScopedRowKey<FieldKey>, EntityVersion> unique = results.next();

      final Field field = parseRowKey(unique.getKey());

      final Iterator<Column<EntityVersion>> columnList = unique.getColumns().iterator();

      // sanity check, nothing to do, skip it
      if (!columnList.hasNext()) {
        continue;
      }

      final EntityVersion entityVersion = columnList.next().getName();

      final UniqueValueImpl uniqueValue =
          new UniqueValueImpl(field, entityVersion.getEntityId(), entityVersion.getEntityVersion());

      uniqueValueSet.addValue(uniqueValue);
    }

    return uniqueValueSet;
  }
  @Test
  public void testTtlOverride() throws Exception {
    final String id = "testTtlAnnotation";
    EntityManager<TtlEntity, String> entityPersister =
        new DefaultEntityManager.Builder<TtlEntity, String>()
            .withEntityType(TtlEntity.class)
            .withKeyspace(keyspace)
            .withColumnFamily(CF_SAMPLE_ENTITY)
            .withTTL(5)
            .build();
    TtlEntity origEntity = createTtlEntity(id);

    entityPersister.put(origEntity);

    // use low-level astyanax API to confirm the write
    {
      ColumnList<String> cl =
          keyspace.prepareQuery(CF_SAMPLE_ENTITY).getKey(id).execute().getResult();
      // test column number
      Assert.assertEquals(1, cl.size());
      // test column value
      Assert.assertEquals(origEntity.getColumn(), cl.getColumnByName("column").getStringValue());
      // custom ttl
      Assert.assertEquals(5, cl.getColumnByName("column").getTtl());
    }

    TtlEntity getEntity = entityPersister.get(id);
    Assert.assertEquals(origEntity, getEntity);

    // entity should still be alive after 3s since TTL is overriden to 5s
    Thread.sleep(1000 * 3);

    getEntity = entityPersister.get(id);
    Assert.assertEquals(origEntity, getEntity);

    // entity should expire after 3s since 6s have passed with 5s TTL
    Thread.sleep(1000 * 3);

    // use low-level astyanax API to confirm the TTL expiration
    {
      ColumnList<String> cl =
          keyspace.prepareQuery(CF_SAMPLE_ENTITY).getKey(id).execute().getResult();
      Assert.assertEquals(0, cl.size());
    }
  }
  @Override
  public boolean deleteMessageByKey(String key) throws MessageQueueException {
    MutationBatch mb = keyspace.prepareMutationBatch().setConsistencyLevel(consistencyLevel);

    String groupRowKey = getCompositeKey(getName(), key);
    try {
      ColumnList<MessageMetadataEntry> columns =
          keyspace
              .prepareQuery(keyIndexColumnFamily)
              .setConsistencyLevel(consistencyLevel)
              .getRow(groupRowKey)
              .withColumnRange(
                  metadataSerializer
                      .buildRange()
                      .greaterThanEquals((byte) MessageMetadataEntryType.MessageId.ordinal())
                      .lessThanEquals((byte) MessageMetadataEntryType.MessageId.ordinal())
                      .build())
              .execute()
              .getResult();

      for (Column<MessageMetadataEntry> entry : columns) {
        String[] parts = splitCompositeKey(entry.getName().getName());

        String shardKey = parts[0];
        MessageQueueEntry queueEntry = new MessageQueueEntry(parts[1]);

        mb.withRow(queueColumnFamily, shardKey).deleteColumn(queueEntry);
      }

      mb.withRow(keyIndexColumnFamily, groupRowKey).delete();
    } catch (NotFoundException e) {
      return false;
    } catch (ConnectionException e) {
      throw new MessageQueueException("Error fetching row " + groupRowKey, e);
    }

    try {
      mb.execute();
    } catch (ConnectionException e) {
      throw new MessageQueueException("Error deleting queue item " + groupRowKey, e);
    }

    return true;
  }
Ejemplo n.º 14
0
  @Override
  public void setConfigurationProperty(final String key, final String value)
      throws StorageException {
    try {
      ensureColumnFamilyExists(SYSTEM_PROPERTIES_CF, "org.apache.cassandra.db.marshal.UTF8Type");

      Keyspace ks = keyspaceContext.getEntity();

      OperationResult<Void> result =
          ks.prepareColumnMutation(PROPERTIES_CF, SYSTEM_PROPERTIES_KEY, key)
              .setConsistencyLevel(ConsistencyLevel.CL_QUORUM)
              .putValue(value, null)
              .execute();

      result.getResult();
    } catch (ConnectionException e) {
      throw new PermanentStorageException(e);
    }
  }
  @Override
  protected RowQuery<String, IndexColumnName> genQuery() {
    RowQuery<String, IndexColumnName> query =
        _keyspace
            .prepareQuery(_field.getIndexCF())
            .getKey(_indexKey.toString())
            .withColumnRange(_field.buildPrefixRange(_prefix, pageCount));

    return query;
  }
Ejemplo n.º 16
0
 private void doPut(Keyspace keyspace, CacheKey key, Integer dataOffset, Point d)
     throws ConnectionException {
   keyspace
       .prepareQuery(columnFamily)
       .withCql(CQL_STMT)
       .asPreparedStatement()
       .withByteBufferValue(key, cacheKeySerializer)
       .withIntegerValue(dataOffset)
       .withDoubleValue(d.getValue())
       .execute();
 }
  @Override
  public void mutateMany(Map<ByteBuffer, Mutation> mutations, TransactionHandle txh)
      throws StorageException {
    // null txh means a Transaction is calling this method
    if (null != txh) {
      // non-null txh -> make sure locks are valid
      AstyanaxTransaction atxh = (AstyanaxTransaction) txh;
      if (!atxh.isMutationStarted()) {
        // This is the first mutate call in the transaction
        atxh.mutationStarted();
        // Verify all blind lock claims now
        atxh.verifyAllLockClaims(); // throws GSE and unlocks everything on any lock failure
      }
    }

    MutationBatch m =
        keyspace
            .prepareMutationBatch()
            .setConsistencyLevel(writeLevel)
            .withRetryPolicy(retryPolicy.duplicate());

    final long delTS = TimestampProvider.getApproxNSSinceEpoch(false);
    final long addTS = TimestampProvider.getApproxNSSinceEpoch(true);

    for (Map.Entry<ByteBuffer, Mutation> ent : mutations.entrySet()) {
      // The CLMs for additions and deletions are separated because
      // Astyanax's operation timestamp cannot be set on a per-delete
      // or per-addition basis.
      ColumnListMutation<ByteBuffer> dels = m.withRow(cf, ent.getKey());
      dels.setTimestamp(delTS);
      ColumnListMutation<ByteBuffer> adds = m.withRow(cf, ent.getKey());
      adds.setTimestamp(addTS);

      Mutation titanMutation = ent.getValue();

      if (titanMutation.hasDeletions()) {
        for (ByteBuffer b : titanMutation.getDeletions()) {
          dels.deleteColumn(b);
        }
      }

      if (titanMutation.hasAdditions()) {
        for (Entry e : titanMutation.getAdditions()) {
          adds.putColumn(e.getColumn(), e.getValue(), null);
        }
      }
    }

    try {
      m.execute();
    } catch (ConnectionException e) {
      throw new TemporaryStorageException(e);
    }
  }
Ejemplo n.º 18
0
 /*
  * (non-Javadoc)
  *
  * @see
  * backtype.storm.contrib.cassandra.client.CassandraClient#writeTuple(backtype
  * .storm.tuple.Tuple,
  * backtype.storm.contrib.cassandra.bolt.mapper.TupleMapper)
  */
 @Override
 public void writeTuple(Tuple input, TupleMapper<T> tupleMapper) throws Exception {
   String columnFamilyName = tupleMapper.mapToColumnFamily(input);
   String rowKey = (String) tupleMapper.mapToRowKey(input);
   MutationBatch mutation = keyspace.prepareMutationBatch();
   ColumnFamily<String, T> columnFamily =
       new ColumnFamily<String, T>(
           columnFamilyName, StringSerializer.get(), this.getColumnNameSerializer(tupleMapper));
   this.addTupleToMutation(input, columnFamily, rowKey, mutation, tupleMapper);
   mutation.execute();
 }
Ejemplo n.º 19
0
  @Override
  public void acquireAndApplyMutation(Function<MutationBatch, Boolean> callback)
      throws NotUniqueException, Exception {
    try {
      // Phase 1: Write a unique column
      MutationBatch m = keyspace.prepareMutationBatch().setConsistencyLevel(consistencyLevel);
      if (data == null) {
        m.withRow(columnFamily, key).putEmptyColumn(uniqueColumn, ttl);
      } else {
        m.withRow(columnFamily, key).putColumn(uniqueColumn, data, ttl);
      }
      m.execute();

      // Phase 2: Read back all columns. There should be only 1
      ColumnList<C> result =
          keyspace
              .prepareQuery(columnFamily)
              .setConsistencyLevel(consistencyLevel)
              .getKey(key)
              .execute()
              .getResult();

      if (result.size() != 1) {
        throw new NotUniqueException(key.toString());
      }

      // Phase 3: Persist the uniqueness with
      m = keyspace.prepareMutationBatch().setConsistencyLevel(consistencyLevel);
      if (callback != null) callback.apply(m);

      if (data == null) {
        m.withRow(columnFamily, key).putEmptyColumn(uniqueColumn, null);
      } else {
        m.withRow(columnFamily, key).putColumn(uniqueColumn, data, null);
      }
      m.execute();
    } catch (Exception e) {
      release();
      throw e;
    }
  }
  private static void createKeyspace() throws Exception {
    keyspaceContext =
        new AstyanaxContext.Builder()
            .forCluster(TEST_CLUSTER_NAME)
            .forKeyspace(TEST_KEYSPACE_NAME)
            .withAstyanaxConfiguration(
                new AstyanaxConfigurationImpl()
                    .setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
                    .setConnectionPoolType(ConnectionPoolType.TOKEN_AWARE))
            .withConnectionPoolConfiguration(
                new ConnectionPoolConfigurationImpl(TEST_CLUSTER_NAME + "_" + TEST_KEYSPACE_NAME)
                    .setSocketTimeout(30000)
                    .setMaxTimeoutWhenExhausted(2000)
                    .setMaxConnsPerHost(20)
                    .setInitConnsPerHost(10)
                    .setSeeds(SEEDS))
            .withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
            .buildKeyspace(ThriftFamilyFactory.getInstance());

    keyspaceContext.start();

    keyspace = keyspaceContext.getEntity();

    try {
      keyspace.dropKeyspace();
    } catch (Exception e) {
      e.printStackTrace();
    }

    keyspace.createKeyspace(
        ImmutableMap.<String, Object>builder()
            .put(
                "strategy_options",
                ImmutableMap.<String, Object>builder().put("replication_factor", "1").build())
            .put("strategy_class", "SimpleStrategy")
            .build());

    keyspace.createColumnFamily(CF_SAMPLE_ENTITY, null);
    keyspace.createColumnFamily(CF_SIMPLE_ENTITY, null);
  }
Ejemplo n.º 21
0
  @Override
  public void clearStorage() throws StorageException {
    try {
      Cluster cluster = clusterContext.getEntity();

      Keyspace ks = cluster.getKeyspace(keySpaceName);

      // Not a big deal if Keyspace doesn't not exist (dropped manually by user or tests).
      // This is called on per test setup basis to make sure that previous test cleaned
      // everything up, so first invocation would always fail as Keyspace doesn't yet exist.
      if (ks == null) return;

      for (ColumnFamilyDefinition cf :
          cluster.describeKeyspace(keySpaceName).getColumnFamilyList()) {
        ks.truncateColumnFamily(new ColumnFamily<Object, Object>(cf.getName(), null, null));
      }
    } catch (ConnectionException e) {
      throw new PermanentStorageException(e);
    } finally {
      close();
    }
  }
Ejemplo n.º 22
0
  @Test
  public void testAcquireAndMutate() throws Exception {
    final String row = "testAcquireAndMutate";
    final String dataColumn = "data";
    final String value = "test";

    ColumnPrefixUniquenessConstraint<String> unique =
        new ColumnPrefixUniquenessConstraint<String>(keyspace, UNIQUE_CF, row)
            .withConsistencyLevel(ConsistencyLevel.CL_ONE)
            .withUniqueId("def");

    try {
      unique.acquireAndApplyMutation(
          new Function<MutationBatch, Boolean>() {
            @Override
            public Boolean apply(MutationBatch m) {
              m.withRow(UNIQUE_CF, row).putColumn(dataColumn, value, null);
              return true;
            }
          });
      String column = unique.readUniqueColumn();
      Assert.assertNotNull(column);
    } catch (Exception e) {
      e.printStackTrace();
      LOG.error("", e);
      Assert.fail();
    } finally {
    }

    ColumnList<String> columns = keyspace.prepareQuery(UNIQUE_CF).getKey(row).execute().getResult();
    Assert.assertEquals(2, columns.size());
    Assert.assertEquals(value, columns.getStringValue(dataColumn, null));

    unique.release();

    columns = keyspace.prepareQuery(UNIQUE_CF).getKey(row).execute().getResult();
    Assert.assertEquals(1, columns.size());
    Assert.assertEquals(value, columns.getStringValue(dataColumn, null));
  }
Ejemplo n.º 23
0
  @Override
  public void map(
      LongWritable key,
      Text value,
      OutputCollector<NullWritable, NullWritable> collector,
      Reporter reporter)
      throws IOException {
    if (value.getLength() == 0) return;

    byte[] raw = value.getBytes();

    Map<String, Object> msg = mapper.readValue(raw, Map.class);
    String rowId = createRowId(msg);

    // System.out.println("rowId:" + rowId.toString());
    if (rowId == null) {
      // TODO ... Error Handler
      return;
    }

    if (mb == null) {
      mb = ks.prepareMutationBatch();
    }
    ColumnListMutation<String> c = mb.withRow(cf, rowId);
    c.putColumn("raw", value.toString(), null);

    if (storeAttirbute) {
      for (String k : msg.keySet()) {
        if (k.startsWith("__")) continue;

        Object v = msg.get(k);

        if (v == null) continue;

        if (v.equals("")) continue;

        c.putColumn(k.toLowerCase(), v.toString(), null);
      }
    }

    try {
      if (mb.getRowCount() > 300) {
        OperationResult<Void> result = mb.execute();
        mb = null;
      }
    } catch (ConnectionException e) {
      e.printStackTrace(); // To change body of catch statement use File | Settings | File
      // Templates.
      mb = null;
    }
  }
  @Test
  public void doubleIdColumnAnnotation() throws Exception {
    final String id = "doubleIdColumnAnnotation";
    EntityManager<DoubleIdColumnEntity, String> entityPersister =
        new DefaultEntityManager.Builder<DoubleIdColumnEntity, String>()
            .withEntityType(DoubleIdColumnEntity.class)
            .withKeyspace(keyspace)
            .withColumnFamily(CF_SAMPLE_ENTITY)
            .build();
    DoubleIdColumnEntity origEntity = createDoubleIdColumnEntity(id);

    entityPersister.put(origEntity);

    // use low-level astyanax API to confirm the write
    {
      ColumnList<String> cl =
          keyspace.prepareQuery(CF_SAMPLE_ENTITY).getKey(id).execute().getResult();
      // test column number
      Assert.assertEquals(3, cl.size());
      // test column value
      Assert.assertEquals(origEntity.getId(), cl.getColumnByName("id").getStringValue());
      Assert.assertEquals(origEntity.getNum(), cl.getColumnByName("num").getIntegerValue());
      Assert.assertEquals(origEntity.getStr(), cl.getColumnByName("str").getStringValue());
    }

    DoubleIdColumnEntity getEntity = entityPersister.get(id);
    Assert.assertEquals(origEntity, getEntity);

    entityPersister.delete(id);

    // use low-level astyanax API to confirm the delete
    {
      ColumnList<String> cl =
          keyspace.prepareQuery(CF_SAMPLE_ENTITY).getKey(id).execute().getResult();
      Assert.assertEquals(0, cl.size());
    }
  }
Ejemplo n.º 25
0
 public void insertToTimeSpanColumnFamily(int userId, int skuId, int priceValue) {
   try {
     OperationResult<CqlResult<Integer, String>> result =
         keyspace
             .prepareQuery(TIMESPAN_CF)
             .withCql(INSERT_STATEMENT)
             .asPreparedStatement()
             .withIntegerValue(userId)
             .withIntegerValue(skuId)
             .withIntegerValue(priceValue)
             .execute();
   } catch (ConnectionException e) {
     throw new RuntimeException("Failed to write to column", e);
   }
 }
  @Override
  public void clearMessages() throws MessageQueueException {
    LOG.info("Clearing messages from '" + getName() + "'");
    MutationBatch mb = keyspace.prepareMutationBatch().setConsistencyLevel(consistencyLevel);

    for (MessageQueueShard partition : shardReaderPolicy.listShards()) {
      mb.withRow(queueColumnFamily, partition.getName()).delete();
    }

    try {
      mb.execute();
    } catch (ConnectionException e) {
      throw new MessageQueueException("Failed to clear messages from queue " + getName(), e);
    }
  }
Ejemplo n.º 27
0
 /*
  * (non-Javadoc)
  *
  * @see
  * backtype.storm.contrib.cassandra.client.CassandraClient#writeTuples(java
  * .util.List, backtype.storm.contrib.cassandra.bolt.mapper.TupleMapper)
  */
 @Override
 public void writeTuples(List<Tuple> inputs, TupleMapper<T> tupleMapper) throws Exception {
   MutationBatch mutation = keyspace.prepareMutationBatch();
   for (Tuple input : inputs) {
     String columnFamilyName = tupleMapper.mapToColumnFamily(input);
     String rowKey = (String) tupleMapper.mapToRowKey(input);
     ColumnFamily<String, T> columnFamily =
         new ColumnFamily<String, T>(
             columnFamilyName, StringSerializer.get(), this.getColumnNameSerializer(tupleMapper));
     this.addTupleToMutation(input, columnFamily, rowKey, mutation, tupleMapper);
   }
   try {
     mutation.execute();
   } catch (Exception e) {
     LOG.error("Could not execute mutation.", e);
   }
 }
  @Override
  public void deleteMessage(String messageId) throws MessageQueueException {
    String[] parts = splitCompositeKey(messageId);

    String shardKey = parts[0];
    MessageQueueEntry entry = new MessageQueueEntry(parts[1]);

    try {
      keyspace
          .prepareColumnMutation(queueColumnFamily, shardKey, entry)
          .setConsistencyLevel(consistencyLevel)
          .deleteColumn()
          .execute();
    } catch (ConnectionException e) {
      throw new MessageQueueException("Error deleting message " + messageId, e);
    }
  }
  @Override
  public void deleteMessages(Collection<String> messageIds) throws MessageQueueException {
    MutationBatch mb = keyspace.prepareMutationBatch().setConsistencyLevel(consistencyLevel);

    for (String messageId : messageIds) {
      String[] parts = splitCompositeKey(messageId);
      String shardKey = parts[0];
      MessageQueueEntry entry = new MessageQueueEntry(parts[1]);

      mb.withRow(queueColumnFamily, shardKey).deleteColumn(entry);
    }

    try {
      mb.execute();
    } catch (ConnectionException e) {
      throw new MessageQueueException("Error deleting messages " + messageIds, e);
    }
  }
 @Override
 public ByteBuffer get(ByteBuffer key, ByteBuffer column, TransactionHandle txh)
     throws StorageException {
   try {
     OperationResult<Column<ByteBuffer>> result =
         keyspace
             .prepareQuery(cf)
             .setConsistencyLevel(readLevel)
             .withRetryPolicy(retryPolicy.duplicate())
             .getKey(key)
             .getColumn(column)
             .execute();
     return result.getResult().getByteBufferValue();
   } catch (NotFoundException e) {
     return null;
   } catch (ConnectionException e) {
     throw new TemporaryStorageException(e);
   }
 }