Esempio n. 1
0
 @Override
 public void close() {
   // Shutdown the Astyanax contexts
   openStores.clear();
   keyspaceContext.shutdown();
   clusterContext.shutdown();
 }
  /**
   * @param _clusterName
   * @param _server
   * @param _keySpaceName
   * @return KeySpace - the Cassandra KeySpace we are going to work in, and that we give back to the
   *     caller ( if everything goes well ).
   * @throws java.lang.Exception
   */
  public static final Keyspace doInit(
      final String _clusterName, final String _server, final String _keySpaceName)
      throws Exception {
    AstyanaxContext<Keyspace> context =
        new AstyanaxContext.Builder()
            .forCluster(_clusterName)
            .forKeyspace(_keySpaceName)
            .withAstyanaxConfiguration(
                new AstyanaxConfigurationImpl()
                    .setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
                    .setCqlVersion("3.0.0")
                    .setTargetCassandraVersion("2.0.4"))
            .withConnectionPoolConfiguration(
                new ConnectionPoolConfigurationImpl("ubicity file loading pool")
                    .setPort(9160)
                    .setMaxConnsPerHost(16)
                    .setSeeds(_server + ":9160")
                    .setMaxOperationsPerConnection(10000000)
                    .setConnectTimeout(10000))
            .withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
            .buildKeyspace(ThriftFamilyFactory.getInstance());

    context.start();

    Keyspace keySpace = context.getClient();

    try {
      keySpace.describeKeyspace();
      logger.log(Level.INFO, "keyspace " + keySpace.getKeyspaceName() + " does exist");
    } catch (BadRequestException bre) {
      logger.log(
          Level.INFO, "keyspace " + keySpace.getKeyspaceName() + " does NOT exist, creating it");
      keySpace.createKeyspace(
          ImmutableMap.<String, Object>builder()
              .put(
                  "strategy_options",
                  ImmutableMap.<String, Object>builder().put("replication_factor", "1").build())
              .put("strategy_class", "SimpleStrategy")
              .build());
    }

    log = CassandraCrawlLogSchema.checkOrBuildMonitrixSchema(keySpace);

    if (log == null) {
      logger.log(Level.SEVERE, "could not create Monitrix ColumnFamily ( table ) ");
    }
    return keySpace;
  }
  public Keyspace getKeyspace() {
    AstyanaxContext<Keyspace> context =
        new AstyanaxContext.Builder()
            .forCluster("Test Cluster")
            .forKeyspace("DEMO")
            .withAstyanaxConfiguration(
                new AstyanaxConfigurationImpl().setDiscoveryType(NodeDiscoveryType.NONE))
            .withConnectionPoolConfiguration(
                new ConnectionPoolConfigurationImpl("MyConnectionPool")
                    .setPort(9160)
                    .setMaxConnsPerHost(1)
                    .setSeeds("127.0.0.1:9160"))
            .withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
            .buildKeyspace(ThriftFamilyFactory.getInstance());

    context.start();
    return context.getEntity();
  }
Esempio n. 4
0
 @Override
 public Partitioner getPartitioner() throws StorageException {
   Cluster cl = clusterContext.getEntity();
   try {
     return Partitioner.getPartitioner(cl.describePartitioner());
   } catch (ConnectionException e) {
     throw new TemporaryStorageException(e);
   }
 }
Esempio n. 5
0
  @Override
  public void configure(JobConf job) {
    this.job = job;

    mapper = new ObjectMapper();
    messagePack = new MessagePack();

    timeFmt = DateTimeFormat.forPattern("YYYYMMddHHmmssSSS");
    oidSerial = new AtomicInteger(0);
    serial = new AtomicInteger(0);

    storeAttirbute = job.getBoolean(CIngest.CONF_INGEST_STORE_ATTR, false);

    Iterator<String> tokens =
        Splitter.on("///").split(job.get(CIngest.CONF_CASSANDRA_CONNECT_TOKEN)).iterator();

    String clusterName = tokens.next();
    String seeds = tokens.next();
    String keyspaceName = tokens.next();
    String columeFamilyName = tokens.next();

    context =
        new AstyanaxContext.Builder()
            .forCluster(clusterName)
            .forKeyspace(keyspaceName)
            .withAstyanaxConfiguration(
                new AstyanaxConfigurationImpl().setDiscoveryType(NodeDiscoveryType.TOKEN_AWARE))
            .withConnectionPoolConfiguration(
                new ConnectionPoolConfigurationImpl("cp")
                    .setPort(9160)
                    .setMaxConnsPerHost(2)
                    .setSeeds(seeds))
            .withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
            .buildKeyspace(ThriftFamilyFactory.getInstance());

    context.start();
    ks = context.getEntity();

    cf =
        new ColumnFamily<String, String>(
            columeFamilyName, StringSerializer.get(), StringSerializer.get());
  }
Esempio n. 6
0
 @Override
 public void close() throws IOException {
   if (mb != null && !mb.isEmpty()) {
     try {
       OperationResult<Void> result = mb.execute();
     } catch (ConnectionException e) {
       e.printStackTrace(); // To change body of catch statement use File | Settings | File
       // Templates.
     }
   }
   context.shutdown();
 }
Esempio n. 7
0
  public static Keyspace getKeySpace(String keySpaceName) {
    AstyanaxContext<Keyspace> context =
        new AstyanaxContext.Builder()
            .forCluster(CLUSTER_NAME)
            .forKeyspace(keySpaceName)
            .withAstyanaxConfiguration(
                new AstyanaxConfigurationImpl().setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
                //                        .setCqlVersion("3.0.0")
                //                        .setTargetCassandraVersion("1.2")
                )
            .withConnectionPoolConfiguration(
                new ConnectionPoolConfigurationImpl("MinderConnectionPool")
                    .setPort(9160)
                    .setMaxConnsPerHost(1)
                    .setSeeds(SEEDS))
            .withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
            .buildKeyspace(ThriftFamilyFactory.getInstance());

    context.start();
    return context.getClient();
  }
  private static void createKeyspace() throws Exception {
    keyspaceContext =
        new AstyanaxContext.Builder()
            .forCluster(TEST_CLUSTER_NAME)
            .forKeyspace(TEST_KEYSPACE_NAME)
            .withAstyanaxConfiguration(
                new AstyanaxConfigurationImpl()
                    .setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
                    .setConnectionPoolType(ConnectionPoolType.TOKEN_AWARE))
            .withConnectionPoolConfiguration(
                new ConnectionPoolConfigurationImpl(TEST_CLUSTER_NAME + "_" + TEST_KEYSPACE_NAME)
                    .setSocketTimeout(30000)
                    .setMaxTimeoutWhenExhausted(2000)
                    .setMaxConnsPerHost(20)
                    .setInitConnsPerHost(10)
                    .setSeeds(SEEDS))
            .withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
            .buildKeyspace(ThriftFamilyFactory.getInstance());

    keyspaceContext.start();

    keyspace = keyspaceContext.getEntity();

    try {
      keyspace.dropKeyspace();
    } catch (Exception e) {
      e.printStackTrace();
    }

    keyspace.createKeyspace(
        ImmutableMap.<String, Object>builder()
            .put(
                "strategy_options",
                ImmutableMap.<String, Object>builder().put("replication_factor", "1").build())
            .put("strategy_class", "SimpleStrategy")
            .build());

    keyspace.createColumnFamily(CF_SAMPLE_ENTITY, null);
    keyspace.createColumnFamily(CF_SIMPLE_ENTITY, null);
  }
Esempio n. 9
0
  @Override
  public void mutateMany(Map<String, Map<ByteBuffer, Mutation>> batch, StoreTransaction txh)
      throws StorageException {
    MutationBatch m =
        keyspaceContext
            .getEntity()
            .prepareMutationBatch()
            .setConsistencyLevel(getTx(txh).getWriteConsistencyLevel().getAstyanaxConsistency())
            .withRetryPolicy(retryPolicy.duplicate());

    final long delTS = TimeUtility.getApproxNSSinceEpoch(false);
    final long addTS = TimeUtility.getApproxNSSinceEpoch(true);

    for (Map.Entry<String, Map<ByteBuffer, Mutation>> batchentry : batch.entrySet()) {
      String storeName = batchentry.getKey();
      Preconditions.checkArgument(
          openStores.containsKey(storeName), "Store cannot be found: " + storeName);
      ColumnFamily<ByteBuffer, ByteBuffer> columnFamily =
          openStores.get(storeName).getColumnFamily();

      Map<ByteBuffer, Mutation> mutations = batchentry.getValue();
      for (Map.Entry<ByteBuffer, Mutation> ent : mutations.entrySet()) {
        // The CLMs for additions and deletions are separated because
        // Astyanax's operation timestamp cannot be set on a per-delete
        // or per-addition basis.
        ColumnListMutation<ByteBuffer> dels = m.withRow(columnFamily, ent.getKey());
        dels.setTimestamp(delTS);
        ColumnListMutation<ByteBuffer> adds = m.withRow(columnFamily, ent.getKey());
        adds.setTimestamp(addTS);

        Mutation titanMutation = ent.getValue();

        if (titanMutation.hasDeletions()) {
          for (ByteBuffer b : titanMutation.getDeletions()) {
            dels.deleteColumn(b);
          }
        }

        if (titanMutation.hasAdditions()) {
          for (Entry e : titanMutation.getAdditions()) {
            adds.putColumn(e.getColumn(), e.getValue(), null);
          }
        }
      }
    }

    try {
      m.execute();
    } catch (ConnectionException e) {
      throw new TemporaryStorageException(e);
    }
  }
Esempio n. 10
0
 @Override
 public synchronized AstyanaxOrderedKeyColumnValueStore openDatabase(String name)
     throws StorageException {
   if (openStores.containsKey(name)) return openStores.get(name);
   else {
     ensureColumnFamilyExists(name);
     AstyanaxOrderedKeyColumnValueStore store =
         new AstyanaxOrderedKeyColumnValueStore(
             name, keyspaceContext.getEntity(), this, retryPolicy);
     openStores.put(name, store);
     return store;
   }
 }
Esempio n. 11
0
  @Override
  public void setConfigurationProperty(final String key, final String value)
      throws StorageException {
    try {
      ensureColumnFamilyExists(SYSTEM_PROPERTIES_CF, "org.apache.cassandra.db.marshal.UTF8Type");

      Keyspace ks = keyspaceContext.getEntity();

      OperationResult<Void> result =
          ks.prepareColumnMutation(PROPERTIES_CF, SYSTEM_PROPERTIES_KEY, key)
              .setConsistencyLevel(ConsistencyLevel.CL_QUORUM)
              .putValue(value, null)
              .execute();

      result.getResult();
    } catch (ConnectionException e) {
      throw new PermanentStorageException(e);
    }
  }
Esempio n. 12
0
  @Override
  public String getConfigurationProperty(final String key) throws StorageException {
    try {
      ensureColumnFamilyExists(SYSTEM_PROPERTIES_CF, "org.apache.cassandra.db.marshal.UTF8Type");

      OperationResult<Column<String>> result =
          keyspaceContext
              .getEntity()
              .prepareQuery(PROPERTIES_CF)
              .setConsistencyLevel(ConsistencyLevel.CL_QUORUM)
              .withRetryPolicy(retryPolicy.duplicate())
              .getKey(SYSTEM_PROPERTIES_KEY)
              .getColumn(key)
              .execute();

      return result.getResult().getStringValue();
    } catch (NotFoundException e) {
      return null;
    } catch (ConnectionException e) {
      throw new PermanentStorageException(e);
    }
  }
Esempio n. 13
0
 private void ensureColumnFamilyExists(String name, String comparator) throws StorageException {
   Cluster cl = clusterContext.getEntity();
   try {
     KeyspaceDefinition ksDef = cl.describeKeyspace(keySpaceName);
     boolean found = false;
     if (null != ksDef) {
       for (ColumnFamilyDefinition cfDef : ksDef.getColumnFamilyList()) {
         found |= cfDef.getName().equals(name);
       }
     }
     if (!found) {
       ColumnFamilyDefinition cfDef =
           cl.makeColumnFamilyDefinition()
               .setName(name)
               .setKeyspace(keySpaceName)
               .setComparatorType(comparator);
       cl.addColumnFamily(cfDef);
     }
   } catch (ConnectionException e) {
     throw new TemporaryStorageException(e);
   }
 }
Esempio n. 14
0
  @Override
  public void clearStorage() throws StorageException {
    try {
      Cluster cluster = clusterContext.getEntity();

      Keyspace ks = cluster.getKeyspace(keySpaceName);

      // Not a big deal if Keyspace doesn't not exist (dropped manually by user or tests).
      // This is called on per test setup basis to make sure that previous test cleaned
      // everything up, so first invocation would always fail as Keyspace doesn't yet exist.
      if (ks == null) return;

      for (ColumnFamilyDefinition cf :
          cluster.describeKeyspace(keySpaceName).getColumnFamilyList()) {
        ks.truncateColumnFamily(new ColumnFamily<Object, Object>(cf.getName(), null, null));
      }
    } catch (ConnectionException e) {
      throw new PermanentStorageException(e);
    } finally {
      close();
    }
  }
Esempio n. 15
0
  public AstyanaxStoreManager(Configuration config) throws StorageException {
    super(config);

    // Check if we have non-default thrift frame size or max message size set and warn users
    // because there is nothing we can do in Astyanax to apply those, warning is good enough here
    // otherwise it would make bad user experience if we don't warn at all or crash on this.
    if (this.thriftFrameSize != THRIFT_DEFAULT_FRAME_SIZE)
      log.warn("Couldn't set custom Thrift Frame Size property, use 'cassandrathrift' instead.");

    if (this.thriftMaxMessageSize != THRIFT_DEFAULT_MAX_MESSAGE_SIZE)
      log.warn(
          "Couldn't set custom Thrift Max Message Size property, use 'cassandrathrift' instead.");

    this.clusterName = config.getString(CLUSTER_KEY, CLUSTER_DEFAULT);

    this.retryPolicy = getRetryPolicy(config.getString(RETRY_POLICY_KEY, RETRY_POLICY_DEFAULT));

    final int maxConnsPerHost =
        config.getInt(MAX_CONNECTIONS_PER_HOST_KEY, MAX_CONNECTIONS_PER_HOST_DEFAULT);

    final int maxClusterConnsPerHost =
        config.getInt(
            MAX_CLUSTER_CONNECTIONS_PER_HOST_KEY, MAX_CLUSTER_CONNECTIONS_PER_HOST_DEFAULT);

    this.clusterContext =
        createCluster(getContextBuilder(config, maxClusterConnsPerHost, "Cluster"));

    ensureKeyspaceExists(clusterContext.getEntity());

    this.keyspaceContext =
        getContextBuilder(config, maxConnsPerHost, "Keyspace")
            .buildKeyspace(ThriftFamilyFactory.getInstance());
    this.keyspaceContext.start();

    openStores = new HashMap<String, AstyanaxOrderedKeyColumnValueStore>(8);
  }
Esempio n. 16
0
  private static AstyanaxContext<Cluster> createCluster(AstyanaxContext.Builder cb) {
    AstyanaxContext<Cluster> clusterCtx = cb.buildCluster(ThriftFamilyFactory.getInstance());
    clusterCtx.start();

    return clusterCtx;
  }
  @AfterClass
  public static void teardown() throws Exception {
    if (keyspaceContext != null) keyspaceContext.shutdown();

    Thread.sleep(1000 * 10);
  }
Esempio n. 18
0
  @AfterClass
  public static void teardown() throws Exception {
    if (keyspaceContext != null) keyspaceContext.shutdown();

    Thread.sleep(CASSANDRA_WAIT_TIME);
  }
Esempio n. 19
0
  public static void createKeyspace() throws Exception {
    keyspaceContext =
        new AstyanaxContext.Builder()
            .forCluster(TEST_CLUSTER_NAME)
            .forKeyspace(TEST_KEYSPACE_NAME)
            .withAstyanaxConfiguration(
                new AstyanaxConfigurationImpl()
                    .setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
                    .setConnectionPoolType(ConnectionPoolType.TOKEN_AWARE)
                    .setDiscoveryDelayInSeconds(60000))
            .withConnectionPoolConfiguration(
                new ConnectionPoolConfigurationImpl(TEST_CLUSTER_NAME + "_" + TEST_KEYSPACE_NAME)
                    .setSocketTimeout(30000)
                    .setMaxTimeoutWhenExhausted(2000)
                    .setMaxConnsPerHost(20)
                    .setInitConnsPerHost(10)
                    .setSeeds(SEEDS))
            .withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
            .buildKeyspace(ThriftFamilyFactory.getInstance());

    keyspaceContext.start();

    keyspace = keyspaceContext.getEntity();

    try {
      keyspace.dropKeyspace();
    } catch (Exception e) {
      LOG.info(e.getMessage());
    }

    keyspace.createKeyspace(
        ImmutableMap.<String, Object>builder()
            .put(
                "strategy_options",
                ImmutableMap.<String, Object>builder().put("replication_factor", "1").build())
            .put("strategy_class", "SimpleStrategy")
            .build());

    keyspace.createColumnFamily(CF_USER_UNIQUE_UUID, null);
    keyspace.createColumnFamily(CF_EMAIL_UNIQUE_UUID, null);
    keyspace.createColumnFamily(CF_ALL_ROWS, null);

    keyspace.createColumnFamily(
        LOCK_CF_LONG,
        ImmutableMap.<String, Object>builder()
            .put("default_validation_class", "LongType")
            .put("key_validation_class", "UTF8Type")
            .put("comparator_type", "UTF8Type")
            .build());

    keyspace.createColumnFamily(
        LOCK_CF_STRING,
        ImmutableMap.<String, Object>builder()
            .put("default_validation_class", "UTF8Type")
            .put("key_validation_class", "UTF8Type")
            .put("comparator_type", "UTF8Type")
            .build());

    keyspace.createColumnFamily(
        CF_STANDARD1,
        ImmutableMap.<String, Object>builder()
            .put(
                "column_metadata",
                ImmutableMap.<String, Object>builder()
                    .put(
                        "Index1",
                        ImmutableMap.<String, Object>builder()
                            .put("validation_class", "UTF8Type")
                            .put("index_name", "Index1")
                            .put("index_type", "KEYS")
                            .build())
                    .put(
                        "Index2",
                        ImmutableMap.<String, Object>builder()
                            .put("validation_class", "UTF8Type")
                            .put("index_name", "Index2")
                            .put("index_type", "KEYS")
                            .build())
                    .build())
            .build());

    keyspace.createColumnFamily(UNIQUE_CF, null);

    KeyspaceDefinition ki = keyspaceContext.getEntity().describeKeyspace();
    System.out.println("Describe Keyspace: " + ki.getName());

    try {
      //
      // CF_Super :
      // 'A' :
      // 'a' :
      // 1 : 'Aa1',
      // 2 : 'Aa2',
      // 'b' :
      // ...
      // 'z' :
      // ...
      // 'B' :
      // ...
      //
      // CF_Standard :
      // 'A' :
      // 'a' : 1,
      // 'b' : 2,
      // ...
      // 'z' : 26,
      // 'B' :
      // ...
      //

      MutationBatch m;
      OperationResult<Void> result;
      m = keyspace.prepareMutationBatch();

      for (char keyName = 'A'; keyName <= 'Z'; keyName++) {
        String rowKey = Character.toString(keyName);
        ColumnListMutation<String> cfmStandard = m.withRow(CF_STANDARD1, rowKey);
        for (char cName = 'a'; cName <= 'z'; cName++) {
          cfmStandard.putColumn(Character.toString(cName), (int) (cName - 'a') + 1, null);
        }
        cfmStandard.putColumn("Index1", (int) (keyName - 'A') + 1, null);
        cfmStandard.putColumn("Index2", 42, null);
        m.execute();
      }

      m.withRow(CF_STANDARD1, "Prefixes")
          .putColumn("Prefix1_a", 1, null)
          .putColumn("Prefix1_b", 2, null)
          .putColumn("prefix2_a", 3, null);

      result = m.execute();

      m.execute();

      m = keyspace.prepareMutationBatch();
      for (int i = 0; i < ALL_ROWS_COUNT; i++) {
        m.withRow(CF_ALL_ROWS, i).putColumn(0, true);
        if (m.getRowCount() == 50) {
          m.execute();
        }
      }
      m.execute();

    } catch (Exception e) {
      System.out.println(e.getMessage());
      Assert.fail();
    }
  }
Esempio n. 20
0
  public static void doDataLoad() {

    Properties properties = new Properties();
    try {
      properties.load(new FileInputStream("cassandraClient.properties"));
    } catch (FileNotFoundException e1) {
      e1.printStackTrace();
    } catch (IOException e1) {
      e1.printStackTrace();
    }

    String cassandraHost = properties.getProperty("cassandraHost");
    String clusterName = properties.getProperty("clusterName");
    String keyspaceName = properties.getProperty("keyspace");

    String targetTableName = "Radio_Plays_1";

    AstyanaxContext<Keyspace> context =
        new AstyanaxContext.Builder()
            .forCluster(clusterName)
            .forKeyspace(keyspaceName)
            .withAstyanaxConfiguration(
                new AstyanaxConfigurationImpl().setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE))
            .withConnectionPoolConfiguration(
                new ConnectionPoolConfigurationImpl("MyConnectionPool")
                    .setPort(9160)
                    .setMaxConnsPerHost(1)
                    .setSeeds(cassandraHost + ":9160"))
            .withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
            .buildKeyspace(ThriftFamilyFactory.getInstance());

    context.start();
    Keyspace keyspace = context.getClient();

    System.out.println("keyspace :" + keyspace.getKeyspaceName());

    // targetTableName
    ColumnFamily<String, String> CF_INFO =
        new ColumnFamily<String, String>(
            // "MUSICMETRIC_CONFIG",              // Column Family Name
            targetTableName,
            StringSerializer.get(), // Key Serializer
            StringSerializer.get()); // Column Serializer

    // load each line data into cassandra
    try {
      // Open the file that is the first command line parameter
      FileInputStream fstream = new FileInputStream("musicmetric-facebook-data.tsv.gz");
      // Get the object of DataInputStream
      DataInputStream in = new DataInputStream(fstream);
      BufferedReader br = new BufferedReader(new InputStreamReader(in));
      String strLine;

      // Read File Line By Line
      while ((strLine = br.readLine()) != null) {
        String[] splitArray = strLine.split("\\s+");
        String key = splitArray[0];
        String value = splitArray[1];

        logger.info("key --> " + key);
        logger.info("value --> " + value);

        // keyspace.prepareColumnMutation(CF_INFO, key, "value").putValue(value, null).execute();
      }
      // Close the input stream
      in.close();
    } catch (Exception e) { // Catch exception if any
      System.err.println("Error: " + e.getMessage());
    } finally {
      context.shutdown();
    }
  }