Exemplo n.º 1
0
  @Override
  public void map(
      LongWritable key,
      Text value,
      OutputCollector<NullWritable, NullWritable> collector,
      Reporter reporter)
      throws IOException {
    if (value.getLength() == 0) return;

    byte[] raw = value.getBytes();

    Map<String, Object> msg = mapper.readValue(raw, Map.class);
    String rowId = createRowId(msg);

    // System.out.println("rowId:" + rowId.toString());
    if (rowId == null) {
      // TODO ... Error Handler
      return;
    }

    if (mb == null) {
      mb = ks.prepareMutationBatch();
    }
    ColumnListMutation<String> c = mb.withRow(cf, rowId);
    c.putColumn("raw", value.toString(), null);

    if (storeAttirbute) {
      for (String k : msg.keySet()) {
        if (k.startsWith("__")) continue;

        Object v = msg.get(k);

        if (v == null) continue;

        if (v.equals("")) continue;

        c.putColumn(k.toLowerCase(), v.toString(), null);
      }
    }

    try {
      if (mb.getRowCount() > 300) {
        OperationResult<Void> result = mb.execute();
        mb = null;
      }
    } catch (ConnectionException e) {
      e.printStackTrace(); // To change body of catch statement use File | Settings | File
      // Templates.
      mb = null;
    }
  }
Exemplo n.º 2
0
  public static void createKeyspace() throws Exception {
    keyspaceContext =
        new AstyanaxContext.Builder()
            .forCluster(TEST_CLUSTER_NAME)
            .forKeyspace(TEST_KEYSPACE_NAME)
            .withAstyanaxConfiguration(
                new AstyanaxConfigurationImpl()
                    .setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
                    .setConnectionPoolType(ConnectionPoolType.TOKEN_AWARE)
                    .setDiscoveryDelayInSeconds(60000))
            .withConnectionPoolConfiguration(
                new ConnectionPoolConfigurationImpl(TEST_CLUSTER_NAME + "_" + TEST_KEYSPACE_NAME)
                    .setSocketTimeout(30000)
                    .setMaxTimeoutWhenExhausted(2000)
                    .setMaxConnsPerHost(20)
                    .setInitConnsPerHost(10)
                    .setSeeds(SEEDS))
            .withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
            .buildKeyspace(ThriftFamilyFactory.getInstance());

    keyspaceContext.start();

    keyspace = keyspaceContext.getEntity();

    try {
      keyspace.dropKeyspace();
    } catch (Exception e) {
      LOG.info(e.getMessage());
    }

    keyspace.createKeyspace(
        ImmutableMap.<String, Object>builder()
            .put(
                "strategy_options",
                ImmutableMap.<String, Object>builder().put("replication_factor", "1").build())
            .put("strategy_class", "SimpleStrategy")
            .build());

    keyspace.createColumnFamily(CF_USER_UNIQUE_UUID, null);
    keyspace.createColumnFamily(CF_EMAIL_UNIQUE_UUID, null);
    keyspace.createColumnFamily(CF_ALL_ROWS, null);

    keyspace.createColumnFamily(
        LOCK_CF_LONG,
        ImmutableMap.<String, Object>builder()
            .put("default_validation_class", "LongType")
            .put("key_validation_class", "UTF8Type")
            .put("comparator_type", "UTF8Type")
            .build());

    keyspace.createColumnFamily(
        LOCK_CF_STRING,
        ImmutableMap.<String, Object>builder()
            .put("default_validation_class", "UTF8Type")
            .put("key_validation_class", "UTF8Type")
            .put("comparator_type", "UTF8Type")
            .build());

    keyspace.createColumnFamily(
        CF_STANDARD1,
        ImmutableMap.<String, Object>builder()
            .put(
                "column_metadata",
                ImmutableMap.<String, Object>builder()
                    .put(
                        "Index1",
                        ImmutableMap.<String, Object>builder()
                            .put("validation_class", "UTF8Type")
                            .put("index_name", "Index1")
                            .put("index_type", "KEYS")
                            .build())
                    .put(
                        "Index2",
                        ImmutableMap.<String, Object>builder()
                            .put("validation_class", "UTF8Type")
                            .put("index_name", "Index2")
                            .put("index_type", "KEYS")
                            .build())
                    .build())
            .build());

    keyspace.createColumnFamily(UNIQUE_CF, null);

    KeyspaceDefinition ki = keyspaceContext.getEntity().describeKeyspace();
    System.out.println("Describe Keyspace: " + ki.getName());

    try {
      //
      // CF_Super :
      // 'A' :
      // 'a' :
      // 1 : 'Aa1',
      // 2 : 'Aa2',
      // 'b' :
      // ...
      // 'z' :
      // ...
      // 'B' :
      // ...
      //
      // CF_Standard :
      // 'A' :
      // 'a' : 1,
      // 'b' : 2,
      // ...
      // 'z' : 26,
      // 'B' :
      // ...
      //

      MutationBatch m;
      OperationResult<Void> result;
      m = keyspace.prepareMutationBatch();

      for (char keyName = 'A'; keyName <= 'Z'; keyName++) {
        String rowKey = Character.toString(keyName);
        ColumnListMutation<String> cfmStandard = m.withRow(CF_STANDARD1, rowKey);
        for (char cName = 'a'; cName <= 'z'; cName++) {
          cfmStandard.putColumn(Character.toString(cName), (int) (cName - 'a') + 1, null);
        }
        cfmStandard.putColumn("Index1", (int) (keyName - 'A') + 1, null);
        cfmStandard.putColumn("Index2", 42, null);
        m.execute();
      }

      m.withRow(CF_STANDARD1, "Prefixes")
          .putColumn("Prefix1_a", 1, null)
          .putColumn("Prefix1_b", 2, null)
          .putColumn("prefix2_a", 3, null);

      result = m.execute();

      m.execute();

      m = keyspace.prepareMutationBatch();
      for (int i = 0; i < ALL_ROWS_COUNT; i++) {
        m.withRow(CF_ALL_ROWS, i).putColumn(0, true);
        if (m.getRowCount() == 50) {
          m.execute();
        }
      }
      m.execute();

    } catch (Exception e) {
      System.out.println(e.getMessage());
      Assert.fail();
    }
  }