@Override
public void map(
LongWritable key,
Text value,
OutputCollector<NullWritable, NullWritable> collector,
Reporter reporter)
throws IOException {
if (value.getLength() == 0) return;
byte[] raw = value.getBytes();
Map<String, Object> msg = mapper.readValue(raw, Map.class);
String rowId = createRowId(msg);
// System.out.println("rowId:" + rowId.toString());
if (rowId == null) {
// TODO ... Error Handler
return;
}
if (mb == null) {
mb = ks.prepareMutationBatch();
}
ColumnListMutation<String> c = mb.withRow(cf, rowId);
c.putColumn("raw", value.toString(), null);
if (storeAttirbute) {
for (String k : msg.keySet()) {
if (k.startsWith("__")) continue;
Object v = msg.get(k);
if (v == null) continue;
if (v.equals("")) continue;
c.putColumn(k.toLowerCase(), v.toString(), null);
}
}
try {
if (mb.getRowCount() > 300) {
OperationResult<Void> result = mb.execute();
mb = null;
}
} catch (ConnectionException e) {
e.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
mb = null;
}
}
public static void createKeyspace() throws Exception {
keyspaceContext =
new AstyanaxContext.Builder()
.forCluster(TEST_CLUSTER_NAME)
.forKeyspace(TEST_KEYSPACE_NAME)
.withAstyanaxConfiguration(
new AstyanaxConfigurationImpl()
.setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
.setConnectionPoolType(ConnectionPoolType.TOKEN_AWARE)
.setDiscoveryDelayInSeconds(60000))
.withConnectionPoolConfiguration(
new ConnectionPoolConfigurationImpl(TEST_CLUSTER_NAME + "_" + TEST_KEYSPACE_NAME)
.setSocketTimeout(30000)
.setMaxTimeoutWhenExhausted(2000)
.setMaxConnsPerHost(20)
.setInitConnsPerHost(10)
.setSeeds(SEEDS))
.withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
.buildKeyspace(ThriftFamilyFactory.getInstance());
keyspaceContext.start();
keyspace = keyspaceContext.getEntity();
try {
keyspace.dropKeyspace();
} catch (Exception e) {
LOG.info(e.getMessage());
}
keyspace.createKeyspace(
ImmutableMap.<String, Object>builder()
.put(
"strategy_options",
ImmutableMap.<String, Object>builder().put("replication_factor", "1").build())
.put("strategy_class", "SimpleStrategy")
.build());
keyspace.createColumnFamily(CF_USER_UNIQUE_UUID, null);
keyspace.createColumnFamily(CF_EMAIL_UNIQUE_UUID, null);
keyspace.createColumnFamily(CF_ALL_ROWS, null);
keyspace.createColumnFamily(
LOCK_CF_LONG,
ImmutableMap.<String, Object>builder()
.put("default_validation_class", "LongType")
.put("key_validation_class", "UTF8Type")
.put("comparator_type", "UTF8Type")
.build());
keyspace.createColumnFamily(
LOCK_CF_STRING,
ImmutableMap.<String, Object>builder()
.put("default_validation_class", "UTF8Type")
.put("key_validation_class", "UTF8Type")
.put("comparator_type", "UTF8Type")
.build());
keyspace.createColumnFamily(
CF_STANDARD1,
ImmutableMap.<String, Object>builder()
.put(
"column_metadata",
ImmutableMap.<String, Object>builder()
.put(
"Index1",
ImmutableMap.<String, Object>builder()
.put("validation_class", "UTF8Type")
.put("index_name", "Index1")
.put("index_type", "KEYS")
.build())
.put(
"Index2",
ImmutableMap.<String, Object>builder()
.put("validation_class", "UTF8Type")
.put("index_name", "Index2")
.put("index_type", "KEYS")
.build())
.build())
.build());
keyspace.createColumnFamily(UNIQUE_CF, null);
KeyspaceDefinition ki = keyspaceContext.getEntity().describeKeyspace();
System.out.println("Describe Keyspace: " + ki.getName());
try {
//
// CF_Super :
// 'A' :
// 'a' :
// 1 : 'Aa1',
// 2 : 'Aa2',
// 'b' :
// ...
// 'z' :
// ...
// 'B' :
// ...
//
// CF_Standard :
// 'A' :
// 'a' : 1,
// 'b' : 2,
// ...
// 'z' : 26,
// 'B' :
// ...
//
MutationBatch m;
OperationResult<Void> result;
m = keyspace.prepareMutationBatch();
for (char keyName = 'A'; keyName <= 'Z'; keyName++) {
String rowKey = Character.toString(keyName);
ColumnListMutation<String> cfmStandard = m.withRow(CF_STANDARD1, rowKey);
for (char cName = 'a'; cName <= 'z'; cName++) {
cfmStandard.putColumn(Character.toString(cName), (int) (cName - 'a') + 1, null);
}
cfmStandard.putColumn("Index1", (int) (keyName - 'A') + 1, null);
cfmStandard.putColumn("Index2", 42, null);
m.execute();
}
m.withRow(CF_STANDARD1, "Prefixes")
.putColumn("Prefix1_a", 1, null)
.putColumn("Prefix1_b", 2, null)
.putColumn("prefix2_a", 3, null);
result = m.execute();
m.execute();
m = keyspace.prepareMutationBatch();
for (int i = 0; i < ALL_ROWS_COUNT; i++) {
m.withRow(CF_ALL_ROWS, i).putColumn(0, true);
if (m.getRowCount() == 50) {
m.execute();
}
}
m.execute();
} catch (Exception e) {
System.out.println(e.getMessage());
Assert.fail();
}
}