@Override
public void close() {
// Shutdown the Astyanax contexts
openStores.clear();
keyspaceContext.shutdown();
clusterContext.shutdown();
}
/**
* @param _clusterName
* @param _server
* @param _keySpaceName
* @return KeySpace - the Cassandra KeySpace we are going to work in, and that we give back to the
* caller ( if everything goes well ).
* @throws java.lang.Exception
*/
public static final Keyspace doInit(
final String _clusterName, final String _server, final String _keySpaceName)
throws Exception {
AstyanaxContext<Keyspace> context =
new AstyanaxContext.Builder()
.forCluster(_clusterName)
.forKeyspace(_keySpaceName)
.withAstyanaxConfiguration(
new AstyanaxConfigurationImpl()
.setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
.setCqlVersion("3.0.0")
.setTargetCassandraVersion("2.0.4"))
.withConnectionPoolConfiguration(
new ConnectionPoolConfigurationImpl("ubicity file loading pool")
.setPort(9160)
.setMaxConnsPerHost(16)
.setSeeds(_server + ":9160")
.setMaxOperationsPerConnection(10000000)
.setConnectTimeout(10000))
.withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
.buildKeyspace(ThriftFamilyFactory.getInstance());
context.start();
Keyspace keySpace = context.getClient();
try {
keySpace.describeKeyspace();
logger.log(Level.INFO, "keyspace " + keySpace.getKeyspaceName() + " does exist");
} catch (BadRequestException bre) {
logger.log(
Level.INFO, "keyspace " + keySpace.getKeyspaceName() + " does NOT exist, creating it");
keySpace.createKeyspace(
ImmutableMap.<String, Object>builder()
.put(
"strategy_options",
ImmutableMap.<String, Object>builder().put("replication_factor", "1").build())
.put("strategy_class", "SimpleStrategy")
.build());
}
log = CassandraCrawlLogSchema.checkOrBuildMonitrixSchema(keySpace);
if (log == null) {
logger.log(Level.SEVERE, "could not create Monitrix ColumnFamily ( table ) ");
}
return keySpace;
}
public Keyspace getKeyspace() {
AstyanaxContext<Keyspace> context =
new AstyanaxContext.Builder()
.forCluster("Test Cluster")
.forKeyspace("DEMO")
.withAstyanaxConfiguration(
new AstyanaxConfigurationImpl().setDiscoveryType(NodeDiscoveryType.NONE))
.withConnectionPoolConfiguration(
new ConnectionPoolConfigurationImpl("MyConnectionPool")
.setPort(9160)
.setMaxConnsPerHost(1)
.setSeeds("127.0.0.1:9160"))
.withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
.buildKeyspace(ThriftFamilyFactory.getInstance());
context.start();
return context.getEntity();
}
@Override
public Partitioner getPartitioner() throws StorageException {
Cluster cl = clusterContext.getEntity();
try {
return Partitioner.getPartitioner(cl.describePartitioner());
} catch (ConnectionException e) {
throw new TemporaryStorageException(e);
}
}
@Override
public void configure(JobConf job) {
this.job = job;
mapper = new ObjectMapper();
messagePack = new MessagePack();
timeFmt = DateTimeFormat.forPattern("YYYYMMddHHmmssSSS");
oidSerial = new AtomicInteger(0);
serial = new AtomicInteger(0);
storeAttirbute = job.getBoolean(CIngest.CONF_INGEST_STORE_ATTR, false);
Iterator<String> tokens =
Splitter.on("///").split(job.get(CIngest.CONF_CASSANDRA_CONNECT_TOKEN)).iterator();
String clusterName = tokens.next();
String seeds = tokens.next();
String keyspaceName = tokens.next();
String columeFamilyName = tokens.next();
context =
new AstyanaxContext.Builder()
.forCluster(clusterName)
.forKeyspace(keyspaceName)
.withAstyanaxConfiguration(
new AstyanaxConfigurationImpl().setDiscoveryType(NodeDiscoveryType.TOKEN_AWARE))
.withConnectionPoolConfiguration(
new ConnectionPoolConfigurationImpl("cp")
.setPort(9160)
.setMaxConnsPerHost(2)
.setSeeds(seeds))
.withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
.buildKeyspace(ThriftFamilyFactory.getInstance());
context.start();
ks = context.getEntity();
cf =
new ColumnFamily<String, String>(
columeFamilyName, StringSerializer.get(), StringSerializer.get());
}
@Override
public void close() throws IOException {
if (mb != null && !mb.isEmpty()) {
try {
OperationResult<Void> result = mb.execute();
} catch (ConnectionException e) {
e.printStackTrace(); // To change body of catch statement use File | Settings | File
// Templates.
}
}
context.shutdown();
}
public static Keyspace getKeySpace(String keySpaceName) {
AstyanaxContext<Keyspace> context =
new AstyanaxContext.Builder()
.forCluster(CLUSTER_NAME)
.forKeyspace(keySpaceName)
.withAstyanaxConfiguration(
new AstyanaxConfigurationImpl().setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
// .setCqlVersion("3.0.0")
// .setTargetCassandraVersion("1.2")
)
.withConnectionPoolConfiguration(
new ConnectionPoolConfigurationImpl("MinderConnectionPool")
.setPort(9160)
.setMaxConnsPerHost(1)
.setSeeds(SEEDS))
.withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
.buildKeyspace(ThriftFamilyFactory.getInstance());
context.start();
return context.getClient();
}
private static void createKeyspace() throws Exception {
keyspaceContext =
new AstyanaxContext.Builder()
.forCluster(TEST_CLUSTER_NAME)
.forKeyspace(TEST_KEYSPACE_NAME)
.withAstyanaxConfiguration(
new AstyanaxConfigurationImpl()
.setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
.setConnectionPoolType(ConnectionPoolType.TOKEN_AWARE))
.withConnectionPoolConfiguration(
new ConnectionPoolConfigurationImpl(TEST_CLUSTER_NAME + "_" + TEST_KEYSPACE_NAME)
.setSocketTimeout(30000)
.setMaxTimeoutWhenExhausted(2000)
.setMaxConnsPerHost(20)
.setInitConnsPerHost(10)
.setSeeds(SEEDS))
.withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
.buildKeyspace(ThriftFamilyFactory.getInstance());
keyspaceContext.start();
keyspace = keyspaceContext.getEntity();
try {
keyspace.dropKeyspace();
} catch (Exception e) {
e.printStackTrace();
}
keyspace.createKeyspace(
ImmutableMap.<String, Object>builder()
.put(
"strategy_options",
ImmutableMap.<String, Object>builder().put("replication_factor", "1").build())
.put("strategy_class", "SimpleStrategy")
.build());
keyspace.createColumnFamily(CF_SAMPLE_ENTITY, null);
keyspace.createColumnFamily(CF_SIMPLE_ENTITY, null);
}
@Override
public void mutateMany(Map<String, Map<ByteBuffer, Mutation>> batch, StoreTransaction txh)
throws StorageException {
MutationBatch m =
keyspaceContext
.getEntity()
.prepareMutationBatch()
.setConsistencyLevel(getTx(txh).getWriteConsistencyLevel().getAstyanaxConsistency())
.withRetryPolicy(retryPolicy.duplicate());
final long delTS = TimeUtility.getApproxNSSinceEpoch(false);
final long addTS = TimeUtility.getApproxNSSinceEpoch(true);
for (Map.Entry<String, Map<ByteBuffer, Mutation>> batchentry : batch.entrySet()) {
String storeName = batchentry.getKey();
Preconditions.checkArgument(
openStores.containsKey(storeName), "Store cannot be found: " + storeName);
ColumnFamily<ByteBuffer, ByteBuffer> columnFamily =
openStores.get(storeName).getColumnFamily();
Map<ByteBuffer, Mutation> mutations = batchentry.getValue();
for (Map.Entry<ByteBuffer, Mutation> ent : mutations.entrySet()) {
// The CLMs for additions and deletions are separated because
// Astyanax's operation timestamp cannot be set on a per-delete
// or per-addition basis.
ColumnListMutation<ByteBuffer> dels = m.withRow(columnFamily, ent.getKey());
dels.setTimestamp(delTS);
ColumnListMutation<ByteBuffer> adds = m.withRow(columnFamily, ent.getKey());
adds.setTimestamp(addTS);
Mutation titanMutation = ent.getValue();
if (titanMutation.hasDeletions()) {
for (ByteBuffer b : titanMutation.getDeletions()) {
dels.deleteColumn(b);
}
}
if (titanMutation.hasAdditions()) {
for (Entry e : titanMutation.getAdditions()) {
adds.putColumn(e.getColumn(), e.getValue(), null);
}
}
}
}
try {
m.execute();
} catch (ConnectionException e) {
throw new TemporaryStorageException(e);
}
}
@Override
public synchronized AstyanaxOrderedKeyColumnValueStore openDatabase(String name)
throws StorageException {
if (openStores.containsKey(name)) return openStores.get(name);
else {
ensureColumnFamilyExists(name);
AstyanaxOrderedKeyColumnValueStore store =
new AstyanaxOrderedKeyColumnValueStore(
name, keyspaceContext.getEntity(), this, retryPolicy);
openStores.put(name, store);
return store;
}
}
@Override
public void setConfigurationProperty(final String key, final String value)
throws StorageException {
try {
ensureColumnFamilyExists(SYSTEM_PROPERTIES_CF, "org.apache.cassandra.db.marshal.UTF8Type");
Keyspace ks = keyspaceContext.getEntity();
OperationResult<Void> result =
ks.prepareColumnMutation(PROPERTIES_CF, SYSTEM_PROPERTIES_KEY, key)
.setConsistencyLevel(ConsistencyLevel.CL_QUORUM)
.putValue(value, null)
.execute();
result.getResult();
} catch (ConnectionException e) {
throw new PermanentStorageException(e);
}
}
@Override
public String getConfigurationProperty(final String key) throws StorageException {
try {
ensureColumnFamilyExists(SYSTEM_PROPERTIES_CF, "org.apache.cassandra.db.marshal.UTF8Type");
OperationResult<Column<String>> result =
keyspaceContext
.getEntity()
.prepareQuery(PROPERTIES_CF)
.setConsistencyLevel(ConsistencyLevel.CL_QUORUM)
.withRetryPolicy(retryPolicy.duplicate())
.getKey(SYSTEM_PROPERTIES_KEY)
.getColumn(key)
.execute();
return result.getResult().getStringValue();
} catch (NotFoundException e) {
return null;
} catch (ConnectionException e) {
throw new PermanentStorageException(e);
}
}
private void ensureColumnFamilyExists(String name, String comparator) throws StorageException {
Cluster cl = clusterContext.getEntity();
try {
KeyspaceDefinition ksDef = cl.describeKeyspace(keySpaceName);
boolean found = false;
if (null != ksDef) {
for (ColumnFamilyDefinition cfDef : ksDef.getColumnFamilyList()) {
found |= cfDef.getName().equals(name);
}
}
if (!found) {
ColumnFamilyDefinition cfDef =
cl.makeColumnFamilyDefinition()
.setName(name)
.setKeyspace(keySpaceName)
.setComparatorType(comparator);
cl.addColumnFamily(cfDef);
}
} catch (ConnectionException e) {
throw new TemporaryStorageException(e);
}
}
@Override
public void clearStorage() throws StorageException {
try {
Cluster cluster = clusterContext.getEntity();
Keyspace ks = cluster.getKeyspace(keySpaceName);
// Not a big deal if Keyspace doesn't not exist (dropped manually by user or tests).
// This is called on per test setup basis to make sure that previous test cleaned
// everything up, so first invocation would always fail as Keyspace doesn't yet exist.
if (ks == null) return;
for (ColumnFamilyDefinition cf :
cluster.describeKeyspace(keySpaceName).getColumnFamilyList()) {
ks.truncateColumnFamily(new ColumnFamily<Object, Object>(cf.getName(), null, null));
}
} catch (ConnectionException e) {
throw new PermanentStorageException(e);
} finally {
close();
}
}
public AstyanaxStoreManager(Configuration config) throws StorageException {
super(config);
// Check if we have non-default thrift frame size or max message size set and warn users
// because there is nothing we can do in Astyanax to apply those, warning is good enough here
// otherwise it would make bad user experience if we don't warn at all or crash on this.
if (this.thriftFrameSize != THRIFT_DEFAULT_FRAME_SIZE)
log.warn("Couldn't set custom Thrift Frame Size property, use 'cassandrathrift' instead.");
if (this.thriftMaxMessageSize != THRIFT_DEFAULT_MAX_MESSAGE_SIZE)
log.warn(
"Couldn't set custom Thrift Max Message Size property, use 'cassandrathrift' instead.");
this.clusterName = config.getString(CLUSTER_KEY, CLUSTER_DEFAULT);
this.retryPolicy = getRetryPolicy(config.getString(RETRY_POLICY_KEY, RETRY_POLICY_DEFAULT));
final int maxConnsPerHost =
config.getInt(MAX_CONNECTIONS_PER_HOST_KEY, MAX_CONNECTIONS_PER_HOST_DEFAULT);
final int maxClusterConnsPerHost =
config.getInt(
MAX_CLUSTER_CONNECTIONS_PER_HOST_KEY, MAX_CLUSTER_CONNECTIONS_PER_HOST_DEFAULT);
this.clusterContext =
createCluster(getContextBuilder(config, maxClusterConnsPerHost, "Cluster"));
ensureKeyspaceExists(clusterContext.getEntity());
this.keyspaceContext =
getContextBuilder(config, maxConnsPerHost, "Keyspace")
.buildKeyspace(ThriftFamilyFactory.getInstance());
this.keyspaceContext.start();
openStores = new HashMap<String, AstyanaxOrderedKeyColumnValueStore>(8);
}
private static AstyanaxContext<Cluster> createCluster(AstyanaxContext.Builder cb) {
AstyanaxContext<Cluster> clusterCtx = cb.buildCluster(ThriftFamilyFactory.getInstance());
clusterCtx.start();
return clusterCtx;
}
@AfterClass
public static void teardown() throws Exception {
if (keyspaceContext != null) keyspaceContext.shutdown();
Thread.sleep(1000 * 10);
}
@AfterClass
public static void teardown() throws Exception {
if (keyspaceContext != null) keyspaceContext.shutdown();
Thread.sleep(CASSANDRA_WAIT_TIME);
}
public static void createKeyspace() throws Exception {
keyspaceContext =
new AstyanaxContext.Builder()
.forCluster(TEST_CLUSTER_NAME)
.forKeyspace(TEST_KEYSPACE_NAME)
.withAstyanaxConfiguration(
new AstyanaxConfigurationImpl()
.setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE)
.setConnectionPoolType(ConnectionPoolType.TOKEN_AWARE)
.setDiscoveryDelayInSeconds(60000))
.withConnectionPoolConfiguration(
new ConnectionPoolConfigurationImpl(TEST_CLUSTER_NAME + "_" + TEST_KEYSPACE_NAME)
.setSocketTimeout(30000)
.setMaxTimeoutWhenExhausted(2000)
.setMaxConnsPerHost(20)
.setInitConnsPerHost(10)
.setSeeds(SEEDS))
.withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
.buildKeyspace(ThriftFamilyFactory.getInstance());
keyspaceContext.start();
keyspace = keyspaceContext.getEntity();
try {
keyspace.dropKeyspace();
} catch (Exception e) {
LOG.info(e.getMessage());
}
keyspace.createKeyspace(
ImmutableMap.<String, Object>builder()
.put(
"strategy_options",
ImmutableMap.<String, Object>builder().put("replication_factor", "1").build())
.put("strategy_class", "SimpleStrategy")
.build());
keyspace.createColumnFamily(CF_USER_UNIQUE_UUID, null);
keyspace.createColumnFamily(CF_EMAIL_UNIQUE_UUID, null);
keyspace.createColumnFamily(CF_ALL_ROWS, null);
keyspace.createColumnFamily(
LOCK_CF_LONG,
ImmutableMap.<String, Object>builder()
.put("default_validation_class", "LongType")
.put("key_validation_class", "UTF8Type")
.put("comparator_type", "UTF8Type")
.build());
keyspace.createColumnFamily(
LOCK_CF_STRING,
ImmutableMap.<String, Object>builder()
.put("default_validation_class", "UTF8Type")
.put("key_validation_class", "UTF8Type")
.put("comparator_type", "UTF8Type")
.build());
keyspace.createColumnFamily(
CF_STANDARD1,
ImmutableMap.<String, Object>builder()
.put(
"column_metadata",
ImmutableMap.<String, Object>builder()
.put(
"Index1",
ImmutableMap.<String, Object>builder()
.put("validation_class", "UTF8Type")
.put("index_name", "Index1")
.put("index_type", "KEYS")
.build())
.put(
"Index2",
ImmutableMap.<String, Object>builder()
.put("validation_class", "UTF8Type")
.put("index_name", "Index2")
.put("index_type", "KEYS")
.build())
.build())
.build());
keyspace.createColumnFamily(UNIQUE_CF, null);
KeyspaceDefinition ki = keyspaceContext.getEntity().describeKeyspace();
System.out.println("Describe Keyspace: " + ki.getName());
try {
//
// CF_Super :
// 'A' :
// 'a' :
// 1 : 'Aa1',
// 2 : 'Aa2',
// 'b' :
// ...
// 'z' :
// ...
// 'B' :
// ...
//
// CF_Standard :
// 'A' :
// 'a' : 1,
// 'b' : 2,
// ...
// 'z' : 26,
// 'B' :
// ...
//
MutationBatch m;
OperationResult<Void> result;
m = keyspace.prepareMutationBatch();
for (char keyName = 'A'; keyName <= 'Z'; keyName++) {
String rowKey = Character.toString(keyName);
ColumnListMutation<String> cfmStandard = m.withRow(CF_STANDARD1, rowKey);
for (char cName = 'a'; cName <= 'z'; cName++) {
cfmStandard.putColumn(Character.toString(cName), (int) (cName - 'a') + 1, null);
}
cfmStandard.putColumn("Index1", (int) (keyName - 'A') + 1, null);
cfmStandard.putColumn("Index2", 42, null);
m.execute();
}
m.withRow(CF_STANDARD1, "Prefixes")
.putColumn("Prefix1_a", 1, null)
.putColumn("Prefix1_b", 2, null)
.putColumn("prefix2_a", 3, null);
result = m.execute();
m.execute();
m = keyspace.prepareMutationBatch();
for (int i = 0; i < ALL_ROWS_COUNT; i++) {
m.withRow(CF_ALL_ROWS, i).putColumn(0, true);
if (m.getRowCount() == 50) {
m.execute();
}
}
m.execute();
} catch (Exception e) {
System.out.println(e.getMessage());
Assert.fail();
}
}
public static void doDataLoad() {
Properties properties = new Properties();
try {
properties.load(new FileInputStream("cassandraClient.properties"));
} catch (FileNotFoundException e1) {
e1.printStackTrace();
} catch (IOException e1) {
e1.printStackTrace();
}
String cassandraHost = properties.getProperty("cassandraHost");
String clusterName = properties.getProperty("clusterName");
String keyspaceName = properties.getProperty("keyspace");
String targetTableName = "Radio_Plays_1";
AstyanaxContext<Keyspace> context =
new AstyanaxContext.Builder()
.forCluster(clusterName)
.forKeyspace(keyspaceName)
.withAstyanaxConfiguration(
new AstyanaxConfigurationImpl().setDiscoveryType(NodeDiscoveryType.RING_DESCRIBE))
.withConnectionPoolConfiguration(
new ConnectionPoolConfigurationImpl("MyConnectionPool")
.setPort(9160)
.setMaxConnsPerHost(1)
.setSeeds(cassandraHost + ":9160"))
.withConnectionPoolMonitor(new CountingConnectionPoolMonitor())
.buildKeyspace(ThriftFamilyFactory.getInstance());
context.start();
Keyspace keyspace = context.getClient();
System.out.println("keyspace :" + keyspace.getKeyspaceName());
// targetTableName
ColumnFamily<String, String> CF_INFO =
new ColumnFamily<String, String>(
// "MUSICMETRIC_CONFIG", // Column Family Name
targetTableName,
StringSerializer.get(), // Key Serializer
StringSerializer.get()); // Column Serializer
// load each line data into cassandra
try {
// Open the file that is the first command line parameter
FileInputStream fstream = new FileInputStream("musicmetric-facebook-data.tsv.gz");
// Get the object of DataInputStream
DataInputStream in = new DataInputStream(fstream);
BufferedReader br = new BufferedReader(new InputStreamReader(in));
String strLine;
// Read File Line By Line
while ((strLine = br.readLine()) != null) {
String[] splitArray = strLine.split("\\s+");
String key = splitArray[0];
String value = splitArray[1];
logger.info("key --> " + key);
logger.info("value --> " + value);
// keyspace.prepareColumnMutation(CF_INFO, key, "value").putValue(value, null).execute();
}
// Close the input stream
in.close();
} catch (Exception e) { // Catch exception if any
System.err.println("Error: " + e.getMessage());
} finally {
context.shutdown();
}
}
