private static AccumuloBackedGraph setupGraph(
Instance instance, Connector conn, String tableName, int numEntries) {
long ageOffTimeInMilliseconds = (30 * 24 * 60 * 60 * 1000L); // 30 days in milliseconds
try {
// Create table
// (this method creates the table, removes the versioning iterator, and adds the
// SetOfStatisticsCombiner iterator,
// and sets the age off iterator to age data off after it is more than
// ageOffTimeInMilliseconds milliseconds old).
TableUtils.createTable(conn, tableName, ageOffTimeInMilliseconds);
// Create numEntries edges and add to Accumulo
BatchWriter writer = conn.createBatchWriter(tableName, 1000000L, 1000L, 1);
for (int i = 0; i < numEntries; i++) {
Edge edge =
new Edge(
"customer",
"" + i,
"product",
"B",
"purchase",
"instore",
true,
visibilityString,
sevenDaysBefore,
sixDaysBefore);
SetOfStatistics statistics = new SetOfStatistics();
statistics.addStatistic("count", new Count(i));
Key key = ConversionUtils.getKeysFromEdge(edge).getFirst();
Value value = ConversionUtils.getValueFromSetOfStatistics(statistics);
Mutation m = new Mutation(key.getRow());
m.put(
key.getColumnFamily(),
key.getColumnQualifier(),
new ColumnVisibility(key.getColumnVisibility()),
key.getTimestamp(),
value);
writer.addMutation(m);
}
writer.close();
// Create Accumulo backed graph
AccumuloBackedGraph graph = new AccumuloBackedGraph(conn, tableName);
return graph;
} catch (AccumuloException e) {
fail("Failed to set up graph in Accumulo with exception: " + e);
} catch (AccumuloSecurityException e) {
fail("Failed to set up graph in Accumulo with exception: " + e);
} catch (TableExistsException e) {
fail("Failed to set up graph in Accumulo with exception: " + e);
} catch (TableNotFoundException e) {
fail("Failed to set up graph in Accumulo with exception: " + e);
}
return null;
}
private static TreeSet<RowColumn> scanAll(
ClientOnDefaultTable opts, ScannerOpts scanOpts, String tableName) throws Exception {
TreeSet<RowColumn> result = new TreeSet<RowColumn>();
Connector conn = opts.getConnector();
Scanner scanner = conn.createScanner(tableName, auths);
scanner.setBatchSize(scanOpts.scanBatchSize);
for (Entry<Key, Value> entry : scanner) {
Key key = entry.getKey();
Column column =
new Column(
TextUtil.getBytes(key.getColumnFamily()),
TextUtil.getBytes(key.getColumnQualifier()),
TextUtil.getBytes(key.getColumnVisibility()));
result.add(new RowColumn(key.getRow(), column, key.getTimestamp()));
}
return result;
}
@Test
public void visibilitySimplified() throws Exception {
// Create a PCJ index within Rya.
final String sparql =
"SELECT ?customer ?worker ?city "
+ "{ "
+ "?customer <"
+ TALKS_TO
+ "> ?worker. "
+ "?worker <"
+ LIVES_IN
+ "> ?city. "
+ "?worker <"
+ WORKS_AT
+ "> <"
+ BURGER_JOINT
+ ">. "
+ "}";
final RyaClient ryaClient =
AccumuloRyaClientFactory.build(
new AccumuloConnectionDetails(
ACCUMULO_USER, ACCUMULO_PASSWORD.toCharArray(), instanceName, zookeepers),
accumuloConn);
final String pcjId = ryaClient.getCreatePCJ().createPCJ(RYA_INSTANCE_NAME, sparql);
// Grant the root user the "u" authorization.
accumuloConn
.securityOperations()
.changeUserAuthorizations(ACCUMULO_USER, new Authorizations("u"));
// Setup a connection to the Rya instance that uses the "u" authorizations. This ensures
// any statements that are inserted will have the "u" authorization on them and that the
// PCJ updating application will have to maintain visibilities.
final AccumuloRdfConfiguration ryaConf = super.makeConfig(instanceName, zookeepers);
ryaConf.set(ConfigUtils.CLOUDBASE_AUTHS, "u");
ryaConf.set(RdfCloudTripleStoreConfiguration.CONF_CV, "u");
Sail sail = null;
RyaSailRepository ryaRepo = null;
RepositoryConnection ryaConn = null;
try {
sail = RyaSailFactory.getInstance(ryaConf);
ryaRepo = new RyaSailRepository(sail);
ryaConn = ryaRepo.getConnection();
// Load a few Statements into Rya.
ryaConn.add(VF.createStatement(ALICE, TALKS_TO, BOB));
ryaConn.add(VF.createStatement(BOB, LIVES_IN, HAPPYVILLE));
ryaConn.add(VF.createStatement(BOB, WORKS_AT, BURGER_JOINT));
// Wait for Fluo to finish processing.
fluo.waitForObservers();
// Fetch the exported result and show that its column visibility has been simplified.
final String pcjTableName = new PcjTableNameFactory().makeTableName(RYA_INSTANCE_NAME, pcjId);
final Scanner scan = accumuloConn.createScanner(pcjTableName, new Authorizations("u"));
scan.fetchColumnFamily(new Text("customer;worker;city"));
final Entry<Key, Value> result = scan.iterator().next();
final Key key = result.getKey();
assertEquals(new Text("u"), key.getColumnVisibility());
} finally {
if (ryaConn != null) {
try {
ryaConn.close();
} finally {
}
}
if (ryaRepo != null) {
try {
ryaRepo.shutDown();
} finally {
}
}
if (sail != null) {
try {
sail.shutDown();
} finally {
}
}
}
}
private void testAggregatingMultiplePropertySetsAcrossRowIDRange(
final AccumuloStore store, final AccumuloElementConverter elementConverter)
throws StoreException, AccumuloElementConversionException, RangeFactoryException {
String visibilityString = "public";
try {
// Create table
// (this method creates the table, removes the versioning iterator, and adds the
// SetOfStatisticsCombiner iterator).
TableUtils.createTable(store);
final Properties properties1 = new Properties();
properties1.put(AccumuloPropertyNames.COUNT, 1);
final Properties properties2 = new Properties();
properties2.put(AccumuloPropertyNames.COUNT, 1);
final Properties properties3 = new Properties();
properties3.put(AccumuloPropertyNames.COUNT, 2);
// Create edge
final Edge edge = new Edge(TestGroups.EDGE);
edge.setSource("2");
edge.setDestination("1");
edge.setDirected(true);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
final Edge edge2 = new Edge(TestGroups.EDGE);
edge2.setSource("B");
edge2.setDestination("Z");
edge2.setDirected(true);
edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge2.putProperty(AccumuloPropertyNames.PROP_1, 1);
edge2.putProperty(AccumuloPropertyNames.PROP_2, 1);
edge2.putProperty(AccumuloPropertyNames.PROP_3, 1);
edge2.putProperty(AccumuloPropertyNames.PROP_4, 1);
final Edge edge3 = new Edge(TestGroups.EDGE);
edge3.setSource("3");
edge3.setDestination("8");
edge3.setDirected(true);
edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
final Edge edge6 = new Edge("BasicEdge2");
edge6.setSource("1");
edge6.setDestination("5");
edge6.setDirected(true);
edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
edge6.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_4, 0);
final Edge edge7 = new Edge("BasicEdge2");
edge7.setSource("2");
edge7.setDestination("6");
edge7.setDirected(true);
edge7.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge7.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge7.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge7.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge7.putProperty(AccumuloPropertyNames.PROP_4, 0);
final Edge edge8 = new Edge("BasicEdge2");
edge8.setSource("4");
edge8.setDestination("8");
edge8.setDirected(true);
edge8.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
edge8.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge8.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge8.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge8.putProperty(AccumuloPropertyNames.PROP_4, 0);
final Edge edge9 = new Edge("BasicEdge2");
edge9.setSource("5");
edge9.setDestination("9");
edge9.setDirected(true);
edge9.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
edge9.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge9.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge9.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge9.putProperty(AccumuloPropertyNames.PROP_4, 0);
// Accumulo key
final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
final Key key4 = elementConverter.getKeysFromEdge(edge6).getFirst();
final Key key5 = elementConverter.getKeysFromEdge(edge7).getFirst();
final Key key6 = elementConverter.getKeysFromEdge(edge8).getFirst();
final Key key7 = elementConverter.getKeysFromEdge(edge9).getFirst();
// Accumulo values
final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties1);
final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties2);
final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties3);
final Value value4 = elementConverter.getValueFromProperties(TestGroups.EDGE_2, properties1);
final Value value5 = elementConverter.getValueFromProperties(TestGroups.EDGE_2, properties2);
// Create mutation
final Mutation m1 = new Mutation(key.getRow());
m1.put(
key.getColumnFamily(),
key.getColumnQualifier(),
new ColumnVisibility(key.getColumnVisibility()),
key.getTimestamp(),
value1);
final Mutation m2 = new Mutation(key.getRow());
m2.put(
key.getColumnFamily(),
key.getColumnQualifier(),
new ColumnVisibility(key.getColumnVisibility()),
key.getTimestamp(),
value2);
final Mutation m3 = new Mutation(key.getRow());
m3.put(
key.getColumnFamily(),
key.getColumnQualifier(),
new ColumnVisibility(key.getColumnVisibility()),
key.getTimestamp(),
value3);
final Mutation m4 = new Mutation(key2.getRow());
m4.put(
key2.getColumnFamily(),
key2.getColumnQualifier(),
new ColumnVisibility(key2.getColumnVisibility()),
key2.getTimestamp(),
value1);
final Mutation m5 = new Mutation(key.getRow());
m5.put(
key3.getColumnFamily(),
key3.getColumnQualifier(),
new ColumnVisibility(key3.getColumnVisibility()),
key3.getTimestamp(),
value1);
final Mutation m6 = new Mutation(key4.getRow());
m6.put(
key4.getColumnFamily(),
key4.getColumnQualifier(),
new ColumnVisibility(key4.getColumnVisibility()),
key4.getTimestamp(),
value4);
final Mutation m7 = new Mutation(key5.getRow());
m7.put(
key5.getColumnFamily(),
key5.getColumnQualifier(),
new ColumnVisibility(key5.getColumnVisibility()),
key5.getTimestamp(),
value5);
final Mutation m8 = new Mutation(key6.getRow());
m8.put(
key6.getColumnFamily(),
key6.getColumnQualifier(),
new ColumnVisibility(key6.getColumnVisibility()),
key6.getTimestamp(),
value5);
final Mutation m9 = new Mutation(key7.getRow());
m9.put(
key7.getColumnFamily(),
key7.getColumnQualifier(),
new ColumnVisibility(key7.getColumnVisibility()),
key7.getTimestamp(),
value5);
// Write mutation
final BatchWriterConfig writerConfig = new BatchWriterConfig();
writerConfig.setMaxMemory(1000000L);
writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
writerConfig.setMaxWriteThreads(1);
final BatchWriter writer =
store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
writer.addMutation(m1);
writer.addMutation(m2);
writer.addMutation(m3);
writer.addMutation(m4);
writer.addMutation(m5);
writer.addMutation(m6);
writer.addMutation(m7);
writer.addMutation(m8);
writer.addMutation(m9);
writer.close();
// Read data back and check we get one merged element
final Authorizations authorizations = new Authorizations(visibilityString);
final BatchScanner scanner =
store
.getConnection()
.createBatchScanner(store.getProperties().getTable(), authorizations, 1000);
try {
scanner.addScanIterator(
store
.getKeyPackage()
.getIteratorFactory()
.getRowIDAggregatorIteratorSetting(store, "BasicEdge2"));
} catch (IteratorSettingException e) {
fail(e.getMessage());
}
final RangeFactory rangeF = store.getKeyPackage().getRangeFactory();
final Range r =
rangeF.getRangeFromPair(
new Pair<ElementSeed>((new EntitySeed("1")), new EntitySeed("4")),
new SummariseGroupOverRanges());
final Range r2 =
rangeF.getRangeFromPair(
new Pair<ElementSeed>((new EntitySeed("5")), new EntitySeed("5")),
new SummariseGroupOverRanges());
scanner.setRanges(Arrays.asList(r, r2));
final Iterator<Map.Entry<Key, Value>> it = scanner.iterator();
Map.Entry<Key, Value> entry = it.next();
Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
Edge expectedEdge = new Edge("BasicEdge2");
expectedEdge.setSource("4");
expectedEdge.setDestination("8");
expectedEdge.setDirected(true);
expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 5);
expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 3);
assertEquals(expectedEdge, readEdge);
assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COUNT));
// Check we get the Result of the second provided range
assertTrue(it.hasNext());
entry = it.next();
readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
expectedEdge = new Edge("BasicEdge2");
expectedEdge.setSource("5");
expectedEdge.setDestination("9");
expectedEdge.setDirected(true);
expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 1);
assertEquals(expectedEdge, readEdge);
// Check no additional rows are found. (For a table of this size we shouldn't see this)
if (it.hasNext()) {
fail("Additional row found.");
}
} catch (AccumuloException | TableExistsException | TableNotFoundException e) {
fail(this.getClass().getSimpleName() + " failed with exception: " + e);
}
}