@Override
public void update(Key k, Value v) {
// SIGNAL from SingleTransposeIterator
if (k.getTimestamp() % 2 != 0) return;
String rStr;
{
ByteSequence rowData = k.getRowData();
rStr = new String(rowData.toArray(), rowData.offset(), rowData.length());
}
int pos = rStr.indexOf(edgeSep);
if (pos == -1) return; // this is a degree row, not an edge row.
// log.debug("edge row "+rStr+" : now "+setNodesReached.toString());
String toNode = rStr.substring(pos + 1);
setNodesReached.add(toNode);
// if (copyDeg) {
// Integer cnt = setNodesReachedCount.get(toNode);
// cnt = cnt == null ? new Integer(1) : new Integer(cnt+1);
// setNodesReachedCount.put(toNode, cnt);
// }
}
@Override
public boolean accept(Key k, Value v) {
long ts = k.getTimestamp();
if ((hasStart && (ts < start)) || (hasEnd && (ts > end))) return false;
if (hasStart && !startInclusive && ts == start) return false;
if (hasEnd && !endInclusive && ts == end) return false;
return true;
}
private static AccumuloBackedGraph setupGraph(
Instance instance, Connector conn, String tableName, int numEntries) {
long ageOffTimeInMilliseconds = (30 * 24 * 60 * 60 * 1000L); // 30 days in milliseconds
try {
// Create table
// (this method creates the table, removes the versioning iterator, and adds the
// SetOfStatisticsCombiner iterator,
// and sets the age off iterator to age data off after it is more than
// ageOffTimeInMilliseconds milliseconds old).
TableUtils.createTable(conn, tableName, ageOffTimeInMilliseconds);
// Create numEntries edges and add to Accumulo
BatchWriter writer = conn.createBatchWriter(tableName, 1000000L, 1000L, 1);
for (int i = 0; i < numEntries; i++) {
Edge edge =
new Edge(
"customer",
"" + i,
"product",
"B",
"purchase",
"instore",
true,
visibilityString,
sevenDaysBefore,
sixDaysBefore);
SetOfStatistics statistics = new SetOfStatistics();
statistics.addStatistic("count", new Count(i));
Key key = ConversionUtils.getKeysFromEdge(edge).getFirst();
Value value = ConversionUtils.getValueFromSetOfStatistics(statistics);
Mutation m = new Mutation(key.getRow());
m.put(
key.getColumnFamily(),
key.getColumnQualifier(),
new ColumnVisibility(key.getColumnVisibility()),
key.getTimestamp(),
value);
writer.addMutation(m);
}
writer.close();
// Create Accumulo backed graph
AccumuloBackedGraph graph = new AccumuloBackedGraph(conn, tableName);
return graph;
} catch (AccumuloException e) {
fail("Failed to set up graph in Accumulo with exception: " + e);
} catch (AccumuloSecurityException e) {
fail("Failed to set up graph in Accumulo with exception: " + e);
} catch (TableExistsException e) {
fail("Failed to set up graph in Accumulo with exception: " + e);
} catch (TableNotFoundException e) {
fail("Failed to set up graph in Accumulo with exception: " + e);
}
return null;
}
private static <M extends Map<Key, V>, V> M transposeMapHelp(Map<Key, V> orig, M neww) {
for (Map.Entry<Key, V> entry : orig.entrySet()) {
Key k0 = entry.getKey();
Key k =
new Key(
k0.getColumnQualifier(),
k0.getColumnFamily(),
k0.getRow(),
k0.getColumnVisibilityParsed(),
k0.getTimestamp());
neww.put(k, entry.getValue());
}
return neww;
}
private static TreeSet<RowColumn> scanAll(
ClientOnDefaultTable opts, ScannerOpts scanOpts, String tableName) throws Exception {
TreeSet<RowColumn> result = new TreeSet<RowColumn>();
Connector conn = opts.getConnector();
Scanner scanner = conn.createScanner(tableName, auths);
scanner.setBatchSize(scanOpts.scanBatchSize);
for (Entry<Key, Value> entry : scanner) {
Key key = entry.getKey();
Column column =
new Column(
TextUtil.getBytes(key.getColumnFamily()),
TextUtil.getBytes(key.getColumnQualifier()),
TextUtil.getBytes(key.getColumnVisibility()));
result.add(new RowColumn(key.getRow(), column, key.getTimestamp()));
}
return result;
}
private void testAggregatingMultiplePropertySetsAcrossRowIDRange(
final AccumuloStore store, final AccumuloElementConverter elementConverter)
throws StoreException, AccumuloElementConversionException, RangeFactoryException {
String visibilityString = "public";
try {
// Create table
// (this method creates the table, removes the versioning iterator, and adds the
// SetOfStatisticsCombiner iterator).
TableUtils.createTable(store);
final Properties properties1 = new Properties();
properties1.put(AccumuloPropertyNames.COUNT, 1);
final Properties properties2 = new Properties();
properties2.put(AccumuloPropertyNames.COUNT, 1);
final Properties properties3 = new Properties();
properties3.put(AccumuloPropertyNames.COUNT, 2);
// Create edge
final Edge edge = new Edge(TestGroups.EDGE);
edge.setSource("2");
edge.setDestination("1");
edge.setDirected(true);
edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge.putProperty(AccumuloPropertyNames.PROP_4, 0);
final Edge edge2 = new Edge(TestGroups.EDGE);
edge2.setSource("B");
edge2.setDestination("Z");
edge2.setDirected(true);
edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge2.putProperty(AccumuloPropertyNames.PROP_1, 1);
edge2.putProperty(AccumuloPropertyNames.PROP_2, 1);
edge2.putProperty(AccumuloPropertyNames.PROP_3, 1);
edge2.putProperty(AccumuloPropertyNames.PROP_4, 1);
final Edge edge3 = new Edge(TestGroups.EDGE);
edge3.setSource("3");
edge3.setDestination("8");
edge3.setDirected(true);
edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);
final Edge edge6 = new Edge("BasicEdge2");
edge6.setSource("1");
edge6.setDestination("5");
edge6.setDirected(true);
edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
edge6.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge6.putProperty(AccumuloPropertyNames.PROP_4, 0);
final Edge edge7 = new Edge("BasicEdge2");
edge7.setSource("2");
edge7.setDestination("6");
edge7.setDirected(true);
edge7.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
edge7.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge7.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge7.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge7.putProperty(AccumuloPropertyNames.PROP_4, 0);
final Edge edge8 = new Edge("BasicEdge2");
edge8.setSource("4");
edge8.setDestination("8");
edge8.setDirected(true);
edge8.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
edge8.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge8.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge8.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge8.putProperty(AccumuloPropertyNames.PROP_4, 0);
final Edge edge9 = new Edge("BasicEdge2");
edge9.setSource("5");
edge9.setDestination("9");
edge9.setDirected(true);
edge9.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
edge9.putProperty(AccumuloPropertyNames.PROP_1, 0);
edge9.putProperty(AccumuloPropertyNames.PROP_2, 0);
edge9.putProperty(AccumuloPropertyNames.PROP_3, 0);
edge9.putProperty(AccumuloPropertyNames.PROP_4, 0);
// Accumulo key
final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
final Key key4 = elementConverter.getKeysFromEdge(edge6).getFirst();
final Key key5 = elementConverter.getKeysFromEdge(edge7).getFirst();
final Key key6 = elementConverter.getKeysFromEdge(edge8).getFirst();
final Key key7 = elementConverter.getKeysFromEdge(edge9).getFirst();
// Accumulo values
final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties1);
final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties2);
final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties3);
final Value value4 = elementConverter.getValueFromProperties(TestGroups.EDGE_2, properties1);
final Value value5 = elementConverter.getValueFromProperties(TestGroups.EDGE_2, properties2);
// Create mutation
final Mutation m1 = new Mutation(key.getRow());
m1.put(
key.getColumnFamily(),
key.getColumnQualifier(),
new ColumnVisibility(key.getColumnVisibility()),
key.getTimestamp(),
value1);
final Mutation m2 = new Mutation(key.getRow());
m2.put(
key.getColumnFamily(),
key.getColumnQualifier(),
new ColumnVisibility(key.getColumnVisibility()),
key.getTimestamp(),
value2);
final Mutation m3 = new Mutation(key.getRow());
m3.put(
key.getColumnFamily(),
key.getColumnQualifier(),
new ColumnVisibility(key.getColumnVisibility()),
key.getTimestamp(),
value3);
final Mutation m4 = new Mutation(key2.getRow());
m4.put(
key2.getColumnFamily(),
key2.getColumnQualifier(),
new ColumnVisibility(key2.getColumnVisibility()),
key2.getTimestamp(),
value1);
final Mutation m5 = new Mutation(key.getRow());
m5.put(
key3.getColumnFamily(),
key3.getColumnQualifier(),
new ColumnVisibility(key3.getColumnVisibility()),
key3.getTimestamp(),
value1);
final Mutation m6 = new Mutation(key4.getRow());
m6.put(
key4.getColumnFamily(),
key4.getColumnQualifier(),
new ColumnVisibility(key4.getColumnVisibility()),
key4.getTimestamp(),
value4);
final Mutation m7 = new Mutation(key5.getRow());
m7.put(
key5.getColumnFamily(),
key5.getColumnQualifier(),
new ColumnVisibility(key5.getColumnVisibility()),
key5.getTimestamp(),
value5);
final Mutation m8 = new Mutation(key6.getRow());
m8.put(
key6.getColumnFamily(),
key6.getColumnQualifier(),
new ColumnVisibility(key6.getColumnVisibility()),
key6.getTimestamp(),
value5);
final Mutation m9 = new Mutation(key7.getRow());
m9.put(
key7.getColumnFamily(),
key7.getColumnQualifier(),
new ColumnVisibility(key7.getColumnVisibility()),
key7.getTimestamp(),
value5);
// Write mutation
final BatchWriterConfig writerConfig = new BatchWriterConfig();
writerConfig.setMaxMemory(1000000L);
writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
writerConfig.setMaxWriteThreads(1);
final BatchWriter writer =
store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
writer.addMutation(m1);
writer.addMutation(m2);
writer.addMutation(m3);
writer.addMutation(m4);
writer.addMutation(m5);
writer.addMutation(m6);
writer.addMutation(m7);
writer.addMutation(m8);
writer.addMutation(m9);
writer.close();
// Read data back and check we get one merged element
final Authorizations authorizations = new Authorizations(visibilityString);
final BatchScanner scanner =
store
.getConnection()
.createBatchScanner(store.getProperties().getTable(), authorizations, 1000);
try {
scanner.addScanIterator(
store
.getKeyPackage()
.getIteratorFactory()
.getRowIDAggregatorIteratorSetting(store, "BasicEdge2"));
} catch (IteratorSettingException e) {
fail(e.getMessage());
}
final RangeFactory rangeF = store.getKeyPackage().getRangeFactory();
final Range r =
rangeF.getRangeFromPair(
new Pair<ElementSeed>((new EntitySeed("1")), new EntitySeed("4")),
new SummariseGroupOverRanges());
final Range r2 =
rangeF.getRangeFromPair(
new Pair<ElementSeed>((new EntitySeed("5")), new EntitySeed("5")),
new SummariseGroupOverRanges());
scanner.setRanges(Arrays.asList(r, r2));
final Iterator<Map.Entry<Key, Value>> it = scanner.iterator();
Map.Entry<Key, Value> entry = it.next();
Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
Edge expectedEdge = new Edge("BasicEdge2");
expectedEdge.setSource("4");
expectedEdge.setDestination("8");
expectedEdge.setDirected(true);
expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 5);
expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 3);
assertEquals(expectedEdge, readEdge);
assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COUNT));
// Check we get the Result of the second provided range
assertTrue(it.hasNext());
entry = it.next();
readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
expectedEdge = new Edge("BasicEdge2");
expectedEdge.setSource("5");
expectedEdge.setDestination("9");
expectedEdge.setDirected(true);
expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 1);
assertEquals(expectedEdge, readEdge);
// Check no additional rows are found. (For a table of this size we shouldn't see this)
if (it.hasNext()) {
fail("Additional row found.");
}
} catch (AccumuloException | TableExistsException | TableNotFoundException e) {
fail(this.getClass().getSimpleName() + " failed with exception: " + e);
}
}
/**
* Create a copy of key with all fields except the ones specified cleared.
*
* @param key The key to copy
* @param pk What fields to retain from the key
* @return A new Key object pointing to new copies of fields specified by pk; other fields are
* empty/default.
*/
public static Key keyCopy(Key key, PartialKey pk) {
if (key == null || pk == null) return null;
switch (pk) {
case ROW:
return new Key(
key.getRowData().getBackingArray(),
EMPTY_BYTES,
EMPTY_BYTES,
EMPTY_BYTES,
Long.MAX_VALUE,
false,
true);
case ROW_COLFAM:
return new Key(
key.getRowData().getBackingArray(),
key.getColumnFamilyData().getBackingArray(),
EMPTY_BYTES,
EMPTY_BYTES,
Long.MAX_VALUE,
false,
true);
case ROW_COLFAM_COLQUAL:
return new Key(
key.getRowData().getBackingArray(),
key.getColumnFamilyData().getBackingArray(),
key.getColumnQualifierData().getBackingArray(),
EMPTY_BYTES,
Long.MAX_VALUE,
false,
true);
case ROW_COLFAM_COLQUAL_COLVIS:
return new Key(
key.getRowData().getBackingArray(),
key.getColumnFamilyData().getBackingArray(),
key.getColumnQualifierData().getBackingArray(),
key.getColumnVisibilityData().getBackingArray(),
Long.MAX_VALUE,
false,
true);
case ROW_COLFAM_COLQUAL_COLVIS_TIME:
return new Key(
key.getRowData().getBackingArray(),
key.getColumnFamilyData().getBackingArray(),
key.getColumnQualifierData().getBackingArray(),
key.getColumnVisibilityData().getBackingArray(),
key.getTimestamp(),
false,
true);
case ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL:
return new Key(
key.getRowData().getBackingArray(),
key.getColumnFamilyData().getBackingArray(),
key.getColumnQualifierData().getBackingArray(),
key.getColumnVisibilityData().getBackingArray(),
key.getTimestamp(),
key.isDeleted(),
true);
default:
throw new AssertionError("unknown pk: " + pk);
}
}
static int checkClone(String srcTableId, String tableId, Connector conn, BatchWriter bw)
throws TableNotFoundException, MutationsRejectedException {
TabletIterator srcIter =
new TabletIterator(
createCloneScanner(srcTableId, conn),
new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(),
true,
true);
TabletIterator cloneIter =
new TabletIterator(
createCloneScanner(tableId, conn),
new KeyExtent(new Text(tableId), null, null).toMetadataRange(),
true,
true);
if (!cloneIter.hasNext() || !srcIter.hasNext())
throw new RuntimeException(
" table deleted during clone? srcTableId = " + srcTableId + " tableId=" + tableId);
int rewrites = 0;
while (cloneIter.hasNext()) {
Map<Key, Value> cloneTablet = cloneIter.next();
Text cloneEndRow =
new KeyExtent(cloneTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
HashSet<String> cloneFiles = new HashSet<String>();
boolean cloneSuccessful = false;
for (Entry<Key, Value> entry : cloneTablet.entrySet()) {
if (entry.getKey().getColumnFamily().equals(ClonedColumnFamily.NAME)) {
cloneSuccessful = true;
break;
}
}
if (!cloneSuccessful) getFiles(cloneFiles, cloneTablet, null);
List<Map<Key, Value>> srcTablets = new ArrayList<Map<Key, Value>>();
Map<Key, Value> srcTablet = srcIter.next();
srcTablets.add(srcTablet);
Text srcEndRow =
new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
int cmp = compareEndRows(cloneEndRow, srcEndRow);
if (cmp < 0)
throw new TabletIterator.TabletDeletedException(
"Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
HashSet<String> srcFiles = new HashSet<String>();
if (!cloneSuccessful) getFiles(srcFiles, srcTablet, srcTableId);
while (cmp > 0) {
srcTablet = srcIter.next();
srcTablets.add(srcTablet);
srcEndRow =
new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
cmp = compareEndRows(cloneEndRow, srcEndRow);
if (cmp < 0)
throw new TabletIterator.TabletDeletedException(
"Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);
if (!cloneSuccessful) getFiles(srcFiles, srcTablet, srcTableId);
}
if (cloneSuccessful) continue;
if (!srcFiles.containsAll(cloneFiles)) {
// delete existing cloned tablet entry
Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
for (Entry<Key, Value> entry : cloneTablet.entrySet()) {
Key k = entry.getKey();
m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), k.getTimestamp());
}
bw.addMutation(m);
for (Map<Key, Value> st : srcTablets)
bw.addMutation(createCloneMutation(srcTableId, tableId, st));
rewrites++;
} else {
// write out marker that this tablet was successfully cloned
Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK".getBytes()));
bw.addMutation(m);
}
}
bw.flush();
return rewrites;
}
