@Override
public void visit(State state, Environment env, Properties props) throws Exception {
Connector conn = env.getConnector();
Random rand = (Random) state.get("rand");
@SuppressWarnings("unchecked")
List<String> tableNames = (List<String>) state.get("tables");
String tableName = tableNames.get(rand.nextInt(tableNames.size()));
try {
BatchScanner bs = conn.createBatchScanner(tableName, Authorizations.EMPTY, 3);
List<Range> ranges = new ArrayList<Range>();
for (int i = 0; i < rand.nextInt(2000) + 1; i++)
ranges.add(new Range(String.format("%016x", rand.nextLong() & 0x7fffffffffffffffl)));
bs.setRanges(ranges);
try {
Iterator<Entry<Key, Value>> iter = bs.iterator();
while (iter.hasNext()) iter.next();
} finally {
bs.close();
}
log.debug("Wrote to " + tableName);
} catch (TableNotFoundException e) {
log.debug("BatchScan " + tableName + " failed, doesnt exist");
} catch (TableDeletedException tde) {
log.debug("BatchScan " + tableName + " failed, table deleted");
} catch (TableOfflineException e) {
log.debug("BatchScan " + tableName + " failed, offline");
} catch (RuntimeException e) {
if (e.getCause() instanceof AccumuloSecurityException) {
log.debug("BatchScan " + tableName + " failed, permission error");
} else {
throw e;
}
}
}
MetaDataTableScanner(
Instance instance,
Credentials credentials,
Range range,
CurrentState state,
String tableName) {
// scan over metadata table, looking for tablets in the wrong state based on the live servers
// and online tables
try {
Connector connector =
instance.getConnector(credentials.getPrincipal(), credentials.getToken());
mdScanner = connector.createBatchScanner(tableName, Authorizations.EMPTY, 8);
configureScanner(mdScanner, state);
mdScanner.setRanges(Collections.singletonList(range));
iter = mdScanner.iterator();
} catch (Exception ex) {
if (mdScanner != null) mdScanner.close();
iter = null;
mdScanner = null;
throw new RuntimeException(ex);
}
}
private long batchScan(Connector c, List<Range> ranges, int threads) throws Exception {
BatchScanner bs = c.createBatchScanner("test_ingest", TestIngest.AUTHS, threads);
bs.setRanges(ranges);
int count = 0;
long t1 = System.currentTimeMillis();
byte rval[] = new byte[50];
Random random = new Random();
for (Entry<Key, Value> entry : bs) {
count++;
int row = VerifyIngest.getRow(entry.getKey());
int col = VerifyIngest.getCol(entry.getKey());
if (row < 0 || row >= NUM_TO_INGEST) {
throw new Exception("unexcepted row " + row);
}
rval = TestIngest.genRandomValue(random, rval, 2, row, col);
if (entry.getValue().compareTo(rval) != 0) {
throw new Exception("unexcepted value row=" + row + " col=" + col);
}
}
long t2 = System.currentTimeMillis();
bs.close();
if (count != NUM_TO_INGEST) {
throw new Exception("Batch Scan did not return expected number of values " + count);
}
return t2 - t1;
}
@Test
public void exampleNMF()
throws FileNotFoundException, TableNotFoundException, AccumuloSecurityException,
AccumuloException {
String Atable = "ex" + SCALE + "A"; // Table base name.
String Etable = "ex" + SCALE + "AEdge"; // Incidence table.
String ETtable = "ex" + SCALE + "AEdgeT"; // Transpose of incidence table.
String EtableSample = "ex" + SCALE + "AEdgeSample"; // Sampled-down version of incidence table.
String ETtableSample =
"ex" + SCALE + "AEdgeTSample"; // Sampled-down version of transpose of incidence table.
String Wtable = "ex" + SCALE + "AEdgeW"; // Output table W.
String WTtable = "ex" + SCALE + "AEdgeWT"; // Transpose of output table W.
String Htable = "ex" + SCALE + "AEdgeH"; // Output table H.
String HTtable = "ex" + SCALE + "AEdgeHT"; // Transpose of output table HT.
int K = 3; // 3 topics
int maxiter = 5; // 3 iterations of NMF maximum
double cutoffThreshold = 0.0; // Threshold to cut off entries with value less than this
int maxColsPerTopic = 10; // Threshold - only keep 10 nodes in H per topic
String newVisibility = ""; // Column Visibility to use for newly created entries.
// In your code, you would connect to an Accumulo instance by writing something similar to:
// ClientConfiguration cc =
// ClientConfiguration.loadDefault().withInstance("instance").withZkHosts("localhost:2181").withZkTimeout(5000);
// Instance instance = new ZooKeeperInstance(cc);
// Connector c = instance.getConnector("root", new PasswordToken("secret"));
// Here, we connect to the Accumulo instance given by TEST_CONFIG.java.
// You can change this by passing the option -DTEST_CONFIG=local or -DTEST_CONFIG=txe1 or
// similar.
Connector conn = tester.getConnector();
// Delete result table if it exists, so that we don't sum in previous runs with our results.
GraphuloUtil.deleteTables(conn, Htable, HTtable, Wtable, WTtable);
if (conn.tableOperations().exists(Htable)) conn.tableOperations().delete(Htable);
// Insert data from the file test/resources/data/10Ar.txt and 10Ac.txt into Accumulo.
// Deletes tables if they already exist.
ExampleUtil.ingestIncidenceSCALE(SCALE, 'A', Atable, conn);
// Create Graphulo executor. Supply the password for your Accumulo user account.
Graphulo graphulo = new Graphulo(conn, tester.getPassword());
// DistributedTrace.enable("NMFExample"); // remove this for no tracing
// Option to use in-memory version
double nmfError =
graphulo.NMF_Client(
Etable,
false,
WTtable,
true,
Htable,
false,
K,
maxiter,
cutoffThreshold,
maxColsPerTopic);
// Sample the graph with 10% uniform sampling and materialize the result in a sampled table
// double probability = 0.1;
// long nnzSample = graphulo.SampleCopy(Etable, EtableSample+"tmp", null, probability,
// trace);
// long nnzSample = graphulo.OneTable(Etable, EtableSample, ETtableSample, null, -1, null,
// null, null, null, null,
// Collections.singletonList(SamplingFilter.iteratorSetting(1, probability)), null,
// Authorizations.EMPTY);
// System.out.println("Sample finished; #entries in sample is "+nnzSample);
// Non-negative matrix factorization.
// This call blocks until the NMF completes.
// double nmfError = graphulo.NMF(EtableSample, ETtableSample, Wtable, WTtable, Htable,
// HTtable, KMER, maxiter, true,
// cutoffThreshold);
System.out.println("Final NMF absolute difference in error: " + nmfError);
// Result is in Htable, HTtable, Wtable, WTtable. Do whatever you like with it.
// For this example we will multiply H*W into a new table that approximates the original
// incidence matrix.
String APtable = "ex" + SCALE + "AEdgeApprox"; // Approximation of the incidence table Etable.
GraphuloUtil.deleteTables(conn, APtable);
graphulo.TableMult(
WTtable,
Htable,
APtable,
null,
-1,
MathTwoScalar.class,
MathTwoScalar.optionMap(
MathTwoScalar.ScalarOp.TIMES, MathTwoScalar.ScalarType.DOUBLE, newVisibility, false),
MathTwoScalar.combinerSetting(
Graphulo.PLUS_ITERATOR_BIGDECIMAL.getPriority(),
null,
MathTwoScalar.ScalarOp.PLUS,
MathTwoScalar.ScalarType.DOUBLE,
false),
null,
null,
null,
false,
false,
-1);
DistributedTrace.disable();
// Now Scanning APtable
BatchScanner bs = conn.createBatchScanner(APtable, Authorizations.EMPTY, 2);
bs.setRanges(Collections.singleton(new Range())); // Scan whole table.
int cnt = 0;
for (Map.Entry<Key, Value> entry : bs) {
cnt++;
// System.out.println(entry.getKey().toStringNoTime()+" -> "+entry.getValue());
}
bs.close();
log.info("# of entries in approximation table " + APtable + ": " + cnt);
}