public static void main(String[] args) throws Exception {
ZooKeeperInstance zki =
new ZooKeeperInstance(
new ClientConfiguration().withInstance("test16").withZkHosts("localhost"));
Connector conn = zki.getConnector("root", new PasswordToken("secret"));
int numlg = Integer.parseInt(args[0]);
ArrayList<byte[]> cfset = new ArrayList<byte[]>();
for (int i = 0; i < 32; i++) {
cfset.add(String.format("%04x", i).getBytes());
}
Map<String, Stat> stats = new TreeMap<String, Stat>();
for (int i = 0; i < 5; i++) {
runTest(conn, numlg, cfset, i > 1 ? stats : null);
System.out.println();
}
for (Entry<String, Stat> entry : stats.entrySet()) {
System.out.printf("%20s : %6.2f\n", entry.getKey(), entry.getValue().getAverage());
}
}
@Test
public void testConfigureAccumuloInputFormatWithIterators() throws Exception {
AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
ColumnMapper columnMapper =
new ColumnMapper(
conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS),
conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE),
columnNames,
columnTypes);
Set<Pair<Text, Text>> cfCqPairs =
inputformat.getPairCollection(columnMapper.getColumnMappings());
List<IteratorSetting> iterators = new ArrayList<IteratorSetting>();
Set<Range> ranges = Collections.singleton(new Range());
String instanceName = "realInstance";
String zookeepers = "host1:2181,host2:2181,host3:2181";
IteratorSetting cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class);
cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "dave");
cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:name");
iterators.add(cfg);
cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class);
cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, IntCompare.class.getName());
cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "50");
cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:age");
iterators.add(cfg);
ZooKeeperInstance zkInstance = Mockito.mock(ZooKeeperInstance.class);
HiveAccumuloTableInputFormat mockInputFormat = Mockito.mock(HiveAccumuloTableInputFormat.class);
// Stub out the ZKI mock
Mockito.when(zkInstance.getInstanceName()).thenReturn(instanceName);
Mockito.when(zkInstance.getZooKeepers()).thenReturn(zookeepers);
// Call out to the real configure method
Mockito.doCallRealMethod()
.when(mockInputFormat)
.configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
// Also compute the correct cf:cq pairs so we can assert the right argument was passed
Mockito.doCallRealMethod()
.when(mockInputFormat)
.getPairCollection(columnMapper.getColumnMappings());
mockInputFormat.configure(
conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
// Verify that the correct methods are invoked on AccumuloInputFormat
Mockito.verify(mockInputFormat).setZooKeeperInstance(conf, instanceName, zookeepers, false);
Mockito.verify(mockInputFormat).setConnectorInfo(conf, USER, new PasswordToken(PASS));
Mockito.verify(mockInputFormat).setInputTableName(conf, TEST_TABLE);
Mockito.verify(mockInputFormat)
.setScanAuthorizations(conf, con.securityOperations().getUserAuthorizations(USER));
Mockito.verify(mockInputFormat).addIterators(conf, iterators);
Mockito.verify(mockInputFormat).setRanges(conf, ranges);
Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
}
@Test
public void testConfigureAccumuloInputFormatWithAuthorizations() throws Exception {
AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
conf.set(AccumuloSerDeParameters.AUTHORIZATIONS_KEY, "foo,bar");
ColumnMapper columnMapper =
new ColumnMapper(
conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS),
conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE),
columnNames,
columnTypes);
Set<Pair<Text, Text>> cfCqPairs =
inputformat.getPairCollection(columnMapper.getColumnMappings());
List<IteratorSetting> iterators = Collections.emptyList();
Set<Range> ranges = Collections.singleton(new Range());
String instanceName = "realInstance";
String zookeepers = "host1:2181,host2:2181,host3:2181";
ZooKeeperInstance zkInstance = Mockito.mock(ZooKeeperInstance.class);
HiveAccumuloTableInputFormat mockInputFormat = Mockito.mock(HiveAccumuloTableInputFormat.class);
// Stub out the ZKI mock
Mockito.when(zkInstance.getInstanceName()).thenReturn(instanceName);
Mockito.when(zkInstance.getZooKeepers()).thenReturn(zookeepers);
// Call out to the real configure method
Mockito.doCallRealMethod()
.when(mockInputFormat)
.configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
// Also compute the correct cf:cq pairs so we can assert the right argument was passed
Mockito.doCallRealMethod()
.when(mockInputFormat)
.getPairCollection(columnMapper.getColumnMappings());
mockInputFormat.configure(
conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
// Verify that the correct methods are invoked on AccumuloInputFormat
Mockito.verify(mockInputFormat).setZooKeeperInstance(conf, instanceName, zookeepers, false);
Mockito.verify(mockInputFormat).setConnectorInfo(conf, USER, new PasswordToken(PASS));
Mockito.verify(mockInputFormat).setInputTableName(conf, TEST_TABLE);
Mockito.verify(mockInputFormat).setScanAuthorizations(conf, new Authorizations("foo,bar"));
Mockito.verify(mockInputFormat).addIterators(conf, iterators);
Mockito.verify(mockInputFormat).setRanges(conf, ranges);
Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
}
public static void main(String[] args)
throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
TableExistsException {
System.out.println("START");
String instanceName = "development";
String zooKeepers = "localhost";
String user = "******";
byte[] pass = "******".getBytes();
String tableName = "users";
ZooKeeperInstance instance = new ZooKeeperInstance(instanceName, zooKeepers);
Connector connector = instance.getConnector(user, pass);
MultiTableBatchWriter writer = connector.createMultiTableBatchWriter(200000l, 300, 4);
if (!connector.tableOperations().exists(tableName)) {
connector.tableOperations().create(tableName);
}
BatchWriter bw = writer.getBatchWriter(tableName);
try {
String userId = "medined";
int age = 48;
int height = 70;
Mutation m = new Mutation(new Text(userId));
m.put(new Text("age"), new Text(""), new Value(new Integer(age).toString().getBytes()));
m.put(new Text("height"), new Text(""), new Value(new Integer(height).toString().getBytes()));
bw.addMutation(m);
} finally {
if (writer != null) {
writer.close();
}
}
System.out.println("END");
}
/**
* Writes a specified number of entries to Accumulo using a {@link BatchWriter}.
*
* @throws AccumuloException
* @throws AccumuloSecurityException
* @throws TableNotFoundException
*/
public static void main(String[] args)
throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
String seed = null;
int index = 0;
String processedArgs[] = new String[13];
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-s")) {
seed = args[++i];
} else {
processedArgs[index++] = args[i];
}
}
if (index != 13) {
System.out.println(
"Usage : RandomBatchWriter [-s <seed>] <instance name> <zoo keepers> <username> <password> <table> <num> <min> <max> <value size> <max memory> <max latency> <num threads> <visibility>");
return;
}
String instanceName = processedArgs[0];
String zooKeepers = processedArgs[1];
String user = processedArgs[2];
byte[] pass = processedArgs[3].getBytes();
String table = processedArgs[4];
int num = Integer.parseInt(processedArgs[5]);
long min = Long.parseLong(processedArgs[6]);
long max = Long.parseLong(processedArgs[7]);
int valueSize = Integer.parseInt(processedArgs[8]);
long maxMemory = Long.parseLong(processedArgs[9]);
long maxLatency =
Long.parseLong(processedArgs[10]) == 0 ? Long.MAX_VALUE : Long.parseLong(processedArgs[10]);
int numThreads = Integer.parseInt(processedArgs[11]);
String visiblity = processedArgs[12];
// Uncomment the following lines for detailed debugging info
// Logger logger = Logger.getLogger(Constants.CORE_PACKAGE_NAME);
// logger.setLevel(Level.TRACE);
Random r;
if (seed == null) r = new Random();
else {
r = new Random(Long.parseLong(seed));
}
ZooKeeperInstance instance = new ZooKeeperInstance(instanceName, zooKeepers);
Connector connector = instance.getConnector(user, pass);
BatchWriter bw = connector.createBatchWriter(table, maxMemory, maxLatency, numThreads);
// reuse the ColumnVisibility object to improve performance
ColumnVisibility cv = new ColumnVisibility(visiblity);
for (int i = 0; i < num; i++) {
long rowid = (Math.abs(r.nextLong()) % (max - min)) + min;
Mutation m = createMutation(rowid, valueSize, cv);
bw.addMutation(m);
}
try {
bw.close();
} catch (MutationsRejectedException e) {
if (e.getAuthorizationFailures().size() > 0) {
HashSet<String> tables = new HashSet<String>();
for (KeyExtent ke : e.getAuthorizationFailures()) {
tables.add(ke.getTableId().toString());
}
System.err.println("ERROR : Not authorized to write to tables : " + tables);
}
if (e.getConstraintViolationSummaries().size() > 0) {
System.err.println(
"ERROR : Constraint violations occurred : " + e.getConstraintViolationSummaries());
}
}
}
@Test(timeout = 40 * 1000)
public void testFilesAreGarbageCollected() throws Exception {
ZooKeeperInstance inst =
new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
Connector c = inst.getConnector("root", new PasswordToken(passwd));
final String table = "foobar";
c.tableOperations().create(table);
final String tableId = c.tableOperations().tableIdMap().get(table);
BatchWriter bw = null;
// Add some data
try {
bw =
c.createBatchWriter(
table,
new BatchWriterConfig()
.setMaxMemory(100000l)
.setMaxLatency(100, TimeUnit.MILLISECONDS)
.setMaxWriteThreads(1));
Mutation m = new Mutation("a");
for (int i = 0; i < 500; i++) {
m.put("colf", Integer.toString(i), "");
}
bw.addMutation(m);
} finally {
if (null != bw) {
bw.close();
}
}
File accumuloDir = new File(testDir, "accumulo");
File tables = new File(accumuloDir.getAbsolutePath(), "tables");
File myTable = new File(tables, tableId);
log.trace(
"Files before compaction: "
+ FileUtils.listFiles(myTable, new SuffixFileFilter(".rf"), TrueFileFilter.TRUE));
final boolean flush = true, wait = true;
// Compact the tables to get some rfiles which we can gc
c.tableOperations().compact(table, null, null, flush, wait);
Collection<File> filesAfterCompaction =
FileUtils.listFiles(myTable, new SuffixFileFilter(".rf"), TrueFileFilter.TRUE);
int fileCountAfterCompaction = filesAfterCompaction.size();
log.trace("Files after compaction: " + filesAfterCompaction);
// Sleep for 10s to let the GC do its thing
for (int i = 1; i < 10; i++) {
Thread.sleep(1000);
filesAfterCompaction =
FileUtils.listFiles(myTable, new SuffixFileFilter(".rf"), TrueFileFilter.TRUE);
log.trace("Files in loop: " + filesAfterCompaction);
int fileCountAfterGCWait = filesAfterCompaction.size();
if (fileCountAfterGCWait < fileCountAfterCompaction) {
return;
}
}
Assert.fail("Expected to find less files after compaction and pause for GC");
}