private static void runTest(
Connector conn, int numlg, ArrayList<byte[]> cfset, Map<String, Stat> stats)
throws Exception {
String table = "immlgb";
try {
conn.tableOperations().delete(table);
} catch (TableNotFoundException tnfe) {
}
conn.tableOperations().create(table);
conn.tableOperations()
.setProperty(table, Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "snappy");
setupLocalityGroups(conn, numlg, cfset, table);
addStat(stats, "write", write(conn, cfset, table));
addStat(stats, "scan cf", scan(conn, cfset, table, false));
addStat(stats, "scan cf:cq", scan(conn, cfset, table, true));
// TODO time reading all data
long t1 = System.currentTimeMillis();
conn.tableOperations().flush(table, null, null, true);
long t2 = System.currentTimeMillis();
addStat(stats, "flush", t2 - t1);
}
@Test(timeout = 5 * 60 * 1000)
public void testConcurrentAssignmentPerformance() throws Exception {
// make a table with a lot of splits
String tableName = getUniqueNames(1)[0];
Connector c = getConnector();
c.tableOperations().create(tableName);
SortedSet<Text> splits = new TreeSet<Text>();
for (int i = 0; i < 4000; i++) {
splits.add(new Text(randomHex(8)));
}
c.tableOperations().addSplits(tableName, splits);
c.tableOperations().offline(tableName, true);
// time how long it takes to load
long now = System.currentTimeMillis();
c.tableOperations().online(tableName, true);
long diff = System.currentTimeMillis() - now;
log.debug("Loaded " + splits.size() + " tablets in " + diff + " ms");
c.instanceOperations().setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT.getKey(), "20");
now = System.currentTimeMillis();
c.tableOperations().offline(tableName, true);
// wait >10 seconds for thread pool to update
UtilWaitThread.sleep(Math.max(0, now + 11 * 1000 - System.currentTimeMillis()));
now = System.currentTimeMillis();
c.tableOperations().online(tableName, true);
long diff2 = System.currentTimeMillis() - now;
log.debug("Loaded " + splits.size() + " tablets in " + diff2 + " ms");
assertTrue(diff2 < diff);
}
@Override
public void setup() {
Connector conn = getConnector();
String tableName = getTestProperty("TABLE");
// delete existing table
if (conn.tableOperations().exists(tableName)) {
System.out.println("Deleting existing table: " + tableName);
try {
conn.tableOperations().delete(tableName);
} catch (Exception e) {
log.error("Failed to delete table '" + tableName + "'.", e);
}
}
// create table
try {
conn.tableOperations().create(tableName);
conn.tableOperations().addSplits(tableName, calculateSplits());
conn.tableOperations().setProperty(tableName, "table.split.threshold", "256M");
} catch (Exception e) {
log.error("Failed to create table '" + tableName + "'.", e);
}
}
@Before
public void init()
throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
TableExistsException {
mock = new MockInstance("accumulo");
PasswordToken pToken = new PasswordToken("pass".getBytes());
conn = mock.getConnector("user", pToken);
config = new BatchWriterConfig();
config.setMaxMemory(1000);
config.setMaxLatency(1000, TimeUnit.SECONDS);
config.setMaxWriteThreads(10);
if (conn.tableOperations().exists("rya_prospects")) {
conn.tableOperations().delete("rya_prospects");
}
if (conn.tableOperations().exists("rya_selectivity")) {
conn.tableOperations().delete("rya_selectivity");
}
arc = new AccumuloRdfConfiguration();
arc.setTableLayoutStrategy(new TablePrefixLayoutStrategy());
arc.setMaxRangesForScanner(300);
res = new ProspectorServiceEvalStatsDAO(conn, arc);
}
public void addTable(Text tableName) throws AccumuloException, AccumuloSecurityException {
if (simulate) {
log.info("Simulating adding table: " + tableName);
return;
}
log.debug("Adding table: " + tableName);
BatchWriter bw = null;
String table = tableName.toString();
if (createTables && !conn.tableOperations().exists(table)) {
try {
conn.tableOperations().create(table);
} catch (AccumuloSecurityException e) {
log.error("Accumulo security violation creating " + table, e);
throw e;
} catch (TableExistsException e) {
// Shouldn't happen
}
}
try {
bw = mtbw.getBatchWriter(table);
} catch (TableNotFoundException e) {
log.error("Accumulo table " + table + " doesn't exist and cannot be created.", e);
throw new AccumuloException(e);
} catch (AccumuloException e) {
throw e;
} catch (AccumuloSecurityException e) {
throw e;
}
if (bw != null) bws.put(tableName, bw);
}
@Test
public void test() throws Exception {
Connector c = getConnector();
String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
Mutation m = new Mutation("row1");
m.put("cf", "col1", "Test");
bw.addMutation(m);
bw.close();
scanCheck(c, tableName, "Test");
FileSystem fs = getCluster().getFileSystem();
Path jarPath = new Path(rootPath + "/lib/ext/Test.jar");
copyStreamToFileSystem(fs, "/TestCombinerX.jar", jarPath);
sleepUninterruptibly(1, TimeUnit.SECONDS);
IteratorSetting is =
new IteratorSetting(10, "TestCombiner", "org.apache.accumulo.test.functional.TestCombiner");
Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf")));
c.tableOperations().attachIterator(tableName, is, EnumSet.of(IteratorScope.scan));
sleepUninterruptibly(ZOOKEEPER_PROPAGATION_TIME, TimeUnit.MILLISECONDS);
scanCheck(c, tableName, "TestX");
fs.delete(jarPath, true);
copyStreamToFileSystem(fs, "/TestCombinerY.jar", jarPath);
sleepUninterruptibly(5, TimeUnit.SECONDS);
scanCheck(c, tableName, "TestY");
fs.delete(jarPath, true);
}
@Test
public void test() throws Exception {
Connector c = getConnector();
// make a table
String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
// write to it
BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
Mutation m = new Mutation("row");
m.put("cf", "cq", "value");
bw.addMutation(m);
bw.close();
// create a fake _tmp file in its directory
String id = c.tableOperations().tableIdMap().get(tableName);
FileSystem fs = getCluster().getFileSystem();
Path tmp = new Path("/accumulo/tables/" + id + "/default_tablet/junk.rf_tmp");
fs.create(tmp).close();
for (ProcessReference tserver : getCluster().getProcesses().get(ServerType.TABLET_SERVER)) {
getCluster().killProcess(ServerType.TABLET_SERVER, tserver);
}
getCluster().start();
Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
FunctionalTestUtils.count(scanner);
assertFalse(fs.exists(tmp));
}
@Override
public int run(String[] args) {
Opts opts = new Opts();
opts.parseArgs(BulkIngestExample.class.getName(), args);
Configuration conf = getConf();
PrintStream out = null;
try {
Job job = JobUtil.getJob(conf);
job.setJobName("bulk ingest example");
job.setJarByClass(this.getClass());
job.setInputFormatClass(TextInputFormat.class);
job.setMapperClass(MapClass.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setReducerClass(ReduceClass.class);
job.setOutputFormatClass(AccumuloFileOutputFormat.class);
opts.setAccumuloConfigs(job);
Connector connector = opts.getConnector();
TextInputFormat.setInputPaths(job, new Path(opts.inputDir));
AccumuloFileOutputFormat.setOutputPath(job, new Path(opts.workDir + "/files"));
FileSystem fs = FileSystem.get(conf);
out =
new PrintStream(
new BufferedOutputStream(fs.create(new Path(opts.workDir + "/splits.txt"))));
Collection<Text> splits = connector.tableOperations().listSplits(opts.getTableName(), 100);
for (Text split : splits)
out.println(new String(Base64.encodeBase64(TextUtil.getBytes(split))));
job.setNumReduceTasks(splits.size() + 1);
out.close();
job.setPartitionerClass(RangePartitioner.class);
RangePartitioner.setSplitFile(job, opts.workDir + "/splits.txt");
job.waitForCompletion(true);
Path failures = new Path(opts.workDir, "failures");
fs.delete(failures, true);
fs.mkdirs(new Path(opts.workDir, "failures"));
connector
.tableOperations()
.importDirectory(
opts.getTableName(), opts.workDir + "/files", opts.workDir + "/failures", false);
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (out != null) out.close();
}
return 0;
}
@Test
public void testBulkSplitOptimization() throws Exception {
final Connector c = getConnector();
final String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "1000");
c.tableOperations().setProperty(tableName, Property.TABLE_FILE_MAX.getKey(), "1000");
c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "1G");
FileSystem fs = cluster.getFileSystem();
Path testDir = new Path(getUsableDir(), "testmf");
FunctionalTestUtils.createRFiles(c, fs, testDir.toString(), ROWS, SPLITS, 8);
FileStatus[] stats = fs.listStatus(testDir);
System.out.println("Number of generated files: " + stats.length);
FunctionalTestUtils.bulkImport(c, fs, tableName, testDir.toString());
FunctionalTestUtils.checkSplits(c, tableName, 0, 0);
FunctionalTestUtils.checkRFiles(c, tableName, 1, 1, 100, 100);
// initiate splits
getConnector()
.tableOperations()
.setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "100K");
sleepUninterruptibly(2, TimeUnit.SECONDS);
// wait until over split threshold -- should be 78 splits
while (getConnector().tableOperations().listSplits(tableName).size() < 75) {
sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
}
FunctionalTestUtils.checkSplits(c, tableName, 50, 100);
VerifyIngest.Opts opts = new VerifyIngest.Opts();
opts.timestamp = 1;
opts.dataSize = 50;
opts.random = 56;
opts.rows = 100000;
opts.startRow = 0;
opts.cols = 1;
opts.setTableName(tableName);
AuthenticationToken adminToken = getAdminToken();
if (adminToken instanceof PasswordToken) {
PasswordToken token = (PasswordToken) getAdminToken();
opts.setPassword(new Password(new String(token.getPassword(), UTF_8)));
opts.setPrincipal(getAdminPrincipal());
} else if (adminToken instanceof KerberosToken) {
ClientConfiguration clientConf = cluster.getClientConfig();
opts.updateKerberosCredentials(clientConf);
} else {
Assert.fail("Unknown token type");
}
VerifyIngest.verifyIngest(c, opts, new ScannerOpts());
// ensure each tablet does not have all map files, should be ~2.5 files per tablet
FunctionalTestUtils.checkRFiles(c, tableName, 50, 100, 1, 4);
}
private void runMergeTest(
Connector conn,
String table,
String[] splits,
String[] expectedSplits,
String[] inserts,
String start,
String end)
throws Exception {
System.out.println(
"Running merge test " + table + " " + Arrays.asList(splits) + " " + start + " " + end);
conn.tableOperations().create(table, new NewTableConfiguration().setTimeType(TimeType.LOGICAL));
TreeSet<Text> splitSet = new TreeSet<Text>();
for (String split : splits) {
splitSet.add(new Text(split));
}
conn.tableOperations().addSplits(table, splitSet);
BatchWriter bw = conn.createBatchWriter(table, null);
HashSet<String> expected = new HashSet<String>();
for (String row : inserts) {
Mutation m = new Mutation(row);
m.put("cf", "cq", row);
bw.addMutation(m);
expected.add(row);
}
bw.close();
conn.tableOperations()
.merge(table, start == null ? null : new Text(start), end == null ? null : new Text(end));
Scanner scanner = conn.createScanner(table, Authorizations.EMPTY);
HashSet<String> observed = new HashSet<String>();
for (Entry<Key, Value> entry : scanner) {
String row = entry.getKey().getRowData().toString();
if (!observed.add(row)) {
throw new Exception("Saw data twice " + table + " " + row);
}
}
if (!observed.equals(expected)) {
throw new Exception("data inconsistency " + table + " " + observed + " != " + expected);
}
HashSet<Text> currentSplits = new HashSet<Text>(conn.tableOperations().listSplits(table));
HashSet<Text> ess = new HashSet<Text>();
for (String es : expectedSplits) {
ess.add(new Text(es));
}
if (!currentSplits.equals(ess)) {
throw new Exception("split inconsistency " + table + " " + currentSplits + " != " + ess);
}
}
@Test
public void interleaveSplit() throws Exception {
Connector c = getConnector();
c.tableOperations().create("test_ingest");
c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
c.tableOperations()
.setProperty("test_ingest", Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none");
ReadWriteIT.interleaveTest(c);
UtilWaitThread.sleep(5 * 1000);
assertTrue(c.tableOperations().listSplits("test_ingest").size() > 20);
}
@Test(timeout = 4 * 60 * 1000)
public void binaryStressTest() throws Exception {
Connector c = getConnector();
c.tableOperations().create("bt");
c.tableOperations().setProperty("bt", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
BinaryIT.runTest(c);
String id = c.tableOperations().tableIdMap().get("bt");
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
FileStatus[] dir =
fs.listStatus(new Path(cluster.getConfig().getDir() + "/accumulo/tables/" + id));
assertTrue(dir.length > 7);
}
@Test
public void deleteSplit() throws Exception {
Connector c = getConnector();
c.tableOperations().create("test_ingest");
c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
DeleteIT.deleteTest(c, cluster);
c.tableOperations().flush("test_ingest", null, null, true);
for (int i = 0; i < 5; i++) {
UtilWaitThread.sleep(10 * 1000);
if (c.tableOperations().listSplits("test_ingest").size() > 20) break;
}
assertTrue(c.tableOperations().listSplits("test_ingest").size() > 20);
}
/**
* Insert the contents of an NDSI data file into an Accumulo table.
*
* @param file Input file
* @param Atable Table to insert into
* @return Nummber of entries inserted into Atable; equal to 2x the number of rows.
* @throws IOException
*/
public long ingestFile(File file, String Atable, boolean deleteIfExists) throws IOException {
if (deleteIfExists && connector.tableOperations().exists(Atable))
try {
connector.tableOperations().delete(Atable);
} catch (AccumuloException | AccumuloSecurityException e) {
log.warn("trouble deleting table " + Atable, e);
throw new RuntimeException(e);
} catch (TableNotFoundException e) {
throw new RuntimeException(e);
}
if (!connector.tableOperations().exists(Atable))
try {
connector.tableOperations().create(Atable);
} catch (AccumuloException | AccumuloSecurityException e) {
log.warn("trouble creating table " + Atable, e);
throw new RuntimeException(e);
} catch (TableExistsException e) {
throw new RuntimeException(e);
}
BatchWriter bw = null;
String line = null;
long entriesProcessed = 0;
try (BufferedReader fo = new BufferedReader(new FileReader(file))) {
BatchWriterConfig config = new BatchWriterConfig();
bw = connector.createBatchWriter(Atable, config);
// Skip header line
fo.readLine();
while ((line = fo.readLine()) != null)
if (!line.isEmpty()) entriesProcessed += ingestLine(bw, line);
} catch (TableNotFoundException e) {
throw new RuntimeException(e);
} catch (MutationsRejectedException e) {
log.warn("Mutation rejected on line " + line, e);
} finally {
if (bw != null)
try {
bw.close();
} catch (MutationsRejectedException e) {
log.warn("Mutation rejected at close() on line " + line, e);
}
}
return entriesProcessed;
}
/** Write entries to a table. */
public static void writeEntries(
Connector connector, Map<Key, Value> map, String table, boolean createIfNotExist) {
if (createIfNotExist && !connector.tableOperations().exists(table))
try {
connector.tableOperations().create(table);
} catch (AccumuloException | AccumuloSecurityException e) {
log.error("trouble creating " + table, e);
throw new RuntimeException(e);
} catch (TableExistsException e) {
log.error("crazy", e);
throw new RuntimeException(e);
}
BatchWriterConfig bwc = new BatchWriterConfig();
BatchWriter bw;
try {
bw = connector.createBatchWriter(table, bwc);
} catch (TableNotFoundException e) {
log.error("tried to write to a non-existant table " + table, e);
throw new RuntimeException(e);
}
try {
for (Map.Entry<Key, Value> entry : map.entrySet()) {
Key k = entry.getKey();
ByteSequence rowData = k.getRowData(),
cfData = k.getColumnFamilyData(),
cqData = k.getColumnQualifierData();
Mutation m = new Mutation(rowData.getBackingArray(), rowData.offset(), rowData.length());
m.put(
cfData.getBackingArray(),
cqData.getBackingArray(),
k.getColumnVisibilityParsed(),
entry.getValue().get());
bw.addMutation(m);
}
} catch (MutationsRejectedException e) {
log.error("mutations rejected", e);
throw new RuntimeException(e);
} finally {
try {
bw.close();
} catch (MutationsRejectedException e) {
log.error("mutations rejected while trying to close BatchWriter", e);
}
}
}
public static synchronized void configureMetadataTable(Connector conn, String tableName) {
TableOperations tops = conn.tableOperations();
Map<String, EnumSet<IteratorScope>> iterators = null;
try {
iterators = tops.listIterators(tableName);
} catch (AccumuloSecurityException | AccumuloException | TableNotFoundException e) {
throw new RuntimeException(e);
}
if (!iterators.containsKey(COMBINER_NAME)) {
// Set our combiner and combine all columns
// Need to set the combiner beneath versioning since we don't want to turn it off
IteratorSetting setting = new IteratorSetting(9, COMBINER_NAME, StatusCombiner.class);
Combiner.setColumns(
setting, Collections.singletonList(new Column(MetadataSchema.ReplicationSection.COLF)));
try {
tops.attachIterator(tableName, setting);
} catch (AccumuloSecurityException | AccumuloException | TableNotFoundException e) {
throw new RuntimeException(e);
}
}
// Make sure the StatusFormatter is set on the metadata table
Iterable<Entry<String, String>> properties;
try {
properties = tops.getProperties(tableName);
} catch (AccumuloException | TableNotFoundException e) {
throw new RuntimeException(e);
}
for (Entry<String, String> property : properties) {
if (Property.TABLE_FORMATTER_CLASS.getKey().equals(property.getKey())) {
if (!STATUS_FORMATTER_CLASS_NAME.equals(property.getValue())) {
log.info(
"Setting formatter for {} from {} to {}",
tableName,
property.getValue(),
STATUS_FORMATTER_CLASS_NAME);
try {
tops.setProperty(
tableName, Property.TABLE_FORMATTER_CLASS.getKey(), STATUS_FORMATTER_CLASS_NAME);
} catch (AccumuloException | AccumuloSecurityException e) {
throw new RuntimeException(e);
}
}
// Don't need to keep iterating over the properties after we found the one we were looking
// for
return;
}
}
// Set the formatter on the table because it wasn't already there
try {
tops.setProperty(
tableName, Property.TABLE_FORMATTER_CLASS.getKey(), STATUS_FORMATTER_CLASS_NAME);
} catch (AccumuloException | AccumuloSecurityException e) {
throw new RuntimeException(e);
}
}
@Override
public void tearDown(State state, Environment env) throws Exception {
// We have resources we need to clean up
if (env.isMultiTableBatchWriterInitialized()) {
MultiTableBatchWriter mtbw = env.getMultiTableBatchWriter();
try {
mtbw.close();
} catch (MutationsRejectedException e) {
log.error("Ignoring mutations that weren't flushed", e);
}
// Reset the MTBW on the state to null
env.resetMultiTableBatchWriter();
}
// Now we can safely delete the tables
log.debug("Dropping tables: " + imageTableName + " " + indexTableName);
Connector conn = env.getConnector();
conn.tableOperations().delete(imageTableName);
conn.tableOperations().delete(indexTableName);
log.debug("Final total of writes: " + state.getLong("totalWrites"));
}
public static void printDiskUsage(
AccumuloConfiguration acuConf,
Collection<String> tables,
VolumeManager fs,
Connector conn,
Printer printer,
boolean humanReadable)
throws TableNotFoundException, IOException {
HashSet<String> tableIds = new HashSet<String>();
// Get table IDs for all tables requested to be 'du'
for (String tableName : tables) {
String tableId = conn.tableOperations().tableIdMap().get(tableName);
if (tableId == null)
throw new TableNotFoundException(null, tableName, "Table " + tableName + " not found");
tableIds.add(tableId);
}
Map<TreeSet<String>, Long> usage = getDiskUsage(acuConf, tableIds, fs, conn);
String valueFormat = humanReadable ? "%9s" : "%,24d";
for (Entry<TreeSet<String>, Long> entry : usage.entrySet()) {
Object value = humanReadable ? NumUtil.bigNumberForSize(entry.getValue()) : entry.getValue();
printer.print(String.format(valueFormat + " %s", value, entry.getKey()));
}
}
@Test(timeout = 2 * 60 * 1000)
public void test() throws Exception {
Connector c = getConnector();
c.tableOperations().create("test_ingest");
deleteTest(c, cluster);
assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
}
@Override
public void visit(State state, Properties props) throws Exception {
Connector conn = state.getConnector();
Random rand = (Random) state.get("rand");
@SuppressWarnings("unchecked")
List<String> tableNames = (List<String>) state.get("tables");
String tableName = tableNames.get(rand.nextInt(tableNames.size()));
// TODO need to sometimes do null start and end ranges
TreeSet<Text> range = new TreeSet<Text>();
range.add(new Text(String.format("%016x", Math.abs(rand.nextLong()))));
range.add(new Text(String.format("%016x", Math.abs(rand.nextLong()))));
try {
boolean wait = rand.nextBoolean();
conn.tableOperations().compact(tableName, range.first(), range.last(), false, wait);
log.debug((wait ? "compacted " : "initiated compaction ") + tableName);
} catch (TableNotFoundException tne) {
log.debug("compact " + tableName + " failed, doesnt exist");
} catch (TableOfflineException toe) {
log.debug("compact " + tableName + " failed, offline");
}
}
public static void run(
String instanceName, String zookeepers, AuthenticationToken rootPassword, String args[])
throws Exception {
// edit this method to play with Accumulo
Instance instance = new ZooKeeperInstance(instanceName, zookeepers);
Connector conn = instance.getConnector("root", rootPassword);
conn.tableOperations().create("foo");
BatchWriterConfig bwConfig = new BatchWriterConfig();
bwConfig.setMaxLatency(60000l, java.util.concurrent.TimeUnit.MILLISECONDS);
bwConfig.setMaxWriteThreads(3);
bwConfig.setMaxMemory(50000000);
BatchWriter bw = conn.createBatchWriter("foo", bwConfig);
Mutation m = new Mutation("r1");
m.put("cf1", "cq1", "v1");
m.put("cf1", "cq2", "v3");
bw.addMutation(m);
bw.close();
Scanner scanner = conn.createScanner("foo", Constants.NO_AUTHS);
for (Entry<Key, Value> entry : scanner) {
System.out.println(entry.getKey() + " " + entry.getValue());
}
}
@Test
public void merge() throws Exception {
Connector c = getConnector();
String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
c.tableOperations().addSplits(tableName, splits("a b c d e f g h i j k".split(" ")));
BatchWriter bw = c.createBatchWriter(tableName, null);
for (String row : "a b c d e f g h i j k".split(" ")) {
Mutation m = new Mutation(row);
m.put("cf", "cq", "value");
bw.addMutation(m);
}
bw.close();
c.tableOperations().flush(tableName, null, null, true);
c.tableOperations().merge(tableName, new Text("c1"), new Text("f1"));
assertEquals(8, c.tableOperations().listSplits(tableName).size());
}
@Test
public void run() throws Exception {
Connector c = getConnector();
String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
for (int i = 0; i < 100000; i++) {
c.createScanner(tableName, Authorizations.EMPTY);
}
}
@Before
public void createMockKeyValues() throws Exception {
// Make a MockInstance here, by setting the instance name to be the same as this mock instance
// we can "trick" the InputFormat into using a MockInstance
mockInstance = new MockInstance(test.getMethodName());
inputformat = new HiveAccumuloTableInputFormat();
conf = new JobConf();
conf.set(AccumuloSerDeParameters.TABLE_NAME, TEST_TABLE);
conf.set(AccumuloSerDeParameters.USE_MOCK_INSTANCE, "true");
conf.set(AccumuloSerDeParameters.INSTANCE_NAME, test.getMethodName());
conf.set(AccumuloSerDeParameters.USER_NAME, USER);
conf.set(AccumuloSerDeParameters.USER_PASS, PASS);
conf.set(AccumuloSerDeParameters.ZOOKEEPERS, "localhost:2181"); // not used for mock, but
// required by input format.
columnNames = Arrays.asList("name", "sid", "dgrs", "mills");
columnTypes =
Arrays.<TypeInfo>asList(
TypeInfoFactory.stringTypeInfo,
TypeInfoFactory.intTypeInfo,
TypeInfoFactory.doubleTypeInfo,
TypeInfoFactory.longTypeInfo);
conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:name,cf:sid,cf:dgrs,cf:mills");
conf.set(serdeConstants.LIST_COLUMNS, "name,sid,dgrs,mills");
conf.set(serdeConstants.LIST_COLUMN_TYPES, "string,int,double,bigint");
con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes()));
con.tableOperations().create(TEST_TABLE);
con.securityOperations().changeUserAuthorizations(USER, new Authorizations("blah"));
BatchWriterConfig writerConf = new BatchWriterConfig();
BatchWriter writer = con.createBatchWriter(TEST_TABLE, writerConf);
Mutation m1 = new Mutation(new Text("r1"));
m1.put(COLUMN_FAMILY, NAME, new Value("brian".getBytes()));
m1.put(COLUMN_FAMILY, SID, new Value(parseIntBytes("1")));
m1.put(COLUMN_FAMILY, DEGREES, new Value(parseDoubleBytes("44.5")));
m1.put(COLUMN_FAMILY, MILLIS, new Value(parseLongBytes("555")));
Mutation m2 = new Mutation(new Text("r2"));
m2.put(COLUMN_FAMILY, NAME, new Value("mark".getBytes()));
m2.put(COLUMN_FAMILY, SID, new Value(parseIntBytes("2")));
m2.put(COLUMN_FAMILY, DEGREES, new Value(parseDoubleBytes("55.5")));
m2.put(COLUMN_FAMILY, MILLIS, new Value(parseLongBytes("666")));
Mutation m3 = new Mutation(new Text("r3"));
m3.put(COLUMN_FAMILY, NAME, new Value("dennis".getBytes()));
m3.put(COLUMN_FAMILY, SID, new Value(parseIntBytes("3")));
m3.put(COLUMN_FAMILY, DEGREES, new Value(parseDoubleBytes("65.5")));
m3.put(COLUMN_FAMILY, MILLIS, new Value(parseLongBytes("777")));
writer.addMutation(m1);
writer.addMutation(m2);
writer.addMutation(m3);
writer.close();
}
@Test
public void tabletShouldSplit() throws Exception {
Connector c = getConnector();
c.tableOperations().create("test_ingest");
c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "256K");
c.tableOperations()
.setProperty("test_ingest", Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1K");
TestIngest.Opts opts = new TestIngest.Opts();
opts.rows = 100000;
TestIngest.ingest(c, opts, new BatchWriterOpts());
VerifyIngest.Opts vopts = new VerifyIngest.Opts();
vopts.rows = opts.rows;
VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
UtilWaitThread.sleep(15 * 1000);
String id = c.tableOperations().tableIdMap().get("test_ingest");
Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
KeyExtent extent = new KeyExtent(new Text(id), null, null);
s.setRange(extent.toMetadataRange());
MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(s);
int count = 0;
int shortened = 0;
for (Entry<Key, Value> entry : s) {
extent = new KeyExtent(entry.getKey().getRow(), entry.getValue());
if (extent.getEndRow() != null && extent.getEndRow().toString().length() < 14) shortened++;
count++;
}
assertTrue(shortened > 0);
assertTrue(count > 10);
assertEquals(
0,
cluster
.exec(
CheckForMetadataProblems.class,
"-i",
cluster.getInstanceName(),
"-u",
"root",
"-p",
ROOT_PASSWORD,
"-z",
cluster.getZooKeepers())
.waitFor());
}
@Test
public void aggregationTest() throws Exception {
Connector c = getConnector();
String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
IteratorSetting setting = new IteratorSetting(10, SummingCombiner.class);
SummingCombiner.setEncodingType(setting, Type.STRING);
SummingCombiner.setColumns(
setting, Collections.singletonList(new IteratorSetting.Column("cf")));
c.tableOperations().attachIterator(tableName, setting);
BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
for (int i = 0; i < 10; i++) {
Mutation m = new Mutation("row1");
m.put("cf".getBytes(), "col1".getBytes(), ("" + i).getBytes());
bw.addMutation(m);
}
bw.close();
checkSum(tableName, c);
}
@Override
public void visit(State state, Properties props) throws Exception {
Connector conn = state.getConnector();
Random rand = (Random) state.get("rand");
@SuppressWarnings("unchecked")
List<String> tableNames = (List<String>) state.get("tables");
String tableName = tableNames.get(rand.nextInt(tableNames.size()));
Configuration conf = CachedConfiguration.getInstance();
FileSystem fs = FileSystem.get(conf);
String bulkDir = "/tmp/concurrent_bulk/b_" + String.format("%016x", Math.abs(rand.nextLong()));
fs.mkdirs(new Path(bulkDir));
fs.mkdirs(new Path(bulkDir + "_f"));
try {
BatchWriter bw = new RFileBatchWriter(conf, fs, bulkDir + "/file01.rf");
try {
TreeSet<Long> rows = new TreeSet<Long>();
int numRows = rand.nextInt(100000);
for (int i = 0; i < numRows; i++) {
rows.add(Math.abs(rand.nextLong()));
}
for (Long row : rows) {
Mutation m = new Mutation(String.format("%016x", row));
long val = Math.abs(rand.nextLong());
for (int j = 0; j < 10; j++) {
m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes()));
}
bw.addMutation(m);
}
} finally {
bw.close();
}
conn.tableOperations()
.importDirectory(tableName, bulkDir, bulkDir + "_f", rand.nextBoolean());
log.debug("BulkImported to " + tableName);
} catch (TableNotFoundException e) {
log.debug("BulkImport " + tableName + " failed, doesnt exist");
} catch (TableOfflineException toe) {
log.debug("BulkImport " + tableName + " failed, offline");
} finally {
fs.delete(new Path(bulkDir), true);
fs.delete(new Path(bulkDir + "_f"), true);
}
}
@Override
public void teardown() {
Connector conn = getConnector();
String tableName = getTestProperty("TABLE");
try {
conn.tableOperations().delete(tableName);
} catch (Exception e) {
log.error("Failed to delete table '" + tableName + "'", e);
}
}
public static void bulkImport(Connector c, FileSystem fs, String table, String dir)
throws Exception {
String failDir = dir + "_failures";
Path failPath = new Path(failDir);
fs.delete(failPath, true);
fs.mkdirs(failPath);
c.tableOperations().importDirectory(table, dir, failDir, false);
if (fs.listStatus(failPath).length > 0) {
throw new Exception("Some files failed to bulk import");
}
}
public static void checkSplits(Connector c, String table, int min, int max) throws Exception {
Collection<Text> splits = c.tableOperations().listSplits(table);
if (splits.size() < min || splits.size() > max) {
throw new Exception(
"# of table splits points out of range, #splits="
+ splits.size()
+ " table="
+ table
+ " min="
+ min
+ " max="
+ max);
}
}
