public static void run(
String instanceName, String zookeepers, AuthenticationToken rootPassword, String args[])
throws Exception {
// edit this method to play with Accumulo
Instance instance = new ZooKeeperInstance(instanceName, zookeepers);
Connector conn = instance.getConnector("root", rootPassword);
conn.tableOperations().create("foo");
BatchWriterConfig bwConfig = new BatchWriterConfig();
bwConfig.setMaxLatency(60000l, java.util.concurrent.TimeUnit.MILLISECONDS);
bwConfig.setMaxWriteThreads(3);
bwConfig.setMaxMemory(50000000);
BatchWriter bw = conn.createBatchWriter("foo", bwConfig);
Mutation m = new Mutation("r1");
m.put("cf1", "cq1", "v1");
m.put("cf1", "cq2", "v3");
bw.addMutation(m);
bw.close();
Scanner scanner = conn.createScanner("foo", Constants.NO_AUTHS);
for (Entry<Key, Value> entry : scanner) {
System.out.println(entry.getKey() + " " + entry.getValue());
}
}
@Override
public void setup() {
Connector conn = getConnector();
String tableName = getTestProperty("TABLE");
// delete existing table
if (conn.tableOperations().exists(tableName)) {
System.out.println("Deleting existing table: " + tableName);
try {
conn.tableOperations().delete(tableName);
} catch (Exception e) {
log.error("Failed to delete table '" + tableName + "'.", e);
}
}
// create table
try {
conn.tableOperations().create(tableName);
conn.tableOperations().addSplits(tableName, calculateSplits());
conn.tableOperations().setProperty(tableName, "table.split.threshold", "256M");
} catch (Exception e) {
log.error("Failed to create table '" + tableName + "'.", e);
}
}
@Override
public void tearDown(State state, Environment env) throws Exception {
// We have resources we need to clean up
if (env.isMultiTableBatchWriterInitialized()) {
MultiTableBatchWriter mtbw = env.getMultiTableBatchWriter();
try {
mtbw.close();
} catch (MutationsRejectedException e) {
log.error("Ignoring mutations that weren't flushed", e);
}
// Reset the MTBW on the state to null
env.resetMultiTableBatchWriter();
}
// Now we can safely delete the tables
log.debug("Dropping tables: " + imageTableName + " " + indexTableName);
Connector conn = env.getConnector();
conn.tableOperations().delete(imageTableName);
conn.tableOperations().delete(indexTableName);
log.debug("Final total of writes: " + state.getLong("totalWrites"));
}
public void addTable(Text tableName) throws AccumuloException, AccumuloSecurityException {
if (simulate) {
log.info("Simulating adding table: " + tableName);
return;
}
log.debug("Adding table: " + tableName);
BatchWriter bw = null;
String table = tableName.toString();
if (createTables && !conn.tableOperations().exists(table)) {
try {
conn.tableOperations().create(table);
} catch (AccumuloSecurityException e) {
log.error("Accumulo security violation creating " + table, e);
throw e;
} catch (TableExistsException e) {
// Shouldn't happen
}
}
try {
bw = mtbw.getBatchWriter(table);
} catch (TableNotFoundException e) {
log.error("Accumulo table " + table + " doesn't exist and cannot be created.", e);
throw new AccumuloException(e);
} catch (AccumuloException e) {
throw e;
} catch (AccumuloSecurityException e) {
throw e;
}
if (bw != null) bws.put(tableName, bw);
}
@Override
public void visit(State state, Properties props) throws Exception {
Connector conn = state.getConnector();
Random rand = (Random) state.get("rand");
@SuppressWarnings("unchecked")
List<String> tableNames = (List<String>) state.get("tables");
String tableName = tableNames.get(rand.nextInt(tableNames.size()));
// TODO need to sometimes do null start and end ranges
TreeSet<Text> range = new TreeSet<Text>();
range.add(new Text(String.format("%016x", Math.abs(rand.nextLong()))));
range.add(new Text(String.format("%016x", Math.abs(rand.nextLong()))));
try {
boolean wait = rand.nextBoolean();
conn.tableOperations().compact(tableName, range.first(), range.last(), false, wait);
log.debug((wait ? "compacted " : "initiated compaction ") + tableName);
} catch (TableNotFoundException tne) {
log.debug("compact " + tableName + " failed, doesnt exist");
} catch (TableOfflineException toe) {
log.debug("compact " + tableName + " failed, offline");
}
}
@Before
public void init()
throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
TableExistsException {
mock = new MockInstance("accumulo");
PasswordToken pToken = new PasswordToken("pass".getBytes());
conn = mock.getConnector("user", pToken);
config = new BatchWriterConfig();
config.setMaxMemory(1000);
config.setMaxLatency(1000, TimeUnit.SECONDS);
config.setMaxWriteThreads(10);
if (conn.tableOperations().exists("rya_prospects")) {
conn.tableOperations().delete("rya_prospects");
}
if (conn.tableOperations().exists("rya_selectivity")) {
conn.tableOperations().delete("rya_selectivity");
}
arc = new AccumuloRdfConfiguration();
arc.setTableLayoutStrategy(new TablePrefixLayoutStrategy());
arc.setMaxRangesForScanner(300);
res = new ProspectorServiceEvalStatsDAO(conn, arc);
}
@Override
public void visit(State state, Properties props) throws Exception {
Connector conn = state.getConnector();
Random rand = (Random) state.get("rand");
@SuppressWarnings("unchecked")
List<String> tableNames = (List<String>) state.get("tables");
String tableName = tableNames.get(rand.nextInt(tableNames.size()));
try {
Scanner scanner = conn.createScanner(tableName, Constants.NO_AUTHS);
Iterator<Entry<Key, Value>> iter = scanner.iterator();
while (iter.hasNext()) {
iter.next();
}
log.debug("Scanned " + tableName);
} catch (TableDeletedException e) {
log.debug("Scan " + tableName + " failed, table deleted");
} catch (TableNotFoundException e) {
log.debug("Scan " + tableName + " failed, doesnt exist");
} catch (TableOfflineException e) {
log.debug("Scan " + tableName + " failed, offline");
} catch (RuntimeException e) {
if (e.getCause() instanceof AccumuloSecurityException) {
log.debug("BatchScan " + tableName + " failed, permission error");
} else {
throw e;
}
}
}
@Test
public void test() throws Exception {
Connector c = getConnector();
// make a table
String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
// write to it
BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
Mutation m = new Mutation("row");
m.put("cf", "cq", "value");
bw.addMutation(m);
bw.close();
// create a fake _tmp file in its directory
String id = c.tableOperations().tableIdMap().get(tableName);
FileSystem fs = getCluster().getFileSystem();
Path tmp = new Path("/accumulo/tables/" + id + "/default_tablet/junk.rf_tmp");
fs.create(tmp).close();
for (ProcessReference tserver : getCluster().getProcesses().get(ServerType.TABLET_SERVER)) {
getCluster().killProcess(ServerType.TABLET_SERVER, tserver);
}
getCluster().start();
Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
FunctionalTestUtils.count(scanner);
assertFalse(fs.exists(tmp));
}
@Test(timeout = 2 * 60 * 1000)
public void test() throws Exception {
Connector c = getConnector();
c.tableOperations().create("test_ingest");
deleteTest(c, cluster);
assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
}
/**
* Sets the connector information needed to communicate with Accumulo in this job.
*
* <p><b>WARNING:</b> Some tokens, when serialized, divulge sensitive information in the
* configuration as a means to pass the token to MapReduce tasks. This information is BASE64
* encoded to provide a charset safe conversion to a string, but this conversion is not intended
* to be secure. {@link PasswordToken} is one example that is insecure in this way; however {@link
* DelegationToken}s, acquired using {@link
* SecurityOperations#getDelegationToken(DelegationTokenConfig)}, is not subject to this concern.
*
* @param job the Hadoop job instance to be configured
* @param principal a valid Accumulo user name (user must have Table.CREATE permission)
* @param token the user's password
* @since 1.5.0
*/
public static void setConnectorInfo(JobConf job, String principal, AuthenticationToken token)
throws AccumuloSecurityException {
if (token instanceof KerberosToken) {
log.info("Received KerberosToken, attempting to fetch DelegationToken");
try {
Instance instance = getInstance(job);
Connector conn = instance.getConnector(principal, token);
token = conn.securityOperations().getDelegationToken(new DelegationTokenConfig());
} catch (Exception e) {
log.warn(
"Failed to automatically obtain DelegationToken, Mappers/Reducers will likely fail to communicate with Accumulo",
e);
}
}
// DelegationTokens can be passed securely from user to task without serializing insecurely in
// the configuration
if (token instanceof DelegationTokenImpl) {
DelegationTokenImpl delegationToken = (DelegationTokenImpl) token;
// Convert it into a Hadoop Token
AuthenticationTokenIdentifier identifier = delegationToken.getIdentifier();
Token<AuthenticationTokenIdentifier> hadoopToken =
new Token<>(
identifier.getBytes(),
delegationToken.getPassword(),
identifier.getKind(),
delegationToken.getServiceName());
// Add the Hadoop Token to the Job so it gets serialized and passed along.
job.getCredentials().addToken(hadoopToken.getService(), hadoopToken);
}
InputConfigurator.setConnectorInfo(CLASS, job, principal, token);
}
@Test
public void test() throws Exception {
Connector c = getConnector();
String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
Mutation m = new Mutation("row1");
m.put("cf", "col1", "Test");
bw.addMutation(m);
bw.close();
scanCheck(c, tableName, "Test");
FileSystem fs = getCluster().getFileSystem();
Path jarPath = new Path(rootPath + "/lib/ext/Test.jar");
copyStreamToFileSystem(fs, "/TestCombinerX.jar", jarPath);
sleepUninterruptibly(1, TimeUnit.SECONDS);
IteratorSetting is =
new IteratorSetting(10, "TestCombiner", "org.apache.accumulo.test.functional.TestCombiner");
Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf")));
c.tableOperations().attachIterator(tableName, is, EnumSet.of(IteratorScope.scan));
sleepUninterruptibly(ZOOKEEPER_PROPAGATION_TIME, TimeUnit.MILLISECONDS);
scanCheck(c, tableName, "TestX");
fs.delete(jarPath, true);
copyStreamToFileSystem(fs, "/TestCombinerY.jar", jarPath);
sleepUninterruptibly(5, TimeUnit.SECONDS);
scanCheck(c, tableName, "TestY");
fs.delete(jarPath, true);
}
private static void runTest(
Connector conn, int numlg, ArrayList<byte[]> cfset, Map<String, Stat> stats)
throws Exception {
String table = "immlgb";
try {
conn.tableOperations().delete(table);
} catch (TableNotFoundException tnfe) {
}
conn.tableOperations().create(table);
conn.tableOperations()
.setProperty(table, Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "snappy");
setupLocalityGroups(conn, numlg, cfset, table);
addStat(stats, "write", write(conn, cfset, table));
addStat(stats, "scan cf", scan(conn, cfset, table, false));
addStat(stats, "scan cf:cq", scan(conn, cfset, table, true));
// TODO time reading all data
long t1 = System.currentTimeMillis();
conn.tableOperations().flush(table, null, null, true);
long t2 = System.currentTimeMillis();
addStat(stats, "flush", t2 - t1);
}
@Override
public int run(String[] args) {
Opts opts = new Opts();
opts.parseArgs(BulkIngestExample.class.getName(), args);
Configuration conf = getConf();
PrintStream out = null;
try {
Job job = JobUtil.getJob(conf);
job.setJobName("bulk ingest example");
job.setJarByClass(this.getClass());
job.setInputFormatClass(TextInputFormat.class);
job.setMapperClass(MapClass.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
job.setReducerClass(ReduceClass.class);
job.setOutputFormatClass(AccumuloFileOutputFormat.class);
opts.setAccumuloConfigs(job);
Connector connector = opts.getConnector();
TextInputFormat.setInputPaths(job, new Path(opts.inputDir));
AccumuloFileOutputFormat.setOutputPath(job, new Path(opts.workDir + "/files"));
FileSystem fs = FileSystem.get(conf);
out =
new PrintStream(
new BufferedOutputStream(fs.create(new Path(opts.workDir + "/splits.txt"))));
Collection<Text> splits = connector.tableOperations().listSplits(opts.getTableName(), 100);
for (Text split : splits)
out.println(new String(Base64.encodeBase64(TextUtil.getBytes(split))));
job.setNumReduceTasks(splits.size() + 1);
out.close();
job.setPartitionerClass(RangePartitioner.class);
RangePartitioner.setSplitFile(job, opts.workDir + "/splits.txt");
job.waitForCompletion(true);
Path failures = new Path(opts.workDir, "failures");
fs.delete(failures, true);
fs.mkdirs(new Path(opts.workDir, "failures"));
connector
.tableOperations()
.importDirectory(
opts.getTableName(), opts.workDir + "/files", opts.workDir + "/failures", false);
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (out != null) out.close();
}
return 0;
}
@Test
public void testBulkSplitOptimization() throws Exception {
final Connector c = getConnector();
final String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "1000");
c.tableOperations().setProperty(tableName, Property.TABLE_FILE_MAX.getKey(), "1000");
c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "1G");
FileSystem fs = cluster.getFileSystem();
Path testDir = new Path(getUsableDir(), "testmf");
FunctionalTestUtils.createRFiles(c, fs, testDir.toString(), ROWS, SPLITS, 8);
FileStatus[] stats = fs.listStatus(testDir);
System.out.println("Number of generated files: " + stats.length);
FunctionalTestUtils.bulkImport(c, fs, tableName, testDir.toString());
FunctionalTestUtils.checkSplits(c, tableName, 0, 0);
FunctionalTestUtils.checkRFiles(c, tableName, 1, 1, 100, 100);
// initiate splits
getConnector()
.tableOperations()
.setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "100K");
sleepUninterruptibly(2, TimeUnit.SECONDS);
// wait until over split threshold -- should be 78 splits
while (getConnector().tableOperations().listSplits(tableName).size() < 75) {
sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
}
FunctionalTestUtils.checkSplits(c, tableName, 50, 100);
VerifyIngest.Opts opts = new VerifyIngest.Opts();
opts.timestamp = 1;
opts.dataSize = 50;
opts.random = 56;
opts.rows = 100000;
opts.startRow = 0;
opts.cols = 1;
opts.setTableName(tableName);
AuthenticationToken adminToken = getAdminToken();
if (adminToken instanceof PasswordToken) {
PasswordToken token = (PasswordToken) getAdminToken();
opts.setPassword(new Password(new String(token.getPassword(), UTF_8)));
opts.setPrincipal(getAdminPrincipal());
} else if (adminToken instanceof KerberosToken) {
ClientConfiguration clientConf = cluster.getClientConfig();
opts.updateKerberosCredentials(clientConf);
} else {
Assert.fail("Unknown token type");
}
VerifyIngest.verifyIngest(c, opts, new ScannerOpts());
// ensure each tablet does not have all map files, should be ~2.5 files per tablet
FunctionalTestUtils.checkRFiles(c, tableName, 50, 100, 1, 4);
}
@After
public void resetConfig() throws Exception {
if (null != majcDelay) {
Connector conn = getConnector();
conn.instanceOperations().setProperty(Property.TSERV_MAJC_DELAY.getKey(), majcDelay);
getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
getClusterControl().startAllServers(ServerType.TABLET_SERVER);
}
}
@Test
public void run() throws Exception {
Connector c = getConnector();
String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
for (int i = 0; i < 100000; i++) {
c.createScanner(tableName, Authorizations.EMPTY);
}
}
@Before
public void createMockKeyValues() throws Exception {
// Make a MockInstance here, by setting the instance name to be the same as this mock instance
// we can "trick" the InputFormat into using a MockInstance
mockInstance = new MockInstance(test.getMethodName());
inputformat = new HiveAccumuloTableInputFormat();
conf = new JobConf();
conf.set(AccumuloSerDeParameters.TABLE_NAME, TEST_TABLE);
conf.set(AccumuloSerDeParameters.USE_MOCK_INSTANCE, "true");
conf.set(AccumuloSerDeParameters.INSTANCE_NAME, test.getMethodName());
conf.set(AccumuloSerDeParameters.USER_NAME, USER);
conf.set(AccumuloSerDeParameters.USER_PASS, PASS);
conf.set(AccumuloSerDeParameters.ZOOKEEPERS, "localhost:2181"); // not used for mock, but
// required by input format.
columnNames = Arrays.asList("name", "sid", "dgrs", "mills");
columnTypes =
Arrays.<TypeInfo>asList(
TypeInfoFactory.stringTypeInfo,
TypeInfoFactory.intTypeInfo,
TypeInfoFactory.doubleTypeInfo,
TypeInfoFactory.longTypeInfo);
conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:name,cf:sid,cf:dgrs,cf:mills");
conf.set(serdeConstants.LIST_COLUMNS, "name,sid,dgrs,mills");
conf.set(serdeConstants.LIST_COLUMN_TYPES, "string,int,double,bigint");
con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes()));
con.tableOperations().create(TEST_TABLE);
con.securityOperations().changeUserAuthorizations(USER, new Authorizations("blah"));
BatchWriterConfig writerConf = new BatchWriterConfig();
BatchWriter writer = con.createBatchWriter(TEST_TABLE, writerConf);
Mutation m1 = new Mutation(new Text("r1"));
m1.put(COLUMN_FAMILY, NAME, new Value("brian".getBytes()));
m1.put(COLUMN_FAMILY, SID, new Value(parseIntBytes("1")));
m1.put(COLUMN_FAMILY, DEGREES, new Value(parseDoubleBytes("44.5")));
m1.put(COLUMN_FAMILY, MILLIS, new Value(parseLongBytes("555")));
Mutation m2 = new Mutation(new Text("r2"));
m2.put(COLUMN_FAMILY, NAME, new Value("mark".getBytes()));
m2.put(COLUMN_FAMILY, SID, new Value(parseIntBytes("2")));
m2.put(COLUMN_FAMILY, DEGREES, new Value(parseDoubleBytes("55.5")));
m2.put(COLUMN_FAMILY, MILLIS, new Value(parseLongBytes("666")));
Mutation m3 = new Mutation(new Text("r3"));
m3.put(COLUMN_FAMILY, NAME, new Value("dennis".getBytes()));
m3.put(COLUMN_FAMILY, SID, new Value(parseIntBytes("3")));
m3.put(COLUMN_FAMILY, DEGREES, new Value(parseDoubleBytes("65.5")));
m3.put(COLUMN_FAMILY, MILLIS, new Value(parseLongBytes("777")));
writer.addMutation(m1);
writer.addMutation(m2);
writer.addMutation(m3);
writer.close();
}
@Override
public Connector getConnector(String principal, AuthenticationToken token)
throws AccumuloException, AccumuloSecurityException {
Connector conn = new MockConnector(new Credentials(principal, token), acu, this);
if (!acu.users.containsKey(principal))
conn.securityOperations().createLocalUser(principal, (PasswordToken) token);
else if (!acu.users.get(principal).token.equals(token))
throw new AccumuloSecurityException(principal, SecurityErrorCode.BAD_CREDENTIALS);
return conn;
}
/** The run method which sets the configuration and starts the MapReduce job */
public int run(String[] args) throws Exception {
if (USE_MINI_ACCUMULO) {
Connector connector = LocalEnvUtil.getConnector(userPass);
userName = "******";
instanceName = connector.getInstance().getInstanceName();
zookeepers = connector.getInstance().getZooKeepers();
}
// Create and initialize a MapReduce Job
Job job = Job.getInstance(getConf(), "tweetIndexer");
job.setJarByClass(IndexedDocIndexer.class);
// Set the AccumuloInputFormat so the mapper can read from Accumulo
AccumuloInputFormat.setConnectorInfo(job, userName, new PasswordToken(userPass));
AccumuloInputFormat.setInputTableName(job, twitterDataTable);
AccumuloInputFormat.setScanAuthorizations(job, new Authorizations());
ClientConfiguration clientConfig = new ClientConfiguration();
clientConfig.withInstance(instanceName);
clientConfig.withZkHosts(zookeepers);
AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
AccumuloOutputFormat.setConnectorInfo(job, userName, new PasswordToken(userPass));
AccumuloOutputFormat.setCreateTables(job, createTables);
AccumuloOutputFormat.setDefaultTableName(job, tweetDocIndex);
AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig);
// Set the map and reduce classes
job.setMapperClass(TweetMapper.class);
job.setReducerClass(TweetReducer.class);
// Set the output key and value class for the mapper
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
// Set the output key and value class for the reducer
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Mutation.class);
// Set the InputFormat and OutputFormat for the job
job.setInputFormatClass(AccumuloInputFormat.class);
job.setOutputFormatClass(AccumuloOutputFormat.class);
// Run the MapReduce job and return 0 for success, 1 otherwise
return job.waitForCompletion(true) ? 0 : 1;
}
@Test
public void interleaveSplit() throws Exception {
Connector c = getConnector();
c.tableOperations().create("test_ingest");
c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
c.tableOperations()
.setProperty("test_ingest", Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none");
ReadWriteIT.interleaveTest(c);
UtilWaitThread.sleep(5 * 1000);
assertTrue(c.tableOperations().listSplits("test_ingest").size() > 20);
}
@Before
public void alterConfig() throws Exception {
Connector conn = getConnector();
majcDelay =
conn.instanceOperations().getSystemConfiguration().get(Property.TSERV_MAJC_DELAY.getKey());
if (!"1s".equals(majcDelay)) {
conn.instanceOperations().setProperty(Property.TSERV_MAJC_DELAY.getKey(), "1s");
getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
getClusterControl().startAllServers(ServerType.TABLET_SERVER);
}
}
@Override
public void visit(State state, Properties props) throws Exception {
Connector conn = state.getConnector();
Random rand = (Random) state.get("rand");
@SuppressWarnings("unchecked")
List<String> tableNames = (List<String>) state.get("tables");
String tableName = tableNames.get(rand.nextInt(tableNames.size()));
Configuration conf = CachedConfiguration.getInstance();
FileSystem fs = FileSystem.get(conf);
String bulkDir = "/tmp/concurrent_bulk/b_" + String.format("%016x", Math.abs(rand.nextLong()));
fs.mkdirs(new Path(bulkDir));
fs.mkdirs(new Path(bulkDir + "_f"));
try {
BatchWriter bw = new RFileBatchWriter(conf, fs, bulkDir + "/file01.rf");
try {
TreeSet<Long> rows = new TreeSet<Long>();
int numRows = rand.nextInt(100000);
for (int i = 0; i < numRows; i++) {
rows.add(Math.abs(rand.nextLong()));
}
for (Long row : rows) {
Mutation m = new Mutation(String.format("%016x", row));
long val = Math.abs(rand.nextLong());
for (int j = 0; j < 10; j++) {
m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes()));
}
bw.addMutation(m);
}
} finally {
bw.close();
}
conn.tableOperations()
.importDirectory(tableName, bulkDir, bulkDir + "_f", rand.nextBoolean());
log.debug("BulkImported to " + tableName);
} catch (TableNotFoundException e) {
log.debug("BulkImport " + tableName + " failed, doesnt exist");
} catch (TableOfflineException toe) {
log.debug("BulkImport " + tableName + " failed, offline");
} finally {
fs.delete(new Path(bulkDir), true);
fs.delete(new Path(bulkDir + "_f"), true);
}
}
@Override
public void teardown() {
Connector conn = getConnector();
String tableName = getTestProperty("TABLE");
try {
conn.tableOperations().delete(tableName);
} catch (Exception e) {
log.error("Failed to delete table '" + tableName + "'", e);
}
}
@Test(timeout = 4 * 60 * 1000)
public void binaryStressTest() throws Exception {
Connector c = getConnector();
c.tableOperations().create("bt");
c.tableOperations().setProperty("bt", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
BinaryIT.runTest(c);
String id = c.tableOperations().tableIdMap().get("bt");
FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
FileStatus[] dir =
fs.listStatus(new Path(cluster.getConfig().getDir() + "/accumulo/tables/" + id));
assertTrue(dir.length > 7);
}
@Test
public void testGetProtectedField() throws Exception {
FileInputFormat.addInputPath(conf, new Path("unused"));
BatchWriterConfig writerConf = new BatchWriterConfig();
BatchWriter writer = con.createBatchWriter(TEST_TABLE, writerConf);
Authorizations origAuths = con.securityOperations().getUserAuthorizations(USER);
con.securityOperations()
.changeUserAuthorizations(USER, new Authorizations(origAuths.toString() + ",foo"));
Mutation m = new Mutation("r4");
m.put(COLUMN_FAMILY, NAME, new ColumnVisibility("foo"), new Value("frank".getBytes()));
m.put(COLUMN_FAMILY, SID, new ColumnVisibility("foo"), new Value(parseIntBytes("4")));
m.put(COLUMN_FAMILY, DEGREES, new ColumnVisibility("foo"), new Value(parseDoubleBytes("60.6")));
m.put(COLUMN_FAMILY, MILLIS, new ColumnVisibility("foo"), new Value(parseLongBytes("777")));
writer.addMutation(m);
writer.close();
conf.set(AccumuloSerDeParameters.AUTHORIZATIONS_KEY, "foo");
InputSplit[] splits = inputformat.getSplits(conf, 0);
assertEquals(splits.length, 1);
RecordReader<Text, AccumuloHiveRow> reader = inputformat.getRecordReader(splits[0], conf, null);
Text rowId = new Text("r1");
AccumuloHiveRow row = new AccumuloHiveRow();
assertTrue(reader.next(rowId, row));
assertEquals(row.getRowId(), rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY, NAME), "brian".getBytes());
rowId = new Text("r2");
assertTrue(reader.next(rowId, row));
assertEquals(row.getRowId(), rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY, NAME), "mark".getBytes());
rowId = new Text("r3");
assertTrue(reader.next(rowId, row));
assertEquals(row.getRowId(), rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY, NAME), "dennis".getBytes());
rowId = new Text("r4");
assertTrue(reader.next(rowId, row));
assertEquals(row.getRowId(), rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY, NAME), "frank".getBytes());
assertFalse(reader.next(rowId, row));
}
@Test
public void deleteSplit() throws Exception {
Connector c = getConnector();
c.tableOperations().create("test_ingest");
c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
DeleteIT.deleteTest(c, cluster);
c.tableOperations().flush("test_ingest", null, null, true);
for (int i = 0; i < 5; i++) {
UtilWaitThread.sleep(10 * 1000);
if (c.tableOperations().listSplits("test_ingest").size() > 20) break;
}
assertTrue(c.tableOperations().listSplits("test_ingest").size() > 20);
}
/** Write entries to a table. */
public static void writeEntries(
Connector connector, Map<Key, Value> map, String table, boolean createIfNotExist) {
if (createIfNotExist && !connector.tableOperations().exists(table))
try {
connector.tableOperations().create(table);
} catch (AccumuloException | AccumuloSecurityException e) {
log.error("trouble creating " + table, e);
throw new RuntimeException(e);
} catch (TableExistsException e) {
log.error("crazy", e);
throw new RuntimeException(e);
}
BatchWriterConfig bwc = new BatchWriterConfig();
BatchWriter bw;
try {
bw = connector.createBatchWriter(table, bwc);
} catch (TableNotFoundException e) {
log.error("tried to write to a non-existant table " + table, e);
throw new RuntimeException(e);
}
try {
for (Map.Entry<Key, Value> entry : map.entrySet()) {
Key k = entry.getKey();
ByteSequence rowData = k.getRowData(),
cfData = k.getColumnFamilyData(),
cqData = k.getColumnQualifierData();
Mutation m = new Mutation(rowData.getBackingArray(), rowData.offset(), rowData.length());
m.put(
cfData.getBackingArray(),
cqData.getBackingArray(),
k.getColumnVisibilityParsed(),
entry.getValue().get());
bw.addMutation(m);
}
} catch (MutationsRejectedException e) {
log.error("mutations rejected", e);
throw new RuntimeException(e);
} finally {
try {
bw.close();
} catch (MutationsRejectedException e) {
log.error("mutations rejected while trying to close BatchWriter", e);
}
}
}
/**
* Insert the contents of an NDSI data file into an Accumulo table.
*
* @param file Input file
* @param Atable Table to insert into
* @return Nummber of entries inserted into Atable; equal to 2x the number of rows.
* @throws IOException
*/
public long ingestFile(File file, String Atable, boolean deleteIfExists) throws IOException {
if (deleteIfExists && connector.tableOperations().exists(Atable))
try {
connector.tableOperations().delete(Atable);
} catch (AccumuloException | AccumuloSecurityException e) {
log.warn("trouble deleting table " + Atable, e);
throw new RuntimeException(e);
} catch (TableNotFoundException e) {
throw new RuntimeException(e);
}
if (!connector.tableOperations().exists(Atable))
try {
connector.tableOperations().create(Atable);
} catch (AccumuloException | AccumuloSecurityException e) {
log.warn("trouble creating table " + Atable, e);
throw new RuntimeException(e);
} catch (TableExistsException e) {
throw new RuntimeException(e);
}
BatchWriter bw = null;
String line = null;
long entriesProcessed = 0;
try (BufferedReader fo = new BufferedReader(new FileReader(file))) {
BatchWriterConfig config = new BatchWriterConfig();
bw = connector.createBatchWriter(Atable, config);
// Skip header line
fo.readLine();
while ((line = fo.readLine()) != null)
if (!line.isEmpty()) entriesProcessed += ingestLine(bw, line);
} catch (TableNotFoundException e) {
throw new RuntimeException(e);
} catch (MutationsRejectedException e) {
log.warn("Mutation rejected on line " + line, e);
} finally {
if (bw != null)
try {
bw.close();
} catch (MutationsRejectedException e) {
log.warn("Mutation rejected at close() on line " + line, e);
}
}
return entriesProcessed;
}
private static long write(Connector conn, ArrayList<byte[]> cfset, String table)
throws TableNotFoundException, MutationsRejectedException {
Random rand = new Random();
byte val[] = new byte[50];
BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
long t1 = System.currentTimeMillis();
for (int i = 0; i < 1 << 15; i++) {
byte[] row = FastFormat.toZeroPaddedString(abs(rand.nextLong()), 16, 16, new byte[0]);
Mutation m = new Mutation(row);
for (byte[] cf : cfset) {
byte[] cq = FastFormat.toZeroPaddedString(rand.nextInt(1 << 16), 4, 16, new byte[0]);
rand.nextBytes(val);
m.put(cf, cq, val);
}
bw.addMutation(m);
}
bw.close();
long t2 = System.currentTimeMillis();
return t2 - t1;
}
private void setupTestUsers(
final Connector accumuloConn, final String ryaInstanceName, final String pcjId)
throws AccumuloException, AccumuloSecurityException {
final PasswordToken pass = new PasswordToken("password");
final SecurityOperations secOps = accumuloConn.securityOperations();
// XXX We need the table name so that we can update security for the users.
final String pcjTableName = new PcjTableNameFactory().makeTableName(ryaInstanceName, pcjId);
// Give the 'roor' user authorizations to see everything.
secOps.changeUserAuthorizations("root", new Authorizations("A", "B", "C", "D", "E"));
// Create a user that can see things with A and B.
secOps.createLocalUser("abUser", pass);
secOps.changeUserAuthorizations("abUser", new Authorizations("A", "B"));
secOps.grantTablePermission("abUser", pcjTableName, TablePermission.READ);
// Create a user that can see things with A, B, and C.
secOps.createLocalUser("abcUser", pass);
secOps.changeUserAuthorizations("abcUser", new Authorizations("A", "B", "C"));
secOps.grantTablePermission("abcUser", pcjTableName, TablePermission.READ);
// Create a user that can see things with A, D, and E.
secOps.createLocalUser("adeUser", pass);
secOps.changeUserAuthorizations("adeUser", new Authorizations("A", "D", "E"));
secOps.grantTablePermission("adeUser", pcjTableName, TablePermission.READ);
// Create a user that can't see anything.
secOps.createLocalUser("noAuth", pass);
secOps.changeUserAuthorizations("noAuth", new Authorizations());
secOps.grantTablePermission("noAuth", pcjTableName, TablePermission.READ);
}
