/** The run method which sets the configuration and starts the MapReduce job */
public int run(String[] args) throws Exception {
if (USE_MINI_ACCUMULO) {
Connector connector = LocalEnvUtil.getConnector(userPass);
userName = "******";
instanceName = connector.getInstance().getInstanceName();
zookeepers = connector.getInstance().getZooKeepers();
}
// Create and initialize a MapReduce Job
Job job = Job.getInstance(getConf(), "tweetIndexer");
job.setJarByClass(IndexedDocIndexer.class);
// Set the AccumuloInputFormat so the mapper can read from Accumulo
AccumuloInputFormat.setConnectorInfo(job, userName, new PasswordToken(userPass));
AccumuloInputFormat.setInputTableName(job, twitterDataTable);
AccumuloInputFormat.setScanAuthorizations(job, new Authorizations());
ClientConfiguration clientConfig = new ClientConfiguration();
clientConfig.withInstance(instanceName);
clientConfig.withZkHosts(zookeepers);
AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);
AccumuloOutputFormat.setConnectorInfo(job, userName, new PasswordToken(userPass));
AccumuloOutputFormat.setCreateTables(job, createTables);
AccumuloOutputFormat.setDefaultTableName(job, tweetDocIndex);
AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig);
// Set the map and reduce classes
job.setMapperClass(TweetMapper.class);
job.setReducerClass(TweetReducer.class);
// Set the output key and value class for the mapper
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Text.class);
// Set the output key and value class for the reducer
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Mutation.class);
// Set the InputFormat and OutputFormat for the job
job.setInputFormatClass(AccumuloInputFormat.class);
job.setOutputFormatClass(AccumuloOutputFormat.class);
// Run the MapReduce job and return 0 for success, 1 otherwise
return job.waitForCompletion(true) ? 0 : 1;
}
static void runTest(Connector c, MiniAccumuloClusterImpl cluster)
throws AccumuloException, AccumuloSecurityException, TableExistsException,
TableNotFoundException, MutationsRejectedException, IOException, InterruptedException,
NoSuchAlgorithmException {
c.tableOperations().create(tablename);
BatchWriter bw = c.createBatchWriter(tablename, new BatchWriterConfig());
for (int i = 0; i < 10; i++) {
Mutation m = new Mutation("" + i);
m.put(input_cf, input_cq, "row" + i);
bw.addMutation(m);
}
bw.close();
Process hash =
cluster.exec(
RowHash.class,
Collections.singletonList(hadoopTmpDirArg),
"-i",
c.getInstance().getInstanceName(),
"-z",
c.getInstance().getZooKeepers(),
"-u",
"root",
"-p",
ROOT_PASSWORD,
"-t",
tablename,
"--column",
input_cfcq);
assertEquals(0, hash.waitFor());
Scanner s = c.createScanner(tablename, Authorizations.EMPTY);
s.fetchColumn(new Text(input_cf), new Text(output_cq));
int i = 0;
for (Entry<Key, Value> entry : s) {
MessageDigest md = MessageDigest.getInstance("MD5");
byte[] check = Base64.encodeBase64(md.digest(("row" + i).getBytes()));
assertEquals(entry.getValue().toString(), new String(check));
i++;
}
}
public static AccumuloConfiguration getTableConfiguration(Connector conn, String tableId)
throws TableNotFoundException, AccumuloException {
String tableName = Tables.getTableName(conn.getInstance(), tableId);
return new ConfigurationCopy(conn.tableOperations().getProperties(tableName));
}
@Test(timeout = 5 * 60 * 1000)
public void test() throws Exception {
// make some tablets, spread 'em around
Connector c = getConnector();
Credentials creds = new Credentials("root", new PasswordToken(ROOT_PASSWORD));
String table = this.getUniqueNames(1)[0];
c.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
c.securityOperations().grantTablePermission("root", RootTable.NAME, TablePermission.WRITE);
c.tableOperations().create(table);
SortedSet<Text> partitions = new TreeSet<Text>();
for (String part : "a b c d e f g h i j k l m n o p q r s t u v w x y z".split(" ")) {
partitions.add(new Text(part));
}
c.tableOperations().addSplits(table, partitions);
// scan the metadata table and get the two table location states
Set<TServerInstance> states = new HashSet<TServerInstance>();
Set<TabletLocationState> oldLocations = new HashSet<TabletLocationState>();
MetaDataStateStore store = new MetaDataStateStore(c.getInstance(), creds, null);
while (states.size() < 2) {
UtilWaitThread.sleep(250);
oldLocations.clear();
for (TabletLocationState tls : store) {
if (tls.current != null) {
states.add(tls.current);
oldLocations.add(tls);
}
}
}
assertEquals(2, states.size());
// Kill a tablet server... we don't care which one... wait for everything to be reassigned
cluster.killProcess(
ServerType.TABLET_SERVER,
cluster.getProcesses().get(ServerType.TABLET_SERVER).iterator().next());
// Find out which tablet server remains
while (true) {
UtilWaitThread.sleep(1000);
states.clear();
boolean allAssigned = true;
for (TabletLocationState tls : store) {
if (tls != null && tls.current != null) {
states.add(tls.current);
} else {
allAssigned = false;
}
}
System.out.println(states + " size " + states.size() + " allAssigned " + allAssigned);
if (states.size() != 2 && allAssigned == true) break;
}
assertEquals(1, states.size());
// pick an assigned tablet and assign it to the old tablet
TabletLocationState moved = null;
for (TabletLocationState old : oldLocations) {
if (!states.contains(old.current)) {
moved = old;
}
}
assertNotEquals(null, moved);
// throw a mutation in as if we were the dying tablet
BatchWriter bw = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
Mutation assignment = new Mutation(moved.extent.getMetadataEntry());
moved.current.putLocation(assignment);
bw.addMutation(assignment);
bw.close();
// wait for the master to fix the problem
waitForCleanStore(store);
// now jam up the metadata table
bw = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
assignment =
new Mutation(new KeyExtent(new Text(MetadataTable.ID), null, null).getMetadataEntry());
moved.current.putLocation(assignment);
bw.addMutation(assignment);
bw.close();
waitForCleanStore(new RootTabletStateStore(c.getInstance(), creds, null));
}
