/** The run method which sets the configuration and starts the MapReduce job */ public int run(String[] args) throws Exception { if (USE_MINI_ACCUMULO) { Connector connector = LocalEnvUtil.getConnector(userPass); userName = "******"; instanceName = connector.getInstance().getInstanceName(); zookeepers = connector.getInstance().getZooKeepers(); } // Create and initialize a MapReduce Job Job job = Job.getInstance(getConf(), "tweetIndexer"); job.setJarByClass(IndexedDocIndexer.class); // Set the AccumuloInputFormat so the mapper can read from Accumulo AccumuloInputFormat.setConnectorInfo(job, userName, new PasswordToken(userPass)); AccumuloInputFormat.setInputTableName(job, twitterDataTable); AccumuloInputFormat.setScanAuthorizations(job, new Authorizations()); ClientConfiguration clientConfig = new ClientConfiguration(); clientConfig.withInstance(instanceName); clientConfig.withZkHosts(zookeepers); AccumuloInputFormat.setZooKeeperInstance(job, clientConfig); AccumuloOutputFormat.setConnectorInfo(job, userName, new PasswordToken(userPass)); AccumuloOutputFormat.setCreateTables(job, createTables); AccumuloOutputFormat.setDefaultTableName(job, tweetDocIndex); AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig); // Set the map and reduce classes job.setMapperClass(TweetMapper.class); job.setReducerClass(TweetReducer.class); // Set the output key and value class for the mapper job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); // Set the output key and value class for the reducer job.setOutputKeyClass(Text.class); job.setOutputValueClass(Mutation.class); // Set the InputFormat and OutputFormat for the job job.setInputFormatClass(AccumuloInputFormat.class); job.setOutputFormatClass(AccumuloOutputFormat.class); // Run the MapReduce job and return 0 for success, 1 otherwise return job.waitForCompletion(true) ? 0 : 1; }
static void runTest(Connector c, MiniAccumuloClusterImpl cluster) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException, MutationsRejectedException, IOException, InterruptedException, NoSuchAlgorithmException { c.tableOperations().create(tablename); BatchWriter bw = c.createBatchWriter(tablename, new BatchWriterConfig()); for (int i = 0; i < 10; i++) { Mutation m = new Mutation("" + i); m.put(input_cf, input_cq, "row" + i); bw.addMutation(m); } bw.close(); Process hash = cluster.exec( RowHash.class, Collections.singletonList(hadoopTmpDirArg), "-i", c.getInstance().getInstanceName(), "-z", c.getInstance().getZooKeepers(), "-u", "root", "-p", ROOT_PASSWORD, "-t", tablename, "--column", input_cfcq); assertEquals(0, hash.waitFor()); Scanner s = c.createScanner(tablename, Authorizations.EMPTY); s.fetchColumn(new Text(input_cf), new Text(output_cq)); int i = 0; for (Entry<Key, Value> entry : s) { MessageDigest md = MessageDigest.getInstance("MD5"); byte[] check = Base64.encodeBase64(md.digest(("row" + i).getBytes())); assertEquals(entry.getValue().toString(), new String(check)); i++; } }
public static AccumuloConfiguration getTableConfiguration(Connector conn, String tableId) throws TableNotFoundException, AccumuloException { String tableName = Tables.getTableName(conn.getInstance(), tableId); return new ConfigurationCopy(conn.tableOperations().getProperties(tableName)); }
@Test(timeout = 5 * 60 * 1000) public void test() throws Exception { // make some tablets, spread 'em around Connector c = getConnector(); Credentials creds = new Credentials("root", new PasswordToken(ROOT_PASSWORD)); String table = this.getUniqueNames(1)[0]; c.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE); c.securityOperations().grantTablePermission("root", RootTable.NAME, TablePermission.WRITE); c.tableOperations().create(table); SortedSet<Text> partitions = new TreeSet<Text>(); for (String part : "a b c d e f g h i j k l m n o p q r s t u v w x y z".split(" ")) { partitions.add(new Text(part)); } c.tableOperations().addSplits(table, partitions); // scan the metadata table and get the two table location states Set<TServerInstance> states = new HashSet<TServerInstance>(); Set<TabletLocationState> oldLocations = new HashSet<TabletLocationState>(); MetaDataStateStore store = new MetaDataStateStore(c.getInstance(), creds, null); while (states.size() < 2) { UtilWaitThread.sleep(250); oldLocations.clear(); for (TabletLocationState tls : store) { if (tls.current != null) { states.add(tls.current); oldLocations.add(tls); } } } assertEquals(2, states.size()); // Kill a tablet server... we don't care which one... wait for everything to be reassigned cluster.killProcess( ServerType.TABLET_SERVER, cluster.getProcesses().get(ServerType.TABLET_SERVER).iterator().next()); // Find out which tablet server remains while (true) { UtilWaitThread.sleep(1000); states.clear(); boolean allAssigned = true; for (TabletLocationState tls : store) { if (tls != null && tls.current != null) { states.add(tls.current); } else { allAssigned = false; } } System.out.println(states + " size " + states.size() + " allAssigned " + allAssigned); if (states.size() != 2 && allAssigned == true) break; } assertEquals(1, states.size()); // pick an assigned tablet and assign it to the old tablet TabletLocationState moved = null; for (TabletLocationState old : oldLocations) { if (!states.contains(old.current)) { moved = old; } } assertNotEquals(null, moved); // throw a mutation in as if we were the dying tablet BatchWriter bw = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig()); Mutation assignment = new Mutation(moved.extent.getMetadataEntry()); moved.current.putLocation(assignment); bw.addMutation(assignment); bw.close(); // wait for the master to fix the problem waitForCleanStore(store); // now jam up the metadata table bw = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig()); assignment = new Mutation(new KeyExtent(new Text(MetadataTable.ID), null, null).getMetadataEntry()); moved.current.putLocation(assignment); bw.addMutation(assignment); bw.close(); waitForCleanStore(new RootTabletStateStore(c.getInstance(), creds, null)); }