/** The run method which sets the configuration and starts the MapReduce job */
  public int run(String[] args) throws Exception {

    if (USE_MINI_ACCUMULO) {
      Connector connector = LocalEnvUtil.getConnector(userPass);
      userName = "******";
      instanceName = connector.getInstance().getInstanceName();
      zookeepers = connector.getInstance().getZooKeepers();
    }

    // Create and initialize a MapReduce Job
    Job job = Job.getInstance(getConf(), "tweetIndexer");
    job.setJarByClass(IndexedDocIndexer.class);

    // Set the AccumuloInputFormat so the mapper can read from Accumulo

    AccumuloInputFormat.setConnectorInfo(job, userName, new PasswordToken(userPass));

    AccumuloInputFormat.setInputTableName(job, twitterDataTable);

    AccumuloInputFormat.setScanAuthorizations(job, new Authorizations());

    ClientConfiguration clientConfig = new ClientConfiguration();
    clientConfig.withInstance(instanceName);
    clientConfig.withZkHosts(zookeepers);

    AccumuloInputFormat.setZooKeeperInstance(job, clientConfig);

    AccumuloOutputFormat.setConnectorInfo(job, userName, new PasswordToken(userPass));

    AccumuloOutputFormat.setCreateTables(job, createTables);

    AccumuloOutputFormat.setDefaultTableName(job, tweetDocIndex);

    AccumuloOutputFormat.setZooKeeperInstance(job, clientConfig);

    // Set the map and reduce classes
    job.setMapperClass(TweetMapper.class);
    job.setReducerClass(TweetReducer.class);

    // Set the output key and value class for the mapper
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(Text.class);

    // Set the output key and value class for the reducer
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Mutation.class);

    // Set the InputFormat and OutputFormat for the job
    job.setInputFormatClass(AccumuloInputFormat.class);
    job.setOutputFormatClass(AccumuloOutputFormat.class);

    // Run the MapReduce job and return 0 for success, 1 otherwise
    return job.waitForCompletion(true) ? 0 : 1;
  }
예제 #2
0
  static void runTest(Connector c, MiniAccumuloClusterImpl cluster)
      throws AccumuloException, AccumuloSecurityException, TableExistsException,
          TableNotFoundException, MutationsRejectedException, IOException, InterruptedException,
          NoSuchAlgorithmException {
    c.tableOperations().create(tablename);
    BatchWriter bw = c.createBatchWriter(tablename, new BatchWriterConfig());
    for (int i = 0; i < 10; i++) {
      Mutation m = new Mutation("" + i);
      m.put(input_cf, input_cq, "row" + i);
      bw.addMutation(m);
    }
    bw.close();
    Process hash =
        cluster.exec(
            RowHash.class,
            Collections.singletonList(hadoopTmpDirArg),
            "-i",
            c.getInstance().getInstanceName(),
            "-z",
            c.getInstance().getZooKeepers(),
            "-u",
            "root",
            "-p",
            ROOT_PASSWORD,
            "-t",
            tablename,
            "--column",
            input_cfcq);
    assertEquals(0, hash.waitFor());

    Scanner s = c.createScanner(tablename, Authorizations.EMPTY);
    s.fetchColumn(new Text(input_cf), new Text(output_cq));
    int i = 0;
    for (Entry<Key, Value> entry : s) {
      MessageDigest md = MessageDigest.getInstance("MD5");
      byte[] check = Base64.encodeBase64(md.digest(("row" + i).getBytes()));
      assertEquals(entry.getValue().toString(), new String(check));
      i++;
    }
  }
예제 #3
0
 public static AccumuloConfiguration getTableConfiguration(Connector conn, String tableId)
     throws TableNotFoundException, AccumuloException {
   String tableName = Tables.getTableName(conn.getInstance(), tableId);
   return new ConfigurationCopy(conn.tableOperations().getProperties(tableName));
 }
예제 #4
0
 @Test(timeout = 5 * 60 * 1000)
 public void test() throws Exception {
   // make some tablets, spread 'em around
   Connector c = getConnector();
   Credentials creds = new Credentials("root", new PasswordToken(ROOT_PASSWORD));
   String table = this.getUniqueNames(1)[0];
   c.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
   c.securityOperations().grantTablePermission("root", RootTable.NAME, TablePermission.WRITE);
   c.tableOperations().create(table);
   SortedSet<Text> partitions = new TreeSet<Text>();
   for (String part : "a b c d e f g h i j k l m n o p q r s t u v w x y z".split(" ")) {
     partitions.add(new Text(part));
   }
   c.tableOperations().addSplits(table, partitions);
   // scan the metadata table and get the two table location states
   Set<TServerInstance> states = new HashSet<TServerInstance>();
   Set<TabletLocationState> oldLocations = new HashSet<TabletLocationState>();
   MetaDataStateStore store = new MetaDataStateStore(c.getInstance(), creds, null);
   while (states.size() < 2) {
     UtilWaitThread.sleep(250);
     oldLocations.clear();
     for (TabletLocationState tls : store) {
       if (tls.current != null) {
         states.add(tls.current);
         oldLocations.add(tls);
       }
     }
   }
   assertEquals(2, states.size());
   // Kill a tablet server... we don't care which one... wait for everything to be reassigned
   cluster.killProcess(
       ServerType.TABLET_SERVER,
       cluster.getProcesses().get(ServerType.TABLET_SERVER).iterator().next());
   // Find out which tablet server remains
   while (true) {
     UtilWaitThread.sleep(1000);
     states.clear();
     boolean allAssigned = true;
     for (TabletLocationState tls : store) {
       if (tls != null && tls.current != null) {
         states.add(tls.current);
       } else {
         allAssigned = false;
       }
     }
     System.out.println(states + " size " + states.size() + " allAssigned " + allAssigned);
     if (states.size() != 2 && allAssigned == true) break;
   }
   assertEquals(1, states.size());
   // pick an assigned tablet and assign it to the old tablet
   TabletLocationState moved = null;
   for (TabletLocationState old : oldLocations) {
     if (!states.contains(old.current)) {
       moved = old;
     }
   }
   assertNotEquals(null, moved);
   // throw a mutation in as if we were the dying tablet
   BatchWriter bw = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
   Mutation assignment = new Mutation(moved.extent.getMetadataEntry());
   moved.current.putLocation(assignment);
   bw.addMutation(assignment);
   bw.close();
   // wait for the master to fix the problem
   waitForCleanStore(store);
   // now jam up the metadata table
   bw = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
   assignment =
       new Mutation(new KeyExtent(new Text(MetadataTable.ID), null, null).getMetadataEntry());
   moved.current.putLocation(assignment);
   bw.addMutation(assignment);
   bw.close();
   waitForCleanStore(new RootTabletStateStore(c.getInstance(), creds, null));
 }