Beispiel #1
0
  public void testCopyDuplication() throws Exception {
    final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
    try {
      MyFile[] files = createFiles(localfs, TEST_ROOT_DIR + "/srcdat");
      ToolRunner.run(
          new DistCpV1(new Configuration()),
          new String[] {
            "file:///" + TEST_ROOT_DIR + "/srcdat", "file:///" + TEST_ROOT_DIR + "/src2/srcdat"
          });
      assertTrue(
          "Source and destination directories do not match.",
          checkFiles(localfs, TEST_ROOT_DIR + "/src2/srcdat", files));

      assertEquals(
          DistCpV1.DuplicationException.ERROR_CODE,
          ToolRunner.run(
              new DistCpV1(new Configuration()),
              new String[] {
                "file:///" + TEST_ROOT_DIR + "/srcdat",
                "file:///" + TEST_ROOT_DIR + "/src2/srcdat",
                "file:///" + TEST_ROOT_DIR + "/destdat",
              }));
    } finally {
      deldir(localfs, TEST_ROOT_DIR + "/destdat");
      deldir(localfs, TEST_ROOT_DIR + "/srcdat");
      deldir(localfs, TEST_ROOT_DIR + "/src2");
    }
  }
  public static void main(String[] args) throws Exception {

    if (args[args.length - 1].equals("history")) {

      try {

        int exitCode = ToolRunner.run(new ImportVTLocationFromFileWithReducer(), args);

      } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
    } else {
      VesselTrackerLoader vl = new VesselTrackerLoader();

      while (true) {

        try {
          if (vl.load()) {

            int exitCode = ToolRunner.run(new ImportVTLocationFromFileWithReducer(), args);
            vl.moveFile();
          }

        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }

        Thread.sleep(2 * 60 * 1000);
      }
    }
  }
Beispiel #3
0
  public void testMapCount() throws Exception {
    String namenode = null;
    MiniDFSCluster dfs = null;
    MiniDFSCluster mr = null;
    try {
      Configuration conf = new Configuration();

      dfs = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).build();

      FileSystem fs = dfs.getFileSystem();
      final FsShell shell = new FsShell(conf);
      namenode = fs.getUri().toString();
      MyFile[] files = createFiles(fs.getUri(), "/srcdat");
      long totsize = 0;
      for (MyFile f : files) {
        totsize += f.getSize();
      }

      Configuration job = new JobConf(conf);
      job.setLong("distcp.bytes.per.map", totsize / 3);
      ToolRunner.run(
          new DistCpV1(job),
          new String[] {
            "-m", "100", "-log", namenode + "/logs", namenode + "/srcdat", namenode + "/destdat"
          });
      assertTrue(
          "Source and destination directories do not match.", checkFiles(fs, "/destdat", files));

      String logdir = namenode + "/logs";
      System.out.println(execCmd(shell, "-lsr", logdir));
      FileStatus[] logs = fs.listStatus(new Path(logdir));
      // rare case where splits are exact, logs.length can be 4
      assertTrue(logs.length == 2);

      deldir(fs, "/destdat");
      deldir(fs, "/logs");
      ToolRunner.run(
          new DistCpV1(job),
          new String[] {
            "-m", "1", "-log", namenode + "/logs", namenode + "/srcdat", namenode + "/destdat"
          });

      System.out.println(execCmd(shell, "-lsr", logdir));
      logs = fs.globStatus(new Path(namenode + "/logs/part*"));
      assertTrue("Unexpected map count, logs.length=" + logs.length, logs.length == 1);
    } finally {
      if (dfs != null) {
        dfs.shutdown();
      }
      if (mr != null) {
        mr.shutdown();
      }
    }
  }
Beispiel #4
0
  private void doCopyTableTest(boolean bulkload) throws Exception {
    final TableName TABLENAME1 = TableName.valueOf("testCopyTable1");
    final TableName TABLENAME2 = TableName.valueOf("testCopyTable2");
    final byte[] FAMILY = Bytes.toBytes("family");
    final byte[] COLUMN1 = Bytes.toBytes("c1");

    try (Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
        Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY); ) {
      // put rows into the first table
      for (int i = 0; i < 10; i++) {
        Put p = new Put(Bytes.toBytes("row" + i));
        p.addColumn(FAMILY, COLUMN1, COLUMN1);
        t1.put(p);
      }

      CopyTable copy = new CopyTable();

      int code;
      if (bulkload) {
        code =
            ToolRunner.run(
                new Configuration(TEST_UTIL.getConfiguration()),
                copy,
                new String[] {
                  "--new.name=" + TABLENAME2.getNameAsString(),
                  "--bulkload",
                  TABLENAME1.getNameAsString()
                });
      } else {
        code =
            ToolRunner.run(
                new Configuration(TEST_UTIL.getConfiguration()),
                copy,
                new String[] {
                  "--new.name=" + TABLENAME2.getNameAsString(), TABLENAME1.getNameAsString()
                });
      }
      assertEquals("copy job failed", 0, code);

      // verify the data was copied into table 2
      for (int i = 0; i < 10; i++) {
        Get g = new Get(Bytes.toBytes("row" + i));
        Result r = t2.get(g);
        assertEquals(1, r.size());
        assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
      }
    } finally {
      TEST_UTIL.deleteTable(TABLENAME1);
      TEST_UTIL.deleteTable(TABLENAME2);
    }
  }
 @BeforeTest
 public void before() throws Exception {
   URL inputSeqFileUrl = this.getClass().getResource("/publications.seq");
   ToolRunner.run(
       new Configuration(),
       new DuplicateGenerator(),
       new String[] {inputSeqFileUrl.getFile(), this.getClass().getResource("/").getFile()});
   FileUtils.deleteDirectory(new File(outputDir));
   URL inputFileUrl = this.getClass().getResource("/generated/ambiguous-publications.seq");
   ToolRunner.run(
       new Configuration(),
       new DuplicateWorkDetector(),
       new String[] {inputFileUrl.getPath(), outputDir});
 }
Beispiel #6
0
 public static void main(String[] args) throws Exception {
   // -files option is also used by GenericOptionsParser
   // Make sure that is not the first argument for fsck
   int res = -1;
   if ((args.length == 0) || ("-files".equals(args[0]))) {
     printUsage(System.err);
     ToolRunner.printGenericCommandUsage(System.err);
   } else if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) {
     res = 0;
   } else {
     res = ToolRunner.run(new DFSck(new HdfsConfiguration()), args);
   }
   System.exit(res);
 }
  public static void main(String[] args) throws Exception {

    // Call the run method to launch the MapReduce Job
    int result =
        ToolRunner.run(new Configuration(USE_MINI_ACCUMULO), new IndexedDocIndexer(), args);
    System.exit(result);
  }
  public static void main(String[] args) throws Exception {
    conf = new Configuration();

    int res = ToolRunner.run(conf, new TestMultipleOutputs4(), args);
    System.out.println("PASS");
    System.exit(res);
  }
Beispiel #9
0
  public static void main(String[] args) throws Exception {
    final Configuration conf = new Configuration();
    initalizeConf(conf, args);

    int res = ToolRunner.run(conf, new FrameworkDriver(), args);
    System.exit(res);
  }
  public static void main(String[] args) throws Exception {

    Configuration conf = new Configuration();

    int exitCode = ToolRunner.run(conf, new ColumnTextDriver(), args);
    System.exit(exitCode);
  }
 /**
  * @param args :
  *     <ol>
  *       <li>table name
  *     </ol>
  */
 public static void main(String[] args) {
   try {
     ToolRunner.run(HBaseConfiguration.create(), new CountRowsMR(), args);
   } catch (Exception e) {
     e.printStackTrace();
   }
 }
 public static void main(String[] args) throws Exception {
   if (ToolRunner.run(new FeatureMatching(), args) == 1) {
     System.out.println(".......Feature Match failure........");
     System.exit(1);
   }
   System.exit(0);
 }
Beispiel #13
0
 /** tests basedir option copying files from dfs file system to dfs file system */
 public void testBasedir() throws Exception {
   String namenode = null;
   MiniDFSCluster cluster = null;
   try {
     Configuration conf = new Configuration();
     cluster = new MiniDFSCluster(conf, 2, true, null);
     final FileSystem hdfs = cluster.getFileSystem();
     namenode = FileSystem.getDefaultUri(conf).toString();
     if (namenode.startsWith("hdfs://")) {
       MyFile[] files = createFiles(URI.create(namenode), "/basedir/middle/srcdat");
       ToolRunner.run(
           new DistCpV1(conf),
           new String[] {
             "-basedir", "/basedir", namenode + "/basedir/middle/srcdat", namenode + "/destdat"
           });
       assertTrue(
           "Source and destination directories do not match.",
           checkFiles(hdfs, "/destdat/middle/srcdat", files));
       deldir(hdfs, "/destdat");
       deldir(hdfs, "/basedir");
       deldir(hdfs, "/logs");
     }
   } finally {
     if (cluster != null) {
       cluster.shutdown();
     }
   }
 }
  /** Print help. */
  private void printHelp() {
    String summary =
        "Usage: bin/hdfs oev [OPTIONS] -i INPUT_FILE -o OUTPUT_FILE\n"
            + "Offline edits viewer\n"
            + "Parse a Hadoop edits log file INPUT_FILE and save results\n"
            + "in OUTPUT_FILE.\n"
            + "Required command line arguments:\n"
            + "-i,--inputFile <arg>   edits file to process, xml (case\n"
            + "                       insensitive) extension means XML format,\n"
            + "                       any other filename means binary format\n"
            + "-o,--outputFile <arg>  Name of output file. If the specified\n"
            + "                       file exists, it will be overwritten,\n"
            + "                       format of the file is determined\n"
            + "                       by -p option\n"
            + "\n"
            + "Optional command line arguments:\n"
            + "-p,--processor <arg>   Select which type of processor to apply\n"
            + "                       against image file, currently supported\n"
            + "                       processors are: binary (native binary format\n"
            + "                       that Hadoop uses), xml (default, XML\n"
            + "                       format), stats (prints statistics about\n"
            + "                       edits file)\n"
            + "-h,--help              Display usage information and exit\n"
            + "-v,--verbose           More verbose output, prints the input and\n"
            + "                       output filenames, for processors that write\n"
            + "                       to a file, also output to screen. On large\n"
            + "                       image files this will dramatically increase\n"
            + "                       processing time (default is false).\n";

    System.out.println(summary);
    System.out.println();
    ToolRunner.printGenericCommandUsage(System.out);
  }
Beispiel #15
0
 /** copy files from dfs file system to local file system */
 public void testCopyFromDfsToLocal() throws Exception {
   MiniDFSCluster cluster = null;
   try {
     Configuration conf = new Configuration();
     final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
     cluster = new MiniDFSCluster(conf, 1, true, null);
     final FileSystem hdfs = cluster.getFileSystem();
     final String namenode = FileSystem.getDefaultUri(conf).toString();
     if (namenode.startsWith("hdfs://")) {
       MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
       ToolRunner.run(
           new DistCpV1(conf),
           new String[] {
             "-log", "/logs", namenode + "/srcdat", "file:///" + TEST_ROOT_DIR + "/destdat"
           });
       assertTrue(
           "Source and destination directories do not match.",
           checkFiles(localfs, TEST_ROOT_DIR + "/destdat", files));
       assertTrue("Log directory does not exist.", hdfs.exists(new Path("/logs")));
       deldir(localfs, TEST_ROOT_DIR + "/destdat");
       deldir(hdfs, "/logs");
       deldir(hdfs, "/srcdat");
     }
   } finally {
     if (cluster != null) {
       cluster.shutdown();
     }
   }
 }
Beispiel #16
0
  /** copy empty directory on dfs file system */
  public void testEmptyDir() throws Exception {
    String namenode = null;
    MiniDFSCluster cluster = null;
    try {
      Configuration conf = new Configuration();
      cluster = new MiniDFSCluster(conf, 2, true, null);
      final FileSystem hdfs = cluster.getFileSystem();
      namenode = FileSystem.getDefaultUri(conf).toString();
      if (namenode.startsWith("hdfs://")) {

        FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
        fs.mkdirs(new Path("/empty"));

        ToolRunner.run(
            new DistCpV1(conf),
            new String[] {"-log", namenode + "/logs", namenode + "/empty", namenode + "/dest"});
        fs = FileSystem.get(URI.create(namenode + "/destdat"), conf);
        assertTrue(
            "Destination directory does not exist.", fs.exists(new Path(namenode + "/dest")));
        deldir(hdfs, "/dest");
        deldir(hdfs, "/empty");
        deldir(hdfs, "/logs");
      }
    } finally {
      if (cluster != null) {
        cluster.shutdown();
      }
    }
  }
  @Override
  public int run(String[] args) throws Exception {
    if (args.length < 2) {
      System.out.println("hitsperurl.DriverHitsPerUrl <inDir> <outDir>");
      ToolRunner.printGenericCommandUsage(System.out);
      System.out.println("");
      return -1;
    }

    System.out.println(Arrays.toString(args));

    Job job = new Job(getConf(), "number of hits per url");
    job.setJarByClass(DriverHitsPerUrl.class);
    job.setMapperClass(ParserMapper.class);
    // Uncomment this to set combiner
    // job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(AggregateURLReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);
    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    job.waitForCompletion(true);

    return 0;
  }
  @Test
  public void testSolver() throws Exception {
    Configuration conf = getConfiguration();
    Path testData = getTestTempDirPath("testdata");
    DistributedRowMatrix matrix =
        new TestDistributedRowMatrix()
            .randomDistributedMatrix(10, 10, 10, 10, 10.0, true, testData.toString());
    matrix.setConf(conf);
    Path output = getTestTempFilePath("output");
    Path vectorPath = getTestTempFilePath("vector");
    Path tempPath = getTestTempDirPath("tmp");

    Vector vector = randomVector(matrix.numCols(), 10.0);
    saveVector(conf, vectorPath, vector);

    String[] args = {
      "-i", matrix.getRowPath().toString(),
      "-o", output.toString(),
      "--tempDir", tempPath.toString(),
      "--vector", vectorPath.toString(),
      "--numRows", "10",
      "--numCols", "10",
      "--symmetric", "true"
    };

    DistributedConjugateGradientSolver solver = new DistributedConjugateGradientSolver();
    ToolRunner.run(getConfiguration(), solver.job(), args);

    Vector x = loadVector(conf, output);

    Vector solvedVector = matrix.times(x);
    double distance = Math.sqrt(vector.getDistanceSquared(solvedVector));
    assertEquals(0.0, distance, EPSILON);
  }
Beispiel #19
0
 /**
  * This is the main driver for recursively copying directories across file systems. It takes at
  * least two cmdline parameters. A source URL and a destination URL. It then essentially does an
  * "ls -lR" on the source URL, and writes the output in a round-robin manner to all the map input
  * files. The mapper actually copies the files allotted to it. The reduce is empty.
  */
 public int run(String[] args) {
   try {
     copy(conf, Arguments.valueOf(args, conf));
     return 0;
   } catch (IllegalArgumentException e) {
     System.err.println(StringUtils.stringifyException(e) + "\n" + usage);
     ToolRunner.printGenericCommandUsage(System.err);
     return -1;
   } catch (DuplicationException e) {
     System.err.println(StringUtils.stringifyException(e));
     return DuplicationException.ERROR_CODE;
   } catch (RemoteException e) {
     final IOException unwrapped =
         e.unwrapRemoteException(
             FileNotFoundException.class,
             AccessControlException.class,
             QuotaExceededException.class);
     System.err.println(StringUtils.stringifyException(unwrapped));
     return -3;
   } catch (Exception e) {
     System.err.println(
         "With failures, global counters are inaccurate; " + "consider running with -i");
     System.err.println("Copy failed: " + StringUtils.stringifyException(e));
     return -999;
   }
 }
  @Override
  public int run(String[] args) throws Exception {
    if (args.length != 2) {
      System.err.printf(
          "Usage: %s [generic options] <input> <output>\n", getClass().getSimpleName());
      ToolRunner.printGenericCommandUsage(System.err);
      return -1;
    }

    JobConf conf = new JobConf(getConf(), getClass());
    conf.setJobName("Max temperature");

    FileInputFormat.addInputPath(conf, new Path(args[0]));
    FileOutputFormat.setOutputPath(conf, new Path(args[1]));

    conf.setOutputKeyClass(Text.class);
    conf.setOutputValueClass(IntWritable.class);

    conf.setMapperClass(MaxTemperatureMapper.class);
    conf.setCombinerClass(MaxTemperatureReducer.class);
    conf.setReducerClass(MaxTemperatureReducer.class);

    // vv MaxTemperatureDriverV6
    conf.setProfileEnabled(true);
    conf.setProfileParams(
        "-agentlib:hprof=cpu=samples,heap=sites,depth=6," + "force=n,thread=y,verbose=n,file=%s");
    conf.setProfileTaskRange(true, "0-2");
    // ^^ MaxTemperatureDriverV6

    JobClient.runJob(conf);
    return 0;
  }
  public static void main(final String[] pArgs) throws Exception {
    TreasuryYieldMulti tym = new TreasuryYieldMulti();

    // Here is an example of how to use multiple collections as the input to
    // a hadoop job, from within Java code directly.
    MultiCollectionSplitBuilder mcsb = new MultiCollectionSplitBuilder();
    mcsb.add(
            new MongoURI("mongodb://localhost:27017/mongo_hadoop.yield_historical.in"),
            (MongoURI) null, // authuri
            true, // notimeout
            (DBObject) null, // fields
            (DBObject) null, // sort
            (DBObject) null, // query
            false)
        .add(
            new MongoURI("mongodb://localhost:27017/mongo_hadoop.yield_historical.in"),
            (MongoURI) null, // authuri
            true, // notimeout
            (DBObject) null, // fields
            (DBObject) null, // sort
            new BasicDBObject("_id", new BasicDBObject("$gt", new Date(883440000000L))),
            false); // range query

    Configuration conf = new Configuration();
    conf.set(MultiMongoCollectionSplitter.MULTI_COLLECTION_CONF_KEY, mcsb.toJSON());
    conf.setSplitterClass(conf, MultiMongoCollectionSplitter.class);

    System.exit(ToolRunner.run(conf, new TreasuryYieldXMLConfig(), pArgs));
  }
 public static void main(String[] args) throws Exception {
   if (args.length != 2) {
     System.err.println("Usage: Class <input> <output>");
     System.exit(2);
   }
   ToolRunner.run(new Configuration(), new SecondarySortingExample(), args);
 }
Beispiel #23
0
 /**
  * Run the RowCounter map reduce job and verify the row count.
  *
  * @param args the command line arguments to be used for rowcounter job.
  * @param expectedCount the expected row count (result of map reduce job).
  * @throws Exception
  */
 private void runRowCount(String[] args, int expectedCount) throws Exception {
   final RowCounter counter = new RowCounter();
   assertEquals(
       "job failed either due to failure or miscount (see log output).",
       0,
       ToolRunner.run(TEST_UTIL.getConfiguration(), counter, args));
 }
 @Test
 public void testRun() throws Exception {
   Path input = getTestTempDirPath("input");
   Path output = getTestTempDirPath("output");
   Path seedsPath = getTestTempDirPath("seeds");
   List<VectorWritable> points = getPointsWritable(REFERENCE);
   List<VectorWritable> seeds = getPointsWritable(SEEDS);
   Configuration conf = new Configuration();
   ClusteringTestUtils.writePointsToFile(points, true, new Path(input, "file1"), fs, conf);
   ClusteringTestUtils.writePointsToFile(seeds, true, new Path(seedsPath, "part-seeds"), fs, conf);
   String[] args = {
     optKey(DefaultOptionCreator.INPUT_OPTION),
     input.toString(),
     optKey(VectorDistanceSimilarityJob.SEEDS),
     seedsPath.toString(),
     optKey(DefaultOptionCreator.OUTPUT_OPTION),
     output.toString(),
     optKey(DefaultOptionCreator.DISTANCE_MEASURE_OPTION),
     EuclideanDistanceMeasure.class.getName()
   };
   ToolRunner.run(new Configuration(), new VectorDistanceSimilarityJob(), args);
   int expect = SEEDS.length * REFERENCE.length;
   DummyOutputCollector<StringTuple, DoubleWritable> collector =
       new DummyOutputCollector<StringTuple, DoubleWritable>();
   //
   for (Pair<StringTuple, DoubleWritable> record :
       new SequenceFileIterable<StringTuple, DoubleWritable>(
           new Path(output, "part-m-00000"), conf)) {
     collector.collect(record.getFirst(), record.getSecond());
   }
   assertEquals(expect, collector.getData().size());
 }
  /**
   * run a distributed job and verify that TokenCache is available
   *
   * @throws IOException
   */
  @Test
  public void testTokenCache() throws IOException {

    System.out.println("running dist job");

    // make sure JT starts
    jConf = mrCluster.createJobConf();

    // provide namenodes names for the job to get the delegation tokens for
    String nnUri = dfsCluster.getURI(0).toString();
    jConf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri);
    // job tracker principla id..
    jConf.set(JTConfig.JT_USER_NAME, "jt_id/foo@BAR");

    // using argument to pass the file name
    String[] args = {
      "-tokenCacheFile", tokenFileName.toString(), "-m", "1", "-r", "1", "-mt", "1", "-rt", "1"
    };

    int res = -1;
    try {
      res = ToolRunner.run(jConf, new MySleepJob(), args);
    } catch (Exception e) {
      System.out.println("Job failed with" + e.getLocalizedMessage());
      e.printStackTrace(System.out);
      fail("Job failed");
    }
    assertEquals("dist job res is not 0", res, 0);
  }
Beispiel #26
0
  @Test
  public void testBracketedCounters() throws Exception {
    final Configuration conf = new Configuration();
    final FileSystem lfs = FileSystem.getLocal(conf);

    final Path rootInputDir =
        new Path(System.getProperty("test.tools.input.dir", "")).makeQualified(lfs);
    final Path rootTempDir =
        new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs);

    final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test");
    final Path tempDir = new Path(rootTempDir, "TestBracketedCounters");
    lfs.delete(tempDir, true);

    final Path topologyPath = new Path(tempDir, "dispatch-topology.json");
    final Path tracePath = new Path(tempDir, "dispatch-trace.json");

    final Path inputPath = new Path(rootInputPath, "counters-format-test-logs");

    System.out.println("topology result file = " + topologyPath);
    System.out.println("testBracketedCounters() trace result file = " + tracePath);

    final Path goldPath = new Path(rootInputPath, "counters-test-trace.json.gz");

    String[] args = {tracePath.toString(), topologyPath.toString(), inputPath.toString()};

    Tool analyzer = new TraceBuilder();
    int result = ToolRunner.run(analyzer, args);
    assertEquals("Non-zero exit", 0, result);

    TestRumenJobTraces.<LoggedJob>jsonFileMatchesGold(
        conf, tracePath, goldPath, LoggedJob.class, "trace");
  }
Beispiel #27
0
  public int run(String[] args) throws Exception {

    if (args.length != 2) {
      System.out.printf(
          "Usage: %s [generic options] <input dir> <output dir>\n", getClass().getSimpleName());
      ToolRunner.printGenericCommandUsage(System.out);
      return -1;
    }

    Job job = new Job(getConf());
    job.setJarByClass(RowFilter.class);
    job.setJobName(this.getClass().getName());

    FileInputFormat.setInputPaths(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    job.setMapperClass(RowMapper.class);
    job.setNumReduceTasks(0);

    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);

    job.setOutputKeyClass(LongWritable.class);
    job.setOutputValueClass(Text.class);

    if (job.waitForCompletion(true)) {
      return 0;
    }
    return 1;
  }
  @Override
  public int run(String[] args) throws Exception {
    if (args.length != 2) {
      System.err.println("Usage: WhiteHouseVisitorDriver <input path> <output path>");
      ToolRunner.printGenericCommandUsage(System.err);
      return -1;
    }

    Configuration conf = new Configuration();
    Job job = new Job(conf);
    job.setJarByClass(WhiteHouseVisitorDriver.class);

    // input file format
    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));

    // map/combine/reduce class definition
    job.setMapperClass(WhiteHouseVisitorMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);

    // Key, Value set type definition
    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(IntWritable.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    return job.waitForCompletion(true) ? 0 : 1;
  }
Beispiel #29
0
  public static void main(String[] args) throws Exception {
    final Configuration configuration = HBaseConfiguration.create();
    configuration.addResource("grade.xml");
    String tables = configuration.get("hbase.cdn.tables");
    if (Strings.isNullOrEmpty(tables)) {
      return;
    }
    List<String> list = Lists.newArrayList(Splitter.on(",").split(tables));
    List<String> results =
        Lists.transform(
            list,
            new Function<String, String>() {
              @Override
              public String apply(@Nullable java.lang.String input) {
                return String.format(
                    configuration.get("hdfs.directory.base.db"), new Date(), input);
              }
            });

    String[] arrays =
        new String[] {
          Joiner.on(",").join(results),
          String.format(configuration.get("hdfs.directory.num.middle"), new Date()),
          String.format(configuration.get("hdfs.directory.num.result"), new Date())
        };
    AbstractJob job = new TopNJob();
    //        job.setStart(true);
    int i = ToolRunner.run(configuration, job, arrays);
    System.exit(i);
  }
Beispiel #30
0
  protected void printUsage(PrintStream out) {
    ToolRunner.printGenericCommandUsage(out);
    out.println("Usage: gridmix [-generate <MiB>] [-users URI] <iopath> <trace>");
    out.println("  e.g. gridmix -generate 100m foo -");
    out.println("Configuration parameters:");
    out.printf("       %-42s : Output directory\n", GRIDMIX_OUT_DIR);
    out.printf("       %-42s : Submitting threads\n", GRIDMIX_SUB_THR);
    out.printf("       %-42s : Queued job desc\n", GRIDMIX_QUE_DEP);
    out.printf("       %-42s : Key fraction of rec\n", AvgRecordFactory.GRIDMIX_KEY_FRC);
    out.printf("       %-42s : User resolution class\n", GRIDMIX_USR_RSV);
    out.printf(
        "       %-42s : Enable/disable using queues in trace\n",
        GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE);
    out.printf("       %-42s : Default queue\n", GridmixJob.GRIDMIX_DEFAULT_QUEUE);

    StringBuilder sb = new StringBuilder();
    String sep = "";
    for (GridmixJobSubmissionPolicy policy : GridmixJobSubmissionPolicy.values()) {
      sb.append(sep);
      sb.append(policy.name());
      sep = "|";
    }
    out.printf(
        "       %-40s : Job submission policy (%s)\n",
        GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, sb.toString());
  }