public static void main(String[] args) throws Exception {

    if (args[args.length - 1].equals("history")) {

      try {

        int exitCode = ToolRunner.run(new ImportVTLocationFromFileWithReducer(), args);

      } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
    } else {
      VesselTrackerLoader vl = new VesselTrackerLoader();

      while (true) {

        try {
          if (vl.load()) {

            int exitCode = ToolRunner.run(new ImportVTLocationFromFileWithReducer(), args);
            vl.moveFile();
          }

        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }

        Thread.sleep(2 * 60 * 1000);
      }
    }
  }
Example #2
0
  public void testCopyDuplication() throws Exception {
    final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
    try {
      MyFile[] files = createFiles(localfs, TEST_ROOT_DIR + "/srcdat");
      ToolRunner.run(
          new DistCpV1(new Configuration()),
          new String[] {
            "file:///" + TEST_ROOT_DIR + "/srcdat", "file:///" + TEST_ROOT_DIR + "/src2/srcdat"
          });
      assertTrue(
          "Source and destination directories do not match.",
          checkFiles(localfs, TEST_ROOT_DIR + "/src2/srcdat", files));

      assertEquals(
          DistCpV1.DuplicationException.ERROR_CODE,
          ToolRunner.run(
              new DistCpV1(new Configuration()),
              new String[] {
                "file:///" + TEST_ROOT_DIR + "/srcdat",
                "file:///" + TEST_ROOT_DIR + "/src2/srcdat",
                "file:///" + TEST_ROOT_DIR + "/destdat",
              }));
    } finally {
      deldir(localfs, TEST_ROOT_DIR + "/destdat");
      deldir(localfs, TEST_ROOT_DIR + "/srcdat");
      deldir(localfs, TEST_ROOT_DIR + "/src2");
    }
  }
Example #3
0
  public void testMapCount() throws Exception {
    String namenode = null;
    MiniDFSCluster dfs = null;
    MiniDFSCluster mr = null;
    try {
      Configuration conf = new Configuration();

      dfs = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).build();

      FileSystem fs = dfs.getFileSystem();
      final FsShell shell = new FsShell(conf);
      namenode = fs.getUri().toString();
      MyFile[] files = createFiles(fs.getUri(), "/srcdat");
      long totsize = 0;
      for (MyFile f : files) {
        totsize += f.getSize();
      }

      Configuration job = new JobConf(conf);
      job.setLong("distcp.bytes.per.map", totsize / 3);
      ToolRunner.run(
          new DistCpV1(job),
          new String[] {
            "-m", "100", "-log", namenode + "/logs", namenode + "/srcdat", namenode + "/destdat"
          });
      assertTrue(
          "Source and destination directories do not match.", checkFiles(fs, "/destdat", files));

      String logdir = namenode + "/logs";
      System.out.println(execCmd(shell, "-lsr", logdir));
      FileStatus[] logs = fs.listStatus(new Path(logdir));
      // rare case where splits are exact, logs.length can be 4
      assertTrue(logs.length == 2);

      deldir(fs, "/destdat");
      deldir(fs, "/logs");
      ToolRunner.run(
          new DistCpV1(job),
          new String[] {
            "-m", "1", "-log", namenode + "/logs", namenode + "/srcdat", namenode + "/destdat"
          });

      System.out.println(execCmd(shell, "-lsr", logdir));
      logs = fs.globStatus(new Path(namenode + "/logs/part*"));
      assertTrue("Unexpected map count, logs.length=" + logs.length, logs.length == 1);
    } finally {
      if (dfs != null) {
        dfs.shutdown();
      }
      if (mr != null) {
        mr.shutdown();
      }
    }
  }
Example #4
0
  private void doCopyTableTest(boolean bulkload) throws Exception {
    final TableName TABLENAME1 = TableName.valueOf("testCopyTable1");
    final TableName TABLENAME2 = TableName.valueOf("testCopyTable2");
    final byte[] FAMILY = Bytes.toBytes("family");
    final byte[] COLUMN1 = Bytes.toBytes("c1");

    try (Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
        Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY); ) {
      // put rows into the first table
      for (int i = 0; i < 10; i++) {
        Put p = new Put(Bytes.toBytes("row" + i));
        p.addColumn(FAMILY, COLUMN1, COLUMN1);
        t1.put(p);
      }

      CopyTable copy = new CopyTable();

      int code;
      if (bulkload) {
        code =
            ToolRunner.run(
                new Configuration(TEST_UTIL.getConfiguration()),
                copy,
                new String[] {
                  "--new.name=" + TABLENAME2.getNameAsString(),
                  "--bulkload",
                  TABLENAME1.getNameAsString()
                });
      } else {
        code =
            ToolRunner.run(
                new Configuration(TEST_UTIL.getConfiguration()),
                copy,
                new String[] {
                  "--new.name=" + TABLENAME2.getNameAsString(), TABLENAME1.getNameAsString()
                });
      }
      assertEquals("copy job failed", 0, code);

      // verify the data was copied into table 2
      for (int i = 0; i < 10; i++) {
        Get g = new Get(Bytes.toBytes("row" + i));
        Result r = t2.get(g);
        assertEquals(1, r.size());
        assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
      }
    } finally {
      TEST_UTIL.deleteTable(TABLENAME1);
      TEST_UTIL.deleteTable(TABLENAME2);
    }
  }
 @BeforeTest
 public void before() throws Exception {
   URL inputSeqFileUrl = this.getClass().getResource("/publications.seq");
   ToolRunner.run(
       new Configuration(),
       new DuplicateGenerator(),
       new String[] {inputSeqFileUrl.getFile(), this.getClass().getResource("/").getFile()});
   FileUtils.deleteDirectory(new File(outputDir));
   URL inputFileUrl = this.getClass().getResource("/generated/ambiguous-publications.seq");
   ToolRunner.run(
       new Configuration(),
       new DuplicateWorkDetector(),
       new String[] {inputFileUrl.getPath(), outputDir});
 }
Example #6
0
  @Test
  public void testBracketedCounters() throws Exception {
    final Configuration conf = new Configuration();
    final FileSystem lfs = FileSystem.getLocal(conf);

    final Path rootInputDir =
        new Path(System.getProperty("test.tools.input.dir", "")).makeQualified(lfs);
    final Path rootTempDir =
        new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs);

    final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test");
    final Path tempDir = new Path(rootTempDir, "TestBracketedCounters");
    lfs.delete(tempDir, true);

    final Path topologyPath = new Path(tempDir, "dispatch-topology.json");
    final Path tracePath = new Path(tempDir, "dispatch-trace.json");

    final Path inputPath = new Path(rootInputPath, "counters-format-test-logs");

    System.out.println("topology result file = " + topologyPath);
    System.out.println("testBracketedCounters() trace result file = " + tracePath);

    final Path goldPath = new Path(rootInputPath, "counters-test-trace.json.gz");

    String[] args = {tracePath.toString(), topologyPath.toString(), inputPath.toString()};

    Tool analyzer = new TraceBuilder();
    int result = ToolRunner.run(analyzer, args);
    assertEquals("Non-zero exit", 0, result);

    TestRumenJobTraces.<LoggedJob>jsonFileMatchesGold(
        conf, tracePath, goldPath, LoggedJob.class, "trace");
  }
 @Test
 public void testRun() throws Exception {
   Path input = getTestTempDirPath("input");
   Path output = getTestTempDirPath("output");
   Path seedsPath = getTestTempDirPath("seeds");
   List<VectorWritable> points = getPointsWritable(REFERENCE);
   List<VectorWritable> seeds = getPointsWritable(SEEDS);
   Configuration conf = new Configuration();
   ClusteringTestUtils.writePointsToFile(points, true, new Path(input, "file1"), fs, conf);
   ClusteringTestUtils.writePointsToFile(seeds, true, new Path(seedsPath, "part-seeds"), fs, conf);
   String[] args = {
     optKey(DefaultOptionCreator.INPUT_OPTION),
     input.toString(),
     optKey(VectorDistanceSimilarityJob.SEEDS),
     seedsPath.toString(),
     optKey(DefaultOptionCreator.OUTPUT_OPTION),
     output.toString(),
     optKey(DefaultOptionCreator.DISTANCE_MEASURE_OPTION),
     EuclideanDistanceMeasure.class.getName()
   };
   ToolRunner.run(new Configuration(), new VectorDistanceSimilarityJob(), args);
   int expect = SEEDS.length * REFERENCE.length;
   DummyOutputCollector<StringTuple, DoubleWritable> collector =
       new DummyOutputCollector<StringTuple, DoubleWritable>();
   //
   for (Pair<StringTuple, DoubleWritable> record :
       new SequenceFileIterable<StringTuple, DoubleWritable>(
           new Path(output, "part-m-00000"), conf)) {
     collector.collect(record.getFirst(), record.getSecond());
   }
   assertEquals(expect, collector.getData().size());
 }
Example #8
0
 /** tests basedir option copying files from dfs file system to dfs file system */
 public void testBasedir() throws Exception {
   String namenode = null;
   MiniDFSCluster cluster = null;
   try {
     Configuration conf = new Configuration();
     cluster = new MiniDFSCluster(conf, 2, true, null);
     final FileSystem hdfs = cluster.getFileSystem();
     namenode = FileSystem.getDefaultUri(conf).toString();
     if (namenode.startsWith("hdfs://")) {
       MyFile[] files = createFiles(URI.create(namenode), "/basedir/middle/srcdat");
       ToolRunner.run(
           new DistCpV1(conf),
           new String[] {
             "-basedir", "/basedir", namenode + "/basedir/middle/srcdat", namenode + "/destdat"
           });
       assertTrue(
           "Source and destination directories do not match.",
           checkFiles(hdfs, "/destdat/middle/srcdat", files));
       deldir(hdfs, "/destdat");
       deldir(hdfs, "/basedir");
       deldir(hdfs, "/logs");
     }
   } finally {
     if (cluster != null) {
       cluster.shutdown();
     }
   }
 }
Example #9
0
 /** copy files from dfs file system to local file system */
 public void testCopyFromDfsToLocal() throws Exception {
   MiniDFSCluster cluster = null;
   try {
     Configuration conf = new Configuration();
     final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
     cluster = new MiniDFSCluster(conf, 1, true, null);
     final FileSystem hdfs = cluster.getFileSystem();
     final String namenode = FileSystem.getDefaultUri(conf).toString();
     if (namenode.startsWith("hdfs://")) {
       MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
       ToolRunner.run(
           new DistCpV1(conf),
           new String[] {
             "-log", "/logs", namenode + "/srcdat", "file:///" + TEST_ROOT_DIR + "/destdat"
           });
       assertTrue(
           "Source and destination directories do not match.",
           checkFiles(localfs, TEST_ROOT_DIR + "/destdat", files));
       assertTrue("Log directory does not exist.", hdfs.exists(new Path("/logs")));
       deldir(localfs, TEST_ROOT_DIR + "/destdat");
       deldir(hdfs, "/logs");
       deldir(hdfs, "/srcdat");
     }
   } finally {
     if (cluster != null) {
       cluster.shutdown();
     }
   }
 }
Example #10
0
  /** copy empty directory on dfs file system */
  public void testEmptyDir() throws Exception {
    String namenode = null;
    MiniDFSCluster cluster = null;
    try {
      Configuration conf = new Configuration();
      cluster = new MiniDFSCluster(conf, 2, true, null);
      final FileSystem hdfs = cluster.getFileSystem();
      namenode = FileSystem.getDefaultUri(conf).toString();
      if (namenode.startsWith("hdfs://")) {

        FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
        fs.mkdirs(new Path("/empty"));

        ToolRunner.run(
            new DistCpV1(conf),
            new String[] {"-log", namenode + "/logs", namenode + "/empty", namenode + "/dest"});
        fs = FileSystem.get(URI.create(namenode + "/destdat"), conf);
        assertTrue(
            "Destination directory does not exist.", fs.exists(new Path(namenode + "/dest")));
        deldir(hdfs, "/dest");
        deldir(hdfs, "/empty");
        deldir(hdfs, "/logs");
      }
    } finally {
      if (cluster != null) {
        cluster.shutdown();
      }
    }
  }
  @Test
  public void testSolver() throws Exception {
    Configuration conf = getConfiguration();
    Path testData = getTestTempDirPath("testdata");
    DistributedRowMatrix matrix =
        new TestDistributedRowMatrix()
            .randomDistributedMatrix(10, 10, 10, 10, 10.0, true, testData.toString());
    matrix.setConf(conf);
    Path output = getTestTempFilePath("output");
    Path vectorPath = getTestTempFilePath("vector");
    Path tempPath = getTestTempDirPath("tmp");

    Vector vector = randomVector(matrix.numCols(), 10.0);
    saveVector(conf, vectorPath, vector);

    String[] args = {
      "-i", matrix.getRowPath().toString(),
      "-o", output.toString(),
      "--tempDir", tempPath.toString(),
      "--vector", vectorPath.toString(),
      "--numRows", "10",
      "--numCols", "10",
      "--symmetric", "true"
    };

    DistributedConjugateGradientSolver solver = new DistributedConjugateGradientSolver();
    ToolRunner.run(getConfiguration(), solver.job(), args);

    Vector x = loadVector(conf, output);

    Vector solvedVector = matrix.times(x);
    double distance = Math.sqrt(vector.getDistanceSquared(solvedVector));
    assertEquals(0.0, distance, EPSILON);
  }
 public static void main(String[] args) throws Exception {
   if (args.length != 2) {
     System.err.println("Usage: Class <input> <output>");
     System.exit(2);
   }
   ToolRunner.run(new Configuration(), new SecondarySortingExample(), args);
 }
Example #13
0
  public static void main(String[] args) throws Exception {
    final Configuration conf = new Configuration();
    initalizeConf(conf, args);

    int res = ToolRunner.run(conf, new FrameworkDriver(), args);
    System.exit(res);
  }
Example #14
0
 public static void main(String[] args) {
   try {
     ToolRunner.run(new MaxTimestamp(), args);
   } catch (Exception e) {
     e.printStackTrace();
   }
 }
Example #15
0
  public static void main(String[] args) throws Exception {
    final Configuration configuration = HBaseConfiguration.create();
    configuration.addResource("grade.xml");
    String tables = configuration.get("hbase.cdn.tables");
    if (Strings.isNullOrEmpty(tables)) {
      return;
    }
    List<String> list = Lists.newArrayList(Splitter.on(",").split(tables));
    List<String> results =
        Lists.transform(
            list,
            new Function<String, String>() {
              @Override
              public String apply(@Nullable java.lang.String input) {
                return String.format(
                    configuration.get("hdfs.directory.base.db"), new Date(), input);
              }
            });

    String[] arrays =
        new String[] {
          Joiner.on(",").join(results),
          String.format(configuration.get("hdfs.directory.num.middle"), new Date()),
          String.format(configuration.get("hdfs.directory.num.result"), new Date())
        };
    AbstractJob job = new TopNJob();
    //        job.setStart(true);
    int i = ToolRunner.run(configuration, job, arrays);
    System.exit(i);
  }
Example #16
0
  @Test
  public void testJob5D() throws Exception {
    final String input = "src/test/resources/data/6d_cuboid/";
    final String output = "target/test-output/5d_cuboid";
    final String cubeName = "test_kylin_cube_with_slr_1_new_segment";
    String segmentName = "20130331080000_20131212080000";
    String jobname = "5d_cuboid";
    String level = "2";

    FileUtil.fullyDelete(new File(output));

    String[] args = {
      "-input",
      input,
      "-cubename",
      cubeName,
      "-segmentname",
      segmentName,
      "-output",
      output,
      "-jobname",
      jobname,
      "-level",
      level
    };
    assertEquals("Job failed", 0, ToolRunner.run(conf, new NDCuboidJob(), args));
  }
  public static void main(String[] args) throws Exception {

    Configuration conf = new Configuration();

    int exitCode = ToolRunner.run(conf, new ColumnTextDriver(), args);
    System.exit(exitCode);
  }
Example #18
0
 /**
  * @param args :
  *     <ol>
  *       <li>table name
  *     </ol>
  */
 public static void main(String[] args) {
   try {
     ToolRunner.run(HBaseConfiguration.create(), new CountRowsMR(), args);
   } catch (Exception e) {
     e.printStackTrace();
   }
 }
Example #19
0
  public static void main(final String[] pArgs) throws Exception {
    TreasuryYieldMulti tym = new TreasuryYieldMulti();

    // Here is an example of how to use multiple collections as the input to
    // a hadoop job, from within Java code directly.
    MultiCollectionSplitBuilder mcsb = new MultiCollectionSplitBuilder();
    mcsb.add(
            new MongoURI("mongodb://localhost:27017/mongo_hadoop.yield_historical.in"),
            (MongoURI) null, // authuri
            true, // notimeout
            (DBObject) null, // fields
            (DBObject) null, // sort
            (DBObject) null, // query
            false)
        .add(
            new MongoURI("mongodb://localhost:27017/mongo_hadoop.yield_historical.in"),
            (MongoURI) null, // authuri
            true, // notimeout
            (DBObject) null, // fields
            (DBObject) null, // sort
            new BasicDBObject("_id", new BasicDBObject("$gt", new Date(883440000000L))),
            false); // range query

    Configuration conf = new Configuration();
    conf.set(MultiMongoCollectionSplitter.MULTI_COLLECTION_CONF_KEY, mcsb.toJSON());
    conf.setSplitterClass(conf, MultiMongoCollectionSplitter.class);

    System.exit(ToolRunner.run(conf, new TreasuryYieldXMLConfig(), pArgs));
  }
  /**
   * run a distributed job and verify that TokenCache is available
   *
   * @throws IOException
   */
  @Test
  public void testTokenCache() throws IOException {

    System.out.println("running dist job");

    // make sure JT starts
    jConf = mrCluster.createJobConf();

    // provide namenodes names for the job to get the delegation tokens for
    String nnUri = dfsCluster.getURI(0).toString();
    jConf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri);
    // job tracker principla id..
    jConf.set(JTConfig.JT_USER_NAME, "jt_id/foo@BAR");

    // using argument to pass the file name
    String[] args = {
      "-tokenCacheFile", tokenFileName.toString(), "-m", "1", "-r", "1", "-mt", "1", "-rt", "1"
    };

    int res = -1;
    try {
      res = ToolRunner.run(jConf, new MySleepJob(), args);
    } catch (Exception e) {
      System.out.println("Job failed with" + e.getLocalizedMessage());
      e.printStackTrace(System.out);
      fail("Job failed");
    }
    assertEquals("dist job res is not 0", res, 0);
  }
Example #21
0
 /**
  * Run the RowCounter map reduce job and verify the row count.
  *
  * @param args the command line arguments to be used for rowcounter job.
  * @param expectedCount the expected row count (result of map reduce job).
  * @throws Exception
  */
 private void runRowCount(String[] args, int expectedCount) throws Exception {
   final RowCounter counter = new RowCounter();
   assertEquals(
       "job failed either due to failure or miscount (see log output).",
       0,
       ToolRunner.run(TEST_UTIL.getConfiguration(), counter, args));
 }
 public static void main(String[] args) throws Exception {
   if (ToolRunner.run(new FeatureMatching(), args) == 1) {
     System.out.println(".......Feature Match failure........");
     System.exit(1);
   }
   System.exit(0);
 }
  public static void main(String[] args) throws Exception {

    // Call the run method to launch the MapReduce Job
    int result =
        ToolRunner.run(new Configuration(USE_MINI_ACCUMULO), new IndexedDocIndexer(), args);
    System.exit(result);
  }
  public static void main(String[] args) throws Exception {
    conf = new Configuration();

    int res = ToolRunner.run(conf, new TestMultipleOutputs4(), args);
    System.out.println("PASS");
    System.exit(res);
  }
  // Configure and run Mapreduce job
  public void runMRJob(
      boolean quiet,
      boolean silent,
      Configuration conf,
      ArrayList<String> D_options,
      String out,
      Logger LOG,
      String field_separator,
      String queue_name,
      String[] args,
      String job,
      Tool tool)
      throws Exception {

    logConsole(quiet, silent, info, "Running Mapreduce job & Calling " + job);

    if (out.equals("-")) {
      // Uncompress results to be able to read to stdout
      D_options.add("-Dmapreduce.output.fileoutputformat.compress=false");
    }

    try {
      conf.set("zk.connect.string", System.getenv("ZK_CONNECT_STRING"));
      conf.setBoolean("mapreduce.output.fileoutputformat.compress", true);
      conf.set("mapred.output.compress.codec", "org.apache.hadoop.io.compress.SnappyCodec");
      conf.setInt("mapred.max.split.size", 256 * 1024 * 1024);
      conf.set("logdriver.output.field.separator", field_separator);
      conf.set("mapred.job.queue.name", StringEscapeUtils.escapeJava(queue_name));

      dOpts(D_options, silent, out, conf);

      // Now run JOB and send arguments
      LOG.info("Sending args to " + job + ": {}", args);
      ToolRunner.run(conf, tool, args);
    } catch (IOException e) {
      if (e.toString().contains("Failed to find any Kerberos")) {
        logConsole(true, true, error, "No/bad Kerberos ticket - please authenticate.");
        System.exit(1);
      } else if (e.toString().contains("Permission denied")) {
        logConsole(true, true, error, "Permission denied.");
        System.err.println(
            "; Please go to https://go/itforms and filled out the Hadoop Onboarding Form "
                + "to get access to the requested data.  Paste the following data into the ticket to help with your request:\n"
                + "Error Message"
                + e);
        System.exit(1);
      } else if (e.toString().contains("quota") && e.toString().contains("exceeded")) {
        logConsole(true, true, error, "Disk quota Exceeded.");
        System.exit(1);
      }
      logConsole(
          true,
          true,
          error,
          "\n\tError running mapreduce job." + generalError() + "\n\tCommand stopped");
      e.printStackTrace();
      System.exit(1);
    }
  }
 public static void main(String[] args) throws Exception {
   if (args.length != 2) {
     System.out.println("usage: [input] [output]");
     System.exit(-1);
   }
   int rc = ToolRunner.run(new MapReduceLauncher(), args);
   System.exit(rc);
 }
Example #27
0
 public static void main(String[] argv) throws Exception {
   int res = -1;
   try {
     res = ToolRunner.run(new Configuration(), new Gridmix(argv), argv);
   } finally {
     System.exit(res);
   }
 }
 /**
  * DoIt.
  *
  * @param args the arguments to do it with
  */
 public static void main(String[] args) {
   try {
     ToolRunner.run(new ProcessingRecordsPrinter(), args);
   } catch (Exception e) {
     // TODO Auto-generated catch block
     e.printStackTrace();
   }
 }
Example #29
0
 public static void main(String[] args) throws Exception {
   if (args.length != 2) {
     System.out.println("Usage: TopKRecords /path/to/citation.txt output_dir");
     System.exit(-1);
   }
   int res = ToolRunner.run(new Configuration(), new TopKRecords(), args);
   System.exit(res);
 }
 /** Dispatches command-line arguments to the tool via the {@code ToolRunner}. */
 public static void main(String[] args) throws Exception {
   LOG.info(
       "Running "
           + TrecDocnoMappingBuilder.class.getCanonicalName()
           + " with args "
           + Arrays.toString(args));
   ToolRunner.run(new Configuration(), new TrecDocnoMappingBuilder(), args);
 }