public static void main(String[] args) throws Exception { if (args[args.length - 1].equals("history")) { try { int exitCode = ToolRunner.run(new ImportVTLocationFromFileWithReducer(), args); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } } else { VesselTrackerLoader vl = new VesselTrackerLoader(); while (true) { try { if (vl.load()) { int exitCode = ToolRunner.run(new ImportVTLocationFromFileWithReducer(), args); vl.moveFile(); } } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } Thread.sleep(2 * 60 * 1000); } } }
public void testCopyDuplication() throws Exception { final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration()); try { MyFile[] files = createFiles(localfs, TEST_ROOT_DIR + "/srcdat"); ToolRunner.run( new DistCpV1(new Configuration()), new String[] { "file:///" + TEST_ROOT_DIR + "/srcdat", "file:///" + TEST_ROOT_DIR + "/src2/srcdat" }); assertTrue( "Source and destination directories do not match.", checkFiles(localfs, TEST_ROOT_DIR + "/src2/srcdat", files)); assertEquals( DistCpV1.DuplicationException.ERROR_CODE, ToolRunner.run( new DistCpV1(new Configuration()), new String[] { "file:///" + TEST_ROOT_DIR + "/srcdat", "file:///" + TEST_ROOT_DIR + "/src2/srcdat", "file:///" + TEST_ROOT_DIR + "/destdat", })); } finally { deldir(localfs, TEST_ROOT_DIR + "/destdat"); deldir(localfs, TEST_ROOT_DIR + "/srcdat"); deldir(localfs, TEST_ROOT_DIR + "/src2"); } }
public void testMapCount() throws Exception { String namenode = null; MiniDFSCluster dfs = null; MiniDFSCluster mr = null; try { Configuration conf = new Configuration(); dfs = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).build(); FileSystem fs = dfs.getFileSystem(); final FsShell shell = new FsShell(conf); namenode = fs.getUri().toString(); MyFile[] files = createFiles(fs.getUri(), "/srcdat"); long totsize = 0; for (MyFile f : files) { totsize += f.getSize(); } Configuration job = new JobConf(conf); job.setLong("distcp.bytes.per.map", totsize / 3); ToolRunner.run( new DistCpV1(job), new String[] { "-m", "100", "-log", namenode + "/logs", namenode + "/srcdat", namenode + "/destdat" }); assertTrue( "Source and destination directories do not match.", checkFiles(fs, "/destdat", files)); String logdir = namenode + "/logs"; System.out.println(execCmd(shell, "-lsr", logdir)); FileStatus[] logs = fs.listStatus(new Path(logdir)); // rare case where splits are exact, logs.length can be 4 assertTrue(logs.length == 2); deldir(fs, "/destdat"); deldir(fs, "/logs"); ToolRunner.run( new DistCpV1(job), new String[] { "-m", "1", "-log", namenode + "/logs", namenode + "/srcdat", namenode + "/destdat" }); System.out.println(execCmd(shell, "-lsr", logdir)); logs = fs.globStatus(new Path(namenode + "/logs/part*")); assertTrue("Unexpected map count, logs.length=" + logs.length, logs.length == 1); } finally { if (dfs != null) { dfs.shutdown(); } if (mr != null) { mr.shutdown(); } } }
private void doCopyTableTest(boolean bulkload) throws Exception { final TableName TABLENAME1 = TableName.valueOf("testCopyTable1"); final TableName TABLENAME2 = TableName.valueOf("testCopyTable2"); final byte[] FAMILY = Bytes.toBytes("family"); final byte[] COLUMN1 = Bytes.toBytes("c1"); try (Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY); Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY); ) { // put rows into the first table for (int i = 0; i < 10; i++) { Put p = new Put(Bytes.toBytes("row" + i)); p.addColumn(FAMILY, COLUMN1, COLUMN1); t1.put(p); } CopyTable copy = new CopyTable(); int code; if (bulkload) { code = ToolRunner.run( new Configuration(TEST_UTIL.getConfiguration()), copy, new String[] { "--new.name=" + TABLENAME2.getNameAsString(), "--bulkload", TABLENAME1.getNameAsString() }); } else { code = ToolRunner.run( new Configuration(TEST_UTIL.getConfiguration()), copy, new String[] { "--new.name=" + TABLENAME2.getNameAsString(), TABLENAME1.getNameAsString() }); } assertEquals("copy job failed", 0, code); // verify the data was copied into table 2 for (int i = 0; i < 10; i++) { Get g = new Get(Bytes.toBytes("row" + i)); Result r = t2.get(g); assertEquals(1, r.size()); assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1)); } } finally { TEST_UTIL.deleteTable(TABLENAME1); TEST_UTIL.deleteTable(TABLENAME2); } }
@BeforeTest public void before() throws Exception { URL inputSeqFileUrl = this.getClass().getResource("/publications.seq"); ToolRunner.run( new Configuration(), new DuplicateGenerator(), new String[] {inputSeqFileUrl.getFile(), this.getClass().getResource("/").getFile()}); FileUtils.deleteDirectory(new File(outputDir)); URL inputFileUrl = this.getClass().getResource("/generated/ambiguous-publications.seq"); ToolRunner.run( new Configuration(), new DuplicateWorkDetector(), new String[] {inputFileUrl.getPath(), outputDir}); }
@Test public void testBracketedCounters() throws Exception { final Configuration conf = new Configuration(); final FileSystem lfs = FileSystem.getLocal(conf); final Path rootInputDir = new Path(System.getProperty("test.tools.input.dir", "")).makeQualified(lfs); final Path rootTempDir = new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs); final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test"); final Path tempDir = new Path(rootTempDir, "TestBracketedCounters"); lfs.delete(tempDir, true); final Path topologyPath = new Path(tempDir, "dispatch-topology.json"); final Path tracePath = new Path(tempDir, "dispatch-trace.json"); final Path inputPath = new Path(rootInputPath, "counters-format-test-logs"); System.out.println("topology result file = " + topologyPath); System.out.println("testBracketedCounters() trace result file = " + tracePath); final Path goldPath = new Path(rootInputPath, "counters-test-trace.json.gz"); String[] args = {tracePath.toString(), topologyPath.toString(), inputPath.toString()}; Tool analyzer = new TraceBuilder(); int result = ToolRunner.run(analyzer, args); assertEquals("Non-zero exit", 0, result); TestRumenJobTraces.<LoggedJob>jsonFileMatchesGold( conf, tracePath, goldPath, LoggedJob.class, "trace"); }
@Test public void testRun() throws Exception { Path input = getTestTempDirPath("input"); Path output = getTestTempDirPath("output"); Path seedsPath = getTestTempDirPath("seeds"); List<VectorWritable> points = getPointsWritable(REFERENCE); List<VectorWritable> seeds = getPointsWritable(SEEDS); Configuration conf = new Configuration(); ClusteringTestUtils.writePointsToFile(points, true, new Path(input, "file1"), fs, conf); ClusteringTestUtils.writePointsToFile(seeds, true, new Path(seedsPath, "part-seeds"), fs, conf); String[] args = { optKey(DefaultOptionCreator.INPUT_OPTION), input.toString(), optKey(VectorDistanceSimilarityJob.SEEDS), seedsPath.toString(), optKey(DefaultOptionCreator.OUTPUT_OPTION), output.toString(), optKey(DefaultOptionCreator.DISTANCE_MEASURE_OPTION), EuclideanDistanceMeasure.class.getName() }; ToolRunner.run(new Configuration(), new VectorDistanceSimilarityJob(), args); int expect = SEEDS.length * REFERENCE.length; DummyOutputCollector<StringTuple, DoubleWritable> collector = new DummyOutputCollector<StringTuple, DoubleWritable>(); // for (Pair<StringTuple, DoubleWritable> record : new SequenceFileIterable<StringTuple, DoubleWritable>( new Path(output, "part-m-00000"), conf)) { collector.collect(record.getFirst(), record.getSecond()); } assertEquals(expect, collector.getData().size()); }
/** tests basedir option copying files from dfs file system to dfs file system */ public void testBasedir() throws Exception { String namenode = null; MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); cluster = new MiniDFSCluster(conf, 2, true, null); final FileSystem hdfs = cluster.getFileSystem(); namenode = FileSystem.getDefaultUri(conf).toString(); if (namenode.startsWith("hdfs://")) { MyFile[] files = createFiles(URI.create(namenode), "/basedir/middle/srcdat"); ToolRunner.run( new DistCpV1(conf), new String[] { "-basedir", "/basedir", namenode + "/basedir/middle/srcdat", namenode + "/destdat" }); assertTrue( "Source and destination directories do not match.", checkFiles(hdfs, "/destdat/middle/srcdat", files)); deldir(hdfs, "/destdat"); deldir(hdfs, "/basedir"); deldir(hdfs, "/logs"); } } finally { if (cluster != null) { cluster.shutdown(); } } }
/** copy files from dfs file system to local file system */ public void testCopyFromDfsToLocal() throws Exception { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); final FileSystem localfs = FileSystem.get(LOCAL_FS, conf); cluster = new MiniDFSCluster(conf, 1, true, null); final FileSystem hdfs = cluster.getFileSystem(); final String namenode = FileSystem.getDefaultUri(conf).toString(); if (namenode.startsWith("hdfs://")) { MyFile[] files = createFiles(URI.create(namenode), "/srcdat"); ToolRunner.run( new DistCpV1(conf), new String[] { "-log", "/logs", namenode + "/srcdat", "file:///" + TEST_ROOT_DIR + "/destdat" }); assertTrue( "Source and destination directories do not match.", checkFiles(localfs, TEST_ROOT_DIR + "/destdat", files)); assertTrue("Log directory does not exist.", hdfs.exists(new Path("/logs"))); deldir(localfs, TEST_ROOT_DIR + "/destdat"); deldir(hdfs, "/logs"); deldir(hdfs, "/srcdat"); } } finally { if (cluster != null) { cluster.shutdown(); } } }
/** copy empty directory on dfs file system */ public void testEmptyDir() throws Exception { String namenode = null; MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); cluster = new MiniDFSCluster(conf, 2, true, null); final FileSystem hdfs = cluster.getFileSystem(); namenode = FileSystem.getDefaultUri(conf).toString(); if (namenode.startsWith("hdfs://")) { FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration()); fs.mkdirs(new Path("/empty")); ToolRunner.run( new DistCpV1(conf), new String[] {"-log", namenode + "/logs", namenode + "/empty", namenode + "/dest"}); fs = FileSystem.get(URI.create(namenode + "/destdat"), conf); assertTrue( "Destination directory does not exist.", fs.exists(new Path(namenode + "/dest"))); deldir(hdfs, "/dest"); deldir(hdfs, "/empty"); deldir(hdfs, "/logs"); } } finally { if (cluster != null) { cluster.shutdown(); } } }
@Test public void testSolver() throws Exception { Configuration conf = getConfiguration(); Path testData = getTestTempDirPath("testdata"); DistributedRowMatrix matrix = new TestDistributedRowMatrix() .randomDistributedMatrix(10, 10, 10, 10, 10.0, true, testData.toString()); matrix.setConf(conf); Path output = getTestTempFilePath("output"); Path vectorPath = getTestTempFilePath("vector"); Path tempPath = getTestTempDirPath("tmp"); Vector vector = randomVector(matrix.numCols(), 10.0); saveVector(conf, vectorPath, vector); String[] args = { "-i", matrix.getRowPath().toString(), "-o", output.toString(), "--tempDir", tempPath.toString(), "--vector", vectorPath.toString(), "--numRows", "10", "--numCols", "10", "--symmetric", "true" }; DistributedConjugateGradientSolver solver = new DistributedConjugateGradientSolver(); ToolRunner.run(getConfiguration(), solver.job(), args); Vector x = loadVector(conf, output); Vector solvedVector = matrix.times(x); double distance = Math.sqrt(vector.getDistanceSquared(solvedVector)); assertEquals(0.0, distance, EPSILON); }
public static void main(String[] args) throws Exception { if (args.length != 2) { System.err.println("Usage: Class <input> <output>"); System.exit(2); } ToolRunner.run(new Configuration(), new SecondarySortingExample(), args); }
public static void main(String[] args) throws Exception { final Configuration conf = new Configuration(); initalizeConf(conf, args); int res = ToolRunner.run(conf, new FrameworkDriver(), args); System.exit(res); }
public static void main(String[] args) { try { ToolRunner.run(new MaxTimestamp(), args); } catch (Exception e) { e.printStackTrace(); } }
public static void main(String[] args) throws Exception { final Configuration configuration = HBaseConfiguration.create(); configuration.addResource("grade.xml"); String tables = configuration.get("hbase.cdn.tables"); if (Strings.isNullOrEmpty(tables)) { return; } List<String> list = Lists.newArrayList(Splitter.on(",").split(tables)); List<String> results = Lists.transform( list, new Function<String, String>() { @Override public String apply(@Nullable java.lang.String input) { return String.format( configuration.get("hdfs.directory.base.db"), new Date(), input); } }); String[] arrays = new String[] { Joiner.on(",").join(results), String.format(configuration.get("hdfs.directory.num.middle"), new Date()), String.format(configuration.get("hdfs.directory.num.result"), new Date()) }; AbstractJob job = new TopNJob(); // job.setStart(true); int i = ToolRunner.run(configuration, job, arrays); System.exit(i); }
@Test public void testJob5D() throws Exception { final String input = "src/test/resources/data/6d_cuboid/"; final String output = "target/test-output/5d_cuboid"; final String cubeName = "test_kylin_cube_with_slr_1_new_segment"; String segmentName = "20130331080000_20131212080000"; String jobname = "5d_cuboid"; String level = "2"; FileUtil.fullyDelete(new File(output)); String[] args = { "-input", input, "-cubename", cubeName, "-segmentname", segmentName, "-output", output, "-jobname", jobname, "-level", level }; assertEquals("Job failed", 0, ToolRunner.run(conf, new NDCuboidJob(), args)); }
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); int exitCode = ToolRunner.run(conf, new ColumnTextDriver(), args); System.exit(exitCode); }
/** * @param args : * <ol> * <li>table name * </ol> */ public static void main(String[] args) { try { ToolRunner.run(HBaseConfiguration.create(), new CountRowsMR(), args); } catch (Exception e) { e.printStackTrace(); } }
public static void main(final String[] pArgs) throws Exception { TreasuryYieldMulti tym = new TreasuryYieldMulti(); // Here is an example of how to use multiple collections as the input to // a hadoop job, from within Java code directly. MultiCollectionSplitBuilder mcsb = new MultiCollectionSplitBuilder(); mcsb.add( new MongoURI("mongodb://localhost:27017/mongo_hadoop.yield_historical.in"), (MongoURI) null, // authuri true, // notimeout (DBObject) null, // fields (DBObject) null, // sort (DBObject) null, // query false) .add( new MongoURI("mongodb://localhost:27017/mongo_hadoop.yield_historical.in"), (MongoURI) null, // authuri true, // notimeout (DBObject) null, // fields (DBObject) null, // sort new BasicDBObject("_id", new BasicDBObject("$gt", new Date(883440000000L))), false); // range query Configuration conf = new Configuration(); conf.set(MultiMongoCollectionSplitter.MULTI_COLLECTION_CONF_KEY, mcsb.toJSON()); conf.setSplitterClass(conf, MultiMongoCollectionSplitter.class); System.exit(ToolRunner.run(conf, new TreasuryYieldXMLConfig(), pArgs)); }
/** * run a distributed job and verify that TokenCache is available * * @throws IOException */ @Test public void testTokenCache() throws IOException { System.out.println("running dist job"); // make sure JT starts jConf = mrCluster.createJobConf(); // provide namenodes names for the job to get the delegation tokens for String nnUri = dfsCluster.getURI(0).toString(); jConf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri); // job tracker principla id.. jConf.set(JTConfig.JT_USER_NAME, "jt_id/foo@BAR"); // using argument to pass the file name String[] args = { "-tokenCacheFile", tokenFileName.toString(), "-m", "1", "-r", "1", "-mt", "1", "-rt", "1" }; int res = -1; try { res = ToolRunner.run(jConf, new MySleepJob(), args); } catch (Exception e) { System.out.println("Job failed with" + e.getLocalizedMessage()); e.printStackTrace(System.out); fail("Job failed"); } assertEquals("dist job res is not 0", res, 0); }
/** * Run the RowCounter map reduce job and verify the row count. * * @param args the command line arguments to be used for rowcounter job. * @param expectedCount the expected row count (result of map reduce job). * @throws Exception */ private void runRowCount(String[] args, int expectedCount) throws Exception { final RowCounter counter = new RowCounter(); assertEquals( "job failed either due to failure or miscount (see log output).", 0, ToolRunner.run(TEST_UTIL.getConfiguration(), counter, args)); }
public static void main(String[] args) throws Exception { if (ToolRunner.run(new FeatureMatching(), args) == 1) { System.out.println(".......Feature Match failure........"); System.exit(1); } System.exit(0); }
public static void main(String[] args) throws Exception { // Call the run method to launch the MapReduce Job int result = ToolRunner.run(new Configuration(USE_MINI_ACCUMULO), new IndexedDocIndexer(), args); System.exit(result); }
public static void main(String[] args) throws Exception { conf = new Configuration(); int res = ToolRunner.run(conf, new TestMultipleOutputs4(), args); System.out.println("PASS"); System.exit(res); }
// Configure and run Mapreduce job public void runMRJob( boolean quiet, boolean silent, Configuration conf, ArrayList<String> D_options, String out, Logger LOG, String field_separator, String queue_name, String[] args, String job, Tool tool) throws Exception { logConsole(quiet, silent, info, "Running Mapreduce job & Calling " + job); if (out.equals("-")) { // Uncompress results to be able to read to stdout D_options.add("-Dmapreduce.output.fileoutputformat.compress=false"); } try { conf.set("zk.connect.string", System.getenv("ZK_CONNECT_STRING")); conf.setBoolean("mapreduce.output.fileoutputformat.compress", true); conf.set("mapred.output.compress.codec", "org.apache.hadoop.io.compress.SnappyCodec"); conf.setInt("mapred.max.split.size", 256 * 1024 * 1024); conf.set("logdriver.output.field.separator", field_separator); conf.set("mapred.job.queue.name", StringEscapeUtils.escapeJava(queue_name)); dOpts(D_options, silent, out, conf); // Now run JOB and send arguments LOG.info("Sending args to " + job + ": {}", args); ToolRunner.run(conf, tool, args); } catch (IOException e) { if (e.toString().contains("Failed to find any Kerberos")) { logConsole(true, true, error, "No/bad Kerberos ticket - please authenticate."); System.exit(1); } else if (e.toString().contains("Permission denied")) { logConsole(true, true, error, "Permission denied."); System.err.println( "; Please go to https://go/itforms and filled out the Hadoop Onboarding Form " + "to get access to the requested data. Paste the following data into the ticket to help with your request:\n" + "Error Message" + e); System.exit(1); } else if (e.toString().contains("quota") && e.toString().contains("exceeded")) { logConsole(true, true, error, "Disk quota Exceeded."); System.exit(1); } logConsole( true, true, error, "\n\tError running mapreduce job." + generalError() + "\n\tCommand stopped"); e.printStackTrace(); System.exit(1); } }
public static void main(String[] args) throws Exception { if (args.length != 2) { System.out.println("usage: [input] [output]"); System.exit(-1); } int rc = ToolRunner.run(new MapReduceLauncher(), args); System.exit(rc); }
public static void main(String[] argv) throws Exception { int res = -1; try { res = ToolRunner.run(new Configuration(), new Gridmix(argv), argv); } finally { System.exit(res); } }
/** * DoIt. * * @param args the arguments to do it with */ public static void main(String[] args) { try { ToolRunner.run(new ProcessingRecordsPrinter(), args); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } }
public static void main(String[] args) throws Exception { if (args.length != 2) { System.out.println("Usage: TopKRecords /path/to/citation.txt output_dir"); System.exit(-1); } int res = ToolRunner.run(new Configuration(), new TopKRecords(), args); System.exit(res); }
/** Dispatches command-line arguments to the tool via the {@code ToolRunner}. */ public static void main(String[] args) throws Exception { LOG.info( "Running " + TrecDocnoMappingBuilder.class.getCanonicalName() + " with args " + Arrays.toString(args)); ToolRunner.run(new Configuration(), new TrecDocnoMappingBuilder(), args); }