public static void main(String[] args) throws Exception {
if (args[args.length - 1].equals("history")) {
try {
int exitCode = ToolRunner.run(new ImportVTLocationFromFileWithReducer(), args);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
} else {
VesselTrackerLoader vl = new VesselTrackerLoader();
while (true) {
try {
if (vl.load()) {
int exitCode = ToolRunner.run(new ImportVTLocationFromFileWithReducer(), args);
vl.moveFile();
}
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
Thread.sleep(2 * 60 * 1000);
}
}
}
public void testCopyDuplication() throws Exception {
final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
try {
MyFile[] files = createFiles(localfs, TEST_ROOT_DIR + "/srcdat");
ToolRunner.run(
new DistCpV1(new Configuration()),
new String[] {
"file:///" + TEST_ROOT_DIR + "/srcdat", "file:///" + TEST_ROOT_DIR + "/src2/srcdat"
});
assertTrue(
"Source and destination directories do not match.",
checkFiles(localfs, TEST_ROOT_DIR + "/src2/srcdat", files));
assertEquals(
DistCpV1.DuplicationException.ERROR_CODE,
ToolRunner.run(
new DistCpV1(new Configuration()),
new String[] {
"file:///" + TEST_ROOT_DIR + "/srcdat",
"file:///" + TEST_ROOT_DIR + "/src2/srcdat",
"file:///" + TEST_ROOT_DIR + "/destdat",
}));
} finally {
deldir(localfs, TEST_ROOT_DIR + "/destdat");
deldir(localfs, TEST_ROOT_DIR + "/srcdat");
deldir(localfs, TEST_ROOT_DIR + "/src2");
}
}
public void testMapCount() throws Exception {
String namenode = null;
MiniDFSCluster dfs = null;
MiniDFSCluster mr = null;
try {
Configuration conf = new Configuration();
dfs = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).build();
FileSystem fs = dfs.getFileSystem();
final FsShell shell = new FsShell(conf);
namenode = fs.getUri().toString();
MyFile[] files = createFiles(fs.getUri(), "/srcdat");
long totsize = 0;
for (MyFile f : files) {
totsize += f.getSize();
}
Configuration job = new JobConf(conf);
job.setLong("distcp.bytes.per.map", totsize / 3);
ToolRunner.run(
new DistCpV1(job),
new String[] {
"-m", "100", "-log", namenode + "/logs", namenode + "/srcdat", namenode + "/destdat"
});
assertTrue(
"Source and destination directories do not match.", checkFiles(fs, "/destdat", files));
String logdir = namenode + "/logs";
System.out.println(execCmd(shell, "-lsr", logdir));
FileStatus[] logs = fs.listStatus(new Path(logdir));
// rare case where splits are exact, logs.length can be 4
assertTrue(logs.length == 2);
deldir(fs, "/destdat");
deldir(fs, "/logs");
ToolRunner.run(
new DistCpV1(job),
new String[] {
"-m", "1", "-log", namenode + "/logs", namenode + "/srcdat", namenode + "/destdat"
});
System.out.println(execCmd(shell, "-lsr", logdir));
logs = fs.globStatus(new Path(namenode + "/logs/part*"));
assertTrue("Unexpected map count, logs.length=" + logs.length, logs.length == 1);
} finally {
if (dfs != null) {
dfs.shutdown();
}
if (mr != null) {
mr.shutdown();
}
}
}
private void doCopyTableTest(boolean bulkload) throws Exception {
final TableName TABLENAME1 = TableName.valueOf("testCopyTable1");
final TableName TABLENAME2 = TableName.valueOf("testCopyTable2");
final byte[] FAMILY = Bytes.toBytes("family");
final byte[] COLUMN1 = Bytes.toBytes("c1");
try (Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY); ) {
// put rows into the first table
for (int i = 0; i < 10; i++) {
Put p = new Put(Bytes.toBytes("row" + i));
p.addColumn(FAMILY, COLUMN1, COLUMN1);
t1.put(p);
}
CopyTable copy = new CopyTable();
int code;
if (bulkload) {
code =
ToolRunner.run(
new Configuration(TEST_UTIL.getConfiguration()),
copy,
new String[] {
"--new.name=" + TABLENAME2.getNameAsString(),
"--bulkload",
TABLENAME1.getNameAsString()
});
} else {
code =
ToolRunner.run(
new Configuration(TEST_UTIL.getConfiguration()),
copy,
new String[] {
"--new.name=" + TABLENAME2.getNameAsString(), TABLENAME1.getNameAsString()
});
}
assertEquals("copy job failed", 0, code);
// verify the data was copied into table 2
for (int i = 0; i < 10; i++) {
Get g = new Get(Bytes.toBytes("row" + i));
Result r = t2.get(g);
assertEquals(1, r.size());
assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
}
} finally {
TEST_UTIL.deleteTable(TABLENAME1);
TEST_UTIL.deleteTable(TABLENAME2);
}
}
@BeforeTest
public void before() throws Exception {
URL inputSeqFileUrl = this.getClass().getResource("/publications.seq");
ToolRunner.run(
new Configuration(),
new DuplicateGenerator(),
new String[] {inputSeqFileUrl.getFile(), this.getClass().getResource("/").getFile()});
FileUtils.deleteDirectory(new File(outputDir));
URL inputFileUrl = this.getClass().getResource("/generated/ambiguous-publications.seq");
ToolRunner.run(
new Configuration(),
new DuplicateWorkDetector(),
new String[] {inputFileUrl.getPath(), outputDir});
}
@Test
public void testBracketedCounters() throws Exception {
final Configuration conf = new Configuration();
final FileSystem lfs = FileSystem.getLocal(conf);
final Path rootInputDir =
new Path(System.getProperty("test.tools.input.dir", "")).makeQualified(lfs);
final Path rootTempDir =
new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs);
final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test");
final Path tempDir = new Path(rootTempDir, "TestBracketedCounters");
lfs.delete(tempDir, true);
final Path topologyPath = new Path(tempDir, "dispatch-topology.json");
final Path tracePath = new Path(tempDir, "dispatch-trace.json");
final Path inputPath = new Path(rootInputPath, "counters-format-test-logs");
System.out.println("topology result file = " + topologyPath);
System.out.println("testBracketedCounters() trace result file = " + tracePath);
final Path goldPath = new Path(rootInputPath, "counters-test-trace.json.gz");
String[] args = {tracePath.toString(), topologyPath.toString(), inputPath.toString()};
Tool analyzer = new TraceBuilder();
int result = ToolRunner.run(analyzer, args);
assertEquals("Non-zero exit", 0, result);
TestRumenJobTraces.<LoggedJob>jsonFileMatchesGold(
conf, tracePath, goldPath, LoggedJob.class, "trace");
}
@Test
public void testRun() throws Exception {
Path input = getTestTempDirPath("input");
Path output = getTestTempDirPath("output");
Path seedsPath = getTestTempDirPath("seeds");
List<VectorWritable> points = getPointsWritable(REFERENCE);
List<VectorWritable> seeds = getPointsWritable(SEEDS);
Configuration conf = new Configuration();
ClusteringTestUtils.writePointsToFile(points, true, new Path(input, "file1"), fs, conf);
ClusteringTestUtils.writePointsToFile(seeds, true, new Path(seedsPath, "part-seeds"), fs, conf);
String[] args = {
optKey(DefaultOptionCreator.INPUT_OPTION),
input.toString(),
optKey(VectorDistanceSimilarityJob.SEEDS),
seedsPath.toString(),
optKey(DefaultOptionCreator.OUTPUT_OPTION),
output.toString(),
optKey(DefaultOptionCreator.DISTANCE_MEASURE_OPTION),
EuclideanDistanceMeasure.class.getName()
};
ToolRunner.run(new Configuration(), new VectorDistanceSimilarityJob(), args);
int expect = SEEDS.length * REFERENCE.length;
DummyOutputCollector<StringTuple, DoubleWritable> collector =
new DummyOutputCollector<StringTuple, DoubleWritable>();
//
for (Pair<StringTuple, DoubleWritable> record :
new SequenceFileIterable<StringTuple, DoubleWritable>(
new Path(output, "part-m-00000"), conf)) {
collector.collect(record.getFirst(), record.getSecond());
}
assertEquals(expect, collector.getData().size());
}
/** tests basedir option copying files from dfs file system to dfs file system */
public void testBasedir() throws Exception {
String namenode = null;
MiniDFSCluster cluster = null;
try {
Configuration conf = new Configuration();
cluster = new MiniDFSCluster(conf, 2, true, null);
final FileSystem hdfs = cluster.getFileSystem();
namenode = FileSystem.getDefaultUri(conf).toString();
if (namenode.startsWith("hdfs://")) {
MyFile[] files = createFiles(URI.create(namenode), "/basedir/middle/srcdat");
ToolRunner.run(
new DistCpV1(conf),
new String[] {
"-basedir", "/basedir", namenode + "/basedir/middle/srcdat", namenode + "/destdat"
});
assertTrue(
"Source and destination directories do not match.",
checkFiles(hdfs, "/destdat/middle/srcdat", files));
deldir(hdfs, "/destdat");
deldir(hdfs, "/basedir");
deldir(hdfs, "/logs");
}
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
/** copy files from dfs file system to local file system */
public void testCopyFromDfsToLocal() throws Exception {
MiniDFSCluster cluster = null;
try {
Configuration conf = new Configuration();
final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
cluster = new MiniDFSCluster(conf, 1, true, null);
final FileSystem hdfs = cluster.getFileSystem();
final String namenode = FileSystem.getDefaultUri(conf).toString();
if (namenode.startsWith("hdfs://")) {
MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
ToolRunner.run(
new DistCpV1(conf),
new String[] {
"-log", "/logs", namenode + "/srcdat", "file:///" + TEST_ROOT_DIR + "/destdat"
});
assertTrue(
"Source and destination directories do not match.",
checkFiles(localfs, TEST_ROOT_DIR + "/destdat", files));
assertTrue("Log directory does not exist.", hdfs.exists(new Path("/logs")));
deldir(localfs, TEST_ROOT_DIR + "/destdat");
deldir(hdfs, "/logs");
deldir(hdfs, "/srcdat");
}
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
/** copy empty directory on dfs file system */
public void testEmptyDir() throws Exception {
String namenode = null;
MiniDFSCluster cluster = null;
try {
Configuration conf = new Configuration();
cluster = new MiniDFSCluster(conf, 2, true, null);
final FileSystem hdfs = cluster.getFileSystem();
namenode = FileSystem.getDefaultUri(conf).toString();
if (namenode.startsWith("hdfs://")) {
FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
fs.mkdirs(new Path("/empty"));
ToolRunner.run(
new DistCpV1(conf),
new String[] {"-log", namenode + "/logs", namenode + "/empty", namenode + "/dest"});
fs = FileSystem.get(URI.create(namenode + "/destdat"), conf);
assertTrue(
"Destination directory does not exist.", fs.exists(new Path(namenode + "/dest")));
deldir(hdfs, "/dest");
deldir(hdfs, "/empty");
deldir(hdfs, "/logs");
}
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
@Test
public void testSolver() throws Exception {
Configuration conf = getConfiguration();
Path testData = getTestTempDirPath("testdata");
DistributedRowMatrix matrix =
new TestDistributedRowMatrix()
.randomDistributedMatrix(10, 10, 10, 10, 10.0, true, testData.toString());
matrix.setConf(conf);
Path output = getTestTempFilePath("output");
Path vectorPath = getTestTempFilePath("vector");
Path tempPath = getTestTempDirPath("tmp");
Vector vector = randomVector(matrix.numCols(), 10.0);
saveVector(conf, vectorPath, vector);
String[] args = {
"-i", matrix.getRowPath().toString(),
"-o", output.toString(),
"--tempDir", tempPath.toString(),
"--vector", vectorPath.toString(),
"--numRows", "10",
"--numCols", "10",
"--symmetric", "true"
};
DistributedConjugateGradientSolver solver = new DistributedConjugateGradientSolver();
ToolRunner.run(getConfiguration(), solver.job(), args);
Vector x = loadVector(conf, output);
Vector solvedVector = matrix.times(x);
double distance = Math.sqrt(vector.getDistanceSquared(solvedVector));
assertEquals(0.0, distance, EPSILON);
}
public static void main(String[] args) throws Exception {
if (args.length != 2) {
System.err.println("Usage: Class <input> <output>");
System.exit(2);
}
ToolRunner.run(new Configuration(), new SecondarySortingExample(), args);
}
public static void main(String[] args) throws Exception {
final Configuration conf = new Configuration();
initalizeConf(conf, args);
int res = ToolRunner.run(conf, new FrameworkDriver(), args);
System.exit(res);
}
public static void main(String[] args) {
try {
ToolRunner.run(new MaxTimestamp(), args);
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws Exception {
final Configuration configuration = HBaseConfiguration.create();
configuration.addResource("grade.xml");
String tables = configuration.get("hbase.cdn.tables");
if (Strings.isNullOrEmpty(tables)) {
return;
}
List<String> list = Lists.newArrayList(Splitter.on(",").split(tables));
List<String> results =
Lists.transform(
list,
new Function<String, String>() {
@Override
public String apply(@Nullable java.lang.String input) {
return String.format(
configuration.get("hdfs.directory.base.db"), new Date(), input);
}
});
String[] arrays =
new String[] {
Joiner.on(",").join(results),
String.format(configuration.get("hdfs.directory.num.middle"), new Date()),
String.format(configuration.get("hdfs.directory.num.result"), new Date())
};
AbstractJob job = new TopNJob();
// job.setStart(true);
int i = ToolRunner.run(configuration, job, arrays);
System.exit(i);
}
@Test
public void testJob5D() throws Exception {
final String input = "src/test/resources/data/6d_cuboid/";
final String output = "target/test-output/5d_cuboid";
final String cubeName = "test_kylin_cube_with_slr_1_new_segment";
String segmentName = "20130331080000_20131212080000";
String jobname = "5d_cuboid";
String level = "2";
FileUtil.fullyDelete(new File(output));
String[] args = {
"-input",
input,
"-cubename",
cubeName,
"-segmentname",
segmentName,
"-output",
output,
"-jobname",
jobname,
"-level",
level
};
assertEquals("Job failed", 0, ToolRunner.run(conf, new NDCuboidJob(), args));
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
int exitCode = ToolRunner.run(conf, new ColumnTextDriver(), args);
System.exit(exitCode);
}
/**
* @param args :
* <ol>
* <li>table name
* </ol>
*/
public static void main(String[] args) {
try {
ToolRunner.run(HBaseConfiguration.create(), new CountRowsMR(), args);
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(final String[] pArgs) throws Exception {
TreasuryYieldMulti tym = new TreasuryYieldMulti();
// Here is an example of how to use multiple collections as the input to
// a hadoop job, from within Java code directly.
MultiCollectionSplitBuilder mcsb = new MultiCollectionSplitBuilder();
mcsb.add(
new MongoURI("mongodb://localhost:27017/mongo_hadoop.yield_historical.in"),
(MongoURI) null, // authuri
true, // notimeout
(DBObject) null, // fields
(DBObject) null, // sort
(DBObject) null, // query
false)
.add(
new MongoURI("mongodb://localhost:27017/mongo_hadoop.yield_historical.in"),
(MongoURI) null, // authuri
true, // notimeout
(DBObject) null, // fields
(DBObject) null, // sort
new BasicDBObject("_id", new BasicDBObject("$gt", new Date(883440000000L))),
false); // range query
Configuration conf = new Configuration();
conf.set(MultiMongoCollectionSplitter.MULTI_COLLECTION_CONF_KEY, mcsb.toJSON());
conf.setSplitterClass(conf, MultiMongoCollectionSplitter.class);
System.exit(ToolRunner.run(conf, new TreasuryYieldXMLConfig(), pArgs));
}
/**
* run a distributed job and verify that TokenCache is available
*
* @throws IOException
*/
@Test
public void testTokenCache() throws IOException {
System.out.println("running dist job");
// make sure JT starts
jConf = mrCluster.createJobConf();
// provide namenodes names for the job to get the delegation tokens for
String nnUri = dfsCluster.getURI(0).toString();
jConf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri);
// job tracker principla id..
jConf.set(JTConfig.JT_USER_NAME, "jt_id/foo@BAR");
// using argument to pass the file name
String[] args = {
"-tokenCacheFile", tokenFileName.toString(), "-m", "1", "-r", "1", "-mt", "1", "-rt", "1"
};
int res = -1;
try {
res = ToolRunner.run(jConf, new MySleepJob(), args);
} catch (Exception e) {
System.out.println("Job failed with" + e.getLocalizedMessage());
e.printStackTrace(System.out);
fail("Job failed");
}
assertEquals("dist job res is not 0", res, 0);
}
/**
* Run the RowCounter map reduce job and verify the row count.
*
* @param args the command line arguments to be used for rowcounter job.
* @param expectedCount the expected row count (result of map reduce job).
* @throws Exception
*/
private void runRowCount(String[] args, int expectedCount) throws Exception {
final RowCounter counter = new RowCounter();
assertEquals(
"job failed either due to failure or miscount (see log output).",
0,
ToolRunner.run(TEST_UTIL.getConfiguration(), counter, args));
}
public static void main(String[] args) throws Exception {
if (ToolRunner.run(new FeatureMatching(), args) == 1) {
System.out.println(".......Feature Match failure........");
System.exit(1);
}
System.exit(0);
}
public static void main(String[] args) throws Exception {
// Call the run method to launch the MapReduce Job
int result =
ToolRunner.run(new Configuration(USE_MINI_ACCUMULO), new IndexedDocIndexer(), args);
System.exit(result);
}
public static void main(String[] args) throws Exception {
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputs4(), args);
System.out.println("PASS");
System.exit(res);
}
// Configure and run Mapreduce job
public void runMRJob(
boolean quiet,
boolean silent,
Configuration conf,
ArrayList<String> D_options,
String out,
Logger LOG,
String field_separator,
String queue_name,
String[] args,
String job,
Tool tool)
throws Exception {
logConsole(quiet, silent, info, "Running Mapreduce job & Calling " + job);
if (out.equals("-")) {
// Uncompress results to be able to read to stdout
D_options.add("-Dmapreduce.output.fileoutputformat.compress=false");
}
try {
conf.set("zk.connect.string", System.getenv("ZK_CONNECT_STRING"));
conf.setBoolean("mapreduce.output.fileoutputformat.compress", true);
conf.set("mapred.output.compress.codec", "org.apache.hadoop.io.compress.SnappyCodec");
conf.setInt("mapred.max.split.size", 256 * 1024 * 1024);
conf.set("logdriver.output.field.separator", field_separator);
conf.set("mapred.job.queue.name", StringEscapeUtils.escapeJava(queue_name));
dOpts(D_options, silent, out, conf);
// Now run JOB and send arguments
LOG.info("Sending args to " + job + ": {}", args);
ToolRunner.run(conf, tool, args);
} catch (IOException e) {
if (e.toString().contains("Failed to find any Kerberos")) {
logConsole(true, true, error, "No/bad Kerberos ticket - please authenticate.");
System.exit(1);
} else if (e.toString().contains("Permission denied")) {
logConsole(true, true, error, "Permission denied.");
System.err.println(
"; Please go to https://go/itforms and filled out the Hadoop Onboarding Form "
+ "to get access to the requested data. Paste the following data into the ticket to help with your request:\n"
+ "Error Message"
+ e);
System.exit(1);
} else if (e.toString().contains("quota") && e.toString().contains("exceeded")) {
logConsole(true, true, error, "Disk quota Exceeded.");
System.exit(1);
}
logConsole(
true,
true,
error,
"\n\tError running mapreduce job." + generalError() + "\n\tCommand stopped");
e.printStackTrace();
System.exit(1);
}
}
public static void main(String[] args) throws Exception {
if (args.length != 2) {
System.out.println("usage: [input] [output]");
System.exit(-1);
}
int rc = ToolRunner.run(new MapReduceLauncher(), args);
System.exit(rc);
}
public static void main(String[] argv) throws Exception {
int res = -1;
try {
res = ToolRunner.run(new Configuration(), new Gridmix(argv), argv);
} finally {
System.exit(res);
}
}
/**
* DoIt.
*
* @param args the arguments to do it with
*/
public static void main(String[] args) {
try {
ToolRunner.run(new ProcessingRecordsPrinter(), args);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static void main(String[] args) throws Exception {
if (args.length != 2) {
System.out.println("Usage: TopKRecords /path/to/citation.txt output_dir");
System.exit(-1);
}
int res = ToolRunner.run(new Configuration(), new TopKRecords(), args);
System.exit(res);
}
/** Dispatches command-line arguments to the tool via the {@code ToolRunner}. */
public static void main(String[] args) throws Exception {
LOG.info(
"Running "
+ TrecDocnoMappingBuilder.class.getCanonicalName()
+ " with args "
+ Arrays.toString(args));
ToolRunner.run(new Configuration(), new TrecDocnoMappingBuilder(), args);
}