public void testCopyDuplication() throws Exception {
final FileSystem localfs = FileSystem.get(LOCAL_FS, new Configuration());
try {
MyFile[] files = createFiles(localfs, TEST_ROOT_DIR + "/srcdat");
ToolRunner.run(
new DistCpV1(new Configuration()),
new String[] {
"file:///" + TEST_ROOT_DIR + "/srcdat", "file:///" + TEST_ROOT_DIR + "/src2/srcdat"
});
assertTrue(
"Source and destination directories do not match.",
checkFiles(localfs, TEST_ROOT_DIR + "/src2/srcdat", files));
assertEquals(
DistCpV1.DuplicationException.ERROR_CODE,
ToolRunner.run(
new DistCpV1(new Configuration()),
new String[] {
"file:///" + TEST_ROOT_DIR + "/srcdat",
"file:///" + TEST_ROOT_DIR + "/src2/srcdat",
"file:///" + TEST_ROOT_DIR + "/destdat",
}));
} finally {
deldir(localfs, TEST_ROOT_DIR + "/destdat");
deldir(localfs, TEST_ROOT_DIR + "/srcdat");
deldir(localfs, TEST_ROOT_DIR + "/src2");
}
}
public static void main(String[] args) throws Exception {
if (args[args.length - 1].equals("history")) {
try {
int exitCode = ToolRunner.run(new ImportVTLocationFromFileWithReducer(), args);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
} else {
VesselTrackerLoader vl = new VesselTrackerLoader();
while (true) {
try {
if (vl.load()) {
int exitCode = ToolRunner.run(new ImportVTLocationFromFileWithReducer(), args);
vl.moveFile();
}
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
Thread.sleep(2 * 60 * 1000);
}
}
}
public void testMapCount() throws Exception {
String namenode = null;
MiniDFSCluster dfs = null;
MiniDFSCluster mr = null;
try {
Configuration conf = new Configuration();
dfs = new MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).build();
FileSystem fs = dfs.getFileSystem();
final FsShell shell = new FsShell(conf);
namenode = fs.getUri().toString();
MyFile[] files = createFiles(fs.getUri(), "/srcdat");
long totsize = 0;
for (MyFile f : files) {
totsize += f.getSize();
}
Configuration job = new JobConf(conf);
job.setLong("distcp.bytes.per.map", totsize / 3);
ToolRunner.run(
new DistCpV1(job),
new String[] {
"-m", "100", "-log", namenode + "/logs", namenode + "/srcdat", namenode + "/destdat"
});
assertTrue(
"Source and destination directories do not match.", checkFiles(fs, "/destdat", files));
String logdir = namenode + "/logs";
System.out.println(execCmd(shell, "-lsr", logdir));
FileStatus[] logs = fs.listStatus(new Path(logdir));
// rare case where splits are exact, logs.length can be 4
assertTrue(logs.length == 2);
deldir(fs, "/destdat");
deldir(fs, "/logs");
ToolRunner.run(
new DistCpV1(job),
new String[] {
"-m", "1", "-log", namenode + "/logs", namenode + "/srcdat", namenode + "/destdat"
});
System.out.println(execCmd(shell, "-lsr", logdir));
logs = fs.globStatus(new Path(namenode + "/logs/part*"));
assertTrue("Unexpected map count, logs.length=" + logs.length, logs.length == 1);
} finally {
if (dfs != null) {
dfs.shutdown();
}
if (mr != null) {
mr.shutdown();
}
}
}
private void doCopyTableTest(boolean bulkload) throws Exception {
final TableName TABLENAME1 = TableName.valueOf("testCopyTable1");
final TableName TABLENAME2 = TableName.valueOf("testCopyTable2");
final byte[] FAMILY = Bytes.toBytes("family");
final byte[] COLUMN1 = Bytes.toBytes("c1");
try (Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY); ) {
// put rows into the first table
for (int i = 0; i < 10; i++) {
Put p = new Put(Bytes.toBytes("row" + i));
p.addColumn(FAMILY, COLUMN1, COLUMN1);
t1.put(p);
}
CopyTable copy = new CopyTable();
int code;
if (bulkload) {
code =
ToolRunner.run(
new Configuration(TEST_UTIL.getConfiguration()),
copy,
new String[] {
"--new.name=" + TABLENAME2.getNameAsString(),
"--bulkload",
TABLENAME1.getNameAsString()
});
} else {
code =
ToolRunner.run(
new Configuration(TEST_UTIL.getConfiguration()),
copy,
new String[] {
"--new.name=" + TABLENAME2.getNameAsString(), TABLENAME1.getNameAsString()
});
}
assertEquals("copy job failed", 0, code);
// verify the data was copied into table 2
for (int i = 0; i < 10; i++) {
Get g = new Get(Bytes.toBytes("row" + i));
Result r = t2.get(g);
assertEquals(1, r.size());
assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
}
} finally {
TEST_UTIL.deleteTable(TABLENAME1);
TEST_UTIL.deleteTable(TABLENAME2);
}
}
@BeforeTest
public void before() throws Exception {
URL inputSeqFileUrl = this.getClass().getResource("/publications.seq");
ToolRunner.run(
new Configuration(),
new DuplicateGenerator(),
new String[] {inputSeqFileUrl.getFile(), this.getClass().getResource("/").getFile()});
FileUtils.deleteDirectory(new File(outputDir));
URL inputFileUrl = this.getClass().getResource("/generated/ambiguous-publications.seq");
ToolRunner.run(
new Configuration(),
new DuplicateWorkDetector(),
new String[] {inputFileUrl.getPath(), outputDir});
}
public static void main(String[] args) throws Exception {
// -files option is also used by GenericOptionsParser
// Make sure that is not the first argument for fsck
int res = -1;
if ((args.length == 0) || ("-files".equals(args[0]))) {
printUsage(System.err);
ToolRunner.printGenericCommandUsage(System.err);
} else if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) {
res = 0;
} else {
res = ToolRunner.run(new DFSck(new HdfsConfiguration()), args);
}
System.exit(res);
}
public static void main(String[] args) throws Exception {
// Call the run method to launch the MapReduce Job
int result =
ToolRunner.run(new Configuration(USE_MINI_ACCUMULO), new IndexedDocIndexer(), args);
System.exit(result);
}
public static void main(String[] args) throws Exception {
conf = new Configuration();
int res = ToolRunner.run(conf, new TestMultipleOutputs4(), args);
System.out.println("PASS");
System.exit(res);
}
public static void main(String[] args) throws Exception {
final Configuration conf = new Configuration();
initalizeConf(conf, args);
int res = ToolRunner.run(conf, new FrameworkDriver(), args);
System.exit(res);
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
int exitCode = ToolRunner.run(conf, new ColumnTextDriver(), args);
System.exit(exitCode);
}
/**
* @param args :
* <ol>
* <li>table name
* </ol>
*/
public static void main(String[] args) {
try {
ToolRunner.run(HBaseConfiguration.create(), new CountRowsMR(), args);
} catch (Exception e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws Exception {
if (ToolRunner.run(new FeatureMatching(), args) == 1) {
System.out.println(".......Feature Match failure........");
System.exit(1);
}
System.exit(0);
}
/** tests basedir option copying files from dfs file system to dfs file system */
public void testBasedir() throws Exception {
String namenode = null;
MiniDFSCluster cluster = null;
try {
Configuration conf = new Configuration();
cluster = new MiniDFSCluster(conf, 2, true, null);
final FileSystem hdfs = cluster.getFileSystem();
namenode = FileSystem.getDefaultUri(conf).toString();
if (namenode.startsWith("hdfs://")) {
MyFile[] files = createFiles(URI.create(namenode), "/basedir/middle/srcdat");
ToolRunner.run(
new DistCpV1(conf),
new String[] {
"-basedir", "/basedir", namenode + "/basedir/middle/srcdat", namenode + "/destdat"
});
assertTrue(
"Source and destination directories do not match.",
checkFiles(hdfs, "/destdat/middle/srcdat", files));
deldir(hdfs, "/destdat");
deldir(hdfs, "/basedir");
deldir(hdfs, "/logs");
}
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
/** Print help. */
private void printHelp() {
String summary =
"Usage: bin/hdfs oev [OPTIONS] -i INPUT_FILE -o OUTPUT_FILE\n"
+ "Offline edits viewer\n"
+ "Parse a Hadoop edits log file INPUT_FILE and save results\n"
+ "in OUTPUT_FILE.\n"
+ "Required command line arguments:\n"
+ "-i,--inputFile <arg> edits file to process, xml (case\n"
+ " insensitive) extension means XML format,\n"
+ " any other filename means binary format\n"
+ "-o,--outputFile <arg> Name of output file. If the specified\n"
+ " file exists, it will be overwritten,\n"
+ " format of the file is determined\n"
+ " by -p option\n"
+ "\n"
+ "Optional command line arguments:\n"
+ "-p,--processor <arg> Select which type of processor to apply\n"
+ " against image file, currently supported\n"
+ " processors are: binary (native binary format\n"
+ " that Hadoop uses), xml (default, XML\n"
+ " format), stats (prints statistics about\n"
+ " edits file)\n"
+ "-h,--help Display usage information and exit\n"
+ "-v,--verbose More verbose output, prints the input and\n"
+ " output filenames, for processors that write\n"
+ " to a file, also output to screen. On large\n"
+ " image files this will dramatically increase\n"
+ " processing time (default is false).\n";
System.out.println(summary);
System.out.println();
ToolRunner.printGenericCommandUsage(System.out);
}
/** copy files from dfs file system to local file system */
public void testCopyFromDfsToLocal() throws Exception {
MiniDFSCluster cluster = null;
try {
Configuration conf = new Configuration();
final FileSystem localfs = FileSystem.get(LOCAL_FS, conf);
cluster = new MiniDFSCluster(conf, 1, true, null);
final FileSystem hdfs = cluster.getFileSystem();
final String namenode = FileSystem.getDefaultUri(conf).toString();
if (namenode.startsWith("hdfs://")) {
MyFile[] files = createFiles(URI.create(namenode), "/srcdat");
ToolRunner.run(
new DistCpV1(conf),
new String[] {
"-log", "/logs", namenode + "/srcdat", "file:///" + TEST_ROOT_DIR + "/destdat"
});
assertTrue(
"Source and destination directories do not match.",
checkFiles(localfs, TEST_ROOT_DIR + "/destdat", files));
assertTrue("Log directory does not exist.", hdfs.exists(new Path("/logs")));
deldir(localfs, TEST_ROOT_DIR + "/destdat");
deldir(hdfs, "/logs");
deldir(hdfs, "/srcdat");
}
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
/** copy empty directory on dfs file system */
public void testEmptyDir() throws Exception {
String namenode = null;
MiniDFSCluster cluster = null;
try {
Configuration conf = new Configuration();
cluster = new MiniDFSCluster(conf, 2, true, null);
final FileSystem hdfs = cluster.getFileSystem();
namenode = FileSystem.getDefaultUri(conf).toString();
if (namenode.startsWith("hdfs://")) {
FileSystem fs = FileSystem.get(URI.create(namenode), new Configuration());
fs.mkdirs(new Path("/empty"));
ToolRunner.run(
new DistCpV1(conf),
new String[] {"-log", namenode + "/logs", namenode + "/empty", namenode + "/dest"});
fs = FileSystem.get(URI.create(namenode + "/destdat"), conf);
assertTrue(
"Destination directory does not exist.", fs.exists(new Path(namenode + "/dest")));
deldir(hdfs, "/dest");
deldir(hdfs, "/empty");
deldir(hdfs, "/logs");
}
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
@Override
public int run(String[] args) throws Exception {
if (args.length < 2) {
System.out.println("hitsperurl.DriverHitsPerUrl <inDir> <outDir>");
ToolRunner.printGenericCommandUsage(System.out);
System.out.println("");
return -1;
}
System.out.println(Arrays.toString(args));
Job job = new Job(getConf(), "number of hits per url");
job.setJarByClass(DriverHitsPerUrl.class);
job.setMapperClass(ParserMapper.class);
// Uncomment this to set combiner
// job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(AggregateURLReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.waitForCompletion(true);
return 0;
}
@Test
public void testSolver() throws Exception {
Configuration conf = getConfiguration();
Path testData = getTestTempDirPath("testdata");
DistributedRowMatrix matrix =
new TestDistributedRowMatrix()
.randomDistributedMatrix(10, 10, 10, 10, 10.0, true, testData.toString());
matrix.setConf(conf);
Path output = getTestTempFilePath("output");
Path vectorPath = getTestTempFilePath("vector");
Path tempPath = getTestTempDirPath("tmp");
Vector vector = randomVector(matrix.numCols(), 10.0);
saveVector(conf, vectorPath, vector);
String[] args = {
"-i", matrix.getRowPath().toString(),
"-o", output.toString(),
"--tempDir", tempPath.toString(),
"--vector", vectorPath.toString(),
"--numRows", "10",
"--numCols", "10",
"--symmetric", "true"
};
DistributedConjugateGradientSolver solver = new DistributedConjugateGradientSolver();
ToolRunner.run(getConfiguration(), solver.job(), args);
Vector x = loadVector(conf, output);
Vector solvedVector = matrix.times(x);
double distance = Math.sqrt(vector.getDistanceSquared(solvedVector));
assertEquals(0.0, distance, EPSILON);
}
/**
* This is the main driver for recursively copying directories across file systems. It takes at
* least two cmdline parameters. A source URL and a destination URL. It then essentially does an
* "ls -lR" on the source URL, and writes the output in a round-robin manner to all the map input
* files. The mapper actually copies the files allotted to it. The reduce is empty.
*/
public int run(String[] args) {
try {
copy(conf, Arguments.valueOf(args, conf));
return 0;
} catch (IllegalArgumentException e) {
System.err.println(StringUtils.stringifyException(e) + "\n" + usage);
ToolRunner.printGenericCommandUsage(System.err);
return -1;
} catch (DuplicationException e) {
System.err.println(StringUtils.stringifyException(e));
return DuplicationException.ERROR_CODE;
} catch (RemoteException e) {
final IOException unwrapped =
e.unwrapRemoteException(
FileNotFoundException.class,
AccessControlException.class,
QuotaExceededException.class);
System.err.println(StringUtils.stringifyException(unwrapped));
return -3;
} catch (Exception e) {
System.err.println(
"With failures, global counters are inaccurate; " + "consider running with -i");
System.err.println("Copy failed: " + StringUtils.stringifyException(e));
return -999;
}
}
@Override
public int run(String[] args) throws Exception {
if (args.length != 2) {
System.err.printf(
"Usage: %s [generic options] <input> <output>\n", getClass().getSimpleName());
ToolRunner.printGenericCommandUsage(System.err);
return -1;
}
JobConf conf = new JobConf(getConf(), getClass());
conf.setJobName("Max temperature");
FileInputFormat.addInputPath(conf, new Path(args[0]));
FileOutputFormat.setOutputPath(conf, new Path(args[1]));
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(IntWritable.class);
conf.setMapperClass(MaxTemperatureMapper.class);
conf.setCombinerClass(MaxTemperatureReducer.class);
conf.setReducerClass(MaxTemperatureReducer.class);
// vv MaxTemperatureDriverV6
conf.setProfileEnabled(true);
conf.setProfileParams(
"-agentlib:hprof=cpu=samples,heap=sites,depth=6," + "force=n,thread=y,verbose=n,file=%s");
conf.setProfileTaskRange(true, "0-2");
// ^^ MaxTemperatureDriverV6
JobClient.runJob(conf);
return 0;
}
public static void main(final String[] pArgs) throws Exception {
TreasuryYieldMulti tym = new TreasuryYieldMulti();
// Here is an example of how to use multiple collections as the input to
// a hadoop job, from within Java code directly.
MultiCollectionSplitBuilder mcsb = new MultiCollectionSplitBuilder();
mcsb.add(
new MongoURI("mongodb://localhost:27017/mongo_hadoop.yield_historical.in"),
(MongoURI) null, // authuri
true, // notimeout
(DBObject) null, // fields
(DBObject) null, // sort
(DBObject) null, // query
false)
.add(
new MongoURI("mongodb://localhost:27017/mongo_hadoop.yield_historical.in"),
(MongoURI) null, // authuri
true, // notimeout
(DBObject) null, // fields
(DBObject) null, // sort
new BasicDBObject("_id", new BasicDBObject("$gt", new Date(883440000000L))),
false); // range query
Configuration conf = new Configuration();
conf.set(MultiMongoCollectionSplitter.MULTI_COLLECTION_CONF_KEY, mcsb.toJSON());
conf.setSplitterClass(conf, MultiMongoCollectionSplitter.class);
System.exit(ToolRunner.run(conf, new TreasuryYieldXMLConfig(), pArgs));
}
public static void main(String[] args) throws Exception {
if (args.length != 2) {
System.err.println("Usage: Class <input> <output>");
System.exit(2);
}
ToolRunner.run(new Configuration(), new SecondarySortingExample(), args);
}
/**
* Run the RowCounter map reduce job and verify the row count.
*
* @param args the command line arguments to be used for rowcounter job.
* @param expectedCount the expected row count (result of map reduce job).
* @throws Exception
*/
private void runRowCount(String[] args, int expectedCount) throws Exception {
final RowCounter counter = new RowCounter();
assertEquals(
"job failed either due to failure or miscount (see log output).",
0,
ToolRunner.run(TEST_UTIL.getConfiguration(), counter, args));
}
@Test
public void testRun() throws Exception {
Path input = getTestTempDirPath("input");
Path output = getTestTempDirPath("output");
Path seedsPath = getTestTempDirPath("seeds");
List<VectorWritable> points = getPointsWritable(REFERENCE);
List<VectorWritable> seeds = getPointsWritable(SEEDS);
Configuration conf = new Configuration();
ClusteringTestUtils.writePointsToFile(points, true, new Path(input, "file1"), fs, conf);
ClusteringTestUtils.writePointsToFile(seeds, true, new Path(seedsPath, "part-seeds"), fs, conf);
String[] args = {
optKey(DefaultOptionCreator.INPUT_OPTION),
input.toString(),
optKey(VectorDistanceSimilarityJob.SEEDS),
seedsPath.toString(),
optKey(DefaultOptionCreator.OUTPUT_OPTION),
output.toString(),
optKey(DefaultOptionCreator.DISTANCE_MEASURE_OPTION),
EuclideanDistanceMeasure.class.getName()
};
ToolRunner.run(new Configuration(), new VectorDistanceSimilarityJob(), args);
int expect = SEEDS.length * REFERENCE.length;
DummyOutputCollector<StringTuple, DoubleWritable> collector =
new DummyOutputCollector<StringTuple, DoubleWritable>();
//
for (Pair<StringTuple, DoubleWritable> record :
new SequenceFileIterable<StringTuple, DoubleWritable>(
new Path(output, "part-m-00000"), conf)) {
collector.collect(record.getFirst(), record.getSecond());
}
assertEquals(expect, collector.getData().size());
}
/**
* run a distributed job and verify that TokenCache is available
*
* @throws IOException
*/
@Test
public void testTokenCache() throws IOException {
System.out.println("running dist job");
// make sure JT starts
jConf = mrCluster.createJobConf();
// provide namenodes names for the job to get the delegation tokens for
String nnUri = dfsCluster.getURI(0).toString();
jConf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri);
// job tracker principla id..
jConf.set(JTConfig.JT_USER_NAME, "jt_id/foo@BAR");
// using argument to pass the file name
String[] args = {
"-tokenCacheFile", tokenFileName.toString(), "-m", "1", "-r", "1", "-mt", "1", "-rt", "1"
};
int res = -1;
try {
res = ToolRunner.run(jConf, new MySleepJob(), args);
} catch (Exception e) {
System.out.println("Job failed with" + e.getLocalizedMessage());
e.printStackTrace(System.out);
fail("Job failed");
}
assertEquals("dist job res is not 0", res, 0);
}
@Test
public void testBracketedCounters() throws Exception {
final Configuration conf = new Configuration();
final FileSystem lfs = FileSystem.getLocal(conf);
final Path rootInputDir =
new Path(System.getProperty("test.tools.input.dir", "")).makeQualified(lfs);
final Path rootTempDir =
new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs);
final Path rootInputPath = new Path(rootInputDir, "rumen/small-trace-test");
final Path tempDir = new Path(rootTempDir, "TestBracketedCounters");
lfs.delete(tempDir, true);
final Path topologyPath = new Path(tempDir, "dispatch-topology.json");
final Path tracePath = new Path(tempDir, "dispatch-trace.json");
final Path inputPath = new Path(rootInputPath, "counters-format-test-logs");
System.out.println("topology result file = " + topologyPath);
System.out.println("testBracketedCounters() trace result file = " + tracePath);
final Path goldPath = new Path(rootInputPath, "counters-test-trace.json.gz");
String[] args = {tracePath.toString(), topologyPath.toString(), inputPath.toString()};
Tool analyzer = new TraceBuilder();
int result = ToolRunner.run(analyzer, args);
assertEquals("Non-zero exit", 0, result);
TestRumenJobTraces.<LoggedJob>jsonFileMatchesGold(
conf, tracePath, goldPath, LoggedJob.class, "trace");
}
public int run(String[] args) throws Exception {
if (args.length != 2) {
System.out.printf(
"Usage: %s [generic options] <input dir> <output dir>\n", getClass().getSimpleName());
ToolRunner.printGenericCommandUsage(System.out);
return -1;
}
Job job = new Job(getConf());
job.setJarByClass(RowFilter.class);
job.setJobName(this.getClass().getName());
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.setMapperClass(RowMapper.class);
job.setNumReduceTasks(0);
job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(LongWritable.class);
job.setOutputValueClass(Text.class);
if (job.waitForCompletion(true)) {
return 0;
}
return 1;
}
@Override
public int run(String[] args) throws Exception {
if (args.length != 2) {
System.err.println("Usage: WhiteHouseVisitorDriver <input path> <output path>");
ToolRunner.printGenericCommandUsage(System.err);
return -1;
}
Configuration conf = new Configuration();
Job job = new Job(conf);
job.setJarByClass(WhiteHouseVisitorDriver.class);
// input file format
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
// map/combine/reduce class definition
job.setMapperClass(WhiteHouseVisitorMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);
// Key, Value set type definition
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
return job.waitForCompletion(true) ? 0 : 1;
}
public static void main(String[] args) throws Exception {
final Configuration configuration = HBaseConfiguration.create();
configuration.addResource("grade.xml");
String tables = configuration.get("hbase.cdn.tables");
if (Strings.isNullOrEmpty(tables)) {
return;
}
List<String> list = Lists.newArrayList(Splitter.on(",").split(tables));
List<String> results =
Lists.transform(
list,
new Function<String, String>() {
@Override
public String apply(@Nullable java.lang.String input) {
return String.format(
configuration.get("hdfs.directory.base.db"), new Date(), input);
}
});
String[] arrays =
new String[] {
Joiner.on(",").join(results),
String.format(configuration.get("hdfs.directory.num.middle"), new Date()),
String.format(configuration.get("hdfs.directory.num.result"), new Date())
};
AbstractJob job = new TopNJob();
// job.setStart(true);
int i = ToolRunner.run(configuration, job, arrays);
System.exit(i);
}
protected void printUsage(PrintStream out) {
ToolRunner.printGenericCommandUsage(out);
out.println("Usage: gridmix [-generate <MiB>] [-users URI] <iopath> <trace>");
out.println(" e.g. gridmix -generate 100m foo -");
out.println("Configuration parameters:");
out.printf(" %-42s : Output directory\n", GRIDMIX_OUT_DIR);
out.printf(" %-42s : Submitting threads\n", GRIDMIX_SUB_THR);
out.printf(" %-42s : Queued job desc\n", GRIDMIX_QUE_DEP);
out.printf(" %-42s : Key fraction of rec\n", AvgRecordFactory.GRIDMIX_KEY_FRC);
out.printf(" %-42s : User resolution class\n", GRIDMIX_USR_RSV);
out.printf(
" %-42s : Enable/disable using queues in trace\n",
GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE);
out.printf(" %-42s : Default queue\n", GridmixJob.GRIDMIX_DEFAULT_QUEUE);
StringBuilder sb = new StringBuilder();
String sep = "";
for (GridmixJobSubmissionPolicy policy : GridmixJobSubmissionPolicy.values()) {
sb.append(sep);
sb.append(policy.name());
sep = "|";
}
out.printf(
" %-40s : Job submission policy (%s)\n",
GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, sb.toString());
}
