public static void main(String[] args) throws ConfigurationException, ParseException {
Configuration config = new PropertiesConfiguration("application.properties");
// Options
Options options = setUpOptions();
CommandLineParser parser = new DefaultParser();
CommandLine cmd = parser.parse(options, args);
// TODO Anatoli: it makes sense to add strict validation
for (Option option : cmd.getOptions()) {
if (StringUtils.isNotBlank(option.getValue())) {
config.addProperty(option.getLongOpt(), option.getValue());
}
}
if (cmd.hasOption('h')) {
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(120);
formatter.printHelp(
"java -Dlogback.configurationFile=logback.xml -jar whatIsOilPriceNow.jar [options]",
options);
return;
}
RockNRoller roller = new RockNRoller(config);
roller.startMainLoop();
}
private void printUsage(Options options, Boolean andExit) {
String HEADER = "";
String USAGE = "java -jar QuickFormatter.jar -i inputfile [-o outputfile] [-d delimiter] [-x]";
String FOOTER = "\n\n";
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.setWidth(100);
helpFormatter.printHelp(USAGE, HEADER, options, FOOTER);
if (andExit) System.exit(0);
}
private void usage(ParseException e, boolean exit) {
if (e != null) {
System.err.println(e.getMessage());
System.err.println();
System.err.println();
}
PrintWriter pw = new PrintWriter(System.err, true);
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(80);
formatter.printHelp(
pw,
formatter.getWidth(),
getCommandUsage(),
null,
getOptions(false),
formatter.getLeftPadding(),
formatter.getDescPadding(),
null);
pw.flush();
System.err.println();
System.err.println("term filter: ");
System.err.println(
" - each f[n], t[n] pair represents a term filter, f[n], t[n] don't have to both exist");
System.err.println(" - f[n] represents a term field name, t[n] represents a term text");
System.err.println(" - the final filter is formed by ORing all term filters together");
System.err.println(" - maximum of " + NUM_TERM_FILTERS + " term filters are allowed");
System.err.println();
System.err.println(" examples:");
System.err.println(" -f1 l.content -t1 foo" + " (term l.content=foo)");
System.err.println(
" -f2 from" + " (all terms with from as the field name)");
System.err.println(" -t3 bar" + " (all terms with bar as the text)");
System.err.println();
System.err.println();
System.err.println("Sample command lines:");
System.err.println(
"zmjava com.zimbra.cs.index.LuceneViewer -a dump -i /opt/zimbra/index/0/2/index/0 -o /tmp/user1-index-dump.txt");
System.err.println(
"zmjava com.zimbra.cs.index.LuceneViewer -a dump -v -f1 l.content -t1 jay -f2 subject -t2 howdy -i /opt/zimbra/index/0/2/index/0 -o /tmp/user1-index-dump.txt");
System.err.println(
"zmjava com.zimbra.cs.index.LuceneViewer -a dump -f1 from [email protected] -i /opt/zimbra/index/0/2/index/0 -o /tmp/user1-index-dump.txt");
if (exit) {
System.exit(1);
}
}
public static CommandLine parseCmdLine(
final String appName, String[] args, Options options, CommandLineParser parser) {
HelpFormatter hf = new HelpFormatter();
hf.setWidth(110);
CommandLine commandLine = null;
try {
commandLine = parser.parse(options, args);
if (commandLine.hasOption('h')) {
hf.printHelp(appName, options, true);
return null;
}
} catch (ParseException e) {
hf.printHelp(appName, options, true);
}
return commandLine;
}
/** Runs this tool. */
@SuppressWarnings({"static-access"})
public int run(String[] args) throws Exception {
Options options = new Options();
options.addOption(
OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INDEX));
options.addOption(
OptionBuilder.withArgName("path")
.hasArg()
.withDescription("output path")
.create(COLLECTION));
CommandLine cmdline = null;
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
System.exit(-1);
}
if (!cmdline.hasOption(INDEX) || !cmdline.hasOption(COLLECTION)) {
System.out.println("args: " + Arrays.toString(args));
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(120);
formatter.printHelp(LookupPostingsCompressed.class.getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
System.exit(-1);
}
String indexPath = cmdline.getOptionValue(INDEX);
String collectionPath = cmdline.getOptionValue(COLLECTION);
if (collectionPath.endsWith(".gz")) {
System.out.println("gzipped collection is not seekable: use compressed version!");
System.exit(-1);
}
Configuration config = new Configuration();
FileSystem fs = FileSystem.get(config);
MapFile.Reader reader = new MapFile.Reader(new Path(indexPath + "/part-r-00000"), config);
FSDataInputStream collection = fs.open(new Path(collectionPath));
BufferedReader d = new BufferedReader(new InputStreamReader(collection));
Text key = new Text();
ArrayListWritable<PairOfInts> postings;
BytesWritable bytesValue = new BytesWritable();
System.out.println("Looking up postings for the term \"starcross'd\"");
key.set("starcross'd");
reader.get(key, bytesValue);
postings = deserializePosting(bytesValue);
// ArrayListWritable<PairOfVInts> postings = value;
for (PairOfInts pair : postings) {
System.out.println(pair);
collection.seek(pair.getLeftElement());
System.out.println(d.readLine());
}
bytesValue = new BytesWritable();
key.set("gold");
reader.get(key, bytesValue);
postings = deserializePosting(bytesValue);
System.out.println(
"Complete postings list for 'gold': (" + postings.size() + ", " + postings + ")");
Int2IntFrequencyDistribution goldHist = new Int2IntFrequencyDistributionEntry();
// postings = value;
for (PairOfInts pair : postings) {
goldHist.increment(pair.getRightElement());
}
System.out.println("histogram of tf values for gold");
for (PairOfInts pair : goldHist) {
System.out.println(pair.getLeftElement() + "\t" + pair.getRightElement());
}
bytesValue = new BytesWritable();
key.set("silver");
reader.get(key, bytesValue);
postings = deserializePosting(bytesValue);
System.out.println(
"Complete postings list for 'silver': (" + postings.size() + ", " + postings + ")");
Int2IntFrequencyDistribution silverHist = new Int2IntFrequencyDistributionEntry();
// postings = value;
for (PairOfInts pair : postings) {
silverHist.increment(pair.getRightElement());
}
System.out.println("histogram of tf values for silver");
for (PairOfInts pair : silverHist) {
System.out.println(pair.getLeftElement() + "\t" + pair.getRightElement());
}
bytesValue = new BytesWritable();
key.set("bronze");
Writable w = reader.get(key, bytesValue);
if (w == null) {
System.out.println("the term bronze does not appear in the collection");
}
collection.close();
reader.close();
return 0;
}
public static void printCommandLineHelp(final String appName, final Options options) {
HelpFormatter hf = new HelpFormatter();
hf.setWidth(110);
hf.printHelp(appName, options, true);
}
@SuppressWarnings("static-access")
@Override
public int run(String[] args) throws Exception {
Options options = new Options();
options.addOption(
OptionBuilder.withArgName("path").hasArg().withDescription("bz2 input path").create(INPUT));
options.addOption(
OptionBuilder.withArgName("path").hasArg().withDescription("output path").create(OUTPUT));
options.addOption(
OptionBuilder.withArgName("integer")
.hasArg()
.withDescription("number of samples")
.create(nSamplesOption));
CommandLine cmdline;
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
return -1;
}
if (!cmdline.hasOption(INPUT)
|| !cmdline.hasOption(OUTPUT)
|| !cmdline.hasOption(nSamplesOption)) {
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(120);
formatter.printHelp(this.getClass().getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
return -1;
}
String inputPath = cmdline.getOptionValue(INPUT);
String outputPath = cmdline.getOptionValue(OUTPUT);
String nSamplesIn = cmdline.getOptionValue(nSamplesOption);
int reduceTasks = 1;
LOG.info("Tool name: " + this.getClass().getName());
LOG.info(" - bz2 file: " + inputPath);
LOG.info(" - output file: " + outputPath);
JobConf conf = new JobConf(getConf(), DedupCLIRMHPairs.class);
conf.setJobName(
String.format("DedupSentencePairs[%s: %s, %s: %s]", INPUT, inputPath, OUTPUT, outputPath));
conf.setNumMapTasks(4);
conf.setNumReduceTasks(reduceTasks);
conf.setInt("nSamples", Integer.parseInt(nSamplesIn));
FileInputFormat.setInputPaths(conf, new Path(inputPath));
FileOutputFormat.setOutputPath(conf, new Path(outputPath));
// Set heap space - using old API
conf.set("mapred.job.map.memory.mb", "2048");
conf.set("mapred.map.child.java.opts", "-Xmx2048m");
conf.set("mapred.job.reduce.memory.mb", "4096");
conf.set("mapred.reduce.child.java.opts", "-Xmx4096m");
conf.setMapperClass(DedupMapper.class);
conf.setReducerClass(DedupReducer.class);
conf.setInputFormat(SequenceFileInputFormat.class);
conf.setOutputFormat(MapFileOutputFormat.class);
conf.setMapOutputKeyClass(PairOfInts.class);
conf.setMapOutputValueClass(IntWritable.class);
conf.setOutputKeyClass(PairOfInts.class);
conf.setOutputValueClass(IntWritable.class);
// Delete the output directory if it exists already.
Path outputDir = new Path(outputPath);
FileSystem.get(conf).delete(outputDir, true);
JobClient.runJob(conf);
return 0;
}
private static void printUsage(@NotNull CliWrapper cliOptions) {
System.out.println(HEADER);
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.setWidth(WIDTH);
helpFormatter.printHelp(SPUTNIK, cliOptions.getOptions(), true);
}
/**
* Prints a usage help message.
*
* @param options command-line options
*/
private void printHelpMessage(Options options) {
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(HELP_FORMATTER_WIDTH);
formatter.setArgName("Test");
formatter.printHelp(getApplicationName(), options, true);
}
/** Runs this tool. */
@SuppressWarnings({"static-access"})
public int run(String[] args) throws Exception {
Options options = new Options();
options.addOption(new Option(RANGE, "use range partitioner"));
options.addOption(
OptionBuilder.withArgName("path").hasArg().withDescription("input path").create(INPUT));
options.addOption(
OptionBuilder.withArgName("path").hasArg().withDescription("output path").create(OUTPUT));
options.addOption(
OptionBuilder.withArgName("num")
.hasArg()
.withDescription("number of nodes")
.create(NUM_NODES));
options.addOption(
OptionBuilder.withArgName("num")
.hasArg()
.withDescription("number of partitions")
.create(NUM_PARTITIONS));
CommandLine cmdline;
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
return -1;
}
if (!cmdline.hasOption(INPUT)
|| !cmdline.hasOption(OUTPUT)
|| !cmdline.hasOption(NUM_NODES)
|| !cmdline.hasOption(NUM_PARTITIONS)) {
System.out.println("args: " + Arrays.toString(args));
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(120);
formatter.printHelp(this.getClass().getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
return -1;
}
String inPath = cmdline.getOptionValue(INPUT);
String outPath = cmdline.getOptionValue(OUTPUT);
int nodeCount = Integer.parseInt(cmdline.getOptionValue(NUM_NODES));
int numParts = Integer.parseInt(cmdline.getOptionValue(NUM_PARTITIONS));
boolean useRange = cmdline.hasOption(RANGE);
LOG.info("Tool name: " + PartitionGraph.class.getSimpleName());
LOG.info(" - input dir: " + inPath);
LOG.info(" - output dir: " + outPath);
LOG.info(" - num partitions: " + numParts);
LOG.info(" - node cnt: " + nodeCount);
LOG.info(" - use range partitioner: " + useRange);
Configuration conf = getConf();
conf.setInt("NodeCount", nodeCount);
Job job = Job.getInstance(conf);
job.setJobName(PartitionGraph.class.getSimpleName() + ":" + inPath);
job.setJarByClass(PartitionGraph.class);
job.setNumReduceTasks(numParts);
FileInputFormat.setInputPaths(job, new Path(inPath));
FileOutputFormat.setOutputPath(job, new Path(outPath));
job.setInputFormatClass(NonSplitableSequenceFileInputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(PageRankNode.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(PageRankNode.class);
FileSystem.get(conf).delete(new Path(outPath), true);
job.waitForCompletion(true);
return 0;
}
@SuppressWarnings("static-access")
@Override
public int run(String[] args) throws Exception {
Options options = new Options();
options.addOption(
OptionBuilder.withArgName("path")
.hasArg()
.withDescription("output path")
.create(matchOutput));
options.addOption(
OptionBuilder.withArgName("path")
.hasArg()
.withDescription("output path")
.create(nomatchOutput));
options.addOption(
OptionBuilder.withArgName("integer")
.hasArg()
.withDescription("number of samples")
.create(nSamplesOption));
CommandLine cmdline;
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
return -1;
}
if (!cmdline.hasOption(matchOutput)
|| !cmdline.hasOption(nomatchOutput)
|| !cmdline.hasOption(nSamplesOption)) {
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(120);
formatter.printHelp(this.getClass().getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
return -1;
}
String matchOutputPath = cmdline.getOptionValue(matchOutput);
String nomatchOutputPath = cmdline.getOptionValue(nomatchOutput);
String nSamplesIn = cmdline.getOptionValue(nSamplesOption);
LOG.info("Tool name: " + this.getClass().getName());
// LOG.info(" - input file: " + inputPath);
// LOG.info(" - output file: " + outputPath);
JobConf conf = new JobConf(getConf(), JaccardCompare.class);
conf.setJobName(String.format("JaccardCompare"));
// FileInputFormat.setInputPaths(conf, new Path(inputPath));
// FileOutputFormat.setOutputPath(conf, new Path(outputPath));
int nSentences = 1000;
int nSamples = Integer.parseInt(nSamplesIn);
try {
File matchFile = new File(matchOutputPath);
File nomatchFile = new File(nomatchOutputPath);
FileOutputStream fosM = null, fosNM = null;
BufferedWriter dosM = null, dosNM = null;
fosM = new FileOutputStream(matchFile);
fosNM = new FileOutputStream(nomatchFile);
dosM = new BufferedWriter(new OutputStreamWriter(fosM));
dosNM = new BufferedWriter(new OutputStreamWriter(fosNM));
MapFile.Reader id2sentenceReader =
new MapFile.Reader(new Path("id2sentence.map/part-00000"), conf);
HashMap<Integer, ArrayListWritable<Text>> id2sentence =
new HashMap<Integer, ArrayListWritable<Text>>();
IntWritable key = new IntWritable();
ArrayListWritable<Text> val = new ArrayListWritable<Text>();
while (id2sentenceReader.next(key, val)) {
id2sentence.put(key.get(), val);
val = new ArrayListWritable<Text>();
}
MapFile.Reader sentence2translationReader =
new MapFile.Reader(new Path("sentence2translation.map/part-00000"), conf);
HashMap<Integer, ArrayListOfIntsWritable> sentence2translation =
new HashMap<Integer, ArrayListOfIntsWritable>();
IntWritable key2 = new IntWritable();
ArrayListOfIntsWritable val2 = new ArrayListOfIntsWritable();
while (sentence2translationReader.next(key2, val2)) {
sentence2translation.put(key2.get(), val2);
val2 = new ArrayListOfIntsWritable();
}
MapFile.Reader sentencematchReader =
new MapFile.Reader(new Path("sentencematchpairs.map/part-00000"), conf);
HashSet<PairOfInts> sentencematchpairs = new HashSet<PairOfInts>();
PairOfInts key3 = new PairOfInts();
IntWritable val3 = new IntWritable();
while (sentencematchReader.next(key3, val3)) {
sentencematchpairs.add(key3);
key3 = new PairOfInts();
}
System.out.println("Done reading");
PairOfInts p = new PairOfInts();
IntWritable match;
IntWritable eLineNum = new IntWritable();
IntWritable eLineId = new IntWritable();
ArrayListWritable<Text> eSentence = new ArrayListWritable<Text>();
for (int i = 0; i < nSentences; i++) {
if (i % 100 == 0) System.out.println("eLine " + i);
// eLineNum.set(2*i);
ArrayListOfIntsWritable transIdList = sentence2translation.get(2 * i);
// ArrayListOfIntsWritable transIdList = new ArrayListOfIntsWritable();
// sentence2translationReader.get(eLineNum, transIdList);
// System.out.println("transIdList " + transIdList);
for (int j = 0; j < nSentences; j++) {
// System.out.println("fLine " + j);
ArrayListWritable<Text> fSentence = id2sentence.get((2 * j + 1) * nSamples);
// ArrayListWritable<Text> fSentence = new ArrayListWritable<Text>();
// IntWritable fLineId = new IntWritable();
// fLineId.set((2*j+1)*nSamples);
// id2sentenceReader.get(fLineId, fSentence);
// System.out.println("fLineId " + (2*j+1)*nSamples + " FSentence " + fSentence);
float jsimMax = -1.0f;
float jsimAvg = 0.0f;
for (int id : transIdList) {
eSentence = id2sentence.get(id);
// eLineId.set(id);
// id2sentenceReader.get(eLineId, eSentence);
float jsim = JaccardSim.jaccardSim(eSentence, fSentence);
// System.out.println("\teSentence " + eSentence + " " + jsim);
jsimAvg += jsim;
if (jsim > jsimMax) {
jsimMax = jsim;
}
}
jsimAvg = jsimAvg / transIdList.size();
if (2 * i < 2 * j + 1) {
p.set(2 * i, 2 * j + 1);
} else {
p.set(2 * j + 1, 2 * i);
}
// match = new IntWritable();
// match = (IntWritable) sentencematchReader.get(p, match);
// if(match != null){
if (sentencematchpairs.contains(p)) {
if (jsimMax < .5) {
System.out.println("Low match: ");
System.out.println("\teSentence: " + i + " " + eSentence);
System.out.println("\tfSentence: " + j + " " + fSentence);
}
// System.out.println("match");
dosM.write(Float.toString(jsimMax));
// dosM.write(Float.toString(jsimAvg));
dosM.write("\n");
} else {
// System.out.println("no match");
dosNM.write(Float.toString(jsimMax));
// dosNM.write(Float.toString(jsimAvg));
dosNM.write("\n");
}
}
}
sentencematchReader.close();
sentence2translationReader.close();
id2sentenceReader.close();
dosM.close();
dosNM.close();
} catch (IOException e2) {
// TODO Auto-generated catch block
e2.printStackTrace();
}
// Delete the output directory if it exists already.
// Path outputDir = new Path(outputPath);
// FileSystem.get(conf).delete(outputDir, true);
// JobClient.runJob(conf);
return 0;
}
/** Runs this tool. */
@SuppressWarnings({"static-access"})
public int run(String[] args) throws Exception {
Options options = new Options();
options.addOption(new Option(COMBINER, "use combiner"));
options.addOption(new Option(INMAPPER_COMBINER, "user in-mapper combiner"));
options.addOption(new Option(RANGE, "use range partitioner"));
options.addOption(
OptionBuilder.withArgName("path").hasArg().withDescription("base path").create(BASE));
options.addOption(
OptionBuilder.withArgName("num").hasArg().withDescription("start iteration").create(START));
options.addOption(
OptionBuilder.withArgName("num").hasArg().withDescription("end iteration").create(END));
options.addOption(
OptionBuilder.withArgName("num")
.hasArg()
.withDescription("number of nodes")
.create(NUM_NODES));
CommandLine cmdline;
CommandLineParser parser = new GnuParser();
try {
cmdline = parser.parse(options, args);
} catch (ParseException exp) {
System.err.println("Error parsing command line: " + exp.getMessage());
return -1;
}
if (!cmdline.hasOption(BASE)
|| !cmdline.hasOption(START)
|| !cmdline.hasOption(END)
|| !cmdline.hasOption(NUM_NODES)) {
System.out.println("args: " + Arrays.toString(args));
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(120);
formatter.printHelp(this.getClass().getName(), options);
ToolRunner.printGenericCommandUsage(System.out);
return -1;
}
String basePath = cmdline.getOptionValue(BASE);
int n = Integer.parseInt(cmdline.getOptionValue(NUM_NODES));
int s = Integer.parseInt(cmdline.getOptionValue(START));
int e = Integer.parseInt(cmdline.getOptionValue(END));
boolean useCombiner = cmdline.hasOption(COMBINER);
boolean useInmapCombiner = cmdline.hasOption(INMAPPER_COMBINER);
boolean useRange = cmdline.hasOption(RANGE);
LOG.info("Tool name: RunPageRank");
LOG.info(" - base path: " + basePath);
LOG.info(" - num nodes: " + n);
LOG.info(" - start iteration: " + s);
LOG.info(" - end iteration: " + e);
LOG.info(" - use combiner: " + useCombiner);
LOG.info(" - use in-mapper combiner: " + useInmapCombiner);
LOG.info(" - user range partitioner: " + useRange);
// Iterate PageRank.
for (int i = s; i < e; i++) {
iteratePageRank(i, i + 1, basePath, n, useCombiner, useInmapCombiner);
}
return 0;
}
public void usage(String msg) {
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.setWidth(80);
helpFormatter.printHelp(syntax, msg + header, options, footer);
}
/** Hadoop {@link Tool} implementation */
@Override
public int run(String[] args) throws Exception {
Options options = new Options();
configureOptions(options);
CommandLineParser parser = new GnuParser();
try {
CommandLine commandLine = parser.parse(options, args);
if (commandLine.hasOption(VERBOSE)) {
Logger.getGlobal().setLevel(Level.FINEST);
}
if (commandLine.hasOption(QUIET)) {
Logger.getGlobal().setLevel(Level.OFF);
}
String transformationLocation = commandLine.getOptionValue(TRANSFORMATION);
String sourcemmLocation = commandLine.getOptionValue(SOURCE_PACKAGE);
String targetmmLocation = commandLine.getOptionValue(TARGET_PACKAGE);
String recordsLocation = commandLine.getOptionValue(RECORDS_FILE);
String inputLocation = commandLine.getOptionValue(INPUT_MODEL);
String outputLocation =
commandLine.getOptionValue(
OUTPUT_MODEL, new Path(inputLocation).suffix(".out.xmi").toString());
int recommendedMappers = 1;
if (commandLine.hasOption(RECOMMENDED_MAPPERS)) {
recommendedMappers =
((Number) commandLine.getParsedOptionValue(RECOMMENDED_MAPPERS)).intValue();
}
Configuration conf = this.getConf();
Job job = Job.getInstance(conf, JOB_NAME);
// Configure classes
job.setJarByClass(ATLMRMaster.class);
job.setMapperClass(ATLMRMapper.class);
job.setReducerClass(ATLMRReducer.class);
job.setInputFormatClass(NLineInputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(Text.class);
job.setNumReduceTasks(1);
// Configure MapReduce input/outputs
Path recordsPath = new Path(recordsLocation);
FileInputFormat.setInputPaths(job, recordsPath);
String timestamp = new SimpleDateFormat("yyyyMMddhhmm").format(new Date());
String outDirName = "atlmr-out-" + timestamp + "-" + UUID.randomUUID();
FileOutputFormat.setOutputPath(
job, new Path(job.getWorkingDirectory().suffix(Path.SEPARATOR + outDirName).toUri()));
// Configure records per map
FileSystem fileSystem = FileSystem.get(recordsPath.toUri(), conf);
InputStream inputStream = fileSystem.open(recordsPath);
long linesPerMap =
(long) Math.ceil((double) countLines(inputStream) / (double) recommendedMappers);
job.getConfiguration().setLong(NLineInputFormat.LINES_PER_MAP, linesPerMap);
// Configure ATL related inputs/outputs
job.getConfiguration().set(TRANSFORMATION, transformationLocation);
job.getConfiguration().set(SOURCE_PACKAGE, sourcemmLocation);
job.getConfiguration().set(TARGET_PACKAGE, targetmmLocation);
job.getConfiguration().set(INPUT_MODEL, inputLocation);
job.getConfiguration().set(OUTPUT_MODEL, outputLocation);
Logger.getGlobal().log(Level.INFO, "Starting Job execution");
long begin = System.currentTimeMillis();
int returnValue = job.waitForCompletion(true) ? STATUS_OK : STATUS_ERROR;
long end = System.currentTimeMillis();
Logger.getGlobal()
.log(
Level.INFO,
MessageFormat.format(
"Job execution ended in {0}s with status code {1}",
(end - begin) / 1000, returnValue));
return returnValue;
} catch (ParseException e) {
System.err.println(e.getLocalizedMessage());
HelpFormatter formatter = new HelpFormatter();
formatter.setOptionComparator(new OptionComarator<>());
try {
formatter.setWidth(Math.max(Terminal.getTerminal().getTerminalWidth(), 80));
} catch (Throwable t) {
// Nothing to do...
}
;
formatter.printHelp("yarn jar <this-file.jar>", options, true);
return STATUS_ERROR;
}
}
@Override
protected void usage(Options options) {
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(120);
formatter.printHelp("config-import-export-tool.sh [file...]", options, true);
}
private void help() {
HelpFormatter formatter = new HelpFormatter();
formatter.setWidth(90);
formatter.printHelp("PerfTimes.jar", options);
System.exit(0);
}