public static void main(String[] args) throws Exception {
if (ToolRunner.run(new FeatureMatching(), args) == 1) {
System.out.println(".......Feature Match failure........");
System.exit(1);
}
System.exit(0);
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
String[] remainArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (remainArgs.length != 2) {
System.err.println("Usage: wordcount <input> <output>");
System.exit(1);
}
Job job = new Job(conf, "wordcount");
job.setJarByClass(WordCount.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setMapperClass(Map.class);
job.setCombinerClass(Reduce.class);
job.setReducerClass(Reduce.class);
job.setNumReduceTasks(4);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileSystem.get(conf).delete(new Path(remainArgs[1]), true);
FileInputFormat.setInputPaths(job, new Path(remainArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(remainArgs[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length != 2) {
System.err.println("Usage: test.icde12.HadoopJoin <in> <out>");
System.exit(2);
}
Job job = new Job(conf, "hadoop join");
job.setJarByClass(HadoopJoin.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);
job.setPartitionerClass(ICDEPartitioner.class);
// WritableComparator.define(Text.class,new ICDEComparator());
job.setSortComparatorClass(ICDEComparator.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setNumReduceTasks(8);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
/** ***********************Driver function****************** */
public static void main(String args[]) throws Exception {
if (args.length != 3) {
System.out.println(
"provide sufficient arguments <input matrix file path> <input vector file path> <output file path>");
System.exit(-1);
}
int res = ToolRunner.run(new Configuration(), new Mat_vect_mul(), args);
System.exit(res);
}
/** Sets up configuration based on params */
private static boolean setup(Hashtable<String, String> curConf, Configuration argConf) {
if (argConf.get("file") == null) {
logger.fatal("Missing file parameter");
System.exit(1);
}
if (argConf.get("hdfs_base_path") == null) {
logger.fatal("Missing HDFS base path, check gestore-conf.xml");
System.exit(1);
}
if (argConf.get("hdfs_temp_path") == null) {
logger.fatal("Missing HDFS temp path, check gestore-conf.xml");
System.exit(1);
}
if (argConf.get("local_temp_path") == null) {
logger.fatal("Missing local temp path, check gestore-conf.xml");
System.exit(1);
}
// Input paramaters
curConf.put("run_id", argConf.get("run", ""));
curConf.put("task_id", argConf.get("task", ""));
curConf.put("file_id", argConf.get("file"));
curConf.put("local_path", argConf.get("path", ""));
curConf.put("type", argConf.get("type", "l2r"));
curConf.put("timestamp_start", argConf.get("timestamp_start", "1"));
curConf.put(
"timestamp_stop", argConf.get("timestamp_stop", Integer.toString(Integer.MAX_VALUE)));
curConf.put("delimiter", argConf.get("regex", "ID=.*"));
curConf.put("taxon", argConf.get("taxon", "all"));
curConf.put("intermediate", argConf.get("full_run", "false"));
curConf.put("quick_add", argConf.get("quick_add", "false"));
Boolean full_run = curConf.get("intermediate").matches("(?i).*true.*");
curConf.put("format", argConf.get("format", "unknown"));
curConf.put("split", argConf.get("split", "1"));
curConf.put("copy", argConf.get("copy", "true"));
// Constants
curConf.put("base_path", argConf.get("hdfs_base_path"));
curConf.put("temp_path", argConf.get("hdfs_temp_path"));
curConf.put("local_temp_path", argConf.get("local_temp_path"));
curConf.put("db_name_files", argConf.get("hbase_file_table"));
curConf.put("db_name_runs", argConf.get("hbase_run_table"));
curConf.put("db_name_updates", argConf.get("hbase_db_update_table"));
// Timestamps
Date currentTime = new Date();
Date endDate = new Date(new Long(curConf.get("timestamp_stop")));
curConf.put("timestamp_real", Long.toString(currentTime.getTime()));
return true;
}
@Override
public int run(String[] args) throws Exception {
JobConf conf = new JobConf(getConf(), getClass());
conf.setJobName("UFO count");
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
if (otherArgs.length != 2) {
System.err.println("Usage: avro UFO counter <in> <out>");
System.exit(2);
}
FileInputFormat.addInputPath(conf, new Path(otherArgs[0]));
Path outputPath = new Path(otherArgs[1]);
FileOutputFormat.setOutputPath(conf, outputPath);
outputPath.getFileSystem(conf).delete(outputPath);
Schema input_schema = Schema.parse(getClass().getResourceAsStream("ufo.avsc"));
AvroJob.setInputSchema(conf, input_schema);
AvroJob.setMapOutputSchema(
conf,
Pair.getPairSchema(Schema.create(Schema.Type.STRING), Schema.create(Schema.Type.LONG)));
AvroJob.setOutputSchema(conf, OUTPUT_SCHEMA);
AvroJob.setMapperClass(conf, AvroRecordMapper.class);
AvroJob.setReducerClass(conf, AvroRecordReducer.class);
conf.setInputFormat(AvroInputFormat.class);
JobClient.runJob(conf);
return 0;
}
public static void main(String args[]) throws Exception {
Configuration c = new Configuration();
if (args.length != 2) {
System.out.println("provide sufficient arguments");
System.exit(-1);
}
Job job = Job.getInstance(c, "Wordcount");
job.setJarByClass(Wordcount.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setMapperClass(Map.class);
job.setReducerClass(Reduce.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
// job.submit();
job.waitForCompletion(true);
}
public void setup(Context context) {
try {
// System.setProperty("java.library.path", "/home/gathors/proj/libs");
// System.loadLibrary(Core.NATIVE_xxx);
// System.loacLibrary("/home/gathors/proj/libs/opencv-300.jar");
} catch (UnsatisfiedLinkError e) {
System.err.println("\nNATIVE LIBRARY failed to load...");
System.err.println("ERROR:" + e);
System.err.println("NATIVE_LIBRARY_NAME:" + Core.NATIVE_LIBRARY_NAME);
System.err.println("#" + System.getProperty("java.library.path"));
System.exit(1);
}
}
public static boolean stopIteration(Configuration conf) throws IOException {
FileSystem fs = FileSystem.get(conf);
Path preFile = new Path("preX/Result");
Path curFile = new Path("curX/part-00000");
if (!(fs.exists(preFile) && fs.exists(curFile))) {
System.exit(1);
}
boolean stop = true;
String line1, line2;
FSDataInputStream in1 = fs.open(preFile);
FSDataInputStream in2 = fs.open(curFile);
InputStreamReader isr1 = new InputStreamReader(in1);
InputStreamReader isr2 = new InputStreamReader(in2);
BufferedReader br1 = new BufferedReader(isr1);
BufferedReader br2 = new BufferedReader(isr2);
while ((line1 = br1.readLine()) != null && (line2 = br2.readLine()) != null) {
String[] str1 = line1.split("\\s+");
String[] str2 = line2.split("\\s+");
double preElem = Double.parseDouble(str1[1]);
double curElem = Double.parseDouble(str2[1]);
if (Math.abs(preElem - curElem) > eps) {
stop = false;
break;
}
}
if (stop == false) {
fs.delete(preFile, true);
if (fs.rename(curFile, preFile) == false) {
System.exit(1);
}
}
return stop;
}
public static void main(String[] args) throws Exception {
Job job = new Job();
job.setJarByClass(Sort.class);
job.setJobName("Sort");
FileInputFormat.addInputPath(job, new Path("hdfs://localhost:9000/input/"));
FileOutputFormat.setOutputPath(job, new Path("hdfs://localhost:9000/output/"));
job.setMapperClass(Map.class);
// job.setCombinerClass(Reduce.class);
job.setReducerClass(Reduce.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setNumReduceTasks(2);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job jobA = Job.getInstance(conf, "delaysum");
jobA.setOutputKeyClass(Text.class);
jobA.setOutputValueClass(IntWritable.class);
jobA.setMapperClass(ArrivalMap.class);
jobA.setReducerClass(ArrivalReduce.class);
FileInputFormat.setInputPaths(jobA, new Path(args[0]));
FileOutputFormat.setOutputPath(jobA, new Path(args[1]));
jobA.setJarByClass(MeanDelayOriginDestination.class);
System.exit(jobA.waitForCompletion(true) ? 0 : 1);
}
public int run(String[] args) throws Exception {
Path tempDir = new Path("/user/akhfa/temp");
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "word count");
job.setJarByClass(AuthorCounter.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(LongWritable.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, tempDir);
System.exit(job.waitForCompletion(true) ? 0 : 1);
return 0;
}
public static void main(String[] args) throws IOException {
if (args.length != 2) {
System.err.println("Usage: OldMaxTemperature <input path> <output path>");
System.exit(-1);
}
/*[*/ JobConf conf = new JobConf(OldMaxTemperature.class); /*]*/
/*[*/ conf /*]*/.setJobName("Max temperature");
FileInputFormat.addInputPath(/*[*/ conf /*]*/, new Path(args[0]));
FileOutputFormat.setOutputPath(/*[*/ conf /*]*/, new Path(args[1]));
/*[*/ conf /*]*/.setMapperClass(OldMaxTemperatureMapper.class);
/*[*/ conf /*]*/.setReducerClass(OldMaxTemperatureReducer.class);
/*[*/ conf /*]*/.setOutputKeyClass(Text.class);
/*[*/ conf /*]*/.setOutputValueClass(IntWritable.class);
/*[*/ JobClient.runJob(conf); /*]*/
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path tmpPath = new Path("/w1/tmp");
fs.delete(tmpPath, true);
Job jobA = Job.getInstance(conf, "delaysum");
jobA.setOutputKeyClass(Text.class);
jobA.setOutputValueClass(IntWritable.class);
jobA.setMapperClass(ArrivalMap.class);
jobA.setReducerClass(ArrivalReduce.class);
FileInputFormat.setInputPaths(jobA, new Path(args[0]));
FileOutputFormat.setOutputPath(jobA, tmpPath);
jobA.setJarByClass(TopArrivalPerformanceCarriersOD.class);
jobA.waitForCompletion(true);
Job jobB = Job.getInstance(conf, "Best Airline");
jobB.setOutputKeyClass(Text.class);
jobB.setOutputValueClass(FloatWritable.class);
jobB.setMapOutputKeyClass(NullWritable.class);
jobB.setMapOutputValueClass(TextArrayWritable.class);
jobB.setMapperClass(DelayMap.class);
jobB.setReducerClass(DelayReduce.class);
jobB.setNumReduceTasks(1);
FileInputFormat.setInputPaths(jobB, tmpPath);
FileOutputFormat.setOutputPath(jobB, new Path(args[1]));
jobB.setInputFormatClass(KeyValueTextInputFormat.class);
jobB.setOutputFormatClass(TextOutputFormat.class);
jobB.setJarByClass(TopArrivalPerformanceCarriersOD.class);
System.exit(jobB.waitForCompletion(true) ? 0 : 1);
}
@Override
public int run(String[] args) throws Exception {
// TODO Auto-generated method stub
Configuration conf = getConf();
if (args.length != 3) {
System.err.println("Usage: WordCountDriver <inpath> <outpath>");
System.exit(0);
}
GenericOptionsParser gop = new GenericOptionsParser(args);
args = gop.getRemainingArgs();
Path inpath = new Path(args[1]);
Path outpath = new Path(args[2]);
Job job = new Job(conf, "WordCountDriver");
job.setJarByClass(WordCountDriver.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
job.setOutputFormatClass(TextOutputFormat.class);
job.setInputFormatClass(TextInputFormat.class);
FileInputFormat.setInputPaths(job, inpath);
FileOutputFormat.setOutputPath(job, outpath);
job.setNumReduceTasks(2);
job.setMapperClass(WordCountMapper.class);
job.setReducerClass(WordCountReducer.class);
job.setPartitionerClass(WordCountPart.class);
job.waitForCompletion(true);
return 0;
}
// Transform the json-type feature to mat-type
public static Mat json2mat(String json) {
JsonParser parser = new JsonParser();
JsonElement parseTree = parser.parse(json);
// Verify the input is JSON type
if (!parseTree.isJsonObject()) {
System.out.println("The input is not a JSON type...\nExiting...");
System.exit(1);
}
JsonObject jobj = parser.parse(json).getAsJsonObject();
if (jobj == null || !jobj.isJsonObject() || jobj.isJsonNull()) {
return null;
}
// Detect broken/null features
JsonElement r = jobj.get("rows");
if (r == null) {
return null;
}
int rows = jobj.get("rows").getAsInt();
int cols = jobj.get("cols").getAsInt();
int type = jobj.get("type").getAsInt();
String data = jobj.get("data").getAsString();
String[] pixs = data.split(",");
Mat descriptor = new Mat(rows, cols, type);
for (String pix : pixs) {
String[] tmp = pix.split(" ");
int r_pos = Integer.valueOf(tmp[0]);
int c_pos = Integer.valueOf(tmp[1]);
double rgb = Double.valueOf(tmp[2]);
descriptor.put(r_pos, c_pos, rgb);
}
return descriptor;
}
public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
if (args.length != 2) {
System.out.println("Usage: FeatureMatching ID <inputName.jpeg/inputName.jpg>");
System.exit(1);
}
SimpleDateFormat sdf = new SimpleDateFormat("", Locale.US);
sdf.applyPattern("yyyy-MM-dd_HH-mm-ss");
String time = sdf.format(new Date());
Job job = Job.getInstance();
ID = "/" + args[0];
String filename = args[1];
filename = filename.toLowerCase();
System.out.println("current filename:" + filename);
// Detect illegal username (if the username dir doesn't exist)
File userPath = new File(LOCAL_USER_DIR + ID);
if (!userPath.exists()) {
System.out.println("Unauthorized username!!!\nExiting......");
System.exit(1);
}
// Preprocess the input image.jpg from local dir: /local.../user/ID/input/image.jpg
// Save the features to local dir: hdfs://.../user/ID/input/image.jpg
extractQueryFeatures2HDFS(filename, job);
// Add the feature file to the hdfs cache
String featureFileName = filename.substring(0, filename.lastIndexOf(".")) + ".json";
// job.addCacheFile(new Path(HDFS_HOME + USER + ID + INPUT + "/" +
// featureFileName).toUri());
job.getConfiguration()
.set("featureFilePath", HDFS_HOME + USER + ID + INPUT + "/" + featureFileName);
// Check the file type. Only support jpeg/jpg type images
String type = filename.substring(args[1].lastIndexOf("."));
if (!(type.equals(".jpg") || type.equals(".jpeg"))) {
System.out.println("Image type not supported!!!\nExiting");
System.exit(1);
}
// Input: hdfs://.../features/
// The feature dir is a location of all features extracted from the database
String inputPathStr = HDFS_HOME + FEATURES;
// Output: hdfs://.../user/ID/output/
String outputPathStr = HDFS_HOME + USER + ID + OUTPUT + "/" + time;
job.setInputFormatClass(KeyValueTextInputFormat.class);
// job.setOutputFormatClass(TextOutputFormat.class);
// Get the lists of all feature files: /.../features/data/part-*
FileSystem fs = FileSystem.get(job.getConfiguration());
FileStatus[] statuses = fs.listStatus(new Path(inputPathStr));
StringBuffer sb = new StringBuffer();
for (FileStatus fileStatus : statuses) {
sb.append(fileStatus.getPath() + ",");
}
sb.deleteCharAt(sb.lastIndexOf(","));
job.setJarByClass(FeatureMatching.class);
job.setMapperClass(FeatureMatchMapper.class);
job.setReducerClass(FeatureMatchReducer.class);
// only need one reducer to collect the result
job.setNumReduceTasks(1);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(DoubleWritable.class);
// Input a directory, so need the recursive input
FileInputFormat.setInputDirRecursive(job, true);
// Set the PathFilter, to omit _SUCCESS files
// (This is not working correctly, as the PathFilter class is an interface rather than a class.
// But the 2nd arg asks me to extend the PathFilter)
// FileInputFormat.setInputPathFilter(job, MyPathFilter.class);
//
// FileInputFormat.setInputPaths(job, new Path(inputPathStr));
FileInputFormat.setInputPaths(job, sb.toString());
FileOutputFormat.setOutputPath(job, new Path(outputPathStr));
boolean success = job.waitForCompletion(true);
return success ? 0 : 1;
}
public static void main(String[] args) throws Exception {
System.out.println("Beginning job...");
// Start phase 1
Configuration conf = new Configuration();
conf.set("mapred.job.tracker", "local");
conf.set("fs.default.name", "local");
String[] inputArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
Path xmlPath = null;
List<String> other_args = new ArrayList<String>();
for (int i = 0; i < args.length; ++i) {
if ("-setup".equals(inputArgs[i])) {
xmlPath = new Path(inputArgs[++i]);
DistributedCache.addCacheFile(xmlPath.toUri(), conf);
conf.setBoolean("minibatch.job.setup", true);
} else {
other_args.add(inputArgs[i]);
}
}
String[] tool_args = other_args.toArray(new String[0]);
int result = ToolRunner.run(conf, new BatchGenerationEngine(), tool_args);
// End phase 1
// get example count and size from xml file
// count = count_size[0];
// size = count_size[1];
int[] count_size = parseJobSetup(xmlPath);
// distribute those output from phase 1 into different directories
String outputPhase1 = tool_args[1];
FileSystem fs = FileSystem.get(new Configuration());
Path outputPhase1Path = new Path(outputPhase1);
fs.setWorkingDirectory(outputPhase1Path);
FileStatus[] outputP1AllFiles = fs.listStatus(outputPhase1Path);
for (int i = 0; i < outputP1AllFiles.length; i++) {
int batchNum = i / count_size[1];
Path batchPath = new Path(outputPhase1 + "/batch" + batchNum);
// if batch# directory not exists, mkdir
if (!fs.exists(batchPath)) FileSystem.mkdirs(fs, batchPath, new FsPermission("777"));
// move file into the batch# directory
fs.rename(
outputP1AllFiles[i].getPath(),
new Path(
outputPhase1 + "/batch" + batchNum + "/" + outputP1AllFiles[i].getPath().getName()));
}
//
// Generate dictionary of jobs
int numberOfJobs = count_size[0] * count_size[1];
JobConfig[] dictionary = new JobConfig[numberOfJobs];
// Add job 0 to dictionary
Configuration conf0 = new Configuration();
DistributedCache.addCacheFile(xmlPath.toUri(), conf0);
JobConfig job0 = new JobConfig(conf0, "input go here", java.util.UUID.randomUUID().toString());
dictionary[0] = job0;
// Add the rest of jobs into dictionary
for (int i = 1; i < dictionary.length; i++) {
Configuration newConf = new Configuration();
DistributedCache.addCacheFile(xmlPath.toUri(), newConf);
JobConfig newJob =
new JobConfig(newConf, dictionary[i - 1].args[1], java.util.UUID.randomUUID().toString());
dictionary[i] = newJob;
}
// running the jobs
logger.info("Start " + dictionary.length + " jobs!");
for (int i = 0; i < dictionary.length; i++) {
int runResult =
ToolRunner.run(dictionary[i].conf, new BatchGenerationEngine(), dictionary[i].args);
if (runResult == 1) {
logger.info("Job " + i + "-th Re-run once!");
dictionary[i].args[1] = java.util.UUID.randomUUID().toString();
runResult =
ToolRunner.run(dictionary[i].conf, new BatchGenerationEngine(), dictionary[i].args);
}
if (runResult == 1) {
logger.info("Job " + i + "-th Re-run twice!");
dictionary[i].args[1] = java.util.UUID.randomUUID().toString();
runResult =
ToolRunner.run(dictionary[i].conf, new BatchGenerationEngine(), dictionary[i].args);
}
if (runResult == 1) {
logger.info("Job " + i + "-th Failed!");
break;
} else {
if (i - 1 < dictionary.length) dictionary[i + 1].args[0] = dictionary[i].args[1];
}
}
System.exit(1);
}
public static void main(String[] args) throws Exception {
int exitCode = ToolRunner.run(new SortByTemperatureUsingTotalOrderPartitioner(), args);
System.exit(exitCode);
}
/**
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
int res = ToolRunner.run(new Configuration(), new WordCountDriver(), args);
System.exit(res);
}
public static void main(String[] args) throws Exception {
KMeansDriver drv = new KMeansDriver();
drv.getConf().set("mapreduce.framework.name", "local");
int exitCode = ToolRunner.run(drv, args);
System.exit(exitCode);
}
public static void main(String[] args) throws Exception {
int exitCode = ToolRunner.run(new MaxTemperatureDriver(), args);
System.exit(exitCode);
}
public static void main(String[] args) throws Exception {
int exitCode = ToolRunner.run(new WhiteHouseVisitorDriver(), args);
System.exit(exitCode);
}
public static void main(String[] args) throws Exception {
JobConf job = new JobConf(DistCp.class);
DistCp distcp = new DistCp(job);
int res = ToolRunner.run(distcp, args);
System.exit(res);
}
public static void main(String[] args) throws Exception {
int rc = ToolRunner.run(new Configuration(), new KnownKeysMRv2(), args);
System.exit(rc);
}
public int run(String[] args) throws Exception {
// printUsage();
/*
* SETUP
*/
Configuration argConf = getConf();
Hashtable<String, String> confArg = new Hashtable<String, String>();
setup(confArg, argConf);
Date currentTime = new Date();
Date endDate = new Date(new Long(confArg.get("timestamp_stop")));
Boolean full_run = confArg.get("intermediate").matches("(?i).*true.*");
Boolean quick_add = confArg.get("quick_add").matches("(?i).*true.*");
logger.info("Running GeStore");
// ZooKeeper setup
Configuration config = HBaseConfiguration.create();
zkWatcher = new ZooKeeperWatcher(config, "Testing", new HBaseAdmin(config));
zkInstance =
new ZooKeeper(
ZKConfig.getZKQuorumServersString(config),
config.getInt("zookeeper.session.timeout", -1),
zkWatcher);
if (!confArg.get("task_id").isEmpty()) {
confArg.put("temp_path", confArg.get("temp_path") + confArg.get("task_id"));
}
String lockRequest = confArg.get("file_id");
if (!confArg.get("run_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("run_id") + "_";
if (!confArg.get("task_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("task_id") + "_";
// Get type of movement
toFrom type_move = checkArgs(confArg);
if (type_move == toFrom.LOCAL2REMOTE && !confArg.get("format").equals("unknown")) {
List<String> arguments = new ArrayList<String>();
arguments.add("-Dinput=" + confArg.get("local_path"));
arguments.add("-Dtable=" + confArg.get("file_id"));
arguments.add("-Dtimestamp=" + confArg.get("timestamp_stop"));
arguments.add("-Dtype=" + confArg.get("format"));
arguments.add("-Dtarget_dir=" + confArg.get("base_path") + "_" + confArg.get("file_id"));
arguments.add("-Dtemp_hdfs_path=" + confArg.get("temp_path"));
arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("run_id").isEmpty()) arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("task_id").isEmpty()) arguments.add("-Dtask_id=" + confArg.get("task_id"));
if (quick_add) arguments.add("-Dquick_add=" + confArg.get("quick_add"));
String lockName = lock(lockRequest);
String[] argumentString = arguments.toArray(new String[arguments.size()]);
adddb.main(argumentString);
unlock(lockName);
System.exit(0);
}
// Database registration
dbutil db_util = new dbutil(config);
db_util.register_database(confArg.get("db_name_files"), true);
db_util.register_database(confArg.get("db_name_runs"), true);
db_util.register_database(confArg.get("db_name_updates"), true);
FileSystem hdfs = FileSystem.get(config);
FileSystem localFS = FileSystem.getLocal(config);
// Get source type
confArg.put("source", getSource(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
confArg.put(
"database", isDatabase(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
if (!confArg.get("source").equals("local")
&& type_move == toFrom.REMOTE2LOCAL
&& !confArg.get("timestamp_stop").equals(Integer.toString(Integer.MAX_VALUE))) {
confArg.put("timestamp_stop", Long.toString(latestVersion(confArg, db_util)));
}
/*
* Get previous timestamp
*/
Get run_id_get = new Get(confArg.get("run_id").getBytes());
Result run_get = db_util.doGet(confArg.get("db_name_runs"), run_id_get);
KeyValue run_file_prev =
run_get.getColumnLatest(
"d".getBytes(), (confArg.get("file_id") + "_db_timestamp").getBytes());
String last_timestamp = new String("0");
if (null != run_file_prev && !confArg.get("source").equals("local")) {
long last_timestamp_real = run_file_prev.getTimestamp();
Long current_timestamp = new Long(confArg.get("timestamp_real"));
if ((current_timestamp - last_timestamp_real) > 36000) {
last_timestamp = new String(run_file_prev.getValue());
Integer lastTimestamp = new Integer(last_timestamp);
lastTimestamp += 1;
last_timestamp = lastTimestamp.toString();
logger.info("Last timestamp: " + last_timestamp + " End data: " + endDate);
Date last_run = new Date(run_file_prev.getTimestamp());
if (last_run.before(endDate) && !full_run) {
confArg.put("timestamp_start", last_timestamp);
}
}
}
Integer tse = new Integer(confArg.get("timestamp_stop"));
Integer tss = new Integer(confArg.get("timestamp_start"));
if (tss > tse) {
logger.info("No new version of requested file.");
return 0;
}
/*
* Generate file
*/
String lockName = lock(lockRequest);
Get file_id_get = new Get(confArg.get("file_id").getBytes());
Result file_get = db_util.doGet(confArg.get("db_name_files"), file_id_get);
if (!file_get.isEmpty()) {
boolean found =
hasFile(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg));
if (confArg.get("source").equals("fullfile")) {
found = false;
}
String filenames_put =
getFileNames(
db_util, confArg.get("db_name_files"), confArg.get("file_id"), getFullPath(confArg));
// Filename not found in file database
if (!found && type_move == toFrom.REMOTE2LOCAL) {
if (!confArg.get("source").equals("local")) {
// Generate intermediate file
if (getFile(hdfs, confArg, db_util) == null) {
unlock(lockName);
return 1;
}
// Put generated file into file database
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
confArg.get("full_file_name"),
confArg.get("source"));
}
} else {
logger.warn("Remote file not found, and cannot be generated! File: " + confArg);
unlock(lockName);
return 1;
}
}
} else {
if (type_move == toFrom.REMOTE2LOCAL) {
logger.warn("Remote file not found, and cannot be generated.");
unlock(lockName);
return 1;
}
}
/*
* Copy file
* Update tables
*/
if (type_move == toFrom.LOCAL2REMOTE) {
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg),
confArg.get("source"));
}
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
hdfs.copyFromLocalFile(new Path(confArg.get("local_path")), new Path(getFullPath(confArg)));
} else if (type_move == toFrom.REMOTE2LOCAL) {
FileStatus[] files = hdfs.globStatus(new Path(getFullPath(confArg) + "*"));
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
unlock(lockName);
for (FileStatus file : files) {
Path cur_file = file.getPath();
Path cur_local_path =
new Path(new String(confArg.get("local_path") + confArg.get("file_id")));
String suffix = getSuffix(getFileName(confArg), cur_file.getName());
if (suffix.length() > 0) {
cur_local_path = cur_local_path.suffix(new String("." + suffix));
}
if (confArg.get("copy").equals("true")) {
String crc = hdfs.getFileChecksum(cur_file).toString();
if (checksumLocalTest(cur_local_path, crc)) {
continue;
} else {
hdfs.copyToLocalFile(cur_file, cur_local_path);
writeChecksum(cur_local_path, crc);
}
} else {
System.out.println(cur_local_path + "\t" + cur_file);
}
}
}
unlock(lockName);
return 0;
}
public static void main(String[] args) throws Exception {
int res = ToolRunner.run(new Configuration(), new XiangLi1_exercise3(), args);
System.exit(res);
}
public static void main(String[] args) throws Exception {
int result = ToolRunner.run(new Configuration(), new move(), args);
System.exit(result);
}
/*
* You shouldn't need to modify this function much. If you think you have a good reason to,
* you might want to discuss with staff.
*
* The skeleton supports several options.
* if you set runJob2 to false, only the first job will run and output will be
* in TextFile format, instead of SequenceFile. This is intended as a debugging aid.
*
* If you set combiner to false, the combiner will not run. This is also
* intended as a debugging aid. Turning on and off the combiner shouldn't alter
* your results. Since the framework doesn't make promises about when it'll
* invoke combiners, it's an error to assume anything about how many times
* values will be combined.
*/
public static void main(String[] rawArgs) throws Exception {
GenericOptionsParser parser = new GenericOptionsParser(rawArgs);
Configuration conf = parser.getConfiguration();
String[] args = parser.getRemainingArgs();
boolean runJob2 = conf.getBoolean("runJob2", true);
boolean combiner = conf.getBoolean("combiner", false);
System.out.println("Target word: " + conf.get("targetWord"));
System.out.println("Function num: " + conf.get("funcNum"));
if (runJob2) System.out.println("running both jobs");
else System.out.println("for debugging, only running job 1");
if (combiner) System.out.println("using combiner");
else System.out.println("NOT using combiner");
Path inputPath = new Path(args[0]);
Path middleOut = new Path(args[1]);
Path finalOut = new Path(args[2]);
FileSystem hdfs = middleOut.getFileSystem(conf);
int reduceCount = conf.getInt("reduces", 32);
if (hdfs.exists(middleOut)) {
System.err.println("can't run: " + middleOut.toUri().toString() + " already exists");
System.exit(1);
}
if (finalOut.getFileSystem(conf).exists(finalOut)) {
System.err.println("can't run: " + finalOut.toUri().toString() + " already exists");
System.exit(1);
}
{
Job firstJob = new Job(conf, "job1");
firstJob.setJarByClass(Map1.class);
/* You may need to change things here */
firstJob.setMapOutputKeyClass(Text.class);
firstJob.setMapOutputValueClass(
DoublePair.class); // Change Map output Value type to DoublePair
firstJob.setOutputKeyClass(DoubleWritable.class);
firstJob.setOutputValueClass(Text.class);
/* End region where we expect you to perhaps need to change things. */
firstJob.setMapperClass(Map1.class);
firstJob.setReducerClass(Reduce1.class);
firstJob.setNumReduceTasks(reduceCount);
if (combiner) firstJob.setCombinerClass(Combine1.class);
firstJob.setInputFormatClass(SequenceFileInputFormat.class);
if (runJob2) firstJob.setOutputFormatClass(SequenceFileOutputFormat.class);
FileInputFormat.addInputPath(firstJob, inputPath);
FileOutputFormat.setOutputPath(firstJob, middleOut);
firstJob.waitForCompletion(true);
}
if (runJob2) {
Job secondJob = new Job(conf, "job2");
secondJob.setJarByClass(Map1.class);
/* You may need to change things here */
secondJob.setMapOutputKeyClass(DoubleWritable.class);
secondJob.setMapOutputValueClass(Text.class);
secondJob.setOutputKeyClass(DoubleWritable.class);
secondJob.setOutputValueClass(Text.class);
/* End region where we expect you to perhaps need to change things. */
secondJob.setMapperClass(Map2.class);
secondJob.setReducerClass(Reduce2.class);
secondJob.setInputFormatClass(SequenceFileInputFormat.class);
secondJob.setOutputFormatClass(TextOutputFormat.class);
secondJob.setNumReduceTasks(1);
FileInputFormat.addInputPath(secondJob, middleOut);
FileOutputFormat.setOutputPath(secondJob, finalOut);
secondJob.waitForCompletion(true);
}
}
public static void main(String[] args) throws Exception {
int result = ToolRunner.run(new HadoopNBFilter(), args);
System.exit(result);
}
