public static void main(String[] args) throws Exception { if (ToolRunner.run(new FeatureMatching(), args) == 1) { System.out.println(".......Feature Match failure........"); System.exit(1); } System.exit(0); }
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); String[] remainArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (remainArgs.length != 2) { System.err.println("Usage: wordcount <input> <output>"); System.exit(1); } Job job = new Job(conf, "wordcount"); job.setJarByClass(WordCount.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setMapperClass(Map.class); job.setCombinerClass(Reduce.class); job.setReducerClass(Reduce.class); job.setNumReduceTasks(4); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); FileSystem.get(conf).delete(new Path(remainArgs[1]), true); FileInputFormat.setInputPaths(job, new Path(remainArgs[0])); FileOutputFormat.setOutputPath(job, new Path(remainArgs[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length != 2) { System.err.println("Usage: test.icde12.HadoopJoin <in> <out>"); System.exit(2); } Job job = new Job(conf, "hadoop join"); job.setJarByClass(HadoopJoin.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setPartitionerClass(ICDEPartitioner.class); // WritableComparator.define(Text.class,new ICDEComparator()); job.setSortComparatorClass(ICDEComparator.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setNumReduceTasks(8); FileInputFormat.addInputPath(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); }
/** ***********************Driver function****************** */ public static void main(String args[]) throws Exception { if (args.length != 3) { System.out.println( "provide sufficient arguments <input matrix file path> <input vector file path> <output file path>"); System.exit(-1); } int res = ToolRunner.run(new Configuration(), new Mat_vect_mul(), args); System.exit(res); }
/** Sets up configuration based on params */ private static boolean setup(Hashtable<String, String> curConf, Configuration argConf) { if (argConf.get("file") == null) { logger.fatal("Missing file parameter"); System.exit(1); } if (argConf.get("hdfs_base_path") == null) { logger.fatal("Missing HDFS base path, check gestore-conf.xml"); System.exit(1); } if (argConf.get("hdfs_temp_path") == null) { logger.fatal("Missing HDFS temp path, check gestore-conf.xml"); System.exit(1); } if (argConf.get("local_temp_path") == null) { logger.fatal("Missing local temp path, check gestore-conf.xml"); System.exit(1); } // Input paramaters curConf.put("run_id", argConf.get("run", "")); curConf.put("task_id", argConf.get("task", "")); curConf.put("file_id", argConf.get("file")); curConf.put("local_path", argConf.get("path", "")); curConf.put("type", argConf.get("type", "l2r")); curConf.put("timestamp_start", argConf.get("timestamp_start", "1")); curConf.put( "timestamp_stop", argConf.get("timestamp_stop", Integer.toString(Integer.MAX_VALUE))); curConf.put("delimiter", argConf.get("regex", "ID=.*")); curConf.put("taxon", argConf.get("taxon", "all")); curConf.put("intermediate", argConf.get("full_run", "false")); curConf.put("quick_add", argConf.get("quick_add", "false")); Boolean full_run = curConf.get("intermediate").matches("(?i).*true.*"); curConf.put("format", argConf.get("format", "unknown")); curConf.put("split", argConf.get("split", "1")); curConf.put("copy", argConf.get("copy", "true")); // Constants curConf.put("base_path", argConf.get("hdfs_base_path")); curConf.put("temp_path", argConf.get("hdfs_temp_path")); curConf.put("local_temp_path", argConf.get("local_temp_path")); curConf.put("db_name_files", argConf.get("hbase_file_table")); curConf.put("db_name_runs", argConf.get("hbase_run_table")); curConf.put("db_name_updates", argConf.get("hbase_db_update_table")); // Timestamps Date currentTime = new Date(); Date endDate = new Date(new Long(curConf.get("timestamp_stop"))); curConf.put("timestamp_real", Long.toString(currentTime.getTime())); return true; }
@Override public int run(String[] args) throws Exception { JobConf conf = new JobConf(getConf(), getClass()); conf.setJobName("UFO count"); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length != 2) { System.err.println("Usage: avro UFO counter <in> <out>"); System.exit(2); } FileInputFormat.addInputPath(conf, new Path(otherArgs[0])); Path outputPath = new Path(otherArgs[1]); FileOutputFormat.setOutputPath(conf, outputPath); outputPath.getFileSystem(conf).delete(outputPath); Schema input_schema = Schema.parse(getClass().getResourceAsStream("ufo.avsc")); AvroJob.setInputSchema(conf, input_schema); AvroJob.setMapOutputSchema( conf, Pair.getPairSchema(Schema.create(Schema.Type.STRING), Schema.create(Schema.Type.LONG))); AvroJob.setOutputSchema(conf, OUTPUT_SCHEMA); AvroJob.setMapperClass(conf, AvroRecordMapper.class); AvroJob.setReducerClass(conf, AvroRecordReducer.class); conf.setInputFormat(AvroInputFormat.class); JobClient.runJob(conf); return 0; }
public static void main(String args[]) throws Exception { Configuration c = new Configuration(); if (args.length != 2) { System.out.println("provide sufficient arguments"); System.exit(-1); } Job job = Job.getInstance(c, "Wordcount"); job.setJarByClass(Wordcount.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); job.setMapperClass(Map.class); job.setReducerClass(Reduce.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path(args[1])); // job.submit(); job.waitForCompletion(true); }
public void setup(Context context) { try { // System.setProperty("java.library.path", "/home/gathors/proj/libs"); // System.loadLibrary(Core.NATIVE_xxx); // System.loacLibrary("/home/gathors/proj/libs/opencv-300.jar"); } catch (UnsatisfiedLinkError e) { System.err.println("\nNATIVE LIBRARY failed to load..."); System.err.println("ERROR:" + e); System.err.println("NATIVE_LIBRARY_NAME:" + Core.NATIVE_LIBRARY_NAME); System.err.println("#" + System.getProperty("java.library.path")); System.exit(1); } }
public static boolean stopIteration(Configuration conf) throws IOException { FileSystem fs = FileSystem.get(conf); Path preFile = new Path("preX/Result"); Path curFile = new Path("curX/part-00000"); if (!(fs.exists(preFile) && fs.exists(curFile))) { System.exit(1); } boolean stop = true; String line1, line2; FSDataInputStream in1 = fs.open(preFile); FSDataInputStream in2 = fs.open(curFile); InputStreamReader isr1 = new InputStreamReader(in1); InputStreamReader isr2 = new InputStreamReader(in2); BufferedReader br1 = new BufferedReader(isr1); BufferedReader br2 = new BufferedReader(isr2); while ((line1 = br1.readLine()) != null && (line2 = br2.readLine()) != null) { String[] str1 = line1.split("\\s+"); String[] str2 = line2.split("\\s+"); double preElem = Double.parseDouble(str1[1]); double curElem = Double.parseDouble(str2[1]); if (Math.abs(preElem - curElem) > eps) { stop = false; break; } } if (stop == false) { fs.delete(preFile, true); if (fs.rename(curFile, preFile) == false) { System.exit(1); } } return stop; }
public static void main(String[] args) throws Exception { Job job = new Job(); job.setJarByClass(Sort.class); job.setJobName("Sort"); FileInputFormat.addInputPath(job, new Path("hdfs://localhost:9000/input/")); FileOutputFormat.setOutputPath(job, new Path("hdfs://localhost:9000/output/")); job.setMapperClass(Map.class); // job.setCombinerClass(Reduce.class); job.setReducerClass(Reduce.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setNumReduceTasks(2); System.exit(job.waitForCompletion(true) ? 0 : 1); }
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Job jobA = Job.getInstance(conf, "delaysum"); jobA.setOutputKeyClass(Text.class); jobA.setOutputValueClass(IntWritable.class); jobA.setMapperClass(ArrivalMap.class); jobA.setReducerClass(ArrivalReduce.class); FileInputFormat.setInputPaths(jobA, new Path(args[0])); FileOutputFormat.setOutputPath(jobA, new Path(args[1])); jobA.setJarByClass(MeanDelayOriginDestination.class); System.exit(jobA.waitForCompletion(true) ? 0 : 1); }
public int run(String[] args) throws Exception { Path tempDir = new Path("/user/akhfa/temp"); Configuration conf = new Configuration(); Job job = Job.getInstance(conf, "word count"); job.setJarByClass(AuthorCounter.class); job.setMapperClass(TokenizerMapper.class); job.setCombinerClass(IntSumReducer.class); job.setReducerClass(IntSumReducer.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, tempDir); System.exit(job.waitForCompletion(true) ? 0 : 1); return 0; }
public static void main(String[] args) throws IOException { if (args.length != 2) { System.err.println("Usage: OldMaxTemperature <input path> <output path>"); System.exit(-1); } /*[*/ JobConf conf = new JobConf(OldMaxTemperature.class); /*]*/ /*[*/ conf /*]*/.setJobName("Max temperature"); FileInputFormat.addInputPath(/*[*/ conf /*]*/, new Path(args[0])); FileOutputFormat.setOutputPath(/*[*/ conf /*]*/, new Path(args[1])); /*[*/ conf /*]*/.setMapperClass(OldMaxTemperatureMapper.class); /*[*/ conf /*]*/.setReducerClass(OldMaxTemperatureReducer.class); /*[*/ conf /*]*/.setOutputKeyClass(Text.class); /*[*/ conf /*]*/.setOutputValueClass(IntWritable.class); /*[*/ JobClient.runJob(conf); /*]*/ }
public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); Path tmpPath = new Path("/w1/tmp"); fs.delete(tmpPath, true); Job jobA = Job.getInstance(conf, "delaysum"); jobA.setOutputKeyClass(Text.class); jobA.setOutputValueClass(IntWritable.class); jobA.setMapperClass(ArrivalMap.class); jobA.setReducerClass(ArrivalReduce.class); FileInputFormat.setInputPaths(jobA, new Path(args[0])); FileOutputFormat.setOutputPath(jobA, tmpPath); jobA.setJarByClass(TopArrivalPerformanceCarriersOD.class); jobA.waitForCompletion(true); Job jobB = Job.getInstance(conf, "Best Airline"); jobB.setOutputKeyClass(Text.class); jobB.setOutputValueClass(FloatWritable.class); jobB.setMapOutputKeyClass(NullWritable.class); jobB.setMapOutputValueClass(TextArrayWritable.class); jobB.setMapperClass(DelayMap.class); jobB.setReducerClass(DelayReduce.class); jobB.setNumReduceTasks(1); FileInputFormat.setInputPaths(jobB, tmpPath); FileOutputFormat.setOutputPath(jobB, new Path(args[1])); jobB.setInputFormatClass(KeyValueTextInputFormat.class); jobB.setOutputFormatClass(TextOutputFormat.class); jobB.setJarByClass(TopArrivalPerformanceCarriersOD.class); System.exit(jobB.waitForCompletion(true) ? 0 : 1); }
@Override public int run(String[] args) throws Exception { // TODO Auto-generated method stub Configuration conf = getConf(); if (args.length != 3) { System.err.println("Usage: WordCountDriver <inpath> <outpath>"); System.exit(0); } GenericOptionsParser gop = new GenericOptionsParser(args); args = gop.getRemainingArgs(); Path inpath = new Path(args[1]); Path outpath = new Path(args[2]); Job job = new Job(conf, "WordCountDriver"); job.setJarByClass(WordCountDriver.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(IntWritable.class); job.setOutputFormatClass(TextOutputFormat.class); job.setInputFormatClass(TextInputFormat.class); FileInputFormat.setInputPaths(job, inpath); FileOutputFormat.setOutputPath(job, outpath); job.setNumReduceTasks(2); job.setMapperClass(WordCountMapper.class); job.setReducerClass(WordCountReducer.class); job.setPartitionerClass(WordCountPart.class); job.waitForCompletion(true); return 0; }
// Transform the json-type feature to mat-type public static Mat json2mat(String json) { JsonParser parser = new JsonParser(); JsonElement parseTree = parser.parse(json); // Verify the input is JSON type if (!parseTree.isJsonObject()) { System.out.println("The input is not a JSON type...\nExiting..."); System.exit(1); } JsonObject jobj = parser.parse(json).getAsJsonObject(); if (jobj == null || !jobj.isJsonObject() || jobj.isJsonNull()) { return null; } // Detect broken/null features JsonElement r = jobj.get("rows"); if (r == null) { return null; } int rows = jobj.get("rows").getAsInt(); int cols = jobj.get("cols").getAsInt(); int type = jobj.get("type").getAsInt(); String data = jobj.get("data").getAsString(); String[] pixs = data.split(","); Mat descriptor = new Mat(rows, cols, type); for (String pix : pixs) { String[] tmp = pix.split(" "); int r_pos = Integer.valueOf(tmp[0]); int c_pos = Integer.valueOf(tmp[1]); double rgb = Double.valueOf(tmp[2]); descriptor.put(r_pos, c_pos, rgb); } return descriptor; }
public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException { if (args.length != 2) { System.out.println("Usage: FeatureMatching ID <inputName.jpeg/inputName.jpg>"); System.exit(1); } SimpleDateFormat sdf = new SimpleDateFormat("", Locale.US); sdf.applyPattern("yyyy-MM-dd_HH-mm-ss"); String time = sdf.format(new Date()); Job job = Job.getInstance(); ID = "/" + args[0]; String filename = args[1]; filename = filename.toLowerCase(); System.out.println("current filename:" + filename); // Detect illegal username (if the username dir doesn't exist) File userPath = new File(LOCAL_USER_DIR + ID); if (!userPath.exists()) { System.out.println("Unauthorized username!!!\nExiting......"); System.exit(1); } // Preprocess the input image.jpg from local dir: /local.../user/ID/input/image.jpg // Save the features to local dir: hdfs://.../user/ID/input/image.jpg extractQueryFeatures2HDFS(filename, job); // Add the feature file to the hdfs cache String featureFileName = filename.substring(0, filename.lastIndexOf(".")) + ".json"; // job.addCacheFile(new Path(HDFS_HOME + USER + ID + INPUT + "/" + // featureFileName).toUri()); job.getConfiguration() .set("featureFilePath", HDFS_HOME + USER + ID + INPUT + "/" + featureFileName); // Check the file type. Only support jpeg/jpg type images String type = filename.substring(args[1].lastIndexOf(".")); if (!(type.equals(".jpg") || type.equals(".jpeg"))) { System.out.println("Image type not supported!!!\nExiting"); System.exit(1); } // Input: hdfs://.../features/ // The feature dir is a location of all features extracted from the database String inputPathStr = HDFS_HOME + FEATURES; // Output: hdfs://.../user/ID/output/ String outputPathStr = HDFS_HOME + USER + ID + OUTPUT + "/" + time; job.setInputFormatClass(KeyValueTextInputFormat.class); // job.setOutputFormatClass(TextOutputFormat.class); // Get the lists of all feature files: /.../features/data/part-* FileSystem fs = FileSystem.get(job.getConfiguration()); FileStatus[] statuses = fs.listStatus(new Path(inputPathStr)); StringBuffer sb = new StringBuffer(); for (FileStatus fileStatus : statuses) { sb.append(fileStatus.getPath() + ","); } sb.deleteCharAt(sb.lastIndexOf(",")); job.setJarByClass(FeatureMatching.class); job.setMapperClass(FeatureMatchMapper.class); job.setReducerClass(FeatureMatchReducer.class); // only need one reducer to collect the result job.setNumReduceTasks(1); job.setMapOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(Text.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(DoubleWritable.class); // Input a directory, so need the recursive input FileInputFormat.setInputDirRecursive(job, true); // Set the PathFilter, to omit _SUCCESS files // (This is not working correctly, as the PathFilter class is an interface rather than a class. // But the 2nd arg asks me to extend the PathFilter) // FileInputFormat.setInputPathFilter(job, MyPathFilter.class); // // FileInputFormat.setInputPaths(job, new Path(inputPathStr)); FileInputFormat.setInputPaths(job, sb.toString()); FileOutputFormat.setOutputPath(job, new Path(outputPathStr)); boolean success = job.waitForCompletion(true); return success ? 0 : 1; }
public static void main(String[] args) throws Exception { System.out.println("Beginning job..."); // Start phase 1 Configuration conf = new Configuration(); conf.set("mapred.job.tracker", "local"); conf.set("fs.default.name", "local"); String[] inputArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); Path xmlPath = null; List<String> other_args = new ArrayList<String>(); for (int i = 0; i < args.length; ++i) { if ("-setup".equals(inputArgs[i])) { xmlPath = new Path(inputArgs[++i]); DistributedCache.addCacheFile(xmlPath.toUri(), conf); conf.setBoolean("minibatch.job.setup", true); } else { other_args.add(inputArgs[i]); } } String[] tool_args = other_args.toArray(new String[0]); int result = ToolRunner.run(conf, new BatchGenerationEngine(), tool_args); // End phase 1 // get example count and size from xml file // count = count_size[0]; // size = count_size[1]; int[] count_size = parseJobSetup(xmlPath); // distribute those output from phase 1 into different directories String outputPhase1 = tool_args[1]; FileSystem fs = FileSystem.get(new Configuration()); Path outputPhase1Path = new Path(outputPhase1); fs.setWorkingDirectory(outputPhase1Path); FileStatus[] outputP1AllFiles = fs.listStatus(outputPhase1Path); for (int i = 0; i < outputP1AllFiles.length; i++) { int batchNum = i / count_size[1]; Path batchPath = new Path(outputPhase1 + "/batch" + batchNum); // if batch# directory not exists, mkdir if (!fs.exists(batchPath)) FileSystem.mkdirs(fs, batchPath, new FsPermission("777")); // move file into the batch# directory fs.rename( outputP1AllFiles[i].getPath(), new Path( outputPhase1 + "/batch" + batchNum + "/" + outputP1AllFiles[i].getPath().getName())); } // // Generate dictionary of jobs int numberOfJobs = count_size[0] * count_size[1]; JobConfig[] dictionary = new JobConfig[numberOfJobs]; // Add job 0 to dictionary Configuration conf0 = new Configuration(); DistributedCache.addCacheFile(xmlPath.toUri(), conf0); JobConfig job0 = new JobConfig(conf0, "input go here", java.util.UUID.randomUUID().toString()); dictionary[0] = job0; // Add the rest of jobs into dictionary for (int i = 1; i < dictionary.length; i++) { Configuration newConf = new Configuration(); DistributedCache.addCacheFile(xmlPath.toUri(), newConf); JobConfig newJob = new JobConfig(newConf, dictionary[i - 1].args[1], java.util.UUID.randomUUID().toString()); dictionary[i] = newJob; } // running the jobs logger.info("Start " + dictionary.length + " jobs!"); for (int i = 0; i < dictionary.length; i++) { int runResult = ToolRunner.run(dictionary[i].conf, new BatchGenerationEngine(), dictionary[i].args); if (runResult == 1) { logger.info("Job " + i + "-th Re-run once!"); dictionary[i].args[1] = java.util.UUID.randomUUID().toString(); runResult = ToolRunner.run(dictionary[i].conf, new BatchGenerationEngine(), dictionary[i].args); } if (runResult == 1) { logger.info("Job " + i + "-th Re-run twice!"); dictionary[i].args[1] = java.util.UUID.randomUUID().toString(); runResult = ToolRunner.run(dictionary[i].conf, new BatchGenerationEngine(), dictionary[i].args); } if (runResult == 1) { logger.info("Job " + i + "-th Failed!"); break; } else { if (i - 1 < dictionary.length) dictionary[i + 1].args[0] = dictionary[i].args[1]; } } System.exit(1); }
public static void main(String[] args) throws Exception { int exitCode = ToolRunner.run(new SortByTemperatureUsingTotalOrderPartitioner(), args); System.exit(exitCode); }
/** * @param args * @throws Exception */ public static void main(String[] args) throws Exception { int res = ToolRunner.run(new Configuration(), new WordCountDriver(), args); System.exit(res); }
public static void main(String[] args) throws Exception { KMeansDriver drv = new KMeansDriver(); drv.getConf().set("mapreduce.framework.name", "local"); int exitCode = ToolRunner.run(drv, args); System.exit(exitCode); }
public static void main(String[] args) throws Exception { int exitCode = ToolRunner.run(new MaxTemperatureDriver(), args); System.exit(exitCode); }
public static void main(String[] args) throws Exception { int exitCode = ToolRunner.run(new WhiteHouseVisitorDriver(), args); System.exit(exitCode); }
public static void main(String[] args) throws Exception { JobConf job = new JobConf(DistCp.class); DistCp distcp = new DistCp(job); int res = ToolRunner.run(distcp, args); System.exit(res); }
public static void main(String[] args) throws Exception { int rc = ToolRunner.run(new Configuration(), new KnownKeysMRv2(), args); System.exit(rc); }
public int run(String[] args) throws Exception { // printUsage(); /* * SETUP */ Configuration argConf = getConf(); Hashtable<String, String> confArg = new Hashtable<String, String>(); setup(confArg, argConf); Date currentTime = new Date(); Date endDate = new Date(new Long(confArg.get("timestamp_stop"))); Boolean full_run = confArg.get("intermediate").matches("(?i).*true.*"); Boolean quick_add = confArg.get("quick_add").matches("(?i).*true.*"); logger.info("Running GeStore"); // ZooKeeper setup Configuration config = HBaseConfiguration.create(); zkWatcher = new ZooKeeperWatcher(config, "Testing", new HBaseAdmin(config)); zkInstance = new ZooKeeper( ZKConfig.getZKQuorumServersString(config), config.getInt("zookeeper.session.timeout", -1), zkWatcher); if (!confArg.get("task_id").isEmpty()) { confArg.put("temp_path", confArg.get("temp_path") + confArg.get("task_id")); } String lockRequest = confArg.get("file_id"); if (!confArg.get("run_id").isEmpty()) lockRequest = lockRequest + "_" + confArg.get("run_id") + "_"; if (!confArg.get("task_id").isEmpty()) lockRequest = lockRequest + "_" + confArg.get("task_id") + "_"; // Get type of movement toFrom type_move = checkArgs(confArg); if (type_move == toFrom.LOCAL2REMOTE && !confArg.get("format").equals("unknown")) { List<String> arguments = new ArrayList<String>(); arguments.add("-Dinput=" + confArg.get("local_path")); arguments.add("-Dtable=" + confArg.get("file_id")); arguments.add("-Dtimestamp=" + confArg.get("timestamp_stop")); arguments.add("-Dtype=" + confArg.get("format")); arguments.add("-Dtarget_dir=" + confArg.get("base_path") + "_" + confArg.get("file_id")); arguments.add("-Dtemp_hdfs_path=" + confArg.get("temp_path")); arguments.add("-Drun_id=" + confArg.get("run_id")); if (!confArg.get("run_id").isEmpty()) arguments.add("-Drun_id=" + confArg.get("run_id")); if (!confArg.get("task_id").isEmpty()) arguments.add("-Dtask_id=" + confArg.get("task_id")); if (quick_add) arguments.add("-Dquick_add=" + confArg.get("quick_add")); String lockName = lock(lockRequest); String[] argumentString = arguments.toArray(new String[arguments.size()]); adddb.main(argumentString); unlock(lockName); System.exit(0); } // Database registration dbutil db_util = new dbutil(config); db_util.register_database(confArg.get("db_name_files"), true); db_util.register_database(confArg.get("db_name_runs"), true); db_util.register_database(confArg.get("db_name_updates"), true); FileSystem hdfs = FileSystem.get(config); FileSystem localFS = FileSystem.getLocal(config); // Get source type confArg.put("source", getSource(db_util, confArg.get("db_name_files"), confArg.get("file_id"))); confArg.put( "database", isDatabase(db_util, confArg.get("db_name_files"), confArg.get("file_id"))); if (!confArg.get("source").equals("local") && type_move == toFrom.REMOTE2LOCAL && !confArg.get("timestamp_stop").equals(Integer.toString(Integer.MAX_VALUE))) { confArg.put("timestamp_stop", Long.toString(latestVersion(confArg, db_util))); } /* * Get previous timestamp */ Get run_id_get = new Get(confArg.get("run_id").getBytes()); Result run_get = db_util.doGet(confArg.get("db_name_runs"), run_id_get); KeyValue run_file_prev = run_get.getColumnLatest( "d".getBytes(), (confArg.get("file_id") + "_db_timestamp").getBytes()); String last_timestamp = new String("0"); if (null != run_file_prev && !confArg.get("source").equals("local")) { long last_timestamp_real = run_file_prev.getTimestamp(); Long current_timestamp = new Long(confArg.get("timestamp_real")); if ((current_timestamp - last_timestamp_real) > 36000) { last_timestamp = new String(run_file_prev.getValue()); Integer lastTimestamp = new Integer(last_timestamp); lastTimestamp += 1; last_timestamp = lastTimestamp.toString(); logger.info("Last timestamp: " + last_timestamp + " End data: " + endDate); Date last_run = new Date(run_file_prev.getTimestamp()); if (last_run.before(endDate) && !full_run) { confArg.put("timestamp_start", last_timestamp); } } } Integer tse = new Integer(confArg.get("timestamp_stop")); Integer tss = new Integer(confArg.get("timestamp_start")); if (tss > tse) { logger.info("No new version of requested file."); return 0; } /* * Generate file */ String lockName = lock(lockRequest); Get file_id_get = new Get(confArg.get("file_id").getBytes()); Result file_get = db_util.doGet(confArg.get("db_name_files"), file_id_get); if (!file_get.isEmpty()) { boolean found = hasFile( db_util, hdfs, confArg.get("db_name_files"), confArg.get("file_id"), getFullPath(confArg)); if (confArg.get("source").equals("fullfile")) { found = false; } String filenames_put = getFileNames( db_util, confArg.get("db_name_files"), confArg.get("file_id"), getFullPath(confArg)); // Filename not found in file database if (!found && type_move == toFrom.REMOTE2LOCAL) { if (!confArg.get("source").equals("local")) { // Generate intermediate file if (getFile(hdfs, confArg, db_util) == null) { unlock(lockName); return 1; } // Put generated file into file database if (!confArg.get("format").equals("fullfile")) { putFileEntry( db_util, hdfs, confArg.get("db_name_files"), confArg.get("file_id"), confArg.get("full_file_name"), confArg.get("source")); } } else { logger.warn("Remote file not found, and cannot be generated! File: " + confArg); unlock(lockName); return 1; } } } else { if (type_move == toFrom.REMOTE2LOCAL) { logger.warn("Remote file not found, and cannot be generated."); unlock(lockName); return 1; } } /* * Copy file * Update tables */ if (type_move == toFrom.LOCAL2REMOTE) { if (!confArg.get("format").equals("fullfile")) { putFileEntry( db_util, hdfs, confArg.get("db_name_files"), confArg.get("file_id"), getFullPath(confArg), confArg.get("source")); } putRunEntry( db_util, confArg.get("db_name_runs"), confArg.get("run_id"), confArg.get("file_id"), confArg.get("type"), confArg.get("timestamp_real"), confArg.get("timestamp_stop"), getFullPath(confArg), confArg.get("delimiter")); hdfs.copyFromLocalFile(new Path(confArg.get("local_path")), new Path(getFullPath(confArg))); } else if (type_move == toFrom.REMOTE2LOCAL) { FileStatus[] files = hdfs.globStatus(new Path(getFullPath(confArg) + "*")); putRunEntry( db_util, confArg.get("db_name_runs"), confArg.get("run_id"), confArg.get("file_id"), confArg.get("type"), confArg.get("timestamp_real"), confArg.get("timestamp_stop"), getFullPath(confArg), confArg.get("delimiter")); unlock(lockName); for (FileStatus file : files) { Path cur_file = file.getPath(); Path cur_local_path = new Path(new String(confArg.get("local_path") + confArg.get("file_id"))); String suffix = getSuffix(getFileName(confArg), cur_file.getName()); if (suffix.length() > 0) { cur_local_path = cur_local_path.suffix(new String("." + suffix)); } if (confArg.get("copy").equals("true")) { String crc = hdfs.getFileChecksum(cur_file).toString(); if (checksumLocalTest(cur_local_path, crc)) { continue; } else { hdfs.copyToLocalFile(cur_file, cur_local_path); writeChecksum(cur_local_path, crc); } } else { System.out.println(cur_local_path + "\t" + cur_file); } } } unlock(lockName); return 0; }
public static void main(String[] args) throws Exception { int res = ToolRunner.run(new Configuration(), new XiangLi1_exercise3(), args); System.exit(res); }
public static void main(String[] args) throws Exception { int result = ToolRunner.run(new Configuration(), new move(), args); System.exit(result); }
/* * You shouldn't need to modify this function much. If you think you have a good reason to, * you might want to discuss with staff. * * The skeleton supports several options. * if you set runJob2 to false, only the first job will run and output will be * in TextFile format, instead of SequenceFile. This is intended as a debugging aid. * * If you set combiner to false, the combiner will not run. This is also * intended as a debugging aid. Turning on and off the combiner shouldn't alter * your results. Since the framework doesn't make promises about when it'll * invoke combiners, it's an error to assume anything about how many times * values will be combined. */ public static void main(String[] rawArgs) throws Exception { GenericOptionsParser parser = new GenericOptionsParser(rawArgs); Configuration conf = parser.getConfiguration(); String[] args = parser.getRemainingArgs(); boolean runJob2 = conf.getBoolean("runJob2", true); boolean combiner = conf.getBoolean("combiner", false); System.out.println("Target word: " + conf.get("targetWord")); System.out.println("Function num: " + conf.get("funcNum")); if (runJob2) System.out.println("running both jobs"); else System.out.println("for debugging, only running job 1"); if (combiner) System.out.println("using combiner"); else System.out.println("NOT using combiner"); Path inputPath = new Path(args[0]); Path middleOut = new Path(args[1]); Path finalOut = new Path(args[2]); FileSystem hdfs = middleOut.getFileSystem(conf); int reduceCount = conf.getInt("reduces", 32); if (hdfs.exists(middleOut)) { System.err.println("can't run: " + middleOut.toUri().toString() + " already exists"); System.exit(1); } if (finalOut.getFileSystem(conf).exists(finalOut)) { System.err.println("can't run: " + finalOut.toUri().toString() + " already exists"); System.exit(1); } { Job firstJob = new Job(conf, "job1"); firstJob.setJarByClass(Map1.class); /* You may need to change things here */ firstJob.setMapOutputKeyClass(Text.class); firstJob.setMapOutputValueClass( DoublePair.class); // Change Map output Value type to DoublePair firstJob.setOutputKeyClass(DoubleWritable.class); firstJob.setOutputValueClass(Text.class); /* End region where we expect you to perhaps need to change things. */ firstJob.setMapperClass(Map1.class); firstJob.setReducerClass(Reduce1.class); firstJob.setNumReduceTasks(reduceCount); if (combiner) firstJob.setCombinerClass(Combine1.class); firstJob.setInputFormatClass(SequenceFileInputFormat.class); if (runJob2) firstJob.setOutputFormatClass(SequenceFileOutputFormat.class); FileInputFormat.addInputPath(firstJob, inputPath); FileOutputFormat.setOutputPath(firstJob, middleOut); firstJob.waitForCompletion(true); } if (runJob2) { Job secondJob = new Job(conf, "job2"); secondJob.setJarByClass(Map1.class); /* You may need to change things here */ secondJob.setMapOutputKeyClass(DoubleWritable.class); secondJob.setMapOutputValueClass(Text.class); secondJob.setOutputKeyClass(DoubleWritable.class); secondJob.setOutputValueClass(Text.class); /* End region where we expect you to perhaps need to change things. */ secondJob.setMapperClass(Map2.class); secondJob.setReducerClass(Reduce2.class); secondJob.setInputFormatClass(SequenceFileInputFormat.class); secondJob.setOutputFormatClass(TextOutputFormat.class); secondJob.setNumReduceTasks(1); FileInputFormat.addInputPath(secondJob, middleOut); FileOutputFormat.setOutputPath(secondJob, finalOut); secondJob.waitForCompletion(true); } }
public static void main(String[] args) throws Exception { int result = ToolRunner.run(new HadoopNBFilter(), args); System.exit(result); }