/** Read a source file and compile it up to the the current job's last barrier. */ public boolean readSource(FileSource source) { // Add a new SourceJob for the given source. If a Job for the source // already exists, then we will be given the existing job. SourceJob job = addJob(source); if (job == null) { // addJob returns null if the job has already been completed, in // which case we can just ignore the request to read in the source. return true; } // Run the new job up to the currentJob's SourceJob's last barrier, to // make sure that dependencies are satisfied. Pass.ID barrier; if (currentJob != null) { if (currentJob.sourceJob().lastBarrier() == null) { throw new InternalCompilerError( "A Source Job which has " + "not reached a barrier cannot read another " + "source file."); } barrier = currentJob.sourceJob().lastBarrier().id(); } else { barrier = Pass.FIRST_BARRIER; } // Make sure we reach at least the first barrier defined // in the base compiler. This forces types to be constructed. // If FIRST_BARRIER is before "barrier", // then the second runToPass will just return true. return runToPass(job, barrier) && runToPass(job, Pass.FIRST_BARRIER); }
/** Run a job up to the <code>goal</code> pass. */ public boolean runToPass(Job job, Pass goal) { if (Report.should_report(Report.frontend, 1)) Report.report(1, "Running " + job + " to pass " + goal); while (!job.pendingPasses().isEmpty()) { Pass pass = (Pass) job.pendingPasses().get(0); try { runPass(job, pass); } catch (CyclicDependencyException e) { // cause the pass to fail. job.finishPass(pass, false); } if (pass == goal) { break; } } if (job.completed()) { if (Report.should_report(Report.frontend, 1)) Report.report(1, "Job " + job + " completed"); } return job.status(); }
public static void main(String[] args) throws Exception { String inputDirectory = "/home/cs246/Desktop/HW2/input"; String outputDirectory = "/home/cs246/Desktop/HW2/output"; String centroidDirectory = "/home/cs246/Desktop/HW2/config"; int iterations = 20; for (int i = 1; i <= iterations; i++) { Configuration conf = new Configuration(); String cFile = centroidDirectory + "/c" + i + ".txt"; String nextCFile = centroidDirectory + "/c" + (i + 1) + ".txt"; conf.set("CFILE", cFile); conf.set("NEXTCFILE", nextCFile); String cFile = centroidDirectory + "/c" + i + ".txt"; String nextCFile = centroidDirectory + "/c" + (i + 1) + ".txt"; conf.set("CFILE", cFile); conf.set("NEXTCFILE", nextCFile); Job job = new Job(conf, "HW2_Q4." + i); job.setJarByClass(HW2_Q4.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(Text.class); job.setMapperClass(Map1.class); job.setReducerClass(Reduce1.class); job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); FileInputFormat.addInputPath(job, new Path(inputDirectory)); FileOutputFormat.setOutputPath(job, new Path(outputDirectory + "/output" + i)); job.waitForCompletion(true); } }
/** Run a job until the <code>goal</code> pass completes. */ public boolean runToPass(Job job, Pass.ID goal) { if (Report.should_report(Report.frontend, 1)) Report.report(1, "Running " + job + " to pass named " + goal); if (job.completed(goal)) { return true; } Pass pass = job.passByID(goal); return runToPass(job, pass); }
/** Adds a dependency from the current job to the given Source. */ public void addDependencyToCurrentJob(Source s) { if (s == null) return; if (currentJob != null) { Object o = jobs.get(s); if (o != COMPLETED_JOB) { if (Report.should_report(Report.frontend, 2)) { Report.report(2, "Adding dependency from " + currentJob.source() + " to " + s); } currentJob.sourceJob().addDependency(s); } } else { throw new InternalCompilerError("No current job!"); } }
void checkFormat(Job job) throws Exception { TaskAttemptContext attemptContext = new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID("123", 0, false, 1, 2)); MyClassMessagePackBase64LineInputFormat format = new MyClassMessagePackBase64LineInputFormat(); FileInputFormat.setInputPaths(job, workDir); List<InputSplit> splits = format.getSplits(job); for (int j = 0; j < splits.size(); j++) { RecordReader<LongWritable, MyClassWritable> reader = format.createRecordReader(splits.get(j), attemptContext); reader.initialize(splits.get(j), attemptContext); int count = 0; try { while (reader.nextKeyValue()) { LongWritable key = reader.getCurrentKey(); MyClassWritable val = reader.getCurrentValue(); MyClass mc = val.get(); assertEquals(mc.v, count); assertEquals(mc.s, Integer.toString(count)); count++; } } finally { reader.close(); } } }
/** Run all pending passes on <code>job</code>. */ public boolean runAllPasses(Job job) { List pending = job.pendingPasses(); // Run until there are no more passes. if (!pending.isEmpty()) { Pass lastPass = (Pass) pending.get(pending.size() - 1); return runToPass(job, lastPass); } return true; }
@Override public int run(String[] args) throws Exception { String locatorHost = args[0]; int locatorPort = Integer.parseInt(args[1]); String hdfsHomeDir = args[2]; System.out.println( "KnownKeysMRv2 invoked with args (locatorHost = " + locatorHost + " locatorPort = " + locatorPort + " hdfsHomeDir = " + hdfsHomeDir); Configuration conf = getConf(); conf.set(GFInputFormat.INPUT_REGION, "partitionedRegion"); conf.set(GFInputFormat.HOME_DIR, hdfsHomeDir); conf.setBoolean(GFInputFormat.CHECKPOINT, false); conf.set(GFOutputFormat.REGION, "validationRegion"); conf.set(GFOutputFormat.LOCATOR_HOST, locatorHost); conf.setInt(GFOutputFormat.LOCATOR_PORT, locatorPort); Job job = Job.getInstance(conf, "knownKeysMRv2"); job.setInputFormatClass(GFInputFormat.class); job.setOutputFormatClass(GFOutputFormat.class); job.setMapperClass(KnownKeysMRv2Mapper.class); job.setMapOutputKeyClass(GFKey.class); job.setMapOutputValueClass(PEIWritable.class); job.setReducerClass(KnownKeysMRv2Reducer.class); // job.setOutputKeyClass(String.class); // job.setOutputValueClass(ValueHolder.class); return job.waitForCompletion(false) ? 0 : 1; }
static void realMain(String[] args) throws Throwable { // jmap doesn't work on Windows if (System.getProperty("os.name").startsWith("Windows")) return; final String childClassName = Job.class.getName(); final String classToCheckForLeaks = Job.classToCheckForLeaks(); final String uniqueID = String.valueOf(new Random().nextInt(Integer.MAX_VALUE)); final String[] jobCmd = { java, "-Xmx8m", "-classpath", System.getProperty("test.classes", "."), childClassName, uniqueID }; final Process p = new ProcessBuilder(jobCmd).start(); final String childPid = match( commandOutputOf(jps, "-m"), "(?m)^ *([0-9]+) +\\Q" + childClassName + "\\E *" + uniqueID + "$", 1); final int n0 = objectsInUse(p, childPid, classToCheckForLeaks); final int n1 = objectsInUse(p, childPid, classToCheckForLeaks); equal(p.waitFor(), 0); equal(p.exitValue(), 0); failed += p.exitValue(); // Check that no objects were leaked. System.out.printf("%d -> %d%n", n0, n1); check(Math.abs(n1 - n0) < 2); // Almost always n0 == n1 check(n1 < 20); drainers.shutdown(); }
/** * Before running <code>Pass pass</code> on <code>SourceJob job</code> make sure that all * appropriate scheduling invariants are satisfied, to ensure that all passes of other jobs that * <code>job</code> depends on will have already been done. */ protected void enforceInvariants(Job job, Pass pass) throws CyclicDependencyException { SourceJob srcJob = job.sourceJob(); if (srcJob == null) { return; } BarrierPass lastBarrier = srcJob.lastBarrier(); if (lastBarrier != null) { // make sure that _all_ dependent jobs have completed at least up to // the last barrier (not just children). // // Ideally the invariant should be that only the source jobs that // job _depends on_ should be brought up to the last barrier. // This is work to be done in the future... List allDependentSrcs = new ArrayList(srcJob.dependencies()); Iterator i = allDependentSrcs.iterator(); while (i.hasNext()) { Source s = (Source) i.next(); Object o = jobs.get(s); if (o == COMPLETED_JOB) continue; if (o == null) { throw new InternalCompilerError("Unknown source " + s); } SourceJob sj = (SourceJob) o; if (sj.pending(lastBarrier.id())) { // Make the job run up to the last barrier. // We ignore the return result, since even if the job // fails, we will keep on going and see // how far we get... if (Report.should_report(Report.frontend, 3)) { Report.report(3, "Running " + sj + " to " + lastBarrier.id() + " for " + srcJob); } runToPass(sj, lastBarrier.id()); } } } if (pass instanceof GlobalBarrierPass) { // need to make sure that _all_ jobs have completed just up to // this global barrier. // If we hit a cyclic dependency, ignore it and run the other // jobs up to that pass. Then try again to run the cyclic // pass. If we hit the cycle again for the same job, stop. LinkedList barrierWorklist = new LinkedList(jobs.values()); while (!barrierWorklist.isEmpty()) { Object o = barrierWorklist.removeFirst(); if (o == COMPLETED_JOB) continue; SourceJob sj = (SourceJob) o; if (sj.completed(pass.id()) || sj.nextPass() == sj.passByID(pass.id())) { // the source job has either done this global pass // (which is possible if the job was loaded late in the // game), or is right up to the global barrier. continue; } // Make the job run up to just before the global barrier. // We ignore the return result, since even if the job // fails, we will keep on going and see // how far we get... Pass beforeGlobal = sj.getPreviousTo(pass.id()); if (Report.should_report(Report.frontend, 3)) { Report.report(3, "Running " + sj + " to " + beforeGlobal.id() + " for " + srcJob); } // Don't use runToPass, since that catches the // CyclicDependencyException that we should report // back to the caller. while (!sj.pendingPasses().isEmpty()) { Pass p = (Pass) sj.pendingPasses().get(0); runPass(sj, p); if (p == beforeGlobal) { break; } } } } }
/** * Run the pass <code>pass</code> on the job. Before running the pass on the job, if the job is a * <code>SourceJob</code>, then this method will ensure that the scheduling invariants are * enforced by calling <code>enforceInvariants</code>. */ protected void runPass(Job job, Pass pass) throws CyclicDependencyException { // make sure that all scheduling invariants are satisfied before running // the next pass. We may thus execute some other passes on other // jobs running the given pass. try { enforceInvariants(job, pass); } catch (CyclicDependencyException e) { // A job that depends on this job is still running // an earlier pass. We cannot continue this pass, // but we can just silently fail since the job we're // that depends on this one will eventually try // to run this pass again when it reaches a barrier. return; } if (getOptions().disable_passes.contains(pass.name())) { if (Report.should_report(Report.frontend, 1)) Report.report(1, "Skipping pass " + pass); job.finishPass(pass, true); return; } if (Report.should_report(Report.frontend, 1)) Report.report(1, "Trying to run pass " + pass + " in " + job); if (job.isRunning()) { // We're currently running. We can't reach the goal. throw new CyclicDependencyException(job + " cannot reach pass " + pass); } pass.resetTimers(); boolean result = false; if (job.status()) { Job oldCurrentJob = this.currentJob; this.currentJob = job; Report.should_report.push(pass.name()); // Stop the timer on the old pass. */ Pass oldPass = oldCurrentJob != null ? oldCurrentJob.runningPass() : null; if (oldPass != null) { oldPass.toggleTimers(true); } job.setRunningPass(pass); pass.toggleTimers(false); result = pass.run(); pass.toggleTimers(false); job.setRunningPass(null); Report.should_report.pop(); this.currentJob = oldCurrentJob; // Restart the timer on the old pass. */ if (oldPass != null) { oldPass.toggleTimers(true); } // pretty-print this pass if we need to. if (getOptions().print_ast.contains(pass.name())) { System.err.println("--------------------------------" + "--------------------------------"); System.err.println("Pretty-printing AST for " + job + " after " + pass.name()); PrettyPrinter pp = new PrettyPrinter(); pp.printAst(job.ast(), new CodeWriter(System.err, 78)); } // dump this pass if we need to. if (getOptions().dump_ast.contains(pass.name())) { System.err.println("--------------------------------" + "--------------------------------"); System.err.println("Dumping AST for " + job + " after " + pass.name()); NodeVisitor dumper = new DumpAst(new CodeWriter(System.err, 78)); dumper = dumper.begin(); job.ast().visit(dumper); dumper.finish(); } // This seems to work around a VM bug on linux with JDK // 1.4.0. The mark-sweep collector will sometimes crash. // Running the GC explicitly here makes the bug go away. // If this fails, maybe run with bigger heap. // System.gc(); } Stats stats = getStats(); stats.accumPassTimes(pass.id(), pass.inclusiveTime(), pass.exclusiveTime()); if (Report.should_report(Report.time, 2)) { Report.report( 2, "Finished " + pass + " status=" + str(result) + " inclusive_time=" + pass.inclusiveTime() + " exclusive_time=" + pass.exclusiveTime()); } else if (Report.should_report(Report.frontend, 1)) { Report.report(1, "Finished " + pass + " status=" + str(result)); } job.finishPass(pass, result); }
@Override void solve(Set<? extends Job> jobs) { Job[] sortedJobs = jobs.toArray(new Job[jobs.size()]); Arrays.sort(sortedJobs, Job::compareArrivalTime); processTime = totWT = 0; long usefulTime = 0; int jobCount = jobs.size(); PriorityQueue<Job> queue = new PriorityQueue<>(Job::compareBurstTime); for (Job job : sortedJobs) { if (job == null) { jobCount--; continue; } while (!queue.isEmpty() && processTime < job.getArrivalTime()) { Job nextJob = queue.poll(); long arrivalTime = nextJob.getArrivalTime(); long burstTime = nextJob.getBurstTime(); if (processTime < nextJob.getArrivalTime()) { processList.add(new RunningProcess("Idle", arrivalTime - processTime)); processTime = arrivalTime; } processList.add(new RunningProcess("P" + nextJob.getId(), burstTime)); usefulTime += burstTime; totWT += processTime - arrivalTime; processTime += burstTime; } queue.add(job); } while (!queue.isEmpty()) { Job nextJob = queue.poll(); long arrivalTime = nextJob.getArrivalTime(); long burstTime = nextJob.getBurstTime(); if (processTime < nextJob.getArrivalTime()) { processList.add(new RunningProcess("Idle", arrivalTime - processTime)); processTime = arrivalTime; } processList.add(new RunningProcess("P" + nextJob.getId(), burstTime)); usefulTime += burstTime; totWT += processTime - arrivalTime; processTime += burstTime; } totRT = totWT; totTAT = totWT + usefulTime; avgRT = avgWT = (double) totWT / (double) jobCount; avgTAT = (double) totTAT / (double) jobCount; utilization = usefulTime * 100.0 / processTime; }
@Override public void addJob( String jobName, byte type, String pathToJar, String className, String pathToData, String[] peers, int parNumber, int numberOfReducers, HashMap<String, Integer> finishedMappers) throws RemoteException { SysLogger.getInstance().info("Job " + jobName + " started"); Logger logger = null; try { logger = new Logger(0, "..\\log\\WorkerNode.log"); } catch (IncorrectLogFileException e) { // TODO Auto-generated catch block e.printStackTrace(); } DFSClient dfs = DFSClient.getInstance(); try { dfs.init("localhost", 20000, logger); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } String localPathToJar = "..\\tasks\\" + jobName + ".jar"; File jarFile = new File(localPathToJar); if (jarFile.exists()) { jarFile.delete(); } String localPathToData = "..\\tasks\\" + jobName + ".dat"; File dataFile = new File(localPathToData); if (dataFile.exists()) { dataFile.delete(); } try { dfs.downloadFile(pathToJar, localPathToJar); if (type == MapReduce.TYPE_MAPPER) { dfs.downloadFile(pathToData, localPathToData); } } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } try { URL url = new URL("file:///" + jarFile.getAbsolutePath()); URLClassLoader classLoader = new URLClassLoader(new URL[] {url}); // todo make this Class c = classLoader.loadClass(className); MapReduce jobObject = (MapReduce) c.newInstance(); Job job = null; if (type == MapReduce.TYPE_MAPPER) { job = new Job( jobName, type, jobObject, dataFile.getAbsolutePath(), peers, parNumber, numberOfReducers, finishedMappers); } else if (type == MapReduce.TYPE_REDUCER) { job = new Job( jobName, type, jobObject, pathToData, peers, parNumber, numberOfReducers, finishedMappers); } jobList.put(jobName, job); job.start(); // start new thread // return control } catch (MalformedURLException e) { e.printStackTrace(); } catch (ClassNotFoundException e) { e.printStackTrace(); } catch (InstantiationException e) { e.printStackTrace(); } catch (IllegalAccessException e) { e.printStackTrace(); } }
public int run(String args[]) throws Exception { Configuration conf = getConf(); /* creates a new job with given job name */ Job job = Job.getInstance(conf, "Mat_vect_mul"); job.setJarByClass(Mat_vect_mul.class); /* adds input-vector file to the cache */ DistributedCache.addCacheFile(new Path(args[1]).toUri(), job.getConfiguration()); /* set the key class and value class for the job output */ job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); /* set mapper and reducer for the job */ job.setMapperClass(Map1.class); job.setReducerClass(Reduce1.class); /* set the input foramt and output format for the job */ job.setInputFormatClass(TextInputFormat.class); job.setOutputFormatClass(TextOutputFormat.class); /* add path to the list of inputs for the map-reduce job */ FileInputFormat.addInputPath(job, new Path(args[0])); /* set path of the output diretcory for the map-reduce job*/ FileOutputFormat.setOutputPath(job, new Path(args[2])); /* Submit the job, then poll for progress until the job is complete */ boolean succ = job.waitForCompletion(true); if (!succ) { System.out.println("Job failed, exiting"); return -1; } return 0; }
public static void main(String[] args) throws Exception { final String NAME_NODE = "hdfs://sandbox.hortonworks.com:8020"; Configuration conf = new Configuration(); Job job = Job.getInstance(conf); job.setJarByClass(WordCount.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(IntWritable.class); job.setOutputKeyClass(IntWritable.class); job.setOutputValueClass(NullWritable.class); if (args.length > 2) { job.setNumReduceTasks(Integer.parseInt(args[2])); } job.setMapperClass(CountMapper.class); job.setReducerClass(CountReducer.class); job.setJarByClass(WordCount.class); job.setNumReduceTasks(1); FileInputFormat.addInputPath(job, new Path(args[0] + "data/plot_summaries.txt")); FileSystem fs = FileSystem.get(conf); // handle (e.g. delete) existing output path Path outputDestination = new Path(args[0] + args[1]); if (fs.exists(outputDestination)) { fs.delete(outputDestination, true); } // set output path & start job1 FileOutputFormat.setOutputPath(job, outputDestination); int jobCompletionStatus = job.waitForCompletion(true) ? 0 : 1; }
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException { Configuration conf = new Configuration(); Job job = new Job(conf, "job"); job.setJarByClass(PVidConvert.class); job.setMapperClass(Map1.class); job.setReducerClass(Reduce1.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); FileInputFormat.addInputPath(job, new Path(args[0])); FileOutputFormat.setOutputPath(job, new Path("/tmp/temporary_execution/")); job.waitForCompletion(true); Configuration conf1 = new Configuration(); Job job1 = new Job(conf1, "job1"); job1.setJarByClass(PVidConvert.class); job1.setMapperClass(Map2.class); job1.setReducerClass(Reduce2.class); job1.setOutputKeyClass(Text.class); job1.setOutputValueClass(Text.class); FileInputFormat.addInputPath(job1, new Path("/tmp/temporary_execution/")); FileOutputFormat.setOutputPath(job1, new Path(args[1])); job1.waitForCompletion(true); }