/** @return true iff the object "passes" the predicate */ public boolean execute(Object o) { if (o instanceof Task) { Task task = (Task) o; return (task.getVerb().equals(MonitoringUtils.Start_Publishing_Cmd)); } return false; }
private void showTaskDetails(Task task) { if (task != null) { taskNLabel.setVisible(true); nextExecutionLabel.setVisible(true); actveLabel.setVisible(true); taskNameLabel.setText(task.getTitle()); if (task.isRepeated()) { taskExecutionTime.setText((DateUtil.format(task.getExecutionDate()))); } else { taskExecutionTime.setText(DateUtil.format(task.getStartTime())); } taskActiveLabel.setText((task.isActive()) ? "Yes" : "No"); chooseLabel.setVisible(false); } else { taskNLabel.setVisible(false); nextExecutionLabel.setVisible(false); actveLabel.setVisible(false); taskNameLabel.setText(""); taskExecutionTime.setText(""); taskActiveLabel.setText(""); } logger.info("Task was set"); }
/* Devolve lista com o tipo de tarefas */ public List<String> listTask() { ArrayList<String> tasklist = new ArrayList<>(); lock.lock(); try { for (Task task : this.tasks.values()) { tasklist.add(task.getType()); } return tasklist; } finally { lock.unlock(); } }
public static void main(String[] args) throws IOException { // InputStream inputStream = new FileInputStream("replaceme.in"); // OutputStream outputStream = new FileOutputStream("replaceme.out"); ///////////////////////////////////////////////////////////////////// InputStream inputStream = System.in; OutputStream outputStream = System.out; InputReader in = new InputReader(inputStream); OutputWriter out = new OutputWriter(outputStream); Task solver = new Task(); solver.solve(1, in, out); out.close(); System.exit(0); }
public static File copyResourceFile( String sourceName, String copyName, Project project, Task task) throws IOException { StudyTaskManager taskManager = StudyTaskManager.getInstance(project); Course course = taskManager.getCourse(); int taskNum = task.getIndex() + 1; int lessonNum = task.getLesson().getIndex() + 1; assert course != null; String pathToResource = FileUtil.join( new File(course.getResourcePath()).getParent(), Lesson.LESSON_DIR + lessonNum, Task.TASK_DIR + taskNum); File resourceFile = new File(pathToResource, copyName); FileUtil.copy(new File(pathToResource, sourceName), resourceFile); return resourceFile; }
public TaskRunner(TaskTracker.TaskInProgress tip, TaskTracker tracker, JobConf conf) { this.tip = tip; this.t = tip.getTask(); this.tracker = tracker; this.conf = conf; this.mapOutputFile = new MapOutputFile(t.getJobID()); this.mapOutputFile.setConf(conf); this.jvmManager = tracker.getJvmManagerInstance(); }
/** * Process "Start_publishing_Cmd" task .It update internal data structure for kind of sensor data * user has requested for. * * @param publichcmdlist Enumeration on Collection of newly Start_publishing_Cmd * @see org.cougaar.core.security.oldmonitoring.util.PublishCmdObj */ protected void process_publishCmd(Enumeration publichcmdlist) { PublishCmdObj pcmd; for (; publichcmdlist.hasMoreElements(); ) { Task tsk = (Task) publichcmdlist.nextElement(); PrepositionalPhrase pp = tsk.getPrepositionalPhrase(MonitoringUtils.Start_publishing_Preposition); if (pp != null) { pcmd = (PublishCmdObj) pp.getIndirectObject(); if (!currentlypublishing.contains(pcmd.Type)) { currentlypublishing.add(pcmd.Type); if (MonitoringUtils.debug > 0) System.out.println( "In sensor Plugin process_publishCmd Add ::::::::" + pcmd.Type + " to current publishing list"); } } } }
public static void main(String[] argv) throws IOException { Task t; boolean home = argv.length > 0 && argv[0].equals("test"); InputStream inputStream; OutputStream outputStream; if (home) { inputStream = new FileInputStream(Task.filename + ".in"); outputStream = new FileOutputStream(Task.filename + ".out"); } else { switch (Task.read) { case 0: inputStream = new FileInputStream(Task.filename + ".in"); outputStream = new FileOutputStream(Task.filename + ".out"); break; case 1: inputStream = new FileInputStream("input.txt"); outputStream = new FileOutputStream("output.txt"); break; default: inputStream = System.in; outputStream = System.out; break; } } InputReader in = new InputReader(inputStream); OutputWriter out = new OutputWriter(outputStream, home); if (home) do { long time = System.currentTimeMillis(); t = new Task(); t.in = in; t.out = out; t.run(); out.writeln(); out.writeln("=====Time:" + (System.currentTimeMillis() - time)); out.flush(); } while (in.toNextTest()); else { t = new Task(); t.in = in; t.out = out; t.run(); } out.close(); }
/** * Returns control when task is complete. * * @param json * @param logger */ private String waitForDeploymentCompletion(JSON json, OctopusApi api, Log logger) { final long WAIT_TIME = 5000; final double WAIT_RANDOM_SCALER = 100.0; JSONObject jsonObj = (JSONObject) json; String id = jsonObj.getString("TaskId"); Task task = null; String lastState = "Unknown"; try { task = api.getTask(id); } catch (IOException ex) { logger.error("Error getting task: " + ex.getMessage()); return null; } logger.info("Task info:"); logger.info("\tId: " + task.getId()); logger.info("\tName: " + task.getName()); logger.info("\tDesc: " + task.getDescription()); logger.info("\tState: " + task.getState()); logger.info("\n\nStarting wait..."); boolean completed = task.getIsCompleted(); while (!completed) { try { task = api.getTask(id); } catch (IOException ex) { logger.error("Error getting task: " + ex.getMessage()); return null; } completed = task.getIsCompleted(); lastState = task.getState(); logger.info("Task state: " + lastState); if (completed) { break; } try { Thread.sleep(WAIT_TIME + (long) (Math.random() * WAIT_RANDOM_SCALER)); } catch (InterruptedException ex) { logger.info("Wait interrupted!"); logger.info(ex.getMessage()); completed = true; // bail out of wait loop } } logger.info("Wait complete!"); return lastState; }
public void run() { StopWatch st = null; if (GPResourceHome.perfProfile.PROFILE) { st = new StopWatch(false, false, true); st.start(); } StopWatch ct = new StopWatch(); int numWorkers = 0; try { if (GPResourceHome.commonState == null) { GPResourceHome.load_common(); } } catch (Exception e) { e.printStackTrace(); } ct.start(); while (true) { if (GPResourceHome.perfProfile.PROFILE) { st.stop(); GPResourceHome.perfProfile.addSample("notification_engine_worker", st.getElapsedTimeMX()); st.reset(); st.start(); } GPResource gpResource = null; try { // logger.debug("gpResource.notificationWorkerQ.remove()..."); gpResource = (GPResource) GPResourceHome.commonState.getNextResource(); // gpResource = // (GPResource)GPResourceHome.commonState.notificationWorkerQ.remove();//GPResourceHome.commonState.notificationQ.waitUntilNotEmpty(); boolean sentOK = false; if (gpResource != null && gpResource.taskQ != null && gpResource.taskPendQ != null && (gpResource.taskQ.size() + gpResource.taskPendQ.size()) > 0) // if (gpResource != null) { Task task = null; // if (gpResource.DATA_AWARE_SCHEDULER && !gpResource.MAX_CACHE_HIT) if (gpResource.DATA_AWARE_SCHEDULER) { // synchronized(gpResource) // { // task = (Task)gpResource.taskQ.remove(); task = (Task) gpResource.taskQ.removeNoWait(); if (task == null) { task = (Task) gpResource.taskPendQ.remove(); } logger.debug( "GPService:NotificationEngine:sendNotification(): removed task " + task.getExecutable().getId() + " from gpResource.taskQ... gpResource.taskQ.size() = " + gpResource.taskQ.size()); gpResource.taskPendQ.insert(task); logger.debug( "GPService:NotificationEngine:sendNotification(): inserted task " + task.getExecutable().getId() + " into gpResource.taskPendQ... gpResource.taskPendQ.size() = " + gpResource.taskPendQ.size()); // } } logger.debug( "GPService:NotificationEngine:sendNotification(): key = " + String.valueOf(gpResource.resourceKey) + " ..."); // if (gpResourceHome.sendNotification(gpResource.resourceKey)) boolean sendNotificationTest = false; if (gpResourceHome.useTCPCore) { sendNotificationTest = gpResourceHome.sendNotificationTCPCore(gpResource); } // else if (gpResource.DATA_AWARE_SCHEDULER && !gpResource.MAX_CACHE_HIT) else if (gpResource.DATA_AWARE_SCHEDULER) { sendNotificationTest = gpResourceHome.sendNotification(gpResource, task); } /* else if (gpResourceHome.localFork) { sendNotificationTest = gpResourceHome.sendNotificationLocal(gpResource); } */ else { sendNotificationTest = gpResourceHome.sendNotification(gpResource); } if (sendNotificationTest) { setNotificationsSent(); logger.debug( "Notification for key " + String.valueOf(gpResource.resourceKey) + " sent successfully!"); // if (gpResource.DATA_AWARE_SCHEDULER) // { // gpResource.taskPendQ.insert(task); // logger.debug("GPService:NotificationEngine:sendNotification(): inserted task into // gpResource.taskPendQ... gpResource.taskPendQ.size() = " + // gpResource.taskPendQ.size()); // } // insert back in to try again later... this is a hack to ensure that we never run out // of notification messages :) // if (!gpResourceHome.useTCPCore) // if (gpResource != null && gpResource.taskQ != null && (gpResource.taskQ.size() + // gpResource.taskPendQ.size()) > 0 && (gpResource.taskQ.size() + // gpResource.taskPendQ.size()) <= GPResourceHome.maxNumNotificationWorkerThreads*2) // { // logger.debug("there is still work to be collected and we have dipped below the // number of notification threads, insert back into the notification queue..."); // GPResourceHome.commonState.notificationWorkerQ.insert(gpResource); // } } else { logger.debug( "Notification for key " + String.valueOf(gpResource.resourceKey) + " failed!"); // if (gpResource.DATA_AWARE_SCHEDULER && !gpResource.MAX_CACHE_HIT) if (gpResource.DATA_AWARE_SCHEDULER) { // synchronized(gpResource) // { // gpResource.taskPendQ.removeTask(task); // should insert back at the front... fix this // gpResource.taskQ.insertFront(task); // } // logger.debug("GPService:NotificationEngine:sendNotification(): inserted task " + // task.getExecutable().getId() + " into gpResource.taskQ... gpResource.taskQ.size() = // " + gpResource.taskQ.size()); } // insert back in to try again later... // if ((gpResource.taskQ.size() + gpResource.taskPendQ.size()) > 0) // { // logger.debug("there is still work to be collected, insert back into the // notification queue..."); // GPResourceHome.commonState.notificationWorkerQ.insert(gpResource); // used for throtling notifications... /* try { System.currentTimeMillis(); Thread.sleep(20); } catch (Exception sss) { } */ // } } } if (MAX_NOT_PER_SEC > 0 && getNotificationsSent() >= MAX_NOT_PER_SEC) { long timeLeft = NOT_TIME_QUANTA - ct.getElapsedTime(); if (timeLeft > 20) try { logger.debug( getNotificationsSent() + " notifications sent in " + ct.getElapsedTime() + " ms, sleeping for " + timeLeft + " ms"); // System.currentTimeMillis(); Thread.sleep(timeLeft); } catch (Exception sss) { } } if (ct.getElapsedTime() >= NOT_TIME_QUANTA) { if (getNotificationsSent() > 0) { logger.debug( "***NotificationEngineWorker(): " + getNotificationsSent() + " notifications sent to workers in " + ct.getElapsedTime() + " ms, maximum allowed notifications " + MAX_NOT_PER_SEC + " per " + NOT_TIME_QUANTA + " ms..."); } ct.reset(); resetNotificationsSent(); ct.start(); } // if (GPResourceHome.commonState.resourceQ.size() > 0 && // GPResourceHome.commonState.getWaitingTasks() <= 0) if (GPResourceHome.commonState.getWaitingTasks() <= 0) { try { Thread.sleep(100); } catch (Exception e) { if (logger.isDebugEnabled()) e.printStackTrace(); } } } catch (Exception eee) { logger.debug("Error in NotificationEngineWorker thread: " + eee); if (logger.isDebugEnabled()) eee.printStackTrace(); // continue; // if (gpResource != null && gpResource.taskQ != null && gpResource.taskQ.size() + // gpResource.taskPendQ.size() > 0) // if (gpResource != null && gpResource.taskQ != null) // { // logger.debug("there is still work to be collected, insert back into the notification // queue..."); // GPResourceHome.commonState.notificationWorkerQ.insert(gpResource); // } } } }
public static void main(String[] args) throws IOException { Scanner keyboard = new Scanner(System.in); String input = null; // for most of the input String fileName = "Tasks.txt"; // file were stuff is written to short menu = 0; // for the short input Date date = new Date(); // for the stupid date int modTask = 0; // for getting a task's position in the array int numTasks = 0; // keeps count of number of tasks added to task list TaskList otherStuff = new TaskList(); // TASK LIST!! System.out.println("Welcome to the Task Manager!"); // A spiff little menu System.out.println("Would you like to: "); System.out.println("1. Load a task list from file"); System.out.println("2. Save a new task list"); System.out.println("3. Add a task"); System.out.println("4. Remove a task"); System.out.println("5. Edit a task"); System.out.println("6. Search tasks"); System.out.println("7. Sort tasks by priority"); System.out.println("8. Print task list"); System.out.println("9. Exit program"); menu = keyboard.nextShort(); switch (menu) { // Switch, switch, switch! case 1: // to load the file try { otherStuff.readFile(fileName); } catch (FileNotFoundException e) { System.out.println("File \"" + fileName + "\" not found!"); System.out.println("Dying...."); e.printStackTrace(); System.exit(-1); } break; case 2: // for making a new Task list and then saving it to disk do { Task stuff = new Task(); // creating a new task System.out.print("Description: "); input = keyboard.nextLine(); stuff.setDescription(input); // setting description System.out.print("Priority(1-3): "); input = keyboard.nextLine(); // setting priority by converting string to short stuff.setPriority(Short.parseShort(input)); System.out.println("Category(1-5): "); System.out.println("1. Other\n2. School\n3. Personal\n4. Chore\n5. Work"); input = keyboard.nextLine(); // setting priority by converting string to short stuff.setCategory(Short.parseShort(input)); System.out.print("Location: "); input = keyboard.nextLine(); stuff.setLocation(input); // setting location /* * Date gets an extra separating space because it is actually a little evil * and might corrupt the rest of the program. */ System.out.print("Due Date (ex. 01/12/2015): "); input = keyboard.nextLine(); // now things get a little weird... // splitting the string where character "/" appears // This needs a string array to hold the three strings String[] dateParts = input.split("/"); // setting first part of split string (which is month in America) int month = Integer.parseInt(dateParts[0]); // setting second part of string to day (America and stuff) int day = Integer.parseInt(dateParts[1]); // setting the third part of the string to year int year = Integer.parseInt(dateParts[2]); // plugging in the newly parsed integers into Gregorian Calendar spiffiness GregorianCalendar gCal = new GregorianCalendar(year, month, day); // converting Gregorian Calendar into date!! date = gCal.getTime(); // SETTING THAT DATE WITH SOME WICKED STYLE stuff.setDate(date); // finally, finally sets the date System.out.print("Completed? : "); // setting completed input = keyboard.nextLine().toLowerCase(); // doesn't matter how you enter the letter stuff.setCompleted(Boolean.parseBoolean(input)); // Parse that string! System.out.print("Add new task?"); input = keyboard.nextLine().toLowerCase(); otherStuff.addTask(stuff); // adds the task to the task list numTasks++; // keeping track of tasks in task list } while (input.equalsIgnoreCase("y")); // ignoring case otherStuff.printTasks(); // writing task list to disk try { otherStuff.writeFile(fileName); } catch (FileNotFoundException e) { // no file = sadness System.out.println("File \"" + fileName + "\" not found!"); System.out.println("Dying..."); e.printStackTrace(); System.exit(-1); } break; case 3: // adding a new task to the pre-existing task list // Same as before... Task stuff = new Task(); System.out.print("Description: "); // setting description input = keyboard.nextLine(); stuff.setDescription(input); System.out.print("Priority(1-3): "); // setting priority input = keyboard.nextLine(); stuff.setPriority(Short.parseShort(input)); // parsing to short System.out.print("Category(1-5): "); // setting category input = keyboard.nextLine(); stuff.setCategory(Short.parseShort(input)); // parse, parse System.out.print("Location: "); // setting location input = keyboard.nextLine(); stuff.setLocation(input); System.out.print("Due Date (ex. 01/12/2015): "); // date..-_- input = keyboard.nextLine(); String[] dateParts = input.split("/"); int month = Integer.parseInt(dateParts[0]); int day = Integer.parseInt(dateParts[1]); int year = Integer.parseInt(dateParts[2]); GregorianCalendar gCal = new GregorianCalendar(year, month, day); date = gCal.getTime(); stuff.setDate(date); // actually really happy with this date stuff in the end :) System.out.print("Completed? : "); // setting completed input = keyboard.nextLine().toLowerCase(); stuff.setCompleted(Boolean.parseBoolean(input)); otherStuff.addTask(stuff); // adding the task to the task list numTasks++; // adding one to number of tasks in task list cause we added a task break; case 4: // remove a task otherStuff.printTasks(); // print tasks so user knows what's where System.out.println( "Which task would you like to remove? " + "(Enter an interger starting with 0)"); // because arrays start with 0 modTask = keyboard.nextInt(); otherStuff.removeTask(modTask); // removing specified task break; case 5: // edit a task otherStuff.printTasks(); // print tasks so user knows what to edit System.out.println("Which task would you like to edit?"); modTask = keyboard.nextInt(); Task task = otherStuff.getTask(modTask); // modify specified task System.out.println("What would you like to edit?"); // getting all specific here System.out.println("1. Description"); System.out.println("2. Priority"); System.out.println("3. Category"); System.out.println("4. Location"); System.out.println("5. Due Date"); System.out.println("6. Completed?"); modTask = keyboard.nextInt(); switch (modTask) { // Modify what you want!! case 1: // change description System.out.print("New Description: "); task.setDescription(input); break; case 2: // change priority System.out.print("New Priority: "); task.setPriority(Short.parseShort(input)); break; case 3: // change category System.out.print("New Category: "); task.setCategory(Short.parseShort(input)); break; case 4: // change location System.out.print("New Location: "); task.setLocation(input); break; case 5: // change due date System.out.print("New Due Date: "); task.setDate(date); break; case 6: // change completed status System.out.print("Completed? "); task.setCompleted(Boolean.parseBoolean(input)); break; } break; case 6: // search tasks System.out.println("Would you like to search by: "); System.out.println("1. Category"); System.out.println("2. Priority"); System.out.println("3. Due date"); System.out.println("4. Location"); System.out.println("5. Completion"); menu = keyboard.nextShort(); // switch in a switch cause that's how I roll... switch (menu) { case 1: // searching by category otherStuff.searchByCategory(menu); break; case 2: // searching by priority otherStuff.searchByPriority(menu); break; case 3: // searching by due date otherStuff.searchByDueDate(date); break; case 4: // searching by location otherStuff.searchByLocation(input); break; case 5: // searching by completed status otherStuff.searchByCompleted(input); break; } break; case 7: // sort tasks // definitely the spiffiest sorting you have ever seen otherStuff.orderByPrio(); // print those amazingly sorted tasks now otherStuff.printTasks(); break; case 8: // print tasks otherStuff.printTasks(); break; case 9: // for those crazy people not absolutely in love with this program System.out.println("GOODBYE!"); keyboard.close(); // closing the keyboard System.exit(0); // actually exiting program } }
@Override public final void run() { try { // before preparing the job localize // all the archives TaskAttemptID taskid = t.getTaskID(); LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir"); File jobCacheDir = null; if (conf.getJar() != null) { jobCacheDir = new File(new Path(conf.getJar()).getParent().toString()); } File workDir = new File( lDirAlloc .getLocalPathToRead( TaskTracker.getJobCacheSubdir() + Path.SEPARATOR + t.getJobID() + Path.SEPARATOR + t.getTaskID() + Path.SEPARATOR + MRConstants.WORKDIR, conf) .toString()); URI[] archives = DistributedCache.getCacheArchives(conf); URI[] files = DistributedCache.getCacheFiles(conf); FileStatus fileStatus; FileSystem fileSystem; Path localPath; String baseDir; if ((archives != null) || (files != null)) { if (archives != null) { String[] archivesTimestamps = DistributedCache.getArchiveTimestamps(conf); Path[] p = new Path[archives.length]; for (int i = 0; i < archives.length; i++) { fileSystem = FileSystem.get(archives[i], conf); fileStatus = fileSystem.getFileStatus(new Path(archives[i].getPath())); String cacheId = DistributedCache.makeRelative(archives[i], conf); String cachePath = TaskTracker.getCacheSubdir() + Path.SEPARATOR + cacheId; if (lDirAlloc.ifExists(cachePath, conf)) { localPath = lDirAlloc.getLocalPathToRead(cachePath, conf); } else { localPath = lDirAlloc.getLocalPathForWrite(cachePath, fileStatus.getLen(), conf); } baseDir = localPath.toString().replace(cacheId, ""); p[i] = DistributedCache.getLocalCache( archives[i], conf, new Path(baseDir), fileStatus, true, Long.parseLong(archivesTimestamps[i]), new Path(workDir.getAbsolutePath()), false); } DistributedCache.setLocalArchives(conf, stringifyPathArray(p)); } if ((files != null)) { String[] fileTimestamps = DistributedCache.getFileTimestamps(conf); Path[] p = new Path[files.length]; for (int i = 0; i < files.length; i++) { fileSystem = FileSystem.get(files[i], conf); fileStatus = fileSystem.getFileStatus(new Path(files[i].getPath())); String cacheId = DistributedCache.makeRelative(files[i], conf); String cachePath = TaskTracker.getCacheSubdir() + Path.SEPARATOR + cacheId; if (lDirAlloc.ifExists(cachePath, conf)) { localPath = lDirAlloc.getLocalPathToRead(cachePath, conf); } else { localPath = lDirAlloc.getLocalPathForWrite(cachePath, fileStatus.getLen(), conf); } baseDir = localPath.toString().replace(cacheId, ""); p[i] = DistributedCache.getLocalCache( files[i], conf, new Path(baseDir), fileStatus, false, Long.parseLong(fileTimestamps[i]), new Path(workDir.getAbsolutePath()), false); } DistributedCache.setLocalFiles(conf, stringifyPathArray(p)); } Path localTaskFile = new Path(t.getJobFile()); FileSystem localFs = FileSystem.getLocal(conf); localFs.delete(localTaskFile, true); OutputStream out = localFs.create(localTaskFile); try { conf.writeXml(out); } finally { out.close(); } } if (!prepare()) { return; } String sep = System.getProperty("path.separator"); StringBuffer classPath = new StringBuffer(); // start with same classpath as parent process classPath.append(System.getProperty("java.class.path")); classPath.append(sep); if (!workDir.mkdirs()) { if (!workDir.isDirectory()) { LOG.fatal("Mkdirs failed to create " + workDir.toString()); } } String jar = conf.getJar(); if (jar != null) { // if jar exists, it into workDir File[] libs = new File(jobCacheDir, "lib").listFiles(); if (libs != null) { for (int i = 0; i < libs.length; i++) { classPath.append(sep); // add libs from jar to classpath classPath.append(libs[i]); } } classPath.append(sep); classPath.append(new File(jobCacheDir, "classes")); classPath.append(sep); classPath.append(jobCacheDir); } // include the user specified classpath // archive paths Path[] archiveClasspaths = DistributedCache.getArchiveClassPaths(conf); if (archiveClasspaths != null && archives != null) { Path[] localArchives = DistributedCache.getLocalCacheArchives(conf); if (localArchives != null) { for (int i = 0; i < archives.length; i++) { for (int j = 0; j < archiveClasspaths.length; j++) { if (archives[i].getPath().equals(archiveClasspaths[j].toString())) { classPath.append(sep); classPath.append(localArchives[i].toString()); } } } } } // file paths Path[] fileClasspaths = DistributedCache.getFileClassPaths(conf); if (fileClasspaths != null && files != null) { Path[] localFiles = DistributedCache.getLocalCacheFiles(conf); if (localFiles != null) { for (int i = 0; i < files.length; i++) { for (int j = 0; j < fileClasspaths.length; j++) { if (files[i].getPath().equals(fileClasspaths[j].toString())) { classPath.append(sep); classPath.append(localFiles[i].toString()); } } } } } classPath.append(sep); classPath.append(workDir); // Build exec child jmv args. Vector<String> vargs = new Vector<String>(8); File jvm = // use same jvm as parent new File(new File(System.getProperty("java.home"), "bin"), "java"); vargs.add(jvm.toString()); // Add child (task) java-vm options. // // The following symbols if present in mapred.child.java.opts value are // replaced: // + @taskid@ is interpolated with value of TaskID. // Other occurrences of @ will not be altered. // // Example with multiple arguments and substitutions, showing // jvm GC logging, and start of a passwordless JVM JMX agent so can // connect with jconsole and the likes to watch child memory, threads // and get thread dumps. // // <property> // <name>mapred.child.java.opts</name> // <value>-verbose:gc -Xloggc:/tmp/@[email protected] \ // -Dcom.sun.management.jmxremote.authenticate=false \ // -Dcom.sun.management.jmxremote.ssl=false \ // </value> // </property> // String javaOpts = conf.get("mapred.child.java.opts", "-Xmx200m"); javaOpts = javaOpts.replace("@taskid@", taskid.toString()); String[] javaOptsSplit = javaOpts.split(" "); // Add java.library.path; necessary for loading native libraries. // // 1. To support native-hadoop library i.e. libhadoop.so, we add the // parent processes' java.library.path to the child. // 2. We also add the 'cwd' of the task to it's java.library.path to help // users distribute native libraries via the DistributedCache. // 3. The user can also specify extra paths to be added to the // java.library.path via mapred.child.java.opts. // String libraryPath = System.getProperty("java.library.path"); if (libraryPath == null) { libraryPath = workDir.getAbsolutePath(); } else { libraryPath += sep + workDir; } boolean hasUserLDPath = false; for (int i = 0; i < javaOptsSplit.length; i++) { if (javaOptsSplit[i].startsWith("-Djava.library.path=")) { javaOptsSplit[i] += sep + libraryPath; hasUserLDPath = true; break; } } if (!hasUserLDPath) { vargs.add("-Djava.library.path=" + libraryPath); } for (int i = 0; i < javaOptsSplit.length; i++) { vargs.add(javaOptsSplit[i]); } // add java.io.tmpdir given by mapred.child.tmp String tmp = conf.get("mapred.child.tmp", "./tmp"); Path tmpDir = new Path(tmp); // if temp directory path is not absolute // prepend it with workDir. if (!tmpDir.isAbsolute()) { tmpDir = new Path(workDir.toString(), tmp); } FileSystem localFs = FileSystem.getLocal(conf); if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) { throw new IOException("Mkdirs failed to create " + tmpDir.toString()); } vargs.add("-Djava.io.tmpdir=" + tmpDir.toString()); // Add classpath. vargs.add("-classpath"); vargs.add(classPath.toString()); // Setup the log4j prop long logSize = TaskLog.getTaskLogLength(conf); vargs.add( "-Dhadoop.log.dir=" + new File(System.getProperty("hadoop.log.dir")).getAbsolutePath()); vargs.add("-Dhadoop.root.logger=INFO,TLA"); vargs.add("-Dhadoop.tasklog.taskid=" + taskid); vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize); if (conf.getProfileEnabled()) { if (conf.getProfileTaskRange(t.isMapTask()).isIncluded(t.getPartition())) { File prof = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.PROFILE); vargs.add(String.format(conf.getProfileParams(), prof.toString())); } } // Add main class and its arguments vargs.add(Child.class.getName()); // main of Child // pass umbilical address InetSocketAddress address = tracker.getTaskTrackerReportAddress(); vargs.add(address.getAddress().getHostAddress()); vargs.add(Integer.toString(address.getPort())); vargs.add(taskid.toString()); // pass task identifier String pidFile = null; if (tracker.isTaskMemoryManagerEnabled()) { pidFile = lDirAlloc .getLocalPathForWrite( (TaskTracker.getPidFilesSubdir() + Path.SEPARATOR + taskid), this.conf) .toString(); } // set memory limit using ulimit if feasible and necessary ... String[] ulimitCmd = Shell.getUlimitMemoryCommand(conf); List<String> setup = null; if (ulimitCmd != null) { setup = new ArrayList<String>(); for (String arg : ulimitCmd) { setup.add(arg); } } // Set up the redirection of the task's stdout and stderr streams File stdout = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDOUT); File stderr = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDERR); stdout.getParentFile().mkdirs(); tracker.getTaskTrackerInstrumentation().reportTaskLaunch(taskid, stdout, stderr); Map<String, String> env = new HashMap<String, String>(); StringBuffer ldLibraryPath = new StringBuffer(); ldLibraryPath.append(workDir.toString()); String oldLdLibraryPath = null; oldLdLibraryPath = System.getenv("LD_LIBRARY_PATH"); if (oldLdLibraryPath != null) { ldLibraryPath.append(sep); ldLibraryPath.append(oldLdLibraryPath); } env.put("LD_LIBRARY_PATH", ldLibraryPath.toString()); jvmManager.launchJvm( this, jvmManager.constructJvmEnv( setup, vargs, stdout, stderr, logSize, workDir, env, pidFile, conf)); synchronized (lock) { while (!done) { lock.wait(); } } tracker.getTaskTrackerInstrumentation().reportTaskEnd(t.getTaskID()); if (exitCodeSet) { if (!killed && exitCode != 0) { if (exitCode == 65) { tracker.getTaskTrackerInstrumentation().taskFailedPing(t.getTaskID()); } throw new IOException("Task process exit with nonzero status of " + exitCode + "."); } } } catch (FSError e) { LOG.fatal("FSError", e); try { tracker.fsError(t.getTaskID(), e.getMessage()); } catch (IOException ie) { LOG.fatal(t.getTaskID() + " reporting FSError", ie); } } catch (Throwable throwable) { LOG.warn(t.getTaskID() + " Child Error", throwable); ByteArrayOutputStream baos = new ByteArrayOutputStream(); throwable.printStackTrace(new PrintStream(baos)); try { tracker.reportDiagnosticInfo(t.getTaskID(), baos.toString()); } catch (IOException e) { LOG.warn(t.getTaskID() + " Reporting Diagnostics", e); } } finally { try { URI[] archives = DistributedCache.getCacheArchives(conf); URI[] files = DistributedCache.getCacheFiles(conf); if (archives != null) { for (int i = 0; i < archives.length; i++) { DistributedCache.releaseCache(archives[i], conf); } } if (files != null) { for (int i = 0; i < files.length; i++) { DistributedCache.releaseCache(files[i], conf); } } } catch (IOException ie) { LOG.warn("Error releasing caches : Cache files might not have been cleaned up"); } tracker.reportTaskFinished(t.getTaskID(), false); if (t.isMapTask()) { tracker.addFreeMapSlot(); } else { tracker.addFreeReduceSlot(); } } }
/** * Creates a flow control element for the given task <element name="controlFlow"> * * @return the xml Element corresponding to the flow control, if the task contains a flow control, * null otherwise */ private Element createFlowControlElement(Document doc, Task task) { Element controlFlowE = null; // <ref name="block"/> if (task.getFlowBlock() != FlowBlock.NONE) { controlFlowE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW.getXMLName()); setAttribute(controlFlowE, XMLAttributes.FLOW_BLOCK, task.getFlowBlock().toString()); } FlowScript flowScript = task.getFlowScript(); if (flowScript != null) { Element flowActionE = null; // flowActionE can be if, loop, replicate or null. // if not null, it contains a script element // <ref name="actionIf"/> // <ref name="actionReplicate"/> // <ref name="actionLoop"/> // </choice> // *** if *** // <element name="if"> if (flowScript.getActionType().equals(FlowActionType.IF.toString())) { flowActionE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW_IF.getXMLName()); setAttribute(flowActionE, XMLAttributes.FLOW_TARGET, flowScript.getActionTarget(), true); setAttribute(flowActionE, XMLAttributes.FLOW_ELSE, flowScript.getActionTargetElse(), true); setAttribute( flowActionE, XMLAttributes.FLOW_CONTINUATION, flowScript.getActionContinuation(), true); } // *** loop *** // <element name="loop"> if (flowScript.getActionType().equals(FlowActionType.LOOP.toString())) { flowActionE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW_LOOP.getXMLName()); setAttribute(flowActionE, XMLAttributes.FLOW_TARGET, flowScript.getActionTarget(), true); } // *** replicate *** // <element name="replicate"> if (flowScript.getActionType().equals(FlowActionType.REPLICATE.toString())) { flowActionE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW_REPLICATE.getXMLName()); } if (flowActionE != null) { if (controlFlowE == null) { controlFlowE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW.getXMLName()); } Element scriptE = createScriptElement(doc, flowScript); flowActionE.appendChild(scriptE); controlFlowE.appendChild(flowActionE); } } // flowScript !=null return controlFlowE; }
/** * Creates the parallel environment element for the given task. Corresponds to <define * name="parallel"> * * @return the {@link XMLTags#PARALLEL_ENV} element if the task has a parallel environment, null * otherwise */ private Element createParallelEnvironment(Document doc, Task task) { ParallelEnvironment penv = task.getParallelEnvironment(); if (penv == null) return null; Element parallelEnvE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.PARALLEL_ENV.getXMLName()); setAttribute( parallelEnvE, XMLAttributes.TASK_NB_NODES, Integer.toString(penv.getNodesNumber())); // <ref name="topology"/> TopologyDescriptor topologyDescr = penv.getTopologyDescriptor(); if (topologyDescr != null) { // <choice> // <ref name="arbitrary"/> // <ref name="bestProximity"/> // <ref name="thresholdProximity"/> // <ref name="singleHost"/> // <ref name="singleHostExclusive"/> // <ref name="multipleHostsExclusive"/> // <ref name="differentHostsExclusive"/> // </choice> Element topologyE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY.getXMLName()); Element topologyDescrE = null; if (topologyDescr instanceof ArbitraryTopologyDescriptor) { topologyDescrE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_ARBITRARY.getXMLName()); } else if (topologyDescr instanceof ThresholdProximityDescriptor) { topologyDescrE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_THRESHOLD_PROXIMITY.getXMLName()); long threshold = ((ThresholdProximityDescriptor) topologyDescr).getThreshold(); topologyDescrE.setAttribute( XMLAttributes.TOPOLOGY_THRESHOLD.getXMLName(), Long.toString(threshold)); } else if (topologyDescr instanceof BestProximityDescriptor) { topologyDescrE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_BEST_PROXIMITY.getXMLName()); } else if (topologyDescr instanceof SingleHostExclusiveDescriptor) { topologyDescrE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_SINGLE_HOST_EXCLUSIVE.getXMLName()); } else if (topologyDescr instanceof SingleHostDescriptor) { topologyDescrE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_SINGLE_HOST.getXMLName()); } else if (topologyDescr instanceof MultipleHostsExclusiveDescriptor) { topologyDescrE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_MULTIPLE_HOSTS_EXCLUSIVE.getXMLName()); } if (topologyDescr instanceof DifferentHostsExclusiveDescriptor) { topologyDescrE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_DIFFERENT_HOSTS_EXCLUSIVE.getXMLName()); } if (topologyDescrE != null) { topologyE.appendChild(topologyDescrE); } parallelEnvE.appendChild(topologyE); } return parallelEnvE; }
/** Creates the task element, corressponding to <define name="task"> */ private Element createTaskElement(Document doc, Task task) { Element taskE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.TASK.getXMLName()); // **** attributes ***** // **** common attributes *** if (task.getOnTaskErrorProperty().isSet()) { setAttribute( taskE, XMLAttributes.COMMON_ON_TASK_ERROR, task.getOnTaskErrorProperty().getValue().toString(), true); } if (task.getMaxNumberOfExecutionProperty().isSet()) { setAttribute( taskE, XMLAttributes.COMMON_MAX_NUMBER_OF_EXECUTION, Integer.toString(task.getMaxNumberOfExecution())); } setAttribute(taskE, XMLAttributes.COMMON_NAME, task.getName(), true); if (task.getRestartTaskOnErrorProperty().isSet()) { setAttribute( taskE, XMLAttributes.COMMON_RESTART_TASK_ON_ERROR, task.getRestartTaskOnError().toString()); } // *** task attributes *** if (task.getWallTime() != 0) { setAttribute(taskE, XMLAttributes.TASK_WALLTIME, formatDate(task.getWallTime())); } if (task.isRunAsMe()) { setAttribute(taskE, XMLAttributes.TASK_RUN_AS_ME, "true"); } if (task.isPreciousResult()) { setAttribute(taskE, XMLAttributes.TASK_PRECIOUS_RESULT, "true"); } if (task.isPreciousLogs()) { setAttribute(taskE, XMLAttributes.TASK_PRECIOUS_LOGS, "true"); } // *** elements **** // <ref name="variables"/> if (task.getVariables() != null && !task.getVariables().isEmpty()) { Element variablesE = createTaskVariablesElement(doc, task.getVariables()); taskE.appendChild(variablesE); } // <ref name="taskDescription"/> if (task.getDescription() != null) { Element descrNode = createElement(doc, XMLTags.COMMON_DESCRIPTION.getXMLName(), task.getDescription()); taskE.appendChild(descrNode); } // <ref name="genericInformation"/> if ((task.getGenericInformation() != null) && (task.getGenericInformation().size() > 0)) { Element genericInfoE = createGenericInformation(doc, task.getGenericInformation()); taskE.appendChild(genericInfoE); } // <ref name="depends"/> List<Task> dependencies = task.getDependencesList(); if ((dependencies != null) && (dependencies.size() > 0)) { Element dependsE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TASK_DEPENDENCES.getXMLName()); for (Task dep : dependencies) { Element dependsTask = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.TASK_DEPENDENCES_TASK.getXMLName()); setAttribute(dependsTask, XMLAttributes.TASK_DEPENDS_REF, dep.getName(), true); dependsE.appendChild(dependsTask); } taskE.appendChild(dependsE); } // if has dependencies // <ref name="inputFiles"/> List<InputSelector> inputFiles = task.getInputFilesList(); if (inputFiles != null) { Element inputFilesE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.DS_INPUT_FILES.getXMLName()); for (InputSelector inputSelector : inputFiles) { FileSelector fs = inputSelector.getInputFiles(); Element filesE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.DS_FILES.getXMLName()); // the xml only supports one value for the includes/excludes // pattern if (!fs.getIncludes().isEmpty()) setAttribute(filesE, XMLAttributes.DS_INCLUDES, fs.getIncludes().iterator().next(), true); if (!fs.getExcludes().isEmpty()) setAttribute(filesE, XMLAttributes.DS_EXCLUDES, fs.getExcludes().iterator().next(), true); if (inputSelector.getMode() != null) { setAttribute( filesE, XMLAttributes.DS_ACCESS_MODE, inputSelector.getMode().toString(), true); } inputFilesE.appendChild(filesE); } taskE.appendChild(inputFilesE); } // <ref name="parallel"/> Element parallelEnvE = createParallelEnvironment(doc, task); if (parallelEnvE != null) taskE.appendChild(parallelEnvE); // <ref name="selection"/> List<SelectionScript> selectionScripts = task.getSelectionScripts(); if (selectionScripts != null && selectionScripts.size() > 0) { Element selectionE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.SCRIPT_SELECTION.getXMLName()); for (SelectionScript selectionScript : selectionScripts) { Element scriptE = createScriptElement(doc, selectionScript); selectionE.appendChild(scriptE); } taskE.appendChild(selectionE); } // <ref name="forkEnvironment"/> if (task.getForkEnvironment() != null) { Element forkEnvE = createForkEnvironmentElement(doc, task.getForkEnvironment()); taskE.appendChild(forkEnvE); } // <ref name="pre"/> Script preScript = task.getPreScript(); if (preScript != null) { Element preE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.SCRIPT_PRE.getXMLName()); Element scriptE = createScriptElement(doc, preScript); preE.appendChild(scriptE); taskE.appendChild(preE); } // <ref name="executable"/> Element executableE = null; if (task instanceof JavaTask) { executableE = createJavaExecutableElement(doc, (JavaTask) task); } else if (task instanceof NativeTask) { executableE = createNativeExecutableElement(doc, (NativeTask) task); } else if (task instanceof ScriptTask) { executableE = createScriptExecutableElement(doc, (ScriptTask) task); } taskE.appendChild(executableE); // <ref name="flow"/> Element controlFlowE = createFlowControlElement(doc, task); if (controlFlowE != null) taskE.appendChild(controlFlowE); // <ref name="post"/> Script postScript = task.getPostScript(); if (postScript != null) { Element postE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.SCRIPT_POST.getXMLName()); Element scriptE = createScriptElement(doc, postScript); postE.appendChild(scriptE); taskE.appendChild(postE); } // <ref name="cleaning"/> Script cleanScript = task.getCleaningScript(); if (cleanScript != null) { Element cleanE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.SCRIPT_CLEANING.getXMLName()); Element scriptE = createScriptElement(doc, cleanScript); cleanE.appendChild(scriptE); taskE.appendChild(cleanE); } // <ref name="outputFiles"/> List<OutputSelector> outputFiles = task.getOutputFilesList(); if (outputFiles != null) { Element outputFilesE = doc.createElementNS( Schemas.SCHEMA_LATEST.namespace, XMLTags.DS_OUTPUT_FILES.getXMLName()); for (OutputSelector outputSelector : outputFiles) { FileSelector fs = outputSelector.getOutputFiles(); Element filesE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.DS_FILES.getXMLName()); // the xml only supports one value for the includes/excludes // pattern if (!fs.getIncludes().isEmpty()) setAttribute(filesE, XMLAttributes.DS_INCLUDES, fs.getIncludes().iterator().next(), true); if (!fs.getExcludes().isEmpty()) setAttribute(filesE, XMLAttributes.DS_EXCLUDES, fs.getExcludes().iterator().next(), true); if (outputSelector.getMode() != null) { setAttribute( filesE, XMLAttributes.DS_ACCESS_MODE, outputSelector.getMode().toString(), true); } outputFilesE.appendChild(filesE); } taskE.appendChild(outputFilesE); } return taskE; }