/** @return true iff the object "passes" the predicate */
public boolean execute(Object o) {
if (o instanceof Task) {
Task task = (Task) o;
return (task.getVerb().equals(MonitoringUtils.Start_Publishing_Cmd));
}
return false;
}
private void showTaskDetails(Task task) {
if (task != null) {
taskNLabel.setVisible(true);
nextExecutionLabel.setVisible(true);
actveLabel.setVisible(true);
taskNameLabel.setText(task.getTitle());
if (task.isRepeated()) {
taskExecutionTime.setText((DateUtil.format(task.getExecutionDate())));
} else {
taskExecutionTime.setText(DateUtil.format(task.getStartTime()));
}
taskActiveLabel.setText((task.isActive()) ? "Yes" : "No");
chooseLabel.setVisible(false);
} else {
taskNLabel.setVisible(false);
nextExecutionLabel.setVisible(false);
actveLabel.setVisible(false);
taskNameLabel.setText("");
taskExecutionTime.setText("");
taskActiveLabel.setText("");
}
logger.info("Task was set");
}
/* Devolve lista com o tipo de tarefas */
public List<String> listTask() {
ArrayList<String> tasklist = new ArrayList<>();
lock.lock();
try {
for (Task task : this.tasks.values()) {
tasklist.add(task.getType());
}
return tasklist;
} finally {
lock.unlock();
}
}
public static void main(String[] args) throws IOException {
// InputStream inputStream = new FileInputStream("replaceme.in");
// OutputStream outputStream = new FileOutputStream("replaceme.out");
/////////////////////////////////////////////////////////////////////
InputStream inputStream = System.in;
OutputStream outputStream = System.out;
InputReader in = new InputReader(inputStream);
OutputWriter out = new OutputWriter(outputStream);
Task solver = new Task();
solver.solve(1, in, out);
out.close();
System.exit(0);
}
public static File copyResourceFile(
String sourceName, String copyName, Project project, Task task) throws IOException {
StudyTaskManager taskManager = StudyTaskManager.getInstance(project);
Course course = taskManager.getCourse();
int taskNum = task.getIndex() + 1;
int lessonNum = task.getLesson().getIndex() + 1;
assert course != null;
String pathToResource =
FileUtil.join(
new File(course.getResourcePath()).getParent(),
Lesson.LESSON_DIR + lessonNum,
Task.TASK_DIR + taskNum);
File resourceFile = new File(pathToResource, copyName);
FileUtil.copy(new File(pathToResource, sourceName), resourceFile);
return resourceFile;
}
public TaskRunner(TaskTracker.TaskInProgress tip, TaskTracker tracker, JobConf conf) {
this.tip = tip;
this.t = tip.getTask();
this.tracker = tracker;
this.conf = conf;
this.mapOutputFile = new MapOutputFile(t.getJobID());
this.mapOutputFile.setConf(conf);
this.jvmManager = tracker.getJvmManagerInstance();
}
/**
* Process "Start_publishing_Cmd" task .It update internal data structure for kind of sensor data
* user has requested for.
*
* @param publichcmdlist Enumeration on Collection of newly Start_publishing_Cmd
* @see org.cougaar.core.security.oldmonitoring.util.PublishCmdObj
*/
protected void process_publishCmd(Enumeration publichcmdlist) {
PublishCmdObj pcmd;
for (; publichcmdlist.hasMoreElements(); ) {
Task tsk = (Task) publichcmdlist.nextElement();
PrepositionalPhrase pp =
tsk.getPrepositionalPhrase(MonitoringUtils.Start_publishing_Preposition);
if (pp != null) {
pcmd = (PublishCmdObj) pp.getIndirectObject();
if (!currentlypublishing.contains(pcmd.Type)) {
currentlypublishing.add(pcmd.Type);
if (MonitoringUtils.debug > 0)
System.out.println(
"In sensor Plugin process_publishCmd Add ::::::::"
+ pcmd.Type
+ " to current publishing list");
}
}
}
}
public static void main(String[] argv) throws IOException
{
Task t;
boolean home = argv.length > 0 && argv[0].equals("test");
InputStream inputStream;
OutputStream outputStream;
if (home) {
inputStream = new FileInputStream(Task.filename + ".in");
outputStream = new FileOutputStream(Task.filename + ".out");
} else {
switch (Task.read) {
case 0:
inputStream = new FileInputStream(Task.filename + ".in");
outputStream = new FileOutputStream(Task.filename + ".out");
break;
case 1:
inputStream = new FileInputStream("input.txt");
outputStream = new FileOutputStream("output.txt");
break;
default:
inputStream = System.in;
outputStream = System.out;
break;
}
}
InputReader in = new InputReader(inputStream);
OutputWriter out = new OutputWriter(outputStream, home);
if (home)
do {
long time = System.currentTimeMillis();
t = new Task();
t.in = in;
t.out = out;
t.run();
out.writeln();
out.writeln("=====Time:" + (System.currentTimeMillis() - time));
out.flush();
} while (in.toNextTest());
else {
t = new Task();
t.in = in;
t.out = out;
t.run();
}
out.close();
}
/**
* Returns control when task is complete.
*
* @param json
* @param logger
*/
private String waitForDeploymentCompletion(JSON json, OctopusApi api, Log logger) {
final long WAIT_TIME = 5000;
final double WAIT_RANDOM_SCALER = 100.0;
JSONObject jsonObj = (JSONObject) json;
String id = jsonObj.getString("TaskId");
Task task = null;
String lastState = "Unknown";
try {
task = api.getTask(id);
} catch (IOException ex) {
logger.error("Error getting task: " + ex.getMessage());
return null;
}
logger.info("Task info:");
logger.info("\tId: " + task.getId());
logger.info("\tName: " + task.getName());
logger.info("\tDesc: " + task.getDescription());
logger.info("\tState: " + task.getState());
logger.info("\n\nStarting wait...");
boolean completed = task.getIsCompleted();
while (!completed) {
try {
task = api.getTask(id);
} catch (IOException ex) {
logger.error("Error getting task: " + ex.getMessage());
return null;
}
completed = task.getIsCompleted();
lastState = task.getState();
logger.info("Task state: " + lastState);
if (completed) {
break;
}
try {
Thread.sleep(WAIT_TIME + (long) (Math.random() * WAIT_RANDOM_SCALER));
} catch (InterruptedException ex) {
logger.info("Wait interrupted!");
logger.info(ex.getMessage());
completed = true; // bail out of wait loop
}
}
logger.info("Wait complete!");
return lastState;
}
public void run() {
StopWatch st = null;
if (GPResourceHome.perfProfile.PROFILE) {
st = new StopWatch(false, false, true);
st.start();
}
StopWatch ct = new StopWatch();
int numWorkers = 0;
try {
if (GPResourceHome.commonState == null) {
GPResourceHome.load_common();
}
} catch (Exception e) {
e.printStackTrace();
}
ct.start();
while (true) {
if (GPResourceHome.perfProfile.PROFILE) {
st.stop();
GPResourceHome.perfProfile.addSample("notification_engine_worker", st.getElapsedTimeMX());
st.reset();
st.start();
}
GPResource gpResource = null;
try {
// logger.debug("gpResource.notificationWorkerQ.remove()...");
gpResource = (GPResource) GPResourceHome.commonState.getNextResource();
// gpResource =
// (GPResource)GPResourceHome.commonState.notificationWorkerQ.remove();//GPResourceHome.commonState.notificationQ.waitUntilNotEmpty();
boolean sentOK = false;
if (gpResource != null
&& gpResource.taskQ != null
&& gpResource.taskPendQ != null
&& (gpResource.taskQ.size() + gpResource.taskPendQ.size()) > 0)
// if (gpResource != null)
{
Task task = null;
// if (gpResource.DATA_AWARE_SCHEDULER && !gpResource.MAX_CACHE_HIT)
if (gpResource.DATA_AWARE_SCHEDULER) {
// synchronized(gpResource)
// {
// task = (Task)gpResource.taskQ.remove();
task = (Task) gpResource.taskQ.removeNoWait();
if (task == null) {
task = (Task) gpResource.taskPendQ.remove();
}
logger.debug(
"GPService:NotificationEngine:sendNotification(): removed task "
+ task.getExecutable().getId()
+ " from gpResource.taskQ... gpResource.taskQ.size() = "
+ gpResource.taskQ.size());
gpResource.taskPendQ.insert(task);
logger.debug(
"GPService:NotificationEngine:sendNotification(): inserted task "
+ task.getExecutable().getId()
+ " into gpResource.taskPendQ... gpResource.taskPendQ.size() = "
+ gpResource.taskPendQ.size());
// }
}
logger.debug(
"GPService:NotificationEngine:sendNotification(): key = "
+ String.valueOf(gpResource.resourceKey)
+ " ...");
// if (gpResourceHome.sendNotification(gpResource.resourceKey))
boolean sendNotificationTest = false;
if (gpResourceHome.useTCPCore) {
sendNotificationTest = gpResourceHome.sendNotificationTCPCore(gpResource);
}
// else if (gpResource.DATA_AWARE_SCHEDULER && !gpResource.MAX_CACHE_HIT)
else if (gpResource.DATA_AWARE_SCHEDULER) {
sendNotificationTest = gpResourceHome.sendNotification(gpResource, task);
} /*
else if (gpResourceHome.localFork)
{
sendNotificationTest = gpResourceHome.sendNotificationLocal(gpResource);
} */ else {
sendNotificationTest = gpResourceHome.sendNotification(gpResource);
}
if (sendNotificationTest) {
setNotificationsSent();
logger.debug(
"Notification for key "
+ String.valueOf(gpResource.resourceKey)
+ " sent successfully!");
// if (gpResource.DATA_AWARE_SCHEDULER)
// {
// gpResource.taskPendQ.insert(task);
// logger.debug("GPService:NotificationEngine:sendNotification(): inserted task into
// gpResource.taskPendQ... gpResource.taskPendQ.size() = " +
// gpResource.taskPendQ.size());
// }
// insert back in to try again later... this is a hack to ensure that we never run out
// of notification messages :)
// if (!gpResourceHome.useTCPCore)
// if (gpResource != null && gpResource.taskQ != null && (gpResource.taskQ.size() +
// gpResource.taskPendQ.size()) > 0 && (gpResource.taskQ.size() +
// gpResource.taskPendQ.size()) <= GPResourceHome.maxNumNotificationWorkerThreads*2)
// {
// logger.debug("there is still work to be collected and we have dipped below the
// number of notification threads, insert back into the notification queue...");
// GPResourceHome.commonState.notificationWorkerQ.insert(gpResource);
// }
} else {
logger.debug(
"Notification for key " + String.valueOf(gpResource.resourceKey) + " failed!");
// if (gpResource.DATA_AWARE_SCHEDULER && !gpResource.MAX_CACHE_HIT)
if (gpResource.DATA_AWARE_SCHEDULER) {
// synchronized(gpResource)
// {
// gpResource.taskPendQ.removeTask(task);
// should insert back at the front... fix this
// gpResource.taskQ.insertFront(task);
// }
// logger.debug("GPService:NotificationEngine:sendNotification(): inserted task " +
// task.getExecutable().getId() + " into gpResource.taskQ... gpResource.taskQ.size() =
// " + gpResource.taskQ.size());
}
// insert back in to try again later...
// if ((gpResource.taskQ.size() + gpResource.taskPendQ.size()) > 0)
// {
// logger.debug("there is still work to be collected, insert back into the
// notification queue...");
// GPResourceHome.commonState.notificationWorkerQ.insert(gpResource);
// used for throtling notifications...
/*
try
{
System.currentTimeMillis();
Thread.sleep(20);
}
catch (Exception sss)
{
} */
// }
}
}
if (MAX_NOT_PER_SEC > 0 && getNotificationsSent() >= MAX_NOT_PER_SEC) {
long timeLeft = NOT_TIME_QUANTA - ct.getElapsedTime();
if (timeLeft > 20)
try {
logger.debug(
getNotificationsSent()
+ " notifications sent in "
+ ct.getElapsedTime()
+ " ms, sleeping for "
+ timeLeft
+ " ms");
// System.currentTimeMillis();
Thread.sleep(timeLeft);
} catch (Exception sss) {
}
}
if (ct.getElapsedTime() >= NOT_TIME_QUANTA) {
if (getNotificationsSent() > 0) {
logger.debug(
"***NotificationEngineWorker(): "
+ getNotificationsSent()
+ " notifications sent to workers in "
+ ct.getElapsedTime()
+ " ms, maximum allowed notifications "
+ MAX_NOT_PER_SEC
+ " per "
+ NOT_TIME_QUANTA
+ " ms...");
}
ct.reset();
resetNotificationsSent();
ct.start();
}
// if (GPResourceHome.commonState.resourceQ.size() > 0 &&
// GPResourceHome.commonState.getWaitingTasks() <= 0)
if (GPResourceHome.commonState.getWaitingTasks() <= 0) {
try {
Thread.sleep(100);
} catch (Exception e) {
if (logger.isDebugEnabled()) e.printStackTrace();
}
}
} catch (Exception eee) {
logger.debug("Error in NotificationEngineWorker thread: " + eee);
if (logger.isDebugEnabled()) eee.printStackTrace();
// continue;
// if (gpResource != null && gpResource.taskQ != null && gpResource.taskQ.size() +
// gpResource.taskPendQ.size() > 0)
// if (gpResource != null && gpResource.taskQ != null)
// {
// logger.debug("there is still work to be collected, insert back into the notification
// queue...");
// GPResourceHome.commonState.notificationWorkerQ.insert(gpResource);
// }
}
}
}
public static void main(String[] args) throws IOException {
Scanner keyboard = new Scanner(System.in);
String input = null; // for most of the input
String fileName = "Tasks.txt"; // file were stuff is written to
short menu = 0; // for the short input
Date date = new Date(); // for the stupid date
int modTask = 0; // for getting a task's position in the array
int numTasks = 0; // keeps count of number of tasks added to task list
TaskList otherStuff = new TaskList(); // TASK LIST!!
System.out.println("Welcome to the Task Manager!");
// A spiff little menu
System.out.println("Would you like to: ");
System.out.println("1. Load a task list from file");
System.out.println("2. Save a new task list");
System.out.println("3. Add a task");
System.out.println("4. Remove a task");
System.out.println("5. Edit a task");
System.out.println("6. Search tasks");
System.out.println("7. Sort tasks by priority");
System.out.println("8. Print task list");
System.out.println("9. Exit program");
menu = keyboard.nextShort();
switch (menu) { // Switch, switch, switch!
case 1: // to load the file
try {
otherStuff.readFile(fileName);
} catch (FileNotFoundException e) {
System.out.println("File \"" + fileName + "\" not found!");
System.out.println("Dying....");
e.printStackTrace();
System.exit(-1);
}
break;
case 2: // for making a new Task list and then saving it to disk
do {
Task stuff = new Task(); // creating a new task
System.out.print("Description: ");
input = keyboard.nextLine();
stuff.setDescription(input); // setting description
System.out.print("Priority(1-3): ");
input = keyboard.nextLine();
// setting priority by converting string to short
stuff.setPriority(Short.parseShort(input));
System.out.println("Category(1-5): ");
System.out.println("1. Other\n2. School\n3. Personal\n4. Chore\n5. Work");
input = keyboard.nextLine();
// setting priority by converting string to short
stuff.setCategory(Short.parseShort(input));
System.out.print("Location: ");
input = keyboard.nextLine();
stuff.setLocation(input); // setting location
/*
* Date gets an extra separating space because it is actually a little evil
* and might corrupt the rest of the program.
*/
System.out.print("Due Date (ex. 01/12/2015): ");
input = keyboard.nextLine();
// now things get a little weird...
// splitting the string where character "/" appears
// This needs a string array to hold the three strings
String[] dateParts = input.split("/");
// setting first part of split string (which is month in America)
int month = Integer.parseInt(dateParts[0]);
// setting second part of string to day (America and stuff)
int day = Integer.parseInt(dateParts[1]);
// setting the third part of the string to year
int year = Integer.parseInt(dateParts[2]);
// plugging in the newly parsed integers into Gregorian Calendar spiffiness
GregorianCalendar gCal = new GregorianCalendar(year, month, day);
// converting Gregorian Calendar into date!!
date = gCal.getTime();
// SETTING THAT DATE WITH SOME WICKED STYLE
stuff.setDate(date);
// finally, finally sets the date
System.out.print("Completed? : "); // setting completed
input = keyboard.nextLine().toLowerCase(); // doesn't matter how you enter the letter
stuff.setCompleted(Boolean.parseBoolean(input)); // Parse that string!
System.out.print("Add new task?");
input = keyboard.nextLine().toLowerCase();
otherStuff.addTask(stuff); // adds the task to the task list
numTasks++; // keeping track of tasks in task list
} while (input.equalsIgnoreCase("y")); // ignoring case
otherStuff.printTasks();
// writing task list to disk
try {
otherStuff.writeFile(fileName);
} catch (FileNotFoundException e) { // no file = sadness
System.out.println("File \"" + fileName + "\" not found!");
System.out.println("Dying...");
e.printStackTrace();
System.exit(-1);
}
break;
case 3: // adding a new task to the pre-existing task list
// Same as before...
Task stuff = new Task();
System.out.print("Description: "); // setting description
input = keyboard.nextLine();
stuff.setDescription(input);
System.out.print("Priority(1-3): "); // setting priority
input = keyboard.nextLine();
stuff.setPriority(Short.parseShort(input)); // parsing to short
System.out.print("Category(1-5): "); // setting category
input = keyboard.nextLine();
stuff.setCategory(Short.parseShort(input)); // parse, parse
System.out.print("Location: "); // setting location
input = keyboard.nextLine();
stuff.setLocation(input);
System.out.print("Due Date (ex. 01/12/2015): "); // date..-_-
input = keyboard.nextLine();
String[] dateParts = input.split("/");
int month = Integer.parseInt(dateParts[0]);
int day = Integer.parseInt(dateParts[1]);
int year = Integer.parseInt(dateParts[2]);
GregorianCalendar gCal = new GregorianCalendar(year, month, day);
date = gCal.getTime();
stuff.setDate(date); // actually really happy with this date stuff in the end :)
System.out.print("Completed? : "); // setting completed
input = keyboard.nextLine().toLowerCase();
stuff.setCompleted(Boolean.parseBoolean(input));
otherStuff.addTask(stuff); // adding the task to the task list
numTasks++; // adding one to number of tasks in task list cause we added a task
break;
case 4: // remove a task
otherStuff.printTasks(); // print tasks so user knows what's where
System.out.println(
"Which task would you like to remove? "
+ "(Enter an interger starting with 0)"); // because arrays start with 0
modTask = keyboard.nextInt();
otherStuff.removeTask(modTask); // removing specified task
break;
case 5: // edit a task
otherStuff.printTasks(); // print tasks so user knows what to edit
System.out.println("Which task would you like to edit?");
modTask = keyboard.nextInt();
Task task = otherStuff.getTask(modTask); // modify specified task
System.out.println("What would you like to edit?"); // getting all specific here
System.out.println("1. Description");
System.out.println("2. Priority");
System.out.println("3. Category");
System.out.println("4. Location");
System.out.println("5. Due Date");
System.out.println("6. Completed?");
modTask = keyboard.nextInt();
switch (modTask) { // Modify what you want!!
case 1: // change description
System.out.print("New Description: ");
task.setDescription(input);
break;
case 2: // change priority
System.out.print("New Priority: ");
task.setPriority(Short.parseShort(input));
break;
case 3: // change category
System.out.print("New Category: ");
task.setCategory(Short.parseShort(input));
break;
case 4: // change location
System.out.print("New Location: ");
task.setLocation(input);
break;
case 5: // change due date
System.out.print("New Due Date: ");
task.setDate(date);
break;
case 6: // change completed status
System.out.print("Completed? ");
task.setCompleted(Boolean.parseBoolean(input));
break;
}
break;
case 6: // search tasks
System.out.println("Would you like to search by: ");
System.out.println("1. Category");
System.out.println("2. Priority");
System.out.println("3. Due date");
System.out.println("4. Location");
System.out.println("5. Completion");
menu = keyboard.nextShort();
// switch in a switch cause that's how I roll...
switch (menu) {
case 1: // searching by category
otherStuff.searchByCategory(menu);
break;
case 2: // searching by priority
otherStuff.searchByPriority(menu);
break;
case 3: // searching by due date
otherStuff.searchByDueDate(date);
break;
case 4: // searching by location
otherStuff.searchByLocation(input);
break;
case 5: // searching by completed status
otherStuff.searchByCompleted(input);
break;
}
break;
case 7: // sort tasks
// definitely the spiffiest sorting you have ever seen
otherStuff.orderByPrio();
// print those amazingly sorted tasks now
otherStuff.printTasks();
break;
case 8: // print tasks
otherStuff.printTasks();
break;
case 9: // for those crazy people not absolutely in love with this program
System.out.println("GOODBYE!");
keyboard.close(); // closing the keyboard
System.exit(0); // actually exiting program
}
}
@Override
public final void run() {
try {
// before preparing the job localize
// all the archives
TaskAttemptID taskid = t.getTaskID();
LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir");
File jobCacheDir = null;
if (conf.getJar() != null) {
jobCacheDir = new File(new Path(conf.getJar()).getParent().toString());
}
File workDir =
new File(
lDirAlloc
.getLocalPathToRead(
TaskTracker.getJobCacheSubdir()
+ Path.SEPARATOR
+ t.getJobID()
+ Path.SEPARATOR
+ t.getTaskID()
+ Path.SEPARATOR
+ MRConstants.WORKDIR,
conf)
.toString());
URI[] archives = DistributedCache.getCacheArchives(conf);
URI[] files = DistributedCache.getCacheFiles(conf);
FileStatus fileStatus;
FileSystem fileSystem;
Path localPath;
String baseDir;
if ((archives != null) || (files != null)) {
if (archives != null) {
String[] archivesTimestamps = DistributedCache.getArchiveTimestamps(conf);
Path[] p = new Path[archives.length];
for (int i = 0; i < archives.length; i++) {
fileSystem = FileSystem.get(archives[i], conf);
fileStatus = fileSystem.getFileStatus(new Path(archives[i].getPath()));
String cacheId = DistributedCache.makeRelative(archives[i], conf);
String cachePath = TaskTracker.getCacheSubdir() + Path.SEPARATOR + cacheId;
if (lDirAlloc.ifExists(cachePath, conf)) {
localPath = lDirAlloc.getLocalPathToRead(cachePath, conf);
} else {
localPath = lDirAlloc.getLocalPathForWrite(cachePath, fileStatus.getLen(), conf);
}
baseDir = localPath.toString().replace(cacheId, "");
p[i] =
DistributedCache.getLocalCache(
archives[i],
conf,
new Path(baseDir),
fileStatus,
true,
Long.parseLong(archivesTimestamps[i]),
new Path(workDir.getAbsolutePath()),
false);
}
DistributedCache.setLocalArchives(conf, stringifyPathArray(p));
}
if ((files != null)) {
String[] fileTimestamps = DistributedCache.getFileTimestamps(conf);
Path[] p = new Path[files.length];
for (int i = 0; i < files.length; i++) {
fileSystem = FileSystem.get(files[i], conf);
fileStatus = fileSystem.getFileStatus(new Path(files[i].getPath()));
String cacheId = DistributedCache.makeRelative(files[i], conf);
String cachePath = TaskTracker.getCacheSubdir() + Path.SEPARATOR + cacheId;
if (lDirAlloc.ifExists(cachePath, conf)) {
localPath = lDirAlloc.getLocalPathToRead(cachePath, conf);
} else {
localPath = lDirAlloc.getLocalPathForWrite(cachePath, fileStatus.getLen(), conf);
}
baseDir = localPath.toString().replace(cacheId, "");
p[i] =
DistributedCache.getLocalCache(
files[i],
conf,
new Path(baseDir),
fileStatus,
false,
Long.parseLong(fileTimestamps[i]),
new Path(workDir.getAbsolutePath()),
false);
}
DistributedCache.setLocalFiles(conf, stringifyPathArray(p));
}
Path localTaskFile = new Path(t.getJobFile());
FileSystem localFs = FileSystem.getLocal(conf);
localFs.delete(localTaskFile, true);
OutputStream out = localFs.create(localTaskFile);
try {
conf.writeXml(out);
} finally {
out.close();
}
}
if (!prepare()) {
return;
}
String sep = System.getProperty("path.separator");
StringBuffer classPath = new StringBuffer();
// start with same classpath as parent process
classPath.append(System.getProperty("java.class.path"));
classPath.append(sep);
if (!workDir.mkdirs()) {
if (!workDir.isDirectory()) {
LOG.fatal("Mkdirs failed to create " + workDir.toString());
}
}
String jar = conf.getJar();
if (jar != null) {
// if jar exists, it into workDir
File[] libs = new File(jobCacheDir, "lib").listFiles();
if (libs != null) {
for (int i = 0; i < libs.length; i++) {
classPath.append(sep); // add libs from jar to classpath
classPath.append(libs[i]);
}
}
classPath.append(sep);
classPath.append(new File(jobCacheDir, "classes"));
classPath.append(sep);
classPath.append(jobCacheDir);
}
// include the user specified classpath
// archive paths
Path[] archiveClasspaths = DistributedCache.getArchiveClassPaths(conf);
if (archiveClasspaths != null && archives != null) {
Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
if (localArchives != null) {
for (int i = 0; i < archives.length; i++) {
for (int j = 0; j < archiveClasspaths.length; j++) {
if (archives[i].getPath().equals(archiveClasspaths[j].toString())) {
classPath.append(sep);
classPath.append(localArchives[i].toString());
}
}
}
}
}
// file paths
Path[] fileClasspaths = DistributedCache.getFileClassPaths(conf);
if (fileClasspaths != null && files != null) {
Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
if (localFiles != null) {
for (int i = 0; i < files.length; i++) {
for (int j = 0; j < fileClasspaths.length; j++) {
if (files[i].getPath().equals(fileClasspaths[j].toString())) {
classPath.append(sep);
classPath.append(localFiles[i].toString());
}
}
}
}
}
classPath.append(sep);
classPath.append(workDir);
// Build exec child jmv args.
Vector<String> vargs = new Vector<String>(8);
File jvm = // use same jvm as parent
new File(new File(System.getProperty("java.home"), "bin"), "java");
vargs.add(jvm.toString());
// Add child (task) java-vm options.
//
// The following symbols if present in mapred.child.java.opts value are
// replaced:
// + @taskid@ is interpolated with value of TaskID.
// Other occurrences of @ will not be altered.
//
// Example with multiple arguments and substitutions, showing
// jvm GC logging, and start of a passwordless JVM JMX agent so can
// connect with jconsole and the likes to watch child memory, threads
// and get thread dumps.
//
// <property>
// <name>mapred.child.java.opts</name>
// <value>-verbose:gc -Xloggc:/tmp/@[email protected] \
// -Dcom.sun.management.jmxremote.authenticate=false \
// -Dcom.sun.management.jmxremote.ssl=false \
// </value>
// </property>
//
String javaOpts = conf.get("mapred.child.java.opts", "-Xmx200m");
javaOpts = javaOpts.replace("@taskid@", taskid.toString());
String[] javaOptsSplit = javaOpts.split(" ");
// Add java.library.path; necessary for loading native libraries.
//
// 1. To support native-hadoop library i.e. libhadoop.so, we add the
// parent processes' java.library.path to the child.
// 2. We also add the 'cwd' of the task to it's java.library.path to help
// users distribute native libraries via the DistributedCache.
// 3. The user can also specify extra paths to be added to the
// java.library.path via mapred.child.java.opts.
//
String libraryPath = System.getProperty("java.library.path");
if (libraryPath == null) {
libraryPath = workDir.getAbsolutePath();
} else {
libraryPath += sep + workDir;
}
boolean hasUserLDPath = false;
for (int i = 0; i < javaOptsSplit.length; i++) {
if (javaOptsSplit[i].startsWith("-Djava.library.path=")) {
javaOptsSplit[i] += sep + libraryPath;
hasUserLDPath = true;
break;
}
}
if (!hasUserLDPath) {
vargs.add("-Djava.library.path=" + libraryPath);
}
for (int i = 0; i < javaOptsSplit.length; i++) {
vargs.add(javaOptsSplit[i]);
}
// add java.io.tmpdir given by mapred.child.tmp
String tmp = conf.get("mapred.child.tmp", "./tmp");
Path tmpDir = new Path(tmp);
// if temp directory path is not absolute
// prepend it with workDir.
if (!tmpDir.isAbsolute()) {
tmpDir = new Path(workDir.toString(), tmp);
}
FileSystem localFs = FileSystem.getLocal(conf);
if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) {
throw new IOException("Mkdirs failed to create " + tmpDir.toString());
}
vargs.add("-Djava.io.tmpdir=" + tmpDir.toString());
// Add classpath.
vargs.add("-classpath");
vargs.add(classPath.toString());
// Setup the log4j prop
long logSize = TaskLog.getTaskLogLength(conf);
vargs.add(
"-Dhadoop.log.dir=" + new File(System.getProperty("hadoop.log.dir")).getAbsolutePath());
vargs.add("-Dhadoop.root.logger=INFO,TLA");
vargs.add("-Dhadoop.tasklog.taskid=" + taskid);
vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize);
if (conf.getProfileEnabled()) {
if (conf.getProfileTaskRange(t.isMapTask()).isIncluded(t.getPartition())) {
File prof = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.PROFILE);
vargs.add(String.format(conf.getProfileParams(), prof.toString()));
}
}
// Add main class and its arguments
vargs.add(Child.class.getName()); // main of Child
// pass umbilical address
InetSocketAddress address = tracker.getTaskTrackerReportAddress();
vargs.add(address.getAddress().getHostAddress());
vargs.add(Integer.toString(address.getPort()));
vargs.add(taskid.toString()); // pass task identifier
String pidFile = null;
if (tracker.isTaskMemoryManagerEnabled()) {
pidFile =
lDirAlloc
.getLocalPathForWrite(
(TaskTracker.getPidFilesSubdir() + Path.SEPARATOR + taskid), this.conf)
.toString();
}
// set memory limit using ulimit if feasible and necessary ...
String[] ulimitCmd = Shell.getUlimitMemoryCommand(conf);
List<String> setup = null;
if (ulimitCmd != null) {
setup = new ArrayList<String>();
for (String arg : ulimitCmd) {
setup.add(arg);
}
}
// Set up the redirection of the task's stdout and stderr streams
File stdout = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDOUT);
File stderr = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDERR);
stdout.getParentFile().mkdirs();
tracker.getTaskTrackerInstrumentation().reportTaskLaunch(taskid, stdout, stderr);
Map<String, String> env = new HashMap<String, String>();
StringBuffer ldLibraryPath = new StringBuffer();
ldLibraryPath.append(workDir.toString());
String oldLdLibraryPath = null;
oldLdLibraryPath = System.getenv("LD_LIBRARY_PATH");
if (oldLdLibraryPath != null) {
ldLibraryPath.append(sep);
ldLibraryPath.append(oldLdLibraryPath);
}
env.put("LD_LIBRARY_PATH", ldLibraryPath.toString());
jvmManager.launchJvm(
this,
jvmManager.constructJvmEnv(
setup, vargs, stdout, stderr, logSize, workDir, env, pidFile, conf));
synchronized (lock) {
while (!done) {
lock.wait();
}
}
tracker.getTaskTrackerInstrumentation().reportTaskEnd(t.getTaskID());
if (exitCodeSet) {
if (!killed && exitCode != 0) {
if (exitCode == 65) {
tracker.getTaskTrackerInstrumentation().taskFailedPing(t.getTaskID());
}
throw new IOException("Task process exit with nonzero status of " + exitCode + ".");
}
}
} catch (FSError e) {
LOG.fatal("FSError", e);
try {
tracker.fsError(t.getTaskID(), e.getMessage());
} catch (IOException ie) {
LOG.fatal(t.getTaskID() + " reporting FSError", ie);
}
} catch (Throwable throwable) {
LOG.warn(t.getTaskID() + " Child Error", throwable);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
throwable.printStackTrace(new PrintStream(baos));
try {
tracker.reportDiagnosticInfo(t.getTaskID(), baos.toString());
} catch (IOException e) {
LOG.warn(t.getTaskID() + " Reporting Diagnostics", e);
}
} finally {
try {
URI[] archives = DistributedCache.getCacheArchives(conf);
URI[] files = DistributedCache.getCacheFiles(conf);
if (archives != null) {
for (int i = 0; i < archives.length; i++) {
DistributedCache.releaseCache(archives[i], conf);
}
}
if (files != null) {
for (int i = 0; i < files.length; i++) {
DistributedCache.releaseCache(files[i], conf);
}
}
} catch (IOException ie) {
LOG.warn("Error releasing caches : Cache files might not have been cleaned up");
}
tracker.reportTaskFinished(t.getTaskID(), false);
if (t.isMapTask()) {
tracker.addFreeMapSlot();
} else {
tracker.addFreeReduceSlot();
}
}
}
/**
* Creates a flow control element for the given task <element name="controlFlow">
*
* @return the xml Element corresponding to the flow control, if the task contains a flow control,
* null otherwise
*/
private Element createFlowControlElement(Document doc, Task task) {
Element controlFlowE = null;
// <ref name="block"/>
if (task.getFlowBlock() != FlowBlock.NONE) {
controlFlowE =
doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW.getXMLName());
setAttribute(controlFlowE, XMLAttributes.FLOW_BLOCK, task.getFlowBlock().toString());
}
FlowScript flowScript = task.getFlowScript();
if (flowScript != null) {
Element flowActionE = null;
// flowActionE can be if, loop, replicate or null.
// if not null, it contains a script element
// <ref name="actionIf"/>
// <ref name="actionReplicate"/>
// <ref name="actionLoop"/>
// </choice>
// *** if ***
// <element name="if">
if (flowScript.getActionType().equals(FlowActionType.IF.toString())) {
flowActionE =
doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW_IF.getXMLName());
setAttribute(flowActionE, XMLAttributes.FLOW_TARGET, flowScript.getActionTarget(), true);
setAttribute(flowActionE, XMLAttributes.FLOW_ELSE, flowScript.getActionTargetElse(), true);
setAttribute(
flowActionE, XMLAttributes.FLOW_CONTINUATION, flowScript.getActionContinuation(), true);
}
// *** loop ***
// <element name="loop">
if (flowScript.getActionType().equals(FlowActionType.LOOP.toString())) {
flowActionE =
doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW_LOOP.getXMLName());
setAttribute(flowActionE, XMLAttributes.FLOW_TARGET, flowScript.getActionTarget(), true);
}
// *** replicate ***
// <element name="replicate">
if (flowScript.getActionType().equals(FlowActionType.REPLICATE.toString())) {
flowActionE =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW_REPLICATE.getXMLName());
}
if (flowActionE != null) {
if (controlFlowE == null) {
controlFlowE =
doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.FLOW.getXMLName());
}
Element scriptE = createScriptElement(doc, flowScript);
flowActionE.appendChild(scriptE);
controlFlowE.appendChild(flowActionE);
}
} // flowScript !=null
return controlFlowE;
}
/**
* Creates the parallel environment element for the given task. Corresponds to <define
* name="parallel">
*
* @return the {@link XMLTags#PARALLEL_ENV} element if the task has a parallel environment, null
* otherwise
*/
private Element createParallelEnvironment(Document doc, Task task) {
ParallelEnvironment penv = task.getParallelEnvironment();
if (penv == null) return null;
Element parallelEnvE =
doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.PARALLEL_ENV.getXMLName());
setAttribute(
parallelEnvE, XMLAttributes.TASK_NB_NODES, Integer.toString(penv.getNodesNumber()));
// <ref name="topology"/>
TopologyDescriptor topologyDescr = penv.getTopologyDescriptor();
if (topologyDescr != null) {
// <choice>
// <ref name="arbitrary"/>
// <ref name="bestProximity"/>
// <ref name="thresholdProximity"/>
// <ref name="singleHost"/>
// <ref name="singleHostExclusive"/>
// <ref name="multipleHostsExclusive"/>
// <ref name="differentHostsExclusive"/>
// </choice>
Element topologyE =
doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY.getXMLName());
Element topologyDescrE = null;
if (topologyDescr instanceof ArbitraryTopologyDescriptor) {
topologyDescrE =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_ARBITRARY.getXMLName());
} else if (topologyDescr instanceof ThresholdProximityDescriptor) {
topologyDescrE =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_THRESHOLD_PROXIMITY.getXMLName());
long threshold = ((ThresholdProximityDescriptor) topologyDescr).getThreshold();
topologyDescrE.setAttribute(
XMLAttributes.TOPOLOGY_THRESHOLD.getXMLName(), Long.toString(threshold));
} else if (topologyDescr instanceof BestProximityDescriptor) {
topologyDescrE =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_BEST_PROXIMITY.getXMLName());
} else if (topologyDescr instanceof SingleHostExclusiveDescriptor) {
topologyDescrE =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace,
XMLTags.TOPOLOGY_SINGLE_HOST_EXCLUSIVE.getXMLName());
} else if (topologyDescr instanceof SingleHostDescriptor) {
topologyDescrE =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace, XMLTags.TOPOLOGY_SINGLE_HOST.getXMLName());
} else if (topologyDescr instanceof MultipleHostsExclusiveDescriptor) {
topologyDescrE =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace,
XMLTags.TOPOLOGY_MULTIPLE_HOSTS_EXCLUSIVE.getXMLName());
}
if (topologyDescr instanceof DifferentHostsExclusiveDescriptor) {
topologyDescrE =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace,
XMLTags.TOPOLOGY_DIFFERENT_HOSTS_EXCLUSIVE.getXMLName());
}
if (topologyDescrE != null) {
topologyE.appendChild(topologyDescrE);
}
parallelEnvE.appendChild(topologyE);
}
return parallelEnvE;
}
/** Creates the task element, corressponding to <define name="task"> */
private Element createTaskElement(Document doc, Task task) {
Element taskE = doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.TASK.getXMLName());
// **** attributes *****
// **** common attributes ***
if (task.getOnTaskErrorProperty().isSet()) {
setAttribute(
taskE,
XMLAttributes.COMMON_ON_TASK_ERROR,
task.getOnTaskErrorProperty().getValue().toString(),
true);
}
if (task.getMaxNumberOfExecutionProperty().isSet()) {
setAttribute(
taskE,
XMLAttributes.COMMON_MAX_NUMBER_OF_EXECUTION,
Integer.toString(task.getMaxNumberOfExecution()));
}
setAttribute(taskE, XMLAttributes.COMMON_NAME, task.getName(), true);
if (task.getRestartTaskOnErrorProperty().isSet()) {
setAttribute(
taskE,
XMLAttributes.COMMON_RESTART_TASK_ON_ERROR,
task.getRestartTaskOnError().toString());
}
// *** task attributes ***
if (task.getWallTime() != 0) {
setAttribute(taskE, XMLAttributes.TASK_WALLTIME, formatDate(task.getWallTime()));
}
if (task.isRunAsMe()) {
setAttribute(taskE, XMLAttributes.TASK_RUN_AS_ME, "true");
}
if (task.isPreciousResult()) {
setAttribute(taskE, XMLAttributes.TASK_PRECIOUS_RESULT, "true");
}
if (task.isPreciousLogs()) {
setAttribute(taskE, XMLAttributes.TASK_PRECIOUS_LOGS, "true");
}
// *** elements ****
// <ref name="variables"/>
if (task.getVariables() != null && !task.getVariables().isEmpty()) {
Element variablesE = createTaskVariablesElement(doc, task.getVariables());
taskE.appendChild(variablesE);
}
// <ref name="taskDescription"/>
if (task.getDescription() != null) {
Element descrNode =
createElement(doc, XMLTags.COMMON_DESCRIPTION.getXMLName(), task.getDescription());
taskE.appendChild(descrNode);
}
// <ref name="genericInformation"/>
if ((task.getGenericInformation() != null) && (task.getGenericInformation().size() > 0)) {
Element genericInfoE = createGenericInformation(doc, task.getGenericInformation());
taskE.appendChild(genericInfoE);
}
// <ref name="depends"/>
List<Task> dependencies = task.getDependencesList();
if ((dependencies != null) && (dependencies.size() > 0)) {
Element dependsE =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace, XMLTags.TASK_DEPENDENCES.getXMLName());
for (Task dep : dependencies) {
Element dependsTask =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace, XMLTags.TASK_DEPENDENCES_TASK.getXMLName());
setAttribute(dependsTask, XMLAttributes.TASK_DEPENDS_REF, dep.getName(), true);
dependsE.appendChild(dependsTask);
}
taskE.appendChild(dependsE);
} // if has dependencies
// <ref name="inputFiles"/>
List<InputSelector> inputFiles = task.getInputFilesList();
if (inputFiles != null) {
Element inputFilesE =
doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.DS_INPUT_FILES.getXMLName());
for (InputSelector inputSelector : inputFiles) {
FileSelector fs = inputSelector.getInputFiles();
Element filesE =
doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.DS_FILES.getXMLName());
// the xml only supports one value for the includes/excludes
// pattern
if (!fs.getIncludes().isEmpty())
setAttribute(filesE, XMLAttributes.DS_INCLUDES, fs.getIncludes().iterator().next(), true);
if (!fs.getExcludes().isEmpty())
setAttribute(filesE, XMLAttributes.DS_EXCLUDES, fs.getExcludes().iterator().next(), true);
if (inputSelector.getMode() != null) {
setAttribute(
filesE, XMLAttributes.DS_ACCESS_MODE, inputSelector.getMode().toString(), true);
}
inputFilesE.appendChild(filesE);
}
taskE.appendChild(inputFilesE);
}
// <ref name="parallel"/>
Element parallelEnvE = createParallelEnvironment(doc, task);
if (parallelEnvE != null) taskE.appendChild(parallelEnvE);
// <ref name="selection"/>
List<SelectionScript> selectionScripts = task.getSelectionScripts();
if (selectionScripts != null && selectionScripts.size() > 0) {
Element selectionE =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace, XMLTags.SCRIPT_SELECTION.getXMLName());
for (SelectionScript selectionScript : selectionScripts) {
Element scriptE = createScriptElement(doc, selectionScript);
selectionE.appendChild(scriptE);
}
taskE.appendChild(selectionE);
}
// <ref name="forkEnvironment"/>
if (task.getForkEnvironment() != null) {
Element forkEnvE = createForkEnvironmentElement(doc, task.getForkEnvironment());
taskE.appendChild(forkEnvE);
}
// <ref name="pre"/>
Script preScript = task.getPreScript();
if (preScript != null) {
Element preE =
doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.SCRIPT_PRE.getXMLName());
Element scriptE = createScriptElement(doc, preScript);
preE.appendChild(scriptE);
taskE.appendChild(preE);
}
// <ref name="executable"/>
Element executableE = null;
if (task instanceof JavaTask) {
executableE = createJavaExecutableElement(doc, (JavaTask) task);
} else if (task instanceof NativeTask) {
executableE = createNativeExecutableElement(doc, (NativeTask) task);
} else if (task instanceof ScriptTask) {
executableE = createScriptExecutableElement(doc, (ScriptTask) task);
}
taskE.appendChild(executableE);
// <ref name="flow"/>
Element controlFlowE = createFlowControlElement(doc, task);
if (controlFlowE != null) taskE.appendChild(controlFlowE);
// <ref name="post"/>
Script postScript = task.getPostScript();
if (postScript != null) {
Element postE =
doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.SCRIPT_POST.getXMLName());
Element scriptE = createScriptElement(doc, postScript);
postE.appendChild(scriptE);
taskE.appendChild(postE);
}
// <ref name="cleaning"/>
Script cleanScript = task.getCleaningScript();
if (cleanScript != null) {
Element cleanE =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace, XMLTags.SCRIPT_CLEANING.getXMLName());
Element scriptE = createScriptElement(doc, cleanScript);
cleanE.appendChild(scriptE);
taskE.appendChild(cleanE);
}
// <ref name="outputFiles"/>
List<OutputSelector> outputFiles = task.getOutputFilesList();
if (outputFiles != null) {
Element outputFilesE =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace, XMLTags.DS_OUTPUT_FILES.getXMLName());
for (OutputSelector outputSelector : outputFiles) {
FileSelector fs = outputSelector.getOutputFiles();
Element filesE =
doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.DS_FILES.getXMLName());
// the xml only supports one value for the includes/excludes
// pattern
if (!fs.getIncludes().isEmpty())
setAttribute(filesE, XMLAttributes.DS_INCLUDES, fs.getIncludes().iterator().next(), true);
if (!fs.getExcludes().isEmpty())
setAttribute(filesE, XMLAttributes.DS_EXCLUDES, fs.getExcludes().iterator().next(), true);
if (outputSelector.getMode() != null) {
setAttribute(
filesE, XMLAttributes.DS_ACCESS_MODE, outputSelector.getMode().toString(), true);
}
outputFilesE.appendChild(filesE);
}
taskE.appendChild(outputFilesE);
}
return taskE;
}
