private void mergeEnvironment(JobSpecification js, Map env) {
Iterator i = js.getEnvironmentVariableNames().iterator();
while (i.hasNext()) {
String envName = (String) i.next();
env.put(envName, js.getEnvironmentVariable(envName));
}
}
private void mergeAttributes(JobSpecification js, Map attrs) {
Iterator i = js.getAttributeNames().iterator();
while (i.hasNext()) {
String attrName = (String) i.next();
if (attrName.equals("maxwalltime")) continue;
attrs.put(attrName, js.getAttribute(attrName));
}
}
private boolean detectEnvironmentConflict(JobSpecification js, Map env) {
Iterator i = js.getEnvironmentVariableNames().iterator();
while (i.hasNext()) {
String envName = (String) i.next();
Object value = env.get(envName);
if (value != null && !value.equals(js.getEnvironmentVariable(envName))) {
return true;
}
}
return false;
}
private boolean detectAttributeConflict(JobSpecification js, Map attrs) {
Iterator ia = js.getAttributeNames().iterator();
while (ia.hasNext()) {
String attrName = (String) ia.next();
if (attrName.equals("maxwalltime")) {
continue;
}
Object value = attrs.get(attrName);
if (value != null && !value.equals(js.getAttribute(attrName))) {
return true;
}
}
return false;
}
/*
* TODO Add maxmemory=max(maxmemory), minmemory=max(minmemory) and all other
* similar attributes
*/
private void processDelayQueue() {
if (logger.isDebugEnabled()) {
logger.debug("Processing clustering queue");
}
synchronized (dq) {
while (!dq.isEmpty()) {
int clusterTime = 0;
LinkedList cluster = new LinkedList();
Map env = new HashMap();
Map attrs = new HashMap();
Object constraints = null;
String dir = null;
Iterator dqi = dq.iterator();
while (clusterTime < minClusterTime && dqi.hasNext()) {
Object[] h = (Object[]) dqi.next();
Task task = (Task) h[0];
JobSpecification js = (JobSpecification) task.getSpecification();
if (constraints == null) {
constraints = ((Object[]) h[1])[0];
} else if (!constraints.equals(((Object[]) h[1])[0])) {
continue;
}
if (dir == null) {
dir = js.getDirectory() == null ? "" : js.getDirectory();
} else if ((js.getDirectory() != null || !dir.equals(""))
&& !dir.equals(js.getDirectory())) {
continue;
}
if (detectConflict(js, env, attrs)) {
continue;
} else {
dqi.remove();
}
merge(js, env, attrs);
clusterTime += getMaxWallTime(task);
cluster.addLast(h);
}
if (logger.isDebugEnabled()) {
logger.debug("Got a cluster with size " + cluster.size());
}
if (cluster.size() == 0) {
continue;
} else if (cluster.size() == 1) {
Object[] h = (Object[]) cluster.removeFirst();
super.enqueue((Task) h[0], h[1]);
} else if (cluster.size() > 1) {
Task t = new TaskImpl();
int thisClusterId = clusterId++;
t.setIdentity(new IdentityImpl("cluster-" + thisClusterId));
t.setType(Task.JOB_SUBMISSION);
t.setRequiredService(1);
JobSpecification js = new JobSpecificationImpl();
t.setSpecification(js);
js.setExecutable("/bin/sh");
js.addArgument("shared/_swiftseq");
js.addArgument("cluster-" + thisClusterId);
js.addArgument("/clusters/"); // slice path more here TODO
js.setDirectory(dir);
js.setAttribute("maxwalltime", secondsToTime(clusterTime));
if (logger.isInfoEnabled()) {
logger.info("Creating cluster " + t.getIdentity() + " with size " + cluster.size());
}
Iterator i = cluster.iterator();
while (i.hasNext()) {
Object[] h = (Object[]) i.next();
Task st = (Task) h[0];
if (logger.isInfoEnabled()) {
logger.info("Task " + st.getIdentity() + " clustered in " + t.getIdentity());
}
JobSpecification sjs = (JobSpecification) st.getSpecification();
js.addArgument(sjs.getExecutable());
List args = sjs.getArgumentsAsList();
Iterator j = args.iterator();
while (j.hasNext()) {
String arg = (String) j.next();
if (arg.equals("|")) {
arg = "||";
}
js.addArgument(arg);
}
js.addArgument("|");
}
i = env.entrySet().iterator();
while (i.hasNext()) {
Map.Entry e = (Map.Entry) i.next();
js.addEnvironmentVariable((String) e.getKey(), (String) e.getValue());
}
i = attrs.entrySet().iterator();
while (i.hasNext()) {
Map.Entry e = (Map.Entry) i.next();
js.setAttribute((String) e.getKey(), (String) e.getValue());
}
synchronized (tasks) {
tasks.put(t, cluster);
}
super.enqueue(t, new Contact[] {(Contact) constraints});
}
}
}
}
private String getJavaHome(Task task) {
JobSpecification js = (JobSpecification) task.getSpecification();
return js.getEnvironmentVariable("JAVA_HOME");
}