private String configureService(CoasterChannel channel, Task task)
throws InterruptedException, ProtocolException, IOException {
String configId = checkConfigured(channel, task);
if (configId == null) {
try {
ServiceConfigurationCommand scc = new ServiceConfigurationCommand(task);
byte[] reply = scc.execute(channel);
configId = new String(reply);
Object rt = task.getService(0).getAttribute("resource-tracker");
if (rt != null) {
if (rt instanceof CoasterResourceTracker) {
LocalService ls = (LocalService) channel.getService();
ls.addResourceTracker(channel, task.getService(0), (CoasterResourceTracker) rt);
} else {
logger.warn(
"Invalid resource tracker specified: "
+ rt.getClass()
+ " does not implement "
+ CoasterResourceTracker.class);
}
}
setConfigured(channel, task, configId);
} catch (Exception e) {
e.printStackTrace();
task.setStatus(new StatusImpl(Status.FAILED, "Failed to configure coaster service", e));
}
}
return configId;
}
public void statusChanged(Status s, String out, String err) {
Task t = getTask();
if (out != null) {
t.setStdOutput(out);
}
if (err != null) {
t.setStdError(err);
}
t.setStatus(s);
}
private CoasterChannel getChannel(Task task)
throws InvalidServiceContactException, IllegalSpecException, TaskSubmissionException,
InvalidSecurityContextException, ChannelException, InterruptedException {
if (autostart) {
String provider = getBootHandlerProvider(task);
url = ServiceManager.getDefault().reserveService(task, provider);
task.getService(0).setAttribute("coaster-url", url);
} else {
url = task.getService(0).getServiceContact().getContact();
}
return ChannelManager.getManager().getOrCreateChannel(url, cred, LocalRequestManager.INSTANCE);
}
private static String checkConfigured(CoasterChannel channel, Task task)
throws InterruptedException {
Service s = task.getService(0);
synchronized (s) {
while (s.getAttribute(ATTR_CONFIGURING) != null) {
s.wait(100);
}
String configId = (String) s.getAttribute(ATTR_CONFIG_ID);
if (configId == null) {
s.setAttribute(ATTR_CONFIGURING, Boolean.TRUE);
} else {
task.setAttribute(ATTR_CONFIG_ID, configId);
}
return configId;
}
}
private static void setConfigured(CoasterChannel channel, Task task, String configId) {
Service s = task.getService(0);
synchronized (s) {
s.removeAttribute(ATTR_CONFIGURING);
s.setAttribute(ATTR_CONFIG_ID, configId);
s.notifyAll();
}
}
private void validateTaskSettings() throws TaskSubmissionException {
synchronized (JobSubmissionTaskHandler.class) {
Task task = getTask();
Service s = task.getService(0);
TaskSubmissionException e = checkedServices.get(s);
if (e != null) {
throw e;
} else if (!checkedServices.containsKey(s)) {
try {
validateTask();
checkedServices.put(s, null);
} catch (IllegalArgumentException ee) {
e = new TaskSubmissionException(ee.getMessage());
checkedServices.put(task.getService(0), e);
throw e;
}
}
}
}
private String getJobManager(Task t) throws IllegalSpecException {
Service s = t.getService(0);
if (s == null) {
throw new IllegalSpecException("Missing service");
}
if (s instanceof ExecutionService) {
return ((ExecutionService) s).getJobManager();
} else {
throw new IllegalSpecException("Service must be an ExecutionService");
}
}
public void enqueue(Task task, Object constraints) {
if (shouldBeClustered(task, constraints)) {
startTimer();
if (logger.isDebugEnabled()) {
logger.debug("Adding task to clustering queue: " + task.getIdentity());
}
synchronized (dq) {
dq.addLast(new Object[] {task, constraints});
}
} else {
super.enqueue(task, constraints);
}
}
public void submit(Task task)
throws IllegalSpecException, InvalidSecurityContextException, InvalidServiceContactException,
TaskSubmissionException {
checkAndSetTask(task);
validateTaskSettings();
task.setStatus(Status.SUBMITTING);
try {
CoasterChannel channel = getChannel(task);
String configId = configureService(channel, task);
submitJob(channel, task, configId);
} catch (Exception e) {
throw new TaskSubmissionException("Could not submit job", e);
}
}
private boolean shouldBeClustered(Task task, Object constraints) {
if (!clusteringEnabled) {
return false;
}
String reason = null;
try {
if (task.getType() != Task.JOB_SUBMISSION) {
reason = "not a job";
return false;
}
if (((JobSpecification) task.getSpecification()).getAttribute("maxwalltime") == null) {
reason = "no maxwalltime";
return false;
}
if (!(constraints instanceof Contact[])) {
reason = "weird constraints";
return false;
}
if (((Contact[]) constraints).length != 1) {
reason = "constraints size != 1";
return false;
}
boolean cluster = getMaxWallTime(task) < minClusterTime;
if (!cluster) {
reason = "not short enough";
}
return cluster;
} finally {
if (reason != null) {
if (logger.isDebugEnabled()) {
logger.debug(
"Task is not suitable for clustering (" + reason + ") " + task.getIdentity());
}
}
}
}
protected void failTask(Task t, String message, Exception e) {
if (logger.isDebugEnabled()) {
logger.debug("Failing task " + t.getIdentity());
}
LinkedList cluster = null;
synchronized (tasks) {
cluster = (LinkedList) tasks.get(t);
}
if (cluster != null) {
Iterator i = cluster.iterator();
while (i.hasNext()) {
Object[] h = (Object[]) i.next();
super.failTask((Task) h[0], message, e);
}
} else {
super.failTask(t, message, e);
}
}
public void statusChanged(StatusEvent e) {
Task t = (Task) e.getSource();
try {
if (logger.isDebugEnabled()) {
logger.debug("Got task status change for " + t.getIdentity());
}
LinkedList cluster = null;
synchronized (tasks) {
cluster = (LinkedList) tasks.get(t);
}
if (cluster == null) {
super.statusChanged(e);
} else {
if (logger.isDebugEnabled()) {
logger.debug("Got cluster status change for " + t.getIdentity());
}
Status clusterMemberStatus = e.getStatus();
if (clusterMemberStatus.getStatusCode() == Status.FAILED) {
clusterMemberStatus = new StatusImpl(Status.COMPLETED);
}
Iterator i = cluster.iterator();
while (i.hasNext()) {
Object[] h = (Object[]) i.next();
Task ct = (Task) h[0];
StatusEvent nse = new StatusEvent(ct, clusterMemberStatus);
ct.setStatus(clusterMemberStatus);
fireJobStatusChangeEvent(nse);
}
if (e.getStatus().isTerminal()) {
if (logger.isInfoEnabled()) {
logger.info("Removing cluster " + t.getIdentity());
}
synchronized (tasks) {
tasks.remove(t);
}
}
}
} catch (Exception ex) {
failTask(t, ex.getMessage(), ex);
}
}
private int getMaxWallTime(Task t) {
return timeToSeconds(
TypeUtil.toString(((JobSpecification) t.getSpecification()).getAttribute("maxwalltime")));
}
/*
* TODO Add maxmemory=max(maxmemory), minmemory=max(minmemory) and all other
* similar attributes
*/
private void processDelayQueue() {
if (logger.isDebugEnabled()) {
logger.debug("Processing clustering queue");
}
synchronized (dq) {
while (!dq.isEmpty()) {
int clusterTime = 0;
LinkedList cluster = new LinkedList();
Map env = new HashMap();
Map attrs = new HashMap();
Object constraints = null;
String dir = null;
Iterator dqi = dq.iterator();
while (clusterTime < minClusterTime && dqi.hasNext()) {
Object[] h = (Object[]) dqi.next();
Task task = (Task) h[0];
JobSpecification js = (JobSpecification) task.getSpecification();
if (constraints == null) {
constraints = ((Object[]) h[1])[0];
} else if (!constraints.equals(((Object[]) h[1])[0])) {
continue;
}
if (dir == null) {
dir = js.getDirectory() == null ? "" : js.getDirectory();
} else if ((js.getDirectory() != null || !dir.equals(""))
&& !dir.equals(js.getDirectory())) {
continue;
}
if (detectConflict(js, env, attrs)) {
continue;
} else {
dqi.remove();
}
merge(js, env, attrs);
clusterTime += getMaxWallTime(task);
cluster.addLast(h);
}
if (logger.isDebugEnabled()) {
logger.debug("Got a cluster with size " + cluster.size());
}
if (cluster.size() == 0) {
continue;
} else if (cluster.size() == 1) {
Object[] h = (Object[]) cluster.removeFirst();
super.enqueue((Task) h[0], h[1]);
} else if (cluster.size() > 1) {
Task t = new TaskImpl();
int thisClusterId = clusterId++;
t.setIdentity(new IdentityImpl("cluster-" + thisClusterId));
t.setType(Task.JOB_SUBMISSION);
t.setRequiredService(1);
JobSpecification js = new JobSpecificationImpl();
t.setSpecification(js);
js.setExecutable("/bin/sh");
js.addArgument("shared/_swiftseq");
js.addArgument("cluster-" + thisClusterId);
js.addArgument("/clusters/"); // slice path more here TODO
js.setDirectory(dir);
js.setAttribute("maxwalltime", secondsToTime(clusterTime));
if (logger.isInfoEnabled()) {
logger.info("Creating cluster " + t.getIdentity() + " with size " + cluster.size());
}
Iterator i = cluster.iterator();
while (i.hasNext()) {
Object[] h = (Object[]) i.next();
Task st = (Task) h[0];
if (logger.isInfoEnabled()) {
logger.info("Task " + st.getIdentity() + " clustered in " + t.getIdentity());
}
JobSpecification sjs = (JobSpecification) st.getSpecification();
js.addArgument(sjs.getExecutable());
List args = sjs.getArgumentsAsList();
Iterator j = args.iterator();
while (j.hasNext()) {
String arg = (String) j.next();
if (arg.equals("|")) {
arg = "||";
}
js.addArgument(arg);
}
js.addArgument("|");
}
i = env.entrySet().iterator();
while (i.hasNext()) {
Map.Entry e = (Map.Entry) i.next();
js.addEnvironmentVariable((String) e.getKey(), (String) e.getValue());
}
i = attrs.entrySet().iterator();
while (i.hasNext()) {
Map.Entry e = (Map.Entry) i.next();
js.setAttribute((String) e.getKey(), (String) e.getValue());
}
synchronized (tasks) {
tasks.put(t, cluster);
}
super.enqueue(t, new Contact[] {(Contact) constraints});
}
}
}
}
private String getJavaHome(Task task) {
JobSpecification js = (JobSpecification) task.getSpecification();
return js.getEnvironmentVariable("JAVA_HOME");
}
private void submitJob(CoasterChannel channel, Task task, String configId)
throws ProtocolException {
NotificationManager.getDefault().registerListener(task.getIdentity(), task, this);
jsc = new SubmitJobCommand(task, configId);
jsc.executeAsync(channel, this);
}