Exemplo n.º 1
0
  private boolean shouldBeClustered(Task task, Object constraints) {
    if (!clusteringEnabled) {
      return false;
    }
    String reason = null;
    try {
      if (task.getType() != Task.JOB_SUBMISSION) {
        reason = "not a job";
        return false;
      }
      if (((JobSpecification) task.getSpecification()).getAttribute("maxwalltime") == null) {
        reason = "no maxwalltime";
        return false;
      }
      if (!(constraints instanceof Contact[])) {
        reason = "weird constraints";
        return false;
      }

      if (((Contact[]) constraints).length != 1) {
        reason = "constraints size != 1";
        return false;
      }
      boolean cluster = getMaxWallTime(task) < minClusterTime;
      if (!cluster) {
        reason = "not short enough";
      }
      return cluster;
    } finally {
      if (reason != null) {
        if (logger.isDebugEnabled()) {
          logger.debug(
              "Task is not suitable for clustering (" + reason + ") " + task.getIdentity());
        }
      }
    }
  }
Exemplo n.º 2
0
 private int getMaxWallTime(Task t) {
   return timeToSeconds(
       TypeUtil.toString(((JobSpecification) t.getSpecification()).getAttribute("maxwalltime")));
 }
Exemplo n.º 3
0
  /*
   * TODO Add maxmemory=max(maxmemory), minmemory=max(minmemory) and all other
   * similar attributes
   */
  private void processDelayQueue() {
    if (logger.isDebugEnabled()) {
      logger.debug("Processing clustering queue");
    }
    synchronized (dq) {
      while (!dq.isEmpty()) {
        int clusterTime = 0;
        LinkedList cluster = new LinkedList();
        Map env = new HashMap();
        Map attrs = new HashMap();
        Object constraints = null;
        String dir = null;

        Iterator dqi = dq.iterator();
        while (clusterTime < minClusterTime && dqi.hasNext()) {
          Object[] h = (Object[]) dqi.next();
          Task task = (Task) h[0];

          JobSpecification js = (JobSpecification) task.getSpecification();

          if (constraints == null) {
            constraints = ((Object[]) h[1])[0];
          } else if (!constraints.equals(((Object[]) h[1])[0])) {
            continue;
          }

          if (dir == null) {
            dir = js.getDirectory() == null ? "" : js.getDirectory();
          } else if ((js.getDirectory() != null || !dir.equals(""))
              && !dir.equals(js.getDirectory())) {
            continue;
          }

          if (detectConflict(js, env, attrs)) {
            continue;
          } else {
            dqi.remove();
          }

          merge(js, env, attrs);

          clusterTime += getMaxWallTime(task);
          cluster.addLast(h);
        }

        if (logger.isDebugEnabled()) {
          logger.debug("Got a cluster with size " + cluster.size());
        }

        if (cluster.size() == 0) {
          continue;
        } else if (cluster.size() == 1) {
          Object[] h = (Object[]) cluster.removeFirst();
          super.enqueue((Task) h[0], h[1]);
        } else if (cluster.size() > 1) {
          Task t = new TaskImpl();
          int thisClusterId = clusterId++;
          t.setIdentity(new IdentityImpl("cluster-" + thisClusterId));
          t.setType(Task.JOB_SUBMISSION);
          t.setRequiredService(1);

          JobSpecification js = new JobSpecificationImpl();
          t.setSpecification(js);
          js.setExecutable("/bin/sh");
          js.addArgument("shared/_swiftseq");
          js.addArgument("cluster-" + thisClusterId);
          js.addArgument("/clusters/"); // slice path more here TODO
          js.setDirectory(dir);
          js.setAttribute("maxwalltime", secondsToTime(clusterTime));

          if (logger.isInfoEnabled()) {
            logger.info("Creating cluster " + t.getIdentity() + " with size " + cluster.size());
          }

          Iterator i = cluster.iterator();
          while (i.hasNext()) {
            Object[] h = (Object[]) i.next();
            Task st = (Task) h[0];
            if (logger.isInfoEnabled()) {
              logger.info("Task " + st.getIdentity() + " clustered in " + t.getIdentity());
            }
            JobSpecification sjs = (JobSpecification) st.getSpecification();
            js.addArgument(sjs.getExecutable());
            List args = sjs.getArgumentsAsList();
            Iterator j = args.iterator();
            while (j.hasNext()) {
              String arg = (String) j.next();
              if (arg.equals("|")) {
                arg = "||";
              }
              js.addArgument(arg);
            }
            js.addArgument("|");
          }

          i = env.entrySet().iterator();
          while (i.hasNext()) {
            Map.Entry e = (Map.Entry) i.next();
            js.addEnvironmentVariable((String) e.getKey(), (String) e.getValue());
          }

          i = attrs.entrySet().iterator();
          while (i.hasNext()) {
            Map.Entry e = (Map.Entry) i.next();
            js.setAttribute((String) e.getKey(), (String) e.getValue());
          }

          synchronized (tasks) {
            tasks.put(t, cluster);
          }
          super.enqueue(t, new Contact[] {(Contact) constraints});
        }
      }
    }
  }
Exemplo n.º 4
0
 private String getJavaHome(Task task) {
   JobSpecification js = (JobSpecification) task.getSpecification();
   return js.getEnvironmentVariable("JAVA_HOME");
 }