Example #1
0
 /**
  * Returns a list of all jobs in a system.
  *
  * @return list of all jobs including running, done, cancelled, crashed jobs.
  */
 public static Job[] all() {
   List list = UKV.get(LIST);
   Job[] jobs = new Job[list == null ? 0 : list._jobs.length];
   int j = 0;
   for (int i = 0; i < jobs.length; i++) {
     Job job = UKV.get(list._jobs[i]);
     if (job != null) jobs[j++] = job;
   }
   if (j < jobs.length) jobs = Arrays.copyOf(jobs, j);
   return jobs;
 }
Example #2
0
  public void set(Argument arg, String input, Object value) {
    if (arg._field.getType() != Key.class && value instanceof Key) value = UKV.get((Key) value);

    try {
      if (arg._field.getType() == Key.class && value instanceof ValueArray)
        value = ((ValueArray) value)._key;
      //
      else if (arg._field.getType() == int.class && value instanceof Long)
        value = ((Long) value).intValue();
      //
      else if (arg._field.getType() == float.class && value instanceof Double)
        value = ((Double) value).floatValue();
      //
      else if (arg._field.getType() == Frame.class && value instanceof ValueArray)
        value = ((ValueArray) value).asFrame(input);
      //
      else if (value instanceof NumberSequence) {
        double[] ds = ((NumberSequence) value)._arr;
        if (arg._field.getType() == int[].class) {
          int[] is = new int[ds.length];
          for (int i = 0; i < is.length; i++) is[i] = (int) ds[i];
          value = is;
        } else value = ds;
      }
      arg._field.set(this, value);
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }
Example #3
0
 @Override
 protected Response serve() {
   Response response = super.serve();
   if (destination_key != null) {
     GridSearch grid = UKV.get(destination_key);
     if (grid != null) jobs = grid.jobs;
   }
   return response;
 }
Example #4
0
 public void onException(Throwable ex) {
   UKV.remove(dest());
   Value v = DKV.get(progressKey());
   if (v != null) {
     ChunkProgress p = v.get();
     p = p.error(ex.getMessage());
     DKV.put(progressKey(), p);
   }
   cancel(ex);
 }
Example #5
0
 protected String[] getVectorDomain(final Vec v) {
   assert v == null || v.isInt() || v.isEnum() : "Cannot get vector domain!";
   if (v == null) return null;
   String[] r;
   if (v.isEnum()) {
     r = v.domain();
   } else {
     Vec tmp = v.toEnum();
     r = tmp.domain();
     UKV.remove(tmp._key);
   }
   return r;
 }
Example #6
0
 @Override
 protected Status exec() {
   UKV.put(destination_key, this);
   int max = jobs[0].gridParallelism();
   int head = 0, tail = 0;
   while (head < jobs.length && !cancelled()) {
     if (tail - head < max && tail < jobs.length) jobs[tail++].fork();
     else {
       try {
         jobs[head++].get();
       } catch (Exception e) {
         throw new RuntimeException(e);
       }
     }
   }
   return Status.Done;
 }
Example #7
0
 /**
  * Start this task based on given top-level fork-join task representing job computation.
  *
  * @param fjtask top-level job computation task.
  * @return this job in {@link JobState#RUNNING} state
  * @see JobState
  * @see H2OCountedCompleter
  */
 public
 /** FIXME: should be final or at least protected */
 Job start(final H2OCountedCompleter fjtask) {
   assert state == JobState.CREATED : "Trying to run job which was already run?";
   assert fjtask != null : "Starting a job with null working task is not permitted! Fix you API";
   _fjtask = fjtask;
   start_time = System.currentTimeMillis();
   state = JobState.RUNNING;
   // Save the full state of the job
   UKV.put(self(), this);
   // Update job list
   new TAtomic<List>() {
     @Override
     public List atomic(List old) {
       if (old == null) old = new List();
       Key[] jobs = old._jobs;
       old._jobs = Arrays.copyOf(jobs, jobs.length + 1);
       old._jobs[jobs.length] = job_key;
       return old;
     }
   }.invoke(LIST);
   return this;
 }
Example #8
0
    @Override
    public boolean toHTML(StringBuilder sb) {
      if (jobs != null) {
        DocGen.HTML.arrayHead(sb);
        sb.append("<tr class='warning'>");
        ArrayList<Argument> args = jobs[0].arguments();
        // Filter some keys to simplify UI
        args = (ArrayList<Argument>) args.clone();
        filter(
            args,
            "destination_key",
            "source",
            "cols",
            "ignored_cols_by_name",
            "response",
            "classification",
            "validation");
        for (int i = 0; i < args.size(); i++)
          sb.append("<td><b>").append(args.get(i)._name).append("</b></td>");
        sb.append("<td><b>").append("run time").append("</b></td>");
        String perf = jobs[0].speedDescription();
        if (perf != null) sb.append("<td><b>").append(perf).append("</b></td>");
        sb.append("<td><b>").append("model key").append("</b></td>");
        sb.append("<td><b>").append("prediction error").append("</b></td>");
        sb.append("<td><b>").append("F1 score").append("</b></td>");
        sb.append("</tr>");

        ArrayList<JobInfo> infos = new ArrayList<JobInfo>();
        for (Job job : jobs) {
          JobInfo info = new JobInfo();
          info._job = job;
          Object value = UKV.get(job.destination_key);
          info._model = value instanceof Model ? (Model) value : null;
          if (info._model != null) info._cm = info._model.cm();
          if (info._cm != null) info._error = info._cm.err();
          infos.add(info);
        }
        Collections.sort(
            infos,
            new Comparator<JobInfo>() {
              @Override
              public int compare(JobInfo a, JobInfo b) {
                return Double.compare(a._error, b._error);
              }
            });

        for (JobInfo info : infos) {
          sb.append("<tr>");
          for (Argument a : args) {
            try {
              Object value = a._field.get(info._job);
              String s;
              if (value instanceof int[]) s = Utils.sampleToString((int[]) value, 20);
              else s = "" + value;
              sb.append("<td>").append(s).append("</td>");
            } catch (Exception e) {
              throw new RuntimeException(e);
            }
          }
          String runTime = "Pending", speed = "";
          if (info._job.start_time != 0) {
            runTime = PrettyPrint.msecs(info._job.runTimeMs(), true);
            speed = perf != null ? PrettyPrint.msecs(info._job.speedValue(), true) : "";
          }
          sb.append("<td>").append(runTime).append("</td>");
          if (perf != null) sb.append("<td>").append(speed).append("</td>");

          String link = info._job.destination_key.toString();
          if (info._job.start_time != 0 && DKV.get(info._job.destination_key) != null) {
            if (info._model instanceof GBMModel)
              link = GBMModelView.link(link, info._job.destination_key);
            else if (info._model instanceof NeuralNetModel)
              link = NeuralNetProgress.link(info._job.self(), info._job.destination_key, link);
            if (info._model instanceof KMeans2Model)
              link = KMeans2ModelView.link(link, info._job.destination_key);
            else link = Inspect.link(link, info._job.destination_key);
          }
          sb.append("<td>").append(link).append("</td>");

          String pct = "", f1 = "";
          if (info._cm != null) {
            pct = String.format("%.2f", 100 * info._error) + "%";
            if (info._cm._arr.length == 2)
              f1 = String.format("%.2f", info._cm.precisionAndRecall());
          }
          sb.append("<td><b>").append(pct).append("</b></td>");
          sb.append("<td><b>").append(f1).append("</b></td>");
          sb.append("</tr>");
        }
        DocGen.HTML.arrayTail(sb);
      }
      return true;
    }
Example #9
0
 @Override
 public void remove() {
   super.remove();
   UKV.remove(_progress);
 }
Example #10
0
 public ChunkProgressJob(long chunksTotal, Key destinationKey) {
   destination_key = destinationKey;
   _progress =
       Key.make(Key.make()._kb, (byte) 0, Key.DFJ_INTERNAL_USER, destinationKey.home_node());
   UKV.put(_progress, new ChunkProgress(chunksTotal));
 }
Example #11
0
 /**
  * Finds a job with given key or returns null.
  *
  * @param jobkey job key
  * @return returns a job with given job key or null if a job is not found.
  */
 public static Job findJob(final Key jobkey) {
   return UKV.get(jobkey);
 }
Example #12
0
 /**
  * Check if given job is running.
  *
  * @param job_key job key
  * @return true if job is still running else returns false.
  */
 public static boolean isRunning(Key job_key) {
   Job j = UKV.get(job_key);
   assert j != null : "Job should be always in DKV!";
   return j.isRunning();
 }
Example #13
0
 /**
  * Blocks and get result of this job.
  *
  * <p>The call blocks on working task which was passed via {@link #start(H2OCountedCompleter)}
  * method and returns the result which is fetched from UKV based on job destination key.
  *
  * @return result of this job fetched from UKV by destination key.
  * @see #start(H2OCountedCompleter)
  * @see UKV
  */
 public <T> T get() {
   _fjtask.join(); // Block until top-level job is done
   T ans = (T) UKV.get(destination_key);
   remove(); // Remove self-job
   return ans;
 }
Example #14
0
 /**
  * Return progress of this job.
  *
  * @return the value in interval &lt;0,1&gt; representing job progress.
  */
 public float progress() {
   Freezable f = UKV.get(destination_key);
   if (f instanceof Progress) return ((Progress) f).progress();
   return 0;
 }