/** * Returns a list of all jobs in a system. * * @return list of all jobs including running, done, cancelled, crashed jobs. */ public static Job[] all() { List list = UKV.get(LIST); Job[] jobs = new Job[list == null ? 0 : list._jobs.length]; int j = 0; for (int i = 0; i < jobs.length; i++) { Job job = UKV.get(list._jobs[i]); if (job != null) jobs[j++] = job; } if (j < jobs.length) jobs = Arrays.copyOf(jobs, j); return jobs; }
public void set(Argument arg, String input, Object value) { if (arg._field.getType() != Key.class && value instanceof Key) value = UKV.get((Key) value); try { if (arg._field.getType() == Key.class && value instanceof ValueArray) value = ((ValueArray) value)._key; // else if (arg._field.getType() == int.class && value instanceof Long) value = ((Long) value).intValue(); // else if (arg._field.getType() == float.class && value instanceof Double) value = ((Double) value).floatValue(); // else if (arg._field.getType() == Frame.class && value instanceof ValueArray) value = ((ValueArray) value).asFrame(input); // else if (value instanceof NumberSequence) { double[] ds = ((NumberSequence) value)._arr; if (arg._field.getType() == int[].class) { int[] is = new int[ds.length]; for (int i = 0; i < is.length; i++) is[i] = (int) ds[i]; value = is; } else value = ds; } arg._field.set(this, value); } catch (Exception e) { throw new RuntimeException(e); } }
@Override protected Response serve() { Response response = super.serve(); if (destination_key != null) { GridSearch grid = UKV.get(destination_key); if (grid != null) jobs = grid.jobs; } return response; }
public void onException(Throwable ex) { UKV.remove(dest()); Value v = DKV.get(progressKey()); if (v != null) { ChunkProgress p = v.get(); p = p.error(ex.getMessage()); DKV.put(progressKey(), p); } cancel(ex); }
protected String[] getVectorDomain(final Vec v) { assert v == null || v.isInt() || v.isEnum() : "Cannot get vector domain!"; if (v == null) return null; String[] r; if (v.isEnum()) { r = v.domain(); } else { Vec tmp = v.toEnum(); r = tmp.domain(); UKV.remove(tmp._key); } return r; }
@Override protected Status exec() { UKV.put(destination_key, this); int max = jobs[0].gridParallelism(); int head = 0, tail = 0; while (head < jobs.length && !cancelled()) { if (tail - head < max && tail < jobs.length) jobs[tail++].fork(); else { try { jobs[head++].get(); } catch (Exception e) { throw new RuntimeException(e); } } } return Status.Done; }
/** * Start this task based on given top-level fork-join task representing job computation. * * @param fjtask top-level job computation task. * @return this job in {@link JobState#RUNNING} state * @see JobState * @see H2OCountedCompleter */ public /** FIXME: should be final or at least protected */ Job start(final H2OCountedCompleter fjtask) { assert state == JobState.CREATED : "Trying to run job which was already run?"; assert fjtask != null : "Starting a job with null working task is not permitted! Fix you API"; _fjtask = fjtask; start_time = System.currentTimeMillis(); state = JobState.RUNNING; // Save the full state of the job UKV.put(self(), this); // Update job list new TAtomic<List>() { @Override public List atomic(List old) { if (old == null) old = new List(); Key[] jobs = old._jobs; old._jobs = Arrays.copyOf(jobs, jobs.length + 1); old._jobs[jobs.length] = job_key; return old; } }.invoke(LIST); return this; }
@Override public boolean toHTML(StringBuilder sb) { if (jobs != null) { DocGen.HTML.arrayHead(sb); sb.append("<tr class='warning'>"); ArrayList<Argument> args = jobs[0].arguments(); // Filter some keys to simplify UI args = (ArrayList<Argument>) args.clone(); filter( args, "destination_key", "source", "cols", "ignored_cols_by_name", "response", "classification", "validation"); for (int i = 0; i < args.size(); i++) sb.append("<td><b>").append(args.get(i)._name).append("</b></td>"); sb.append("<td><b>").append("run time").append("</b></td>"); String perf = jobs[0].speedDescription(); if (perf != null) sb.append("<td><b>").append(perf).append("</b></td>"); sb.append("<td><b>").append("model key").append("</b></td>"); sb.append("<td><b>").append("prediction error").append("</b></td>"); sb.append("<td><b>").append("F1 score").append("</b></td>"); sb.append("</tr>"); ArrayList<JobInfo> infos = new ArrayList<JobInfo>(); for (Job job : jobs) { JobInfo info = new JobInfo(); info._job = job; Object value = UKV.get(job.destination_key); info._model = value instanceof Model ? (Model) value : null; if (info._model != null) info._cm = info._model.cm(); if (info._cm != null) info._error = info._cm.err(); infos.add(info); } Collections.sort( infos, new Comparator<JobInfo>() { @Override public int compare(JobInfo a, JobInfo b) { return Double.compare(a._error, b._error); } }); for (JobInfo info : infos) { sb.append("<tr>"); for (Argument a : args) { try { Object value = a._field.get(info._job); String s; if (value instanceof int[]) s = Utils.sampleToString((int[]) value, 20); else s = "" + value; sb.append("<td>").append(s).append("</td>"); } catch (Exception e) { throw new RuntimeException(e); } } String runTime = "Pending", speed = ""; if (info._job.start_time != 0) { runTime = PrettyPrint.msecs(info._job.runTimeMs(), true); speed = perf != null ? PrettyPrint.msecs(info._job.speedValue(), true) : ""; } sb.append("<td>").append(runTime).append("</td>"); if (perf != null) sb.append("<td>").append(speed).append("</td>"); String link = info._job.destination_key.toString(); if (info._job.start_time != 0 && DKV.get(info._job.destination_key) != null) { if (info._model instanceof GBMModel) link = GBMModelView.link(link, info._job.destination_key); else if (info._model instanceof NeuralNetModel) link = NeuralNetProgress.link(info._job.self(), info._job.destination_key, link); if (info._model instanceof KMeans2Model) link = KMeans2ModelView.link(link, info._job.destination_key); else link = Inspect.link(link, info._job.destination_key); } sb.append("<td>").append(link).append("</td>"); String pct = "", f1 = ""; if (info._cm != null) { pct = String.format("%.2f", 100 * info._error) + "%"; if (info._cm._arr.length == 2) f1 = String.format("%.2f", info._cm.precisionAndRecall()); } sb.append("<td><b>").append(pct).append("</b></td>"); sb.append("<td><b>").append(f1).append("</b></td>"); sb.append("</tr>"); } DocGen.HTML.arrayTail(sb); } return true; }
@Override public void remove() { super.remove(); UKV.remove(_progress); }
public ChunkProgressJob(long chunksTotal, Key destinationKey) { destination_key = destinationKey; _progress = Key.make(Key.make()._kb, (byte) 0, Key.DFJ_INTERNAL_USER, destinationKey.home_node()); UKV.put(_progress, new ChunkProgress(chunksTotal)); }
/** * Finds a job with given key or returns null. * * @param jobkey job key * @return returns a job with given job key or null if a job is not found. */ public static Job findJob(final Key jobkey) { return UKV.get(jobkey); }
/** * Check if given job is running. * * @param job_key job key * @return true if job is still running else returns false. */ public static boolean isRunning(Key job_key) { Job j = UKV.get(job_key); assert j != null : "Job should be always in DKV!"; return j.isRunning(); }
/** * Blocks and get result of this job. * * <p>The call blocks on working task which was passed via {@link #start(H2OCountedCompleter)} * method and returns the result which is fetched from UKV based on job destination key. * * @return result of this job fetched from UKV by destination key. * @see #start(H2OCountedCompleter) * @see UKV */ public <T> T get() { _fjtask.join(); // Block until top-level job is done T ans = (T) UKV.get(destination_key); remove(); // Remove self-job return ans; }
/** * Return progress of this job. * * @return the value in interval <0,1> representing job progress. */ public float progress() { Freezable f = UKV.get(destination_key); if (f instanceof Progress) return ((Progress) f).progress(); return 0; }