public static Response redirect(
JsonObject fromPageResponse, Key rfModel, Key dataKey, boolean oobee) {
JsonObject redir = new JsonObject();
redir.addProperty(MODEL_KEY, rfModel.toString());
redir.addProperty(DATA_KEY, dataKey.toString());
redir.addProperty(OOBEE, oobee);
return Response.redirect(fromPageResponse, RFView.class, redir);
}
private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
try {
if (fs == null) return;
for (FileStatus file : fs.listStatus(p)) {
Path pfs = file.getPath();
if (file.isDir()) {
addFolder(fs, pfs, succeeded, failed);
} else {
Key k = Key.make(pfs.toString());
long size = file.getLen();
Value val = null;
if (pfs.getName().endsWith(Extensions.JSON)) {
JsonParser parser = new JsonParser();
JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
JsonElement v = json.get(Constants.VERSION);
if (v == null) throw new RuntimeException("Missing version");
JsonElement type = json.get(Constants.TYPE);
if (type == null) throw new RuntimeException("Missing type");
Class c = Class.forName(type.getAsString());
OldModel model = (OldModel) c.newInstance();
model.fromJson(json);
} else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
FSDataInputStream s = fs.open(pfs);
int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
byte[] mem = MemoryManager.malloc1(sz);
s.readFully(mem);
// Convert to a ValueArray (hope it fits in 1Meg!)
ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
val = new Value(k, ary, Value.HDFS);
} else if (size >= 2 * ValueArray.CHUNK_SZ) {
val =
new Value(
k,
new ValueArray(k, size),
Value.HDFS); // ValueArray byte wrapper over a large file
} else {
val = new Value(k, (int) size, Value.HDFS); // Plain Value
val.setdsk();
}
DKV.put(k, val);
Log.info("PersistHdfs: DKV.put(" + k + ")");
JsonObject o = new JsonObject();
o.addProperty(Constants.KEY, k.toString());
o.addProperty(Constants.FILE, pfs.toString());
o.addProperty(Constants.VALUE_SIZE, file.getLen());
succeeded.add(o);
}
}
} catch (Exception e) {
Log.err(e);
JsonObject o = new JsonObject();
o.addProperty(Constants.FILE, p.toString());
o.addProperty(Constants.ERROR, e.getMessage());
failed.add(o);
}
}
public JsonObject toJson() {
JsonObject res = new JsonObject();
JsonArray rows = new JsonArray();
for (int i = 0; i < _rows.length; ++i) rows.add(new JsonPrimitive(_rows[i]));
JsonArray dist = new JsonArray();
for (int i = 0; i < _dist.length; ++i) dist.add(new JsonPrimitive(_dist[i]));
res.add("rows_per_cluster", rows);
res.add("sqr_error_per_cluster", dist);
return res;
}
public static void addFolder(Path p, JsonArray succeeded, JsonArray failed) throws IOException {
FileSystem fs = FileSystem.get(p.toUri(), PersistHdfs.CONF);
if (!fs.exists(p)) {
JsonObject o = new JsonObject();
o.addProperty(Constants.FILE, p.toString());
o.addProperty(Constants.ERROR, "Path does not exist!");
failed.add(o);
return;
}
addFolder(fs, p, succeeded, failed);
}
private StringBuilder stats(StringBuilder sb, JsonElement json) {
if (json == null) {
return sb.append(" / / ");
} else {
JsonObject obj = json.getAsJsonObject();
return sb.append(
String.format(
"%4.1f / %4.1f / %4.1f",
obj.get(MIN).getAsDouble(), obj.get(MAX).getAsDouble(), obj.get(MEAN).getAsDouble()));
}
}
@Override
public JsonObject toJson() {
JsonObject res = new JsonObject();
res.addProperty(Constants.VERSION, H2O.VERSION);
res.addProperty(Constants.TYPE, KMeansModel.class.getName());
res.addProperty(Constants.ERROR, _error);
JsonArray ary = new JsonArray();
for (double[] dd : clusters()) {
JsonArray ary2 = new JsonArray();
for (double d : dd) ary2.add(new JsonPrimitive(d));
ary.add(ary2);
}
res.add(Constants.CLUSTERS, ary);
return res;
}
protected JsonObject defaultJsonResponse() {
// This will be shown every request
JsonObject r = new JsonObject();
RFModel model = _modelKey.value();
r.addProperty(DATA_KEY, _dataKey.originalValue());
r.addProperty(MODEL_KEY, _modelKey.originalValue());
r.addProperty(CLASS, _classCol.specified() ? _classCol.value() : findResponseIdx(model));
r.addProperty(NUM_TREES, model._totalTrees);
r.addProperty(MTRY, model._splitFeatures);
r.addProperty(MTRY_NODES, Arrays.toString(model._nodesSplitFeatures));
r.addProperty(OOBEE, _oobee.value());
// CM specific options
r.addProperty(NO_CM, _noCM.value());
r.addProperty(JSON_REFRESH_THRESHOLD_CM, _refreshThresholdCM.value());
return r;
}
@Override
public String build(Response response, JsonObject t, String contextName) {
int n = t.get(Constants.TREE_COUNT).getAsInt();
StringBuilder sb = new StringBuilder();
if (n > 0) {
sb.append("<h3>Trees</h3>");
sb.append(t.get(Constants.TREE_COUNT)).append(" trees with min/max/mean depth of ");
stats(sb, t.get(TREE_DEPTH)).append(" and leaf of ");
stats(sb, t.get(TREE_LEAVES)).append(".<br>");
for (int i = 0; i < n; ++i) {
sb.append(
RFTreeView.link(_modelKey.value(), i, _dataKey.value(), Integer.toString(i + 1)))
.append(" ");
}
} else {
sb.append("<h3>No trees yet...</h3>");
}
return sb.toString();
}
public static Response redirect(
JsonObject fromPageResponse,
Key jobKey,
Key modelKey,
Key dataKey,
int ntree,
int classCol,
String weights,
boolean oobee,
boolean iterativeCM) {
JsonObject redirect = new JsonObject();
if (jobKey != null) redirect.addProperty(JOB, jobKey.toString());
redirect.addProperty(MODEL_KEY, modelKey.toString());
redirect.addProperty(DEST_KEY, modelKey.toString());
redirect.addProperty(DATA_KEY, dataKey.toString());
redirect.addProperty(NUM_TREES, ntree);
redirect.addProperty(CLASS, classCol);
if (weights != null) redirect.addProperty(WEIGHTS, weights);
redirect.addProperty(OOBEE, oobee);
redirect.addProperty(ITERATIVE_CM, iterativeCM);
return Response.redirect(fromPageResponse, RFView.class, redirect);
}
@Override
public String build(Response response, JsonObject cm, String contextName) {
StringBuilder sb = new StringBuilder();
if (cm.has(JSON_CM_MATRIX)) {
sb.append("<h3>Confusion matrix - ")
.append(cm.get(JSON_CM_TYPE).getAsString())
.append("</h3>");
sb.append("<dl class='dl-horizontal'>");
sb.append("<dt>classification error</dt><dd>")
.append(String.format("%5.3f %%", 100 * cm.get(JSON_CM_CLASS_ERR).getAsFloat()))
.append("</dd>");
long rows = cm.get(JSON_CM_ROWS).getAsLong();
long skippedRows = cm.get(JSON_CM_ROWS_SKIPPED).getAsLong();
sb.append("<dt>used / skipped rows </dt><dd>")
.append(
String.format(
"%d / %d (%3.1f %%)",
rows, skippedRows, (double) skippedRows * 100 / (skippedRows + rows)))
.append("</dd>");
sb.append("<dt>trees used</dt><dd>" + cm.get(JSON_CM_TREES).getAsInt()).append("</dd>");
sb.append("</dl>");
sb.append("<table class='table table-striped table-bordered table-condensed'>");
sb.append("<tr><th>Actual \\ Predicted</th>");
JsonArray header = (JsonArray) cm.get(JSON_CM_HEADER);
for (JsonElement e : header) sb.append("<th>" + e.getAsString() + "</th>");
sb.append("<th>Error</th></tr>");
int classes = header.size();
long[] totals = new long[classes];
JsonArray matrix = (JsonArray) cm.get(JSON_CM_MATRIX);
long sumTotal = 0;
long sumError = 0;
for (int crow = 0; crow < classes; ++crow) {
JsonArray row = (JsonArray) matrix.get(crow);
long total = 0;
long error = 0;
sb.append("<tr><th>" + header.get(crow).getAsString() + "</th>");
for (int ccol = 0; ccol < classes; ++ccol) {
long num = row.get(ccol).getAsLong();
total += num;
totals[ccol] += num;
if (ccol == crow) {
sb.append("<td style='background-color:LightGreen'>");
} else {
sb.append("<td>");
error += num;
}
sb.append(num);
sb.append("</td>");
}
sb.append("<td>");
sb.append(String.format("%5.3f = %d / %d", (double) error / total, error, total));
sb.append("</td></tr>");
sumTotal += total;
sumError += error;
}
sb.append("<tr><th>Totals</th>");
for (int i = 0; i < totals.length; ++i) sb.append("<td>" + totals[i] + "</td>");
sb.append("<td><b>");
sb.append(
String.format("%5.3f = %d / %d", (double) sumError / sumTotal, sumError, sumTotal));
sb.append("</b></td></tr>");
sb.append("</table>");
} else {
sb.append("<div class='alert alert-info'>");
sb.append("Confusion matrix is being computed into the key:</br>");
sb.append(cm.get(JSON_CONFUSION_KEY).getAsString());
sb.append("</div>");
}
return sb.toString();
}
@Override
protected Response serve() {
int tasks = 0;
int finished = 0;
RFModel model = _modelKey.value();
double[] weights = _weights.value();
// Finish refresh after rf model is done and confusion matrix for all trees is computed
boolean done = false;
int classCol = _classCol.specified() ? _classCol.value() : findResponseIdx(model);
tasks = model._totalTrees;
finished = model.size();
// Handle cancelled/aborted jobs
if (_job.value() != null) {
Job jjob = Job.findJob(_job.value());
if (jjob != null && jjob.isCancelled())
return Response.error(
jjob.exception == null ? "Job was cancelled by user!" : jjob.exception);
}
JsonObject response = defaultJsonResponse();
// CM return and possible computation is requested
if (!_noCM.value() && (finished == tasks || _iterativeCM.value()) && finished > 0) {
// Compute the highest number of trees which is less then a threshold
int modelSize = tasks * _refreshThresholdCM.value() / 100;
modelSize =
modelSize == 0 || finished == tasks ? finished : modelSize * (finished / modelSize);
// Get the computing the matrix - if no job is computing, then start a new job
Job cmJob =
ConfusionTask.make(
model, modelSize, _dataKey.value()._key, classCol, weights, _oobee.value());
// Here the the job is running - it saved a CM which can be already finished or in invalid
// state.
CMFinal confusion = UKV.get(cmJob.dest());
// if the matrix is valid, report it in the JSON
if (confusion != null && confusion.valid() && modelSize > 0) {
// finished += 1;
JsonObject cm = new JsonObject();
JsonArray cmHeader = new JsonArray();
JsonArray matrix = new JsonArray();
cm.addProperty(JSON_CM_TYPE, _oobee.value() ? "OOB error estimate" : "full scoring");
cm.addProperty(JSON_CM_CLASS_ERR, confusion.classError());
cm.addProperty(JSON_CM_ROWS_SKIPPED, confusion.skippedRows());
cm.addProperty(JSON_CM_ROWS, confusion.rows());
// create the header
for (String s : cfDomain(confusion, 1024)) cmHeader.add(new JsonPrimitive(s));
cm.add(JSON_CM_HEADER, cmHeader);
// add the matrix
final int nclasses = confusion.dimension();
JsonArray classErrors = new JsonArray();
for (int crow = 0; crow < nclasses; ++crow) {
JsonArray row = new JsonArray();
int classHitScore = 0;
for (int ccol = 0; ccol < nclasses; ++ccol) {
row.add(new JsonPrimitive(confusion.matrix(crow, ccol)));
if (crow != ccol) classHitScore += confusion.matrix(crow, ccol);
}
// produce infinity members in case of 0.f/0
classErrors.add(
new JsonPrimitive(
(float) classHitScore / (classHitScore + confusion.matrix(crow, crow))));
matrix.add(row);
}
cm.add(JSON_CM_CLASSES_ERRORS, classErrors);
cm.add(JSON_CM_MATRIX, matrix);
cm.addProperty(JSON_CM_TREES, modelSize);
response.add(JSON_CM, cm);
// Signal end only and only if all trees were generated and confusion matrix is valid
done = finished == tasks;
}
} else if (_noCM.value() && finished == tasks) done = true;
// Trees
JsonObject trees = new JsonObject();
trees.addProperty(Constants.TREE_COUNT, model.size());
if (model.size() > 0) {
trees.add(Constants.TREE_DEPTH, model.depth().toJson());
trees.add(Constants.TREE_LEAVES, model.leaves().toJson());
}
response.add(Constants.TREES, trees);
// Build a response
Response r;
if (done) {
r = jobDone(response);
r.addHeader(
"<div class='alert'>"
+ /*RFScore.link(MODEL_KEY, model._key, "Use this model for scoring.") */ GeneratePredictionsPage
.link(model._key, "Predict!")
+ " </div>");
} else {
r = Response.poll(response, finished, tasks);
}
r.setBuilder(JSON_CM, new ConfusionMatrixBuilder());
r.setBuilder(TREES, new TreeListBuilder());
return r;
}