public static Response redirect(
JsonObject fromPageResponse, Key rfModel, Key dataKey, boolean oobee) {
JsonObject redir = new JsonObject();
redir.addProperty(MODEL_KEY, rfModel.toString());
redir.addProperty(DATA_KEY, dataKey.toString());
redir.addProperty(OOBEE, oobee);
return Response.redirect(fromPageResponse, RFView.class, redir);
}
private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
try {
if (fs == null) return;
for (FileStatus file : fs.listStatus(p)) {
Path pfs = file.getPath();
if (file.isDir()) {
addFolder(fs, pfs, succeeded, failed);
} else {
Key k = Key.make(pfs.toString());
long size = file.getLen();
Value val = null;
if (pfs.getName().endsWith(Extensions.JSON)) {
JsonParser parser = new JsonParser();
JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
JsonElement v = json.get(Constants.VERSION);
if (v == null) throw new RuntimeException("Missing version");
JsonElement type = json.get(Constants.TYPE);
if (type == null) throw new RuntimeException("Missing type");
Class c = Class.forName(type.getAsString());
OldModel model = (OldModel) c.newInstance();
model.fromJson(json);
} else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
FSDataInputStream s = fs.open(pfs);
int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
byte[] mem = MemoryManager.malloc1(sz);
s.readFully(mem);
// Convert to a ValueArray (hope it fits in 1Meg!)
ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
val = new Value(k, ary, Value.HDFS);
} else if (size >= 2 * ValueArray.CHUNK_SZ) {
val =
new Value(
k,
new ValueArray(k, size),
Value.HDFS); // ValueArray byte wrapper over a large file
} else {
val = new Value(k, (int) size, Value.HDFS); // Plain Value
val.setdsk();
}
DKV.put(k, val);
Log.info("PersistHdfs: DKV.put(" + k + ")");
JsonObject o = new JsonObject();
o.addProperty(Constants.KEY, k.toString());
o.addProperty(Constants.FILE, pfs.toString());
o.addProperty(Constants.VALUE_SIZE, file.getLen());
succeeded.add(o);
}
}
} catch (Exception e) {
Log.err(e);
JsonObject o = new JsonObject();
o.addProperty(Constants.FILE, p.toString());
o.addProperty(Constants.ERROR, e.getMessage());
failed.add(o);
}
}
public static void addFolder(Path p, JsonArray succeeded, JsonArray failed) throws IOException {
FileSystem fs = FileSystem.get(p.toUri(), PersistHdfs.CONF);
if (!fs.exists(p)) {
JsonObject o = new JsonObject();
o.addProperty(Constants.FILE, p.toString());
o.addProperty(Constants.ERROR, "Path does not exist!");
failed.add(o);
return;
}
addFolder(fs, p, succeeded, failed);
}
@Override
public JsonObject toJson() {
JsonObject res = new JsonObject();
res.addProperty(Constants.VERSION, H2O.VERSION);
res.addProperty(Constants.TYPE, KMeansModel.class.getName());
res.addProperty(Constants.ERROR, _error);
JsonArray ary = new JsonArray();
for (double[] dd : clusters()) {
JsonArray ary2 = new JsonArray();
for (double d : dd) ary2.add(new JsonPrimitive(d));
ary.add(ary2);
}
res.add(Constants.CLUSTERS, ary);
return res;
}
protected JsonObject defaultJsonResponse() {
// This will be shown every request
JsonObject r = new JsonObject();
RFModel model = _modelKey.value();
r.addProperty(DATA_KEY, _dataKey.originalValue());
r.addProperty(MODEL_KEY, _modelKey.originalValue());
r.addProperty(CLASS, _classCol.specified() ? _classCol.value() : findResponseIdx(model));
r.addProperty(NUM_TREES, model._totalTrees);
r.addProperty(MTRY, model._splitFeatures);
r.addProperty(MTRY_NODES, Arrays.toString(model._nodesSplitFeatures));
r.addProperty(OOBEE, _oobee.value());
// CM specific options
r.addProperty(NO_CM, _noCM.value());
r.addProperty(JSON_REFRESH_THRESHOLD_CM, _refreshThresholdCM.value());
return r;
}
public static Response redirect(
JsonObject fromPageResponse,
Key jobKey,
Key modelKey,
Key dataKey,
int ntree,
int classCol,
String weights,
boolean oobee,
boolean iterativeCM) {
JsonObject redirect = new JsonObject();
if (jobKey != null) redirect.addProperty(JOB, jobKey.toString());
redirect.addProperty(MODEL_KEY, modelKey.toString());
redirect.addProperty(DEST_KEY, modelKey.toString());
redirect.addProperty(DATA_KEY, dataKey.toString());
redirect.addProperty(NUM_TREES, ntree);
redirect.addProperty(CLASS, classCol);
if (weights != null) redirect.addProperty(WEIGHTS, weights);
redirect.addProperty(OOBEE, oobee);
redirect.addProperty(ITERATIVE_CM, iterativeCM);
return Response.redirect(fromPageResponse, RFView.class, redirect);
}
@Override
protected Response serve() {
int tasks = 0;
int finished = 0;
RFModel model = _modelKey.value();
double[] weights = _weights.value();
// Finish refresh after rf model is done and confusion matrix for all trees is computed
boolean done = false;
int classCol = _classCol.specified() ? _classCol.value() : findResponseIdx(model);
tasks = model._totalTrees;
finished = model.size();
// Handle cancelled/aborted jobs
if (_job.value() != null) {
Job jjob = Job.findJob(_job.value());
if (jjob != null && jjob.isCancelled())
return Response.error(
jjob.exception == null ? "Job was cancelled by user!" : jjob.exception);
}
JsonObject response = defaultJsonResponse();
// CM return and possible computation is requested
if (!_noCM.value() && (finished == tasks || _iterativeCM.value()) && finished > 0) {
// Compute the highest number of trees which is less then a threshold
int modelSize = tasks * _refreshThresholdCM.value() / 100;
modelSize =
modelSize == 0 || finished == tasks ? finished : modelSize * (finished / modelSize);
// Get the computing the matrix - if no job is computing, then start a new job
Job cmJob =
ConfusionTask.make(
model, modelSize, _dataKey.value()._key, classCol, weights, _oobee.value());
// Here the the job is running - it saved a CM which can be already finished or in invalid
// state.
CMFinal confusion = UKV.get(cmJob.dest());
// if the matrix is valid, report it in the JSON
if (confusion != null && confusion.valid() && modelSize > 0) {
// finished += 1;
JsonObject cm = new JsonObject();
JsonArray cmHeader = new JsonArray();
JsonArray matrix = new JsonArray();
cm.addProperty(JSON_CM_TYPE, _oobee.value() ? "OOB error estimate" : "full scoring");
cm.addProperty(JSON_CM_CLASS_ERR, confusion.classError());
cm.addProperty(JSON_CM_ROWS_SKIPPED, confusion.skippedRows());
cm.addProperty(JSON_CM_ROWS, confusion.rows());
// create the header
for (String s : cfDomain(confusion, 1024)) cmHeader.add(new JsonPrimitive(s));
cm.add(JSON_CM_HEADER, cmHeader);
// add the matrix
final int nclasses = confusion.dimension();
JsonArray classErrors = new JsonArray();
for (int crow = 0; crow < nclasses; ++crow) {
JsonArray row = new JsonArray();
int classHitScore = 0;
for (int ccol = 0; ccol < nclasses; ++ccol) {
row.add(new JsonPrimitive(confusion.matrix(crow, ccol)));
if (crow != ccol) classHitScore += confusion.matrix(crow, ccol);
}
// produce infinity members in case of 0.f/0
classErrors.add(
new JsonPrimitive(
(float) classHitScore / (classHitScore + confusion.matrix(crow, crow))));
matrix.add(row);
}
cm.add(JSON_CM_CLASSES_ERRORS, classErrors);
cm.add(JSON_CM_MATRIX, matrix);
cm.addProperty(JSON_CM_TREES, modelSize);
response.add(JSON_CM, cm);
// Signal end only and only if all trees were generated and confusion matrix is valid
done = finished == tasks;
}
} else if (_noCM.value() && finished == tasks) done = true;
// Trees
JsonObject trees = new JsonObject();
trees.addProperty(Constants.TREE_COUNT, model.size());
if (model.size() > 0) {
trees.add(Constants.TREE_DEPTH, model.depth().toJson());
trees.add(Constants.TREE_LEAVES, model.leaves().toJson());
}
response.add(Constants.TREES, trees);
// Build a response
Response r;
if (done) {
r = jobDone(response);
r.addHeader(
"<div class='alert'>"
+ /*RFScore.link(MODEL_KEY, model._key, "Use this model for scoring.") */ GeneratePredictionsPage
.link(model._key, "Predict!")
+ " </div>");
} else {
r = Response.poll(response, finished, tasks);
}
r.setBuilder(JSON_CM, new ConfusionMatrixBuilder());
r.setBuilder(TREES, new TreeListBuilder());
return r;
}
