private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
try {
if (fs == null) return;
for (FileStatus file : fs.listStatus(p)) {
Path pfs = file.getPath();
if (file.isDir()) {
addFolder(fs, pfs, succeeded, failed);
} else {
Key k = Key.make(pfs.toString());
long size = file.getLen();
Value val = null;
if (pfs.getName().endsWith(Extensions.JSON)) {
JsonParser parser = new JsonParser();
JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
JsonElement v = json.get(Constants.VERSION);
if (v == null) throw new RuntimeException("Missing version");
JsonElement type = json.get(Constants.TYPE);
if (type == null) throw new RuntimeException("Missing type");
Class c = Class.forName(type.getAsString());
OldModel model = (OldModel) c.newInstance();
model.fromJson(json);
} else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
FSDataInputStream s = fs.open(pfs);
int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
byte[] mem = MemoryManager.malloc1(sz);
s.readFully(mem);
// Convert to a ValueArray (hope it fits in 1Meg!)
ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
val = new Value(k, ary, Value.HDFS);
} else if (size >= 2 * ValueArray.CHUNK_SZ) {
val =
new Value(
k,
new ValueArray(k, size),
Value.HDFS); // ValueArray byte wrapper over a large file
} else {
val = new Value(k, (int) size, Value.HDFS); // Plain Value
val.setdsk();
}
DKV.put(k, val);
Log.info("PersistHdfs: DKV.put(" + k + ")");
JsonObject o = new JsonObject();
o.addProperty(Constants.KEY, k.toString());
o.addProperty(Constants.FILE, pfs.toString());
o.addProperty(Constants.VALUE_SIZE, file.getLen());
succeeded.add(o);
}
}
} catch (Exception e) {
Log.err(e);
JsonObject o = new JsonObject();
o.addProperty(Constants.FILE, p.toString());
o.addProperty(Constants.ERROR, e.getMessage());
failed.add(o);
}
}
private StringBuilder stats(StringBuilder sb, JsonElement json) {
if (json == null) {
return sb.append(" / / ");
} else {
JsonObject obj = json.getAsJsonObject();
return sb.append(
String.format(
"%4.1f / %4.1f / %4.1f",
obj.get(MIN).getAsDouble(), obj.get(MAX).getAsDouble(), obj.get(MEAN).getAsDouble()));
}
}
@Override
public String build(Response response, JsonObject cm, String contextName) {
StringBuilder sb = new StringBuilder();
if (cm.has(JSON_CM_MATRIX)) {
sb.append("<h3>Confusion matrix - ")
.append(cm.get(JSON_CM_TYPE).getAsString())
.append("</h3>");
sb.append("<dl class='dl-horizontal'>");
sb.append("<dt>classification error</dt><dd>")
.append(String.format("%5.3f %%", 100 * cm.get(JSON_CM_CLASS_ERR).getAsFloat()))
.append("</dd>");
long rows = cm.get(JSON_CM_ROWS).getAsLong();
long skippedRows = cm.get(JSON_CM_ROWS_SKIPPED).getAsLong();
sb.append("<dt>used / skipped rows </dt><dd>")
.append(
String.format(
"%d / %d (%3.1f %%)",
rows, skippedRows, (double) skippedRows * 100 / (skippedRows + rows)))
.append("</dd>");
sb.append("<dt>trees used</dt><dd>" + cm.get(JSON_CM_TREES).getAsInt()).append("</dd>");
sb.append("</dl>");
sb.append("<table class='table table-striped table-bordered table-condensed'>");
sb.append("<tr><th>Actual \\ Predicted</th>");
JsonArray header = (JsonArray) cm.get(JSON_CM_HEADER);
for (JsonElement e : header) sb.append("<th>" + e.getAsString() + "</th>");
sb.append("<th>Error</th></tr>");
int classes = header.size();
long[] totals = new long[classes];
JsonArray matrix = (JsonArray) cm.get(JSON_CM_MATRIX);
long sumTotal = 0;
long sumError = 0;
for (int crow = 0; crow < classes; ++crow) {
JsonArray row = (JsonArray) matrix.get(crow);
long total = 0;
long error = 0;
sb.append("<tr><th>" + header.get(crow).getAsString() + "</th>");
for (int ccol = 0; ccol < classes; ++ccol) {
long num = row.get(ccol).getAsLong();
total += num;
totals[ccol] += num;
if (ccol == crow) {
sb.append("<td style='background-color:LightGreen'>");
} else {
sb.append("<td>");
error += num;
}
sb.append(num);
sb.append("</td>");
}
sb.append("<td>");
sb.append(String.format("%5.3f = %d / %d", (double) error / total, error, total));
sb.append("</td></tr>");
sumTotal += total;
sumError += error;
}
sb.append("<tr><th>Totals</th>");
for (int i = 0; i < totals.length; ++i) sb.append("<td>" + totals[i] + "</td>");
sb.append("<td><b>");
sb.append(
String.format("%5.3f = %d / %d", (double) sumError / sumTotal, sumError, sumTotal));
sb.append("</b></td></tr>");
sb.append("</table>");
} else {
sb.append("<div class='alert alert-info'>");
sb.append("Confusion matrix is being computed into the key:</br>");
sb.append(cm.get(JSON_CONFUSION_KEY).getAsString());
sb.append("</div>");
}
return sb.toString();
}