@Override
protected void checkMemoryFootPrint() {
if (_model._output._ntrees == 0) return;
int trees_so_far = _model._output._ntrees; // existing trees
long model_mem_size =
new ComputeModelSize(trees_so_far, _model._output._treeKeys).doAllNodes()._model_mem_size;
_model._output._treeStats._byte_size = model_mem_size;
double avg_tree_mem_size = (double) model_mem_size / trees_so_far;
Log.debug(
"Average tree size (for all classes): " + PrettyPrint.bytes((long) avg_tree_mem_size));
// all the compressed trees are stored on the driver node
long max_mem = H2O.SELF.get_max_mem();
if (_parms._ntrees * avg_tree_mem_size > max_mem) {
String msg =
"The tree model will not fit in the driver node's memory ("
+ PrettyPrint.bytes((long) avg_tree_mem_size)
+ " per tree x "
+ _parms._ntrees
+ " > "
+ PrettyPrint.bytes(max_mem)
+ ") - try decreasing ntrees and/or max_depth or increasing min_rows!";
error("_ntrees", msg);
cancel(msg);
}
}
protected void checkMemoryFootPrint() {
long mem_usage = 8 /*doubles*/ * _parms._k * _train.numCols() * (_parms._standardize ? 2 : 1);
long max_mem = H2O.SELF._heartbeat.get_free_mem();
if (mem_usage > max_mem) {
String msg =
"Centroids won't fit in the driver node's memory ("
+ PrettyPrint.bytes(mem_usage)
+ " > "
+ PrettyPrint.bytes(max_mem)
+ ") - try reducing the number of columns and/or the number of categorical factors.";
error("_train", msg);
cancel(msg);
}
}
private TwoDimTable createScoringHistoryTable(KMeansModel.KMeansOutput output) {
List<String> colHeaders = new ArrayList<>();
List<String> colTypes = new ArrayList<>();
List<String> colFormat = new ArrayList<>();
colHeaders.add("Timestamp");
colTypes.add("string");
colFormat.add("%s");
colHeaders.add("Duration");
colTypes.add("string");
colFormat.add("%s");
colHeaders.add("Iteration");
colTypes.add("long");
colFormat.add("%d");
colHeaders.add("Avg. Change of Std. Centroids");
colTypes.add("double");
colFormat.add("%.5f");
colHeaders.add("Within Cluster Sum Of Squares");
colTypes.add("double");
colFormat.add("%.5f");
final int rows = output._avg_centroids_chg.length;
TwoDimTable table =
new TwoDimTable(
"Scoring History",
null,
new String[rows],
colHeaders.toArray(new String[0]),
colTypes.toArray(new String[0]),
colFormat.toArray(new String[0]),
"");
int row = 0;
for (int i = 0; i < rows; i++) {
int col = 0;
assert (row < table.getRowDim());
assert (col < table.getColDim());
DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");
table.set(row, col++, fmt.print(output._training_time_ms[i]));
table.set(row, col++, PrettyPrint.msecs(output._training_time_ms[i] - _start_time, true));
table.set(row, col++, i);
table.set(row, col++, output._avg_centroids_chg[i]);
table.set(row, col++, output._history_withinss[i]);
row++;
}
return table;
}
@Override
public String toString() {
// Across
String s = "{" + _names[0];
long bs = _vecs[0].byteSize();
for (int i = 1; i < _names.length; i++) {
s += "," + _names[i];
bs += _vecs[i].byteSize();
}
s += "}, " + PrettyPrint.bytes(bs) + "\n";
// Down
Vec v0 = firstReadable();
if (v0 == null) return s;
int nc = v0.nChunks();
s += "Chunk starts: {";
for (int i = 0; i < nc; i++) s += v0.elem2BV(i)._start + ",";
s += "}";
return s;
}
/**
* Create a summary table
*
* @return
*/
TwoDimTable createSummaryTable() {
Neurons[] neurons = DeepLearningTask.makeNeuronsForTesting(this);
long byte_size = new AutoBuffer().put(this).buf().length;
TwoDimTable table =
new TwoDimTable(
"Status of Neuron Layers",
(get_params()._diagnostics ? "" : "diagnostics disabled, ")
+ (!get_params()._autoencoder ? ("predicting " + _train.lastVecName() + ", ") : "")
+ (get_params()._autoencoder
? "auto-encoder"
: _classification
? (units[units.length - 1] + "-class classification")
: "regression")
+ ", "
+ get_params()._distribution
+ " distribution, "
+ get_params()._loss
+ " loss, "
+ String.format("%,d", size())
+ " weights/biases, "
+ PrettyPrint.bytes(byte_size)
+ ", "
+ String.format("%,d", get_processed_global())
+ " training samples, "
+ "mini-batch size "
+ String.format("%,d", get_params()._mini_batch_size),
new String[neurons.length],
new String[] {
"Layer",
"Units",
"Type",
"Dropout",
"L1",
"L2",
"Mean Rate",
"Rate RMS",
"Momentum",
"Mean Weight",
"Weight RMS",
"Mean Bias",
"Bias RMS"
},
new String[] {
"int", "int", "string", "double", "double", "double", "double", "double", "double",
"double", "double", "double", "double"
},
new String[] {
"%d",
"%d",
"%s",
"%2.2f %%",
"%5f",
"%5f",
"%5f",
"%5f",
"%5f",
"%5f",
"%5f",
"%5f",
"%5f"
},
"");
for (int i = 0; i < neurons.length; ++i) {
table.set(i, 0, i + 1);
table.set(i, 1, neurons[i].units);
table.set(i, 2, neurons[i].getClass().getSimpleName());
if (i == 0) {
table.set(i, 3, neurons[i].params._input_dropout_ratio * 100);
continue;
} else if (i < neurons.length - 1) {
if (neurons[i].params._hidden_dropout_ratios == null) {
table.set(i, 3, 0);
} else {
table.set(i, 3, neurons[i].params._hidden_dropout_ratios[i - 1] * 100);
}
}
table.set(i, 4, neurons[i].params._l1);
table.set(i, 5, neurons[i].params._l2);
table.set(
i,
6,
(get_params()._adaptive_rate ? mean_rate[i] : neurons[i].rate(get_processed_total())));
table.set(i, 7, (get_params()._adaptive_rate ? rms_rate[i] : 0));
table.set(i, 8, get_params()._adaptive_rate ? 0 : neurons[i].momentum(get_processed_total()));
table.set(i, 9, mean_weight[i]);
table.set(i, 10, rms_weight[i]);
table.set(i, 11, mean_bias[i]);
table.set(i, 12, rms_bias[i]);
}
summaryTable = table;
return summaryTable;
}
private TwoDimTable createScoringHistoryTable(SharedTreeModel.SharedTreeOutput _output) {
List<String> colHeaders = new ArrayList<>();
List<String> colTypes = new ArrayList<>();
List<String> colFormat = new ArrayList<>();
colHeaders.add("Timestamp");
colTypes.add("string");
colFormat.add("%s");
colHeaders.add("Duration");
colTypes.add("string");
colFormat.add("%s");
colHeaders.add("Number of Trees");
colTypes.add("long");
colFormat.add("%d");
colHeaders.add("Training MSE");
colTypes.add("double");
colFormat.add("%.5f");
if (_output.isClassifier()) {
colHeaders.add("Training LogLoss");
colTypes.add("double");
colFormat.add("%.5f");
}
if (_output.getModelCategory() == ModelCategory.Binomial) {
colHeaders.add("Training AUC");
colTypes.add("double");
colFormat.add("%.5f");
}
if (_output.getModelCategory() == ModelCategory.Binomial
|| _output.getModelCategory() == ModelCategory.Multinomial) {
colHeaders.add("Training Classification Error");
colTypes.add("double");
colFormat.add("%.5f");
}
if (valid() != null) {
colHeaders.add("Validation MSE");
colTypes.add("double");
colFormat.add("%.5f");
if (_output.isClassifier()) {
colHeaders.add("Validation LogLoss");
colTypes.add("double");
colFormat.add("%.5f");
}
if (_output.getModelCategory() == ModelCategory.Binomial) {
colHeaders.add("Validation AUC");
colTypes.add("double");
colFormat.add("%.5f");
}
if (_output.isClassifier()) {
colHeaders.add("Validation Classification Error");
colTypes.add("double");
colFormat.add("%.5f");
}
}
int rows = 0;
for (int i = 1; i < _output._scored_train.length; i++) {
if (!Double.isNaN(_output._scored_train[i]._mse)) ++rows;
}
TwoDimTable table =
new TwoDimTable(
"Scoring History",
null,
new String[rows],
colHeaders.toArray(new String[0]),
colTypes.toArray(new String[0]),
colFormat.toArray(new String[0]),
"");
int row = 0;
for (int i = 1; i < _output._scored_train.length; i++) {
if (Double.isNaN(_output._scored_train[i]._mse)) continue;
int col = 0;
assert (row < table.getRowDim());
assert (col < table.getColDim());
DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");
table.set(row, col++, fmt.print(_output._training_time_ms[i]));
table.set(row, col++, PrettyPrint.msecs(_output._training_time_ms[i] - _start_time, true));
table.set(row, col++, i);
ScoreKeeper st = _output._scored_train[i];
table.set(row, col++, st._mse);
if (_output.isClassifier()) table.set(row, col++, st._logloss);
if (_output.getModelCategory() == ModelCategory.Binomial) table.set(row, col++, st._AUC);
if (_output.isClassifier()) table.set(row, col++, st._classError);
if (_valid != null) {
st = _output._scored_valid[i];
table.set(row, col++, st._mse);
if (_output.isClassifier()) table.set(row, col++, st._logloss);
if (_output.getModelCategory() == ModelCategory.Binomial) table.set(row, col++, st._AUC);
if (_output.isClassifier()) table.set(row, col++, st._classError);
}
row++;
}
return table;
}
@Override
public boolean toHTML(StringBuilder sb) {
if (jobs != null) {
DocGen.HTML.arrayHead(sb);
sb.append("<tr class='warning'>");
ArrayList<Argument> args = jobs[0].arguments();
// Filter some keys to simplify UI
args = (ArrayList<Argument>) args.clone();
filter(
args,
"destination_key",
"source",
"cols",
"ignored_cols_by_name",
"response",
"classification",
"validation");
for (int i = 0; i < args.size(); i++)
sb.append("<td><b>").append(args.get(i)._name).append("</b></td>");
sb.append("<td><b>").append("run time").append("</b></td>");
String perf = jobs[0].speedDescription();
if (perf != null) sb.append("<td><b>").append(perf).append("</b></td>");
sb.append("<td><b>").append("model key").append("</b></td>");
sb.append("<td><b>").append("prediction error").append("</b></td>");
sb.append("<td><b>").append("F1 score").append("</b></td>");
sb.append("</tr>");
ArrayList<JobInfo> infos = new ArrayList<JobInfo>();
for (Job job : jobs) {
JobInfo info = new JobInfo();
info._job = job;
Object value = UKV.get(job.destination_key);
info._model = value instanceof Model ? (Model) value : null;
if (info._model != null) info._cm = info._model.cm();
if (info._cm != null) info._error = info._cm.err();
infos.add(info);
}
Collections.sort(
infos,
new Comparator<JobInfo>() {
@Override
public int compare(JobInfo a, JobInfo b) {
return Double.compare(a._error, b._error);
}
});
for (JobInfo info : infos) {
sb.append("<tr>");
for (Argument a : args) {
try {
Object value = a._field.get(info._job);
String s;
if (value instanceof int[]) s = Utils.sampleToString((int[]) value, 20);
else s = "" + value;
sb.append("<td>").append(s).append("</td>");
} catch (Exception e) {
throw new RuntimeException(e);
}
}
String runTime = "Pending", speed = "";
if (info._job.start_time != 0) {
runTime = PrettyPrint.msecs(info._job.runTimeMs(), true);
speed = perf != null ? PrettyPrint.msecs(info._job.speedValue(), true) : "";
}
sb.append("<td>").append(runTime).append("</td>");
if (perf != null) sb.append("<td>").append(speed).append("</td>");
String link = info._job.destination_key.toString();
if (info._job.start_time != 0 && DKV.get(info._job.destination_key) != null) {
if (info._model instanceof GBMModel)
link = GBMModelView.link(link, info._job.destination_key);
else if (info._model instanceof NeuralNetModel)
link = NeuralNetProgress.link(info._job.self(), info._job.destination_key, link);
if (info._model instanceof KMeans2Model)
link = KMeans2ModelView.link(link, info._job.destination_key);
else link = Inspect.link(link, info._job.destination_key);
}
sb.append("<td>").append(link).append("</td>");
String pct = "", f1 = "";
if (info._cm != null) {
pct = String.format("%.2f", 100 * info._error) + "%";
if (info._cm._arr.length == 2)
f1 = String.format("%.2f", info._cm.precisionAndRecall());
}
sb.append("<td><b>").append(pct).append("</b></td>");
sb.append("<td><b>").append(f1).append("</b></td>");
sb.append("</tr>");
}
DocGen.HTML.arrayTail(sb);
}
return true;
}
