protected double doScoringAndSaveModel(
boolean finalScoring, boolean oob, boolean build_tree_one_node) {
double training_r2 = Double.NaN; // Training R^2 value, if computed
long now = System.currentTimeMillis();
if (_firstScore == 0) _firstScore = now;
long sinceLastScore = now - _timeLastScoreStart;
boolean updated = false;
new ProgressUpdate(
"Built " + _model._output._ntrees + " trees so far (out of " + _parms._ntrees + ").")
.fork(_progressKey);
// Now model already contains tid-trees in serialized form
if (_parms._score_each_iteration
|| finalScoring
|| (now - _firstScore < 4000)
|| // Score every time for 4 secs
// Throttle scoring to keep the cost sane; limit to a 10% duty cycle & every 4 secs
(sinceLastScore > 4000
&& // Limit scoring updates to every 4sec
(double) (_timeLastScoreEnd - _timeLastScoreStart) / sinceLastScore
< 0.1)) { // 10% duty cycle
checkMemoryFootPrint();
// If validation is specified we use a model for scoring, so we need to
// update it! First we save model with trees (i.e., make them available
// for scoring) and then update it with resulting error
_model.update(_key);
updated = true;
Log.info("============================================================== ");
SharedTreeModel.SharedTreeOutput out = _model._output;
_timeLastScoreStart = now;
// Score on training data
new ProgressUpdate("Scoring the model.").fork(_progressKey);
Score sc =
new Score(this, true, oob, _model._output.getModelCategory())
.doAll(train(), build_tree_one_node);
ModelMetrics mm = sc.makeModelMetrics(_model, _parms.train());
out._training_metrics = mm;
if (oob)
out._training_metrics._description = "Metrics reported on Out-Of-Bag training samples";
out._scored_train[out._ntrees].fillFrom(mm);
if (out._ntrees > 0) Log.info("Training " + out._scored_train[out._ntrees].toString());
// Score again on validation data
if (_parms._valid != null) {
Score scv =
new Score(this, false, false, _model._output.getModelCategory())
.doAll(valid(), build_tree_one_node);
ModelMetrics mmv = scv.makeModelMetrics(_model, _parms.valid());
out._validation_metrics = mmv;
out._scored_valid[out._ntrees].fillFrom(mmv);
if (out._ntrees > 0) Log.info("Validation " + out._scored_valid[out._ntrees].toString());
}
if (out._ntrees > 0) { // Compute variable importances
out._model_summary = createModelSummaryTable(out);
out._scoring_history = createScoringHistoryTable(out);
out._varimp = new hex.VarImp(_improvPerVar, out._names);
out._variable_importances = hex.ModelMetrics.calcVarImp(out._varimp);
Log.info(out._model_summary.toString());
// For Debugging:
// Log.info(out._scoring_history.toString());
// Log.info(out._variable_importances.toString());
}
ConfusionMatrix cm = mm.cm();
if (cm != null) {
if (cm._cm.length <= _parms._max_confusion_matrix_size) {
Log.info(cm.toASCII());
} else {
Log.info(
"Confusion Matrix is too large (max_confusion_matrix_size="
+ _parms._max_confusion_matrix_size
+ "): "
+ _nclass
+ " classes.");
}
}
_timeLastScoreEnd = System.currentTimeMillis();
}
// Double update - after either scoring or variable importance
if (updated) _model.update(_key);
return training_r2;
}
