// Handle the case where some centers go dry. Rescue only 1 cluster
// per iteration ('cause we only tracked the 1 worst row)
boolean cleanupBadClusters(
Lloyds task,
final Vec[] vecs,
final double[][] centers,
final double[] means,
final double[] mults,
final int[] modes) {
// Find any bad clusters
int clu;
for (clu = 0; clu < _parms._k; clu++) if (task._size[clu] == 0) break;
if (clu == _parms._k) return false; // No bad clusters
long row = task._worst_row;
Log.warn("KMeans: Re-initializing cluster " + clu + " to row " + row);
data(centers[clu] = task._cMeans[clu], vecs, row, means, mults, modes);
task._size[clu] =
1; // FIXME: PUBDEV-871 Some other cluster had their membership count reduced by one!
// (which one?)
// Find any MORE bad clusters; we only fixed the first one
for (clu = 0; clu < _parms._k; clu++) if (task._size[clu] == 0) break;
if (clu == _parms._k) return false; // No MORE bad clusters
// If we see 2 or more bad rows, just re-run Lloyds to get the
// next-worst row. We don't count this as an iteration, because
// we're not really adjusting the centers, we're trying to get
// some centers *at-all*.
Log.warn("KMeans: Re-running Lloyds to re-init another cluster");
if (_reinit_attempts++ < _parms._k) {
return true; // Rerun Lloyds, and assign points to centroids
} else {
_reinit_attempts = 0;
return false;
}
}
@Override
protected void checkMemoryFootPrint() {
if (_model._output._ntrees == 0) return;
int trees_so_far = _model._output._ntrees; // existing trees
long model_mem_size =
new ComputeModelSize(trees_so_far, _model._output._treeKeys).doAllNodes()._model_mem_size;
_model._output._treeStats._byte_size = model_mem_size;
double avg_tree_mem_size = (double) model_mem_size / trees_so_far;
Log.debug(
"Average tree size (for all classes): " + PrettyPrint.bytes((long) avg_tree_mem_size));
// all the compressed trees are stored on the driver node
long max_mem = H2O.SELF.get_max_mem();
if (_parms._ntrees * avg_tree_mem_size > max_mem) {
String msg =
"The tree model will not fit in the driver node's memory ("
+ PrettyPrint.bytes((long) avg_tree_mem_size)
+ " per tree x "
+ _parms._ntrees
+ " > "
+ PrettyPrint.bytes(max_mem)
+ ") - try decreasing ntrees and/or max_depth or increasing min_rows!";
error("_ntrees", msg);
cancel(msg);
}
}
// Main worker thread
@Override
protected void compute2() {
KMeansModel model = null;
try {
init(true);
// Do lock even before checking the errors, since this block is finalized by unlock
// (not the best solution, but the code is more readable)
_parms.read_lock_frames(KMeans.this); // Fetch & read-lock input frames
// Something goes wrong
if (error_count() > 0)
throw H2OModelBuilderIllegalArgumentException.makeFromBuilder(KMeans.this);
// The model to be built
model = new KMeansModel(dest(), _parms, new KMeansModel.KMeansOutput(KMeans.this));
model.delete_and_lock(_key);
//
final Vec vecs[] = _train.vecs();
// mults & means for standardization
final double[] means = _train.means(); // means are used to impute NAs
final double[] mults = _parms._standardize ? _train.mults() : null;
final int[] impute_cat = new int[vecs.length];
for (int i = 0; i < vecs.length; i++)
impute_cat[i] = vecs[i].isNumeric() ? -1 : DataInfo.imputeCat(vecs[i]);
model._output._normSub = means;
model._output._normMul = mults;
// Initialize cluster centers and standardize if requested
double[][] centers = initial_centers(model, vecs, means, mults, impute_cat);
if (centers == null) return; // Stopped/cancelled during center-finding
double[][] oldCenters = null;
// ---
// Run the main KMeans Clustering loop
// Stop after enough iterations or average_change < TOLERANCE
model._output._iterations =
0; // Loop ends only when iterations > max_iterations with strict inequality
while (!isDone(model, centers, oldCenters)) {
Lloyds task =
new Lloyds(centers, means, mults, impute_cat, _isCats, _parms._k, hasWeightCol())
.doAll(vecs);
// Pick the max categorical level for cluster center
max_cats(task._cMeans, task._cats, _isCats);
// Handle the case where some centers go dry. Rescue only 1 cluster
// per iteration ('cause we only tracked the 1 worst row)
if (cleanupBadClusters(task, vecs, centers, means, mults, impute_cat)) continue;
// Compute model stats; update standardized cluster centers
oldCenters = centers;
centers = computeStatsFillModel(task, model, vecs, means, mults, impute_cat);
model.update(_key); // Update model in K/V store
update(1); // One unit of work
if (model._parms._score_each_iteration) Log.info(model._output._model_summary);
}
Log.info(model._output._model_summary);
// Log.info(model._output._scoring_history);
//
// Log.info(((ModelMetricsClustering)model._output._training_metrics).createCentroidStatsTable().toString());
// At the end: validation scoring (no need to gather scoring history)
if (_valid != null) {
model.score(_parms.valid()).delete(); // this appends a ModelMetrics on the validation set
model._output._validation_metrics = ModelMetrics.getFromDKV(model, _parms.valid());
model.update(_key); // Update model in K/V store
}
done(); // Job done!
} catch (Throwable t) {
Job thisJob = DKV.getGet(_key);
if (thisJob._state == JobState.CANCELLED) {
Log.info("Job cancelled by user.");
} else {
t.printStackTrace();
failed(t);
throw t;
}
} finally {
updateModelOutput();
if (model != null) model.unlock(_key);
_parms.read_unlock_frames(KMeans.this);
}
tryComplete();
}
protected double doScoringAndSaveModel(
boolean finalScoring, boolean oob, boolean build_tree_one_node) {
double training_r2 = Double.NaN; // Training R^2 value, if computed
long now = System.currentTimeMillis();
if (_firstScore == 0) _firstScore = now;
long sinceLastScore = now - _timeLastScoreStart;
boolean updated = false;
new ProgressUpdate(
"Built " + _model._output._ntrees + " trees so far (out of " + _parms._ntrees + ").")
.fork(_progressKey);
// Now model already contains tid-trees in serialized form
if (_parms._score_each_iteration
|| finalScoring
|| (now - _firstScore < 4000)
|| // Score every time for 4 secs
// Throttle scoring to keep the cost sane; limit to a 10% duty cycle & every 4 secs
(sinceLastScore > 4000
&& // Limit scoring updates to every 4sec
(double) (_timeLastScoreEnd - _timeLastScoreStart) / sinceLastScore
< 0.1)) { // 10% duty cycle
checkMemoryFootPrint();
// If validation is specified we use a model for scoring, so we need to
// update it! First we save model with trees (i.e., make them available
// for scoring) and then update it with resulting error
_model.update(_key);
updated = true;
Log.info("============================================================== ");
SharedTreeModel.SharedTreeOutput out = _model._output;
_timeLastScoreStart = now;
// Score on training data
new ProgressUpdate("Scoring the model.").fork(_progressKey);
Score sc =
new Score(this, true, oob, _model._output.getModelCategory())
.doAll(train(), build_tree_one_node);
ModelMetrics mm = sc.makeModelMetrics(_model, _parms.train());
out._training_metrics = mm;
if (oob)
out._training_metrics._description = "Metrics reported on Out-Of-Bag training samples";
out._scored_train[out._ntrees].fillFrom(mm);
if (out._ntrees > 0) Log.info("Training " + out._scored_train[out._ntrees].toString());
// Score again on validation data
if (_parms._valid != null) {
Score scv =
new Score(this, false, false, _model._output.getModelCategory())
.doAll(valid(), build_tree_one_node);
ModelMetrics mmv = scv.makeModelMetrics(_model, _parms.valid());
out._validation_metrics = mmv;
out._scored_valid[out._ntrees].fillFrom(mmv);
if (out._ntrees > 0) Log.info("Validation " + out._scored_valid[out._ntrees].toString());
}
if (out._ntrees > 0) { // Compute variable importances
out._model_summary = createModelSummaryTable(out);
out._scoring_history = createScoringHistoryTable(out);
out._varimp = new hex.VarImp(_improvPerVar, out._names);
out._variable_importances = hex.ModelMetrics.calcVarImp(out._varimp);
Log.info(out._model_summary.toString());
// For Debugging:
// Log.info(out._scoring_history.toString());
// Log.info(out._variable_importances.toString());
}
ConfusionMatrix cm = mm.cm();
if (cm != null) {
if (cm._cm.length <= _parms._max_confusion_matrix_size) {
Log.info(cm.toASCII());
} else {
Log.info(
"Confusion Matrix is too large (max_confusion_matrix_size="
+ _parms._max_confusion_matrix_size
+ "): "
+ _nclass
+ " classes.");
}
}
_timeLastScoreEnd = System.currentTimeMillis();
}
// Double update - after either scoring or variable importance
if (updated) _model.update(_key);
return training_r2;
}
@Override
protected void compute2() {
_model = null; // Resulting model!
try {
Scope.enter(); // Cleanup temp keys
init(true); // Do any expensive tests & conversions now
// Do lock even before checking the errors, since this block is finalized by unlock
// (not the best solution, but the code is more readable)
_parms.read_lock_frames(SharedTree.this); // Fetch & read-lock input frames
if (error_count() > 0)
throw H2OModelBuilderIllegalArgumentException.makeFromBuilder(SharedTree.this);
// New Model? Or continuing from a checkpoint?
if (_parms._checkpoint && DKV.get(_parms._model_id) != null) {
_model = DKV.get(_dest).get();
_model.write_lock(_key); // do not delete previous model; we are extending it
} else { // New Model
// Compute the zero-tree error - guessing only the class distribution.
// MSE is stddev squared when guessing for regression.
// For classification, guess the largest class.
_model =
makeModel(
_dest,
_parms,
initial_MSE(_response, _response),
_valid == null
? Double.NaN
: initial_MSE(_response, _vresponse)); // Make a fresh model
_model.delete_and_lock(_key); // and clear & write-lock it (smashing any prior)
_model._output._init_f = _initialPrediction;
}
// Compute the response domain; makes for nicer printouts
String[] domain = _response.domain();
assert (_nclass > 1 && domain != null) || (_nclass == 1 && domain == null);
if (_nclass == 1) domain = new String[] {"r"}; // For regression, give a name to class 0
// Compute class distribution, used to for initial guesses and to
// upsample minority classes (if asked for).
if (_nclass > 1) { // Classification?
// Handle imbalanced classes by stratified over/under-sampling.
// initWorkFrame sets the modeled class distribution, and
// model.score() corrects the probabilities back using the
// distribution ratios
if (_model._output.isClassifier() && _parms._balance_classes) {
float[] trainSamplingFactors =
new float
[_train
.lastVec()
.domain()
.length]; // leave initialized to 0 -> will be filled up below
if (_parms._class_sampling_factors != null) {
if (_parms._class_sampling_factors.length != _train.lastVec().domain().length)
throw new IllegalArgumentException(
"class_sampling_factors must have "
+ _train.lastVec().domain().length
+ " elements");
trainSamplingFactors =
_parms._class_sampling_factors.clone(); // clone: don't modify the original
}
Frame stratified =
water.util.MRUtils.sampleFrameStratified(
_train,
_train.lastVec(),
_train.vec(_model._output.weightsName()),
trainSamplingFactors,
(long) (_parms._max_after_balance_size * _train.numRows()),
_parms._seed,
true,
false);
if (stratified != _train) {
_train = stratified;
_response = stratified.vec(_parms._response_column);
_weights = stratified.vec(_parms._weights_column);
// Recompute distribution since the input frame was modified
MRUtils.ClassDist cdmt2 =
_weights != null
? new MRUtils.ClassDist(_nclass).doAll(_response, _weights)
: new MRUtils.ClassDist(_nclass).doAll(_response);
_model._output._distribution = cdmt2.dist();
_model._output._modelClassDist = cdmt2.rel_dist();
}
}
Log.info("Prior class distribution: " + Arrays.toString(_model._output._priorClassDist));
Log.info("Model class distribution: " + Arrays.toString(_model._output._modelClassDist));
}
// Also add to the basic working Frame these sets:
// nclass Vecs of current forest results (sum across all trees)
// nclass Vecs of working/temp data
// nclass Vecs of NIDs, allowing 1 tree per class
// Current forest values: results of summing the prior M trees
for (int i = 0; i < _nclass; i++) _train.add("Tree_" + domain[i], _response.makeZero());
// Initial work columns. Set-before-use in the algos.
for (int i = 0; i < _nclass; i++) _train.add("Work_" + domain[i], _response.makeZero());
// One Tree per class, each tree needs a NIDs. For empty classes use a -1
// NID signifying an empty regression tree.
for (int i = 0; i < _nclass; i++)
_train.add(
"NIDs_" + domain[i],
_response.makeCon(
_model._output._distribution == null
? 0
: (_model._output._distribution[i] == 0 ? -1 : 0)));
// Tag out rows missing the response column
new ExcludeNAResponse().doAll(_train);
// Variable importance: squared-error-improvement-per-variable-per-split
_improvPerVar = new float[_ncols];
// Sub-class tree-model-builder specific build code
buildModel();
done(); // Job done!
} catch (Throwable t) {
Job thisJob = DKV.getGet(_key);
if (thisJob._state == JobState.CANCELLED) {
Log.info("Job cancelled by user.");
} else {
t.printStackTrace();
failed(t);
throw t;
}
} finally {
if (_model != null) _model.unlock(_key);
_parms.read_unlock_frames(SharedTree.this);
if (_model == null) Scope.exit();
else {
Scope.exit(
_model._key,
ModelMetrics.buildKey(_model, _parms.train()),
ModelMetrics.buildKey(_model, _parms.valid()));
}
}
tryComplete();
}
/**
* Simple GLM wrapper to enable launching GLM from command line.
*
* <p>Example input: java -jar target/h2o.jar -name=test -runMethod water.util.GLMRunner
* -file=smalldata/logreg/prostate.csv -y=CAPSULE -family=binomial
*
* @param args
* @throws InterruptedException
*/
public static void main(String[] args) throws InterruptedException {
try {
GLMArgs ARGS = new GLMArgs();
new Arguments(args).extract(ARGS);
System.out.println("==================<GLMRunner START>===================");
ValueArray ary = Utils.loadAndParseKey(ARGS.file);
int ycol;
try {
ycol = Integer.parseInt(ARGS.y);
} catch (NumberFormatException e) {
ycol = ary.getColumnIds(new String[] {ARGS.y})[0];
}
int ncols = ary.numCols();
if (ycol < 0 || ycol >= ary.numCols()) {
System.err.println("invalid y column: " + ycol);
H2O.exit(-1);
}
int[] xcols;
if (ARGS.xs.equalsIgnoreCase("all")) {
xcols = new int[ncols - 1];
for (int i = 0; i < ycol; ++i) xcols[i] = i;
for (int i = ycol; i < ncols - 1; ++i) xcols[i] = i + 1;
} else {
System.out.println("xs = " + ARGS.xs);
String[] names = ARGS.xs.split(",");
xcols = new int[names.length];
try {
for (int i = 0; i < names.length; ++i) xcols[i] = Integer.valueOf(names[i]);
} catch (NumberFormatException e) {
xcols = ary.getColumnIds(ARGS.xs.split(","));
}
}
for (int x : xcols)
if (x < 0) {
System.err.println("Invalid predictor specification " + ARGS.xs);
H2O.exit(-1);
}
GLMJob j =
DGLM.startGLMJob(
DGLM.getData(ary, xcols, ycol, null, true),
new ADMMSolver(ARGS.lambda, ARGS._alpha),
new GLMParams(Family.valueOf(ARGS.family)),
null,
ARGS.xval,
true);
System.out.print("[GLM] computing model...");
int progress = 0;
while (!j.isDone()) {
int p = (int) (100 * j.progress());
int dots = p - progress;
progress = p;
for (int i = 0; i < dots; ++i) System.out.print('.');
Thread.sleep(250);
}
Log.debug(Sys.GENLM, "DONE.");
GLMModel m = j.get();
String[] colnames = ary.colNames();
System.out.println("Intercept" + " = " + m._beta[ncols - 1]);
for (int i = 0; i < xcols.length; ++i) {
System.out.println(colnames[i] + " = " + m._beta[i]);
}
} catch (Throwable t) {
Log.err(t);
} finally { // we're done. shutdown the cloud
Log.debug(Sys.GENLM, "==================<GLMRunner DONE>===================");
UDPRebooted.suicide(UDPRebooted.T.shutdown, H2O.SELF);
}
}
@Override
protected void compute2() {
CoxPHModel model = null;
try {
Scope.enter();
_parms.read_lock_frames(CoxPH.this);
init(true);
applyScoringFrameSideEffects();
// The model to be built
model = new CoxPHModel(dest(), _parms, new CoxPHModel.CoxPHOutput(CoxPH.this));
model.delete_and_lock(_key);
applyTrainingFrameSideEffects();
int nResponses = 1;
boolean useAllFactorLevels = false;
final DataInfo dinfo =
new DataInfo(
Key.make(),
_modelBuilderTrain,
null,
nResponses,
useAllFactorLevels,
DataInfo.TransformType.DEMEAN,
TransformType.NONE,
true,
false,
false,
false,
false,
false);
initStats(model, dinfo);
final int n_offsets =
(model._parms.offset_columns == null) ? 0 : model._parms.offset_columns.length;
final int n_coef = dinfo.fullN() - n_offsets;
final double[] step = MemoryManager.malloc8d(n_coef);
final double[] oldCoef = MemoryManager.malloc8d(n_coef);
final double[] newCoef = MemoryManager.malloc8d(n_coef);
Arrays.fill(step, Double.NaN);
Arrays.fill(oldCoef, Double.NaN);
for (int j = 0; j < n_coef; ++j) newCoef[j] = model._parms.init;
double oldLoglik = -Double.MAX_VALUE;
final int n_time = (int) (model._output.max_time - model._output.min_time + 1);
final boolean has_start_column = (model._parms.start_column != null);
final boolean has_weights_column = (model._parms.weights_column != null);
for (int i = 0; i <= model._parms.iter_max; ++i) {
model._output.iter = i;
final CoxPHTask coxMR =
new CoxPHTask(
self(),
dinfo,
newCoef,
model._output.min_time,
n_time,
n_offsets,
has_start_column,
has_weights_column)
.doAll(dinfo._adaptedFrame);
final double newLoglik = calcLoglik(model, coxMR);
if (newLoglik > oldLoglik) {
if (i == 0) calcCounts(model, coxMR);
calcModelStats(model, newCoef, newLoglik);
calcCumhaz_0(model, coxMR);
if (newLoglik == 0) model._output.lre = -Math.log10(Math.abs(oldLoglik - newLoglik));
else model._output.lre = -Math.log10(Math.abs((oldLoglik - newLoglik) / newLoglik));
if (model._output.lre >= model._parms.lre_min) break;
Arrays.fill(step, 0);
for (int j = 0; j < n_coef; ++j)
for (int k = 0; k < n_coef; ++k)
step[j] -= model._output.var_coef[j][k] * model._output.gradient[k];
for (int j = 0; j < n_coef; ++j)
if (Double.isNaN(step[j]) || Double.isInfinite(step[j])) break;
oldLoglik = newLoglik;
System.arraycopy(newCoef, 0, oldCoef, 0, oldCoef.length);
} else {
for (int j = 0; j < n_coef; ++j) step[j] /= 2;
}
for (int j = 0; j < n_coef; ++j) newCoef[j] = oldCoef[j] - step[j];
}
model.update(_key);
} catch (Throwable t) {
Job thisJob = DKV.getGet(_key);
if (thisJob._state == JobState.CANCELLED) {
Log.info("Job cancelled by user.");
} else {
t.printStackTrace();
failed(t);
throw t;
}
} finally {
updateModelOutput();
_parms.read_unlock_frames(CoxPH.this);
Scope.exit();
done(); // Job done!
}
tryComplete();
}
