/**
* Train a Deep Learning neural net model
*
* @param model Input model (e.g., from initModel(), or from a previous training run)
* @return Trained model
*/
public final DeepLearningModel trainModel(DeepLearningModel model) {
Frame validScoreFrame = null;
Frame train, trainScoreFrame;
try {
// if (checkpoint == null && !quiet_mode) logStart(); //if checkpoint is given, some
// Job's params might be uninitialized (but the restarted model's parameters are correct)
if (model == null) {
model = DKV.get(dest()).get();
}
Log.info(
"Model category: "
+ (_parms._autoencoder
? "Auto-Encoder"
: isClassifier() ? "Classification" : "Regression"));
final long model_size = model.model_info().size();
Log.info(
"Number of model parameters (weights/biases): " + String.format("%,d", model_size));
model.write_lock(_job);
_job.update(0, "Setting up training data...");
final DeepLearningParameters mp = model.model_info().get_params();
// temporary frames of the same "name" as the orig _train/_valid (asking the parameter's
// Key, not the actual frame)
// Note: don't put into DKV or they would overwrite the _train/_valid frames!
Frame tra_fr = new Frame(mp._train, _train.names(), _train.vecs());
Frame val_fr = _valid != null ? new Frame(mp._valid, _valid.names(), _valid.vecs()) : null;
train = tra_fr;
if (model._output.isClassifier() && mp._balance_classes) {
_job.update(0, "Balancing class distribution of training data...");
float[] trainSamplingFactors =
new float
[train
.lastVec()
.domain()
.length]; // leave initialized to 0 -> will be filled up below
if (mp._class_sampling_factors != null) {
if (mp._class_sampling_factors.length != train.lastVec().domain().length)
throw new IllegalArgumentException(
"class_sampling_factors must have "
+ train.lastVec().domain().length
+ " elements");
trainSamplingFactors =
mp._class_sampling_factors.clone(); // clone: don't modify the original
}
train =
sampleFrameStratified(
train,
train.lastVec(),
train.vec(model._output.weightsName()),
trainSamplingFactors,
(long) (mp._max_after_balance_size * train.numRows()),
mp._seed,
true,
false);
Vec l = train.lastVec();
Vec w = train.vec(model._output.weightsName());
MRUtils.ClassDist cd = new MRUtils.ClassDist(l);
model._output._modelClassDist =
_weights != null ? cd.doAll(l, w).rel_dist() : cd.doAll(l).rel_dist();
}
model.training_rows = train.numRows();
if (_weights != null && _weights.min() == 0 && _weights.max() == 1 && _weights.isInt()) {
model.training_rows = Math.round(train.numRows() * _weights.mean());
Log.warn(
"Not counting "
+ (train.numRows() - model.training_rows)
+ " rows with weight=0 towards an epoch.");
}
Log.info("One epoch corresponds to " + model.training_rows + " training data rows.");
trainScoreFrame =
sampleFrame(
train,
mp._score_training_samples,
mp._seed); // training scoring dataset is always sampled uniformly from the training
// dataset
if (trainScoreFrame != train) Scope.track(trainScoreFrame);
if (!_parms._quiet_mode)
Log.info("Number of chunks of the training data: " + train.anyVec().nChunks());
if (val_fr != null) {
model.validation_rows = val_fr.numRows();
// validation scoring dataset can be sampled in multiple ways from the given validation
// dataset
if (model._output.isClassifier()
&& mp._balance_classes
&& mp._score_validation_sampling
== DeepLearningParameters.ClassSamplingMethod.Stratified) {
_job.update(0, "Sampling validation data (stratified)...");
validScoreFrame =
sampleFrameStratified(
val_fr,
val_fr.lastVec(),
val_fr.vec(model._output.weightsName()),
null,
mp._score_validation_samples > 0
? mp._score_validation_samples
: val_fr.numRows(),
mp._seed + 1,
false /* no oversampling */,
false);
} else {
_job.update(0, "Sampling validation data...");
validScoreFrame = sampleFrame(val_fr, mp._score_validation_samples, mp._seed + 1);
if (validScoreFrame != val_fr) Scope.track(validScoreFrame);
}
if (!_parms._quiet_mode)
Log.info(
"Number of chunks of the validation data: " + validScoreFrame.anyVec().nChunks());
}
// Set train_samples_per_iteration size (cannot be done earlier since this depends on
// whether stratified sampling is done)
model.actual_train_samples_per_iteration =
computeTrainSamplesPerIteration(mp, model.training_rows, model);
// Determine whether shuffling is enforced
if (mp._replicate_training_data
&& (model.actual_train_samples_per_iteration
== model.training_rows * (mp._single_node_mode ? 1 : H2O.CLOUD.size()))
&& !mp._shuffle_training_data
&& H2O.CLOUD.size() > 1
&& !mp._reproducible) {
if (!mp._quiet_mode)
Log.info(
"Enabling training data shuffling, because all nodes train on the full dataset (replicated training data).");
mp._shuffle_training_data = true;
}
if (!mp._shuffle_training_data
&& model.actual_train_samples_per_iteration == model.training_rows
&& train.anyVec().nChunks() == 1) {
if (!mp._quiet_mode)
Log.info(
"Enabling training data shuffling to avoid training rows in the same order over and over (no Hogwild since there's only 1 chunk).");
mp._shuffle_training_data = true;
}
// if (!mp._quiet_mode) Log.info("Initial model:\n" + model.model_info());
long now = System.currentTimeMillis();
model._timeLastIterationEnter = now;
if (_parms._autoencoder) {
_job.update(0, "Scoring null model of autoencoder...");
if (!mp._quiet_mode) Log.info("Scoring the null model of the autoencoder.");
model.doScoring(
trainScoreFrame,
validScoreFrame,
_job._key,
0,
false); // get the null model reconstruction error
}
// put the initial version of the model into DKV
model.update(_job);
model.total_setup_time_ms += now - _job.start_time();
Log.info("Total setup time: " + PrettyPrint.msecs(model.total_setup_time_ms, true));
Log.info("Starting to train the Deep Learning model.");
_job.update(0, "Training...");
// main loop
for (; ; ) {
model.iterations++;
model.set_model_info(
mp._epochs == 0
? model.model_info()
: H2O.CLOUD.size() > 1 && mp._replicate_training_data
? (mp._single_node_mode
? new DeepLearningTask2(
_job._key,
train,
model.model_info(),
rowFraction(train, mp, model),
model.iterations)
.doAll(Key.make(H2O.SELF))
.model_info()
: // replicated data + single node mode
new DeepLearningTask2(
_job._key,
train,
model.model_info(),
rowFraction(train, mp, model),
model.iterations)
.doAllNodes()
.model_info())
: // replicated data + multi-node mode
new DeepLearningTask(
_job._key,
model.model_info(),
rowFraction(train, mp, model),
model.iterations)
.doAll(train)
.model_info()); // distributed data (always in multi-node mode)
if (stop_requested() && !timeout()) break; // cancellation
if (!model.doScoring(
trainScoreFrame, validScoreFrame, _job._key, model.iterations, false))
break; // finished training (or early stopping or convergence)
if (timeout()) break; // stop after scoring
}
// replace the model with the best model so far (if it's better)
if (!stop_requested()
&& _parms._overwrite_with_best_model
&& model.actual_best_model_key != null
&& _parms._nfolds == 0) {
DeepLearningModel best_model = DKV.getGet(model.actual_best_model_key);
if (best_model != null
&& best_model.loss() < model.loss()
&& Arrays.equals(best_model.model_info().units, model.model_info().units)) {
if (!_parms._quiet_mode)
Log.info("Setting the model to be the best model so far (based on scoring history).");
DeepLearningModelInfo mi = best_model.model_info().deep_clone();
// Don't cheat - count full amount of training samples, since that's the amount of
// training it took to train (without finding anything better)
mi.set_processed_global(model.model_info().get_processed_global());
mi.set_processed_local(model.model_info().get_processed_local());
model.set_model_info(mi);
model.update(_job);
model.doScoring(trainScoreFrame, validScoreFrame, _job._key, model.iterations, true);
assert (best_model.loss() == model.loss());
}
}
// store coefficient names for future use
// possibly change
model.model_info().data_info().coefNames();
if (!_parms._quiet_mode) {
Log.info(
"==============================================================================================================================================================================");
if (stop_requested()) {
Log.info("Deep Learning model training was interrupted.");
} else {
Log.info("Finished training the Deep Learning model.");
Log.info(model);
}
Log.info(
"==============================================================================================================================================================================");
}
} finally {
if (model != null) {
model.deleteElasticAverageModels();
model.unlock(_job);
if (model.actual_best_model_key != null) {
assert (model.actual_best_model_key != model._key);
DKV.remove(model.actual_best_model_key);
}
}
}
return model;
}
/**
* Compute the actual train_samples_per_iteration size from the user-given parameter
*
* @param mp Model parameter (DeepLearning object)
* @param numRows number of training rows
* @param model DL model
* @return The total number of training rows to be processed per iteration (summed over on all
* nodes)
*/
private long computeTrainSamplesPerIteration(
final DeepLearningParameters mp, final long numRows, final DeepLearningModel model) {
long tspi = mp._train_samples_per_iteration;
assert (tspi == 0 || tspi == -1 || tspi == -2 || tspi >= 1);
if (tspi == 0 || (!mp._replicate_training_data && tspi == -1)) {
tspi = numRows;
if (!mp._quiet_mode)
Log.info(
"Setting train_samples_per_iteration ("
+ mp._train_samples_per_iteration
+ ") to one epoch: #rows ("
+ tspi
+ ").");
} else if (tspi == -1) {
tspi = (mp._single_node_mode ? 1 : H2O.CLOUD.size()) * numRows;
if (!mp._quiet_mode)
Log.info(
"Setting train_samples_per_iteration ("
+ mp._train_samples_per_iteration
+ ") to #nodes x #rows ("
+ tspi
+ ").");
} else if (tspi == -2) {
// automatic tuning based on CPU speed, network speed and model size
// measure cpu speed
double total_gflops = 0;
for (H2ONode h2o : H2O.CLOUD._memary) {
HeartBeat hb = h2o._heartbeat;
total_gflops += hb._gflops; // can be NaN if not yet run
}
if (mp._single_node_mode) total_gflops /= H2O.CLOUD.size();
if (Double.isNaN(total_gflops)) {
total_gflops =
Linpack.run(H2O.SELF._heartbeat._cpus_allowed)
* (mp._single_node_mode ? 1 : H2O.CLOUD.size());
}
assert (!Double.isNaN(total_gflops));
final long model_size = model.model_info().size();
int[] msg_sizes =
new int[] {
1,
(int) (model_size * 4) == (model_size * 4)
? (int) (model_size * 4)
: Integer.MAX_VALUE
};
double[] microseconds_collective = new double[msg_sizes.length];
NetworkTest.NetworkTester nt =
new NetworkTest.NetworkTester(
msg_sizes,
null,
microseconds_collective,
model_size > 1e6 ? 1 : 5 /*repeats*/,
false,
true /*only collectives*/);
nt.compute2();
// length of the network traffic queue based on log-tree rollup (2 log(nodes))
int network_queue_length =
mp._single_node_mode || H2O.CLOUD.size() == 1
? 1
: 2 * (int) Math.floor(Math.log(H2O.CLOUD.size()) / Math.log(2));
// heuristics
double flops_overhead_per_row = 50;
if (mp._activation == DeepLearningParameters.Activation.Maxout
|| mp._activation == DeepLearningParameters.Activation.MaxoutWithDropout) {
flops_overhead_per_row *= 8;
} else if (mp._activation == DeepLearningParameters.Activation.Tanh
|| mp._activation == DeepLearningParameters.Activation.TanhWithDropout) {
flops_overhead_per_row *= 5;
}
// target fraction of comm vs cpu time: 5%
double fraction =
mp._single_node_mode || H2O.CLOUD.size() == 1
? 1e-3
: mp._target_ratio_comm_to_comp; // one single node mode, there's no model averaging
// effect, so less need to shorten the M/R
// iteration
// estimate the time for communication (network) and training (compute)
model.time_for_communication_us =
(H2O.CLOUD.size() == 1
? 1e4 /* add 10ms for single-node */
: 1e5 /* add 100ms for multi-node MR overhead */)
+ network_queue_length * microseconds_collective[1];
double time_per_row_us =
(flops_overhead_per_row * model_size + 10000 * model.model_info().units[0])
/ (total_gflops * 1e9)
/ H2O.SELF._heartbeat._cpus_allowed
* 1e6;
assert (!Double.isNaN(time_per_row_us));
// compute the optimal number of training rows per iteration
// fraction := time_comm_us / (time_comm_us + tspi * time_per_row_us) ==> tspi =
// (time_comm_us/fraction - time_comm_us)/time_per_row_us
tspi =
(long)
((model.time_for_communication_us / fraction - model.time_for_communication_us)
/ time_per_row_us);
tspi =
Math.min(
tspi,
(mp._single_node_mode ? 1 : H2O.CLOUD.size())
* numRows
* 10); // not more than 10x of what train_samples_per_iteration=-1 would do
// If the number is close to a multiple of epochs, use that -> prettier scoring
if (tspi > numRows && Math.abs(tspi % numRows) / (double) numRows < 0.2)
tspi -= tspi % numRows;
tspi =
Math.min(
tspi,
(long)
(mp._epochs
* numRows
/ 10)); // limit to number of epochs desired, but at least 10 iterations
// total
if (H2O.CLOUD.size() == 1 || mp._single_node_mode) {
tspi =
Math.min(
tspi,
10
* (int)
(1e6
/ time_per_row_us)); // in single-node mode, only run for at most 10
// seconds
}
tspi = Math.max(1, tspi); // at least 1 row
tspi =
Math.min(
100000 * H2O.CLOUD.size(),
tspi); // at most 100k rows per node for initial guess - can always relax later on
if (!mp._quiet_mode) {
Log.info("Auto-tuning parameter 'train_samples_per_iteration':");
Log.info("Estimated compute power : " + Math.round(total_gflops * 100) / 100 + " GFlops");
Log.info(
"Estimated time for comm : "
+ PrettyPrint.usecs((long) model.time_for_communication_us));
Log.info(
"Estimated time per row : "
+ ((long) time_per_row_us > 0
? PrettyPrint.usecs((long) time_per_row_us)
: time_per_row_us + " usecs"));
Log.info("Estimated training speed: " + (int) (1e6 / time_per_row_us) + " rows/sec");
Log.info(
"Setting train_samples_per_iteration ("
+ mp._train_samples_per_iteration
+ ") to auto-tuned value: "
+ tspi);
}
} else {
// limit user-given value to number of epochs desired
tspi = Math.max(1, Math.min(tspi, (long) (mp._epochs * numRows)));
}
assert (tspi != 0 && tspi != -1 && tspi != -2 && tspi >= 1);
model.tspiGuess = tspi;
return tspi;
}
/**
* Train a Deep Learning model, assumes that all members are populated If checkpoint == null,
* then start training a new model, otherwise continue from a checkpoint
*/
public final void buildModel() {
DeepLearningModel cp = null;
if (_parms._checkpoint == null) {
cp =
new DeepLearningModel(
dest(),
_parms,
new DeepLearningModel.DeepLearningModelOutput(DeepLearning.this),
_train,
_valid,
nclasses());
cp.model_info().initializeMembers();
} else {
final DeepLearningModel previous = DKV.getGet(_parms._checkpoint);
if (previous == null) throw new IllegalArgumentException("Checkpoint not found.");
Log.info("Resuming from checkpoint.");
_job.update(0, "Resuming from checkpoint");
if (isClassifier() != previous._output.isClassifier())
throw new H2OIllegalArgumentException(
"Response type must be the same as for the checkpointed model.");
if (isSupervised() != previous._output.isSupervised())
throw new H2OIllegalArgumentException(
"Model type must be the same as for the checkpointed model.");
// check the user-given arguments for consistency
DeepLearningParameters oldP =
previous._parms; // sanitized parameters for checkpointed model
DeepLearningParameters newP = _parms; // user-given parameters for restart
DeepLearningParameters oldP2 = (DeepLearningParameters) oldP.clone();
DeepLearningParameters newP2 = (DeepLearningParameters) newP.clone();
DeepLearningParameters.Sanity.modifyParms(
oldP, oldP2, nclasses()); // sanitize the user-given parameters
DeepLearningParameters.Sanity.modifyParms(
newP, newP2, nclasses()); // sanitize the user-given parameters
DeepLearningParameters.Sanity.checkpoint(oldP2, newP2);
DataInfo dinfo;
try {
// PUBDEV-2513: Adapt _train and _valid (in-place) to match the frames that were used for
// the previous model
// This can add or remove dummy columns (can happen if the dataset is sparse and datasets
// have different non-const columns)
for (String st : previous.adaptTestForTrain(_train, true, false)) Log.warn(st);
for (String st : previous.adaptTestForTrain(_valid, true, false)) Log.warn(st);
dinfo = makeDataInfo(_train, _valid, _parms, nclasses());
DKV.put(dinfo);
cp = new DeepLearningModel(dest(), _parms, previous, false, dinfo);
cp.write_lock(_job);
if (!Arrays.equals(cp._output._names, previous._output._names)) {
throw new H2OIllegalArgumentException(
"The columns of the training data must be the same as for the checkpointed model. Check ignored columns (or disable ignore_const_cols).");
}
if (!Arrays.deepEquals(cp._output._domains, previous._output._domains)) {
throw new H2OIllegalArgumentException(
"Categorical factor levels of the training data must be the same as for the checkpointed model.");
}
if (dinfo.fullN() != previous.model_info().data_info().fullN()) {
throw new H2OIllegalArgumentException(
"Total number of predictors is different than for the checkpointed model.");
}
if (_parms._epochs <= previous.epoch_counter) {
throw new H2OIllegalArgumentException(
"Total number of epochs must be larger than the number of epochs already trained for the checkpointed model ("
+ previous.epoch_counter
+ ").");
}
// these are the mutable parameters that are to be used by the model (stored in
// model_info._parms)
final DeepLearningParameters actualNewP =
cp.model_info()
.get_params(); // actually used parameters for model building (defaults filled in,
// etc.)
assert (actualNewP != previous.model_info().get_params());
assert (actualNewP != newP);
assert (actualNewP != oldP);
DeepLearningParameters.Sanity.update(actualNewP, newP, nclasses());
Log.info(
"Continuing training after "
+ String.format("%.3f", previous.epoch_counter)
+ " epochs from the checkpointed model.");
cp.update(_job);
} catch (H2OIllegalArgumentException ex) {
if (cp != null) {
cp.unlock(_job);
cp.delete();
cp = null;
}
throw ex;
} finally {
if (cp != null) cp.unlock(_job);
}
}
trainModel(cp);
// clean up, but don't delete weights and biases if user asked for export
List<Key> keep = new ArrayList<>();
try {
if (_parms._export_weights_and_biases
&& cp._output.weights != null
&& cp._output.biases != null) {
for (Key k : Arrays.asList(cp._output.weights)) {
keep.add(k);
for (Vec vk : ((Frame) DKV.getGet(k)).vecs()) {
keep.add(vk._key);
}
}
for (Key k : Arrays.asList(cp._output.biases)) {
keep.add(k);
for (Vec vk : ((Frame) DKV.getGet(k)).vecs()) {
keep.add(vk._key);
}
}
}
} finally {
Scope.exit(keep.toArray(new Key[keep.size()]));
}
}