@Test
public void testDomains() {
Frame frame = parse_test_file("smalldata/junit/weather.csv");
for (String s : new String[] {"MaxWindSpeed", "RelHumid9am", "Cloud9am"}) {
Vec v = frame.vec(s);
Vec newV = v.toCategoricalVec();
frame.remove(s);
frame.add(s, newV);
v.remove();
}
DKV.put(frame);
AggregatorModel.AggregatorParameters parms = new AggregatorModel.AggregatorParameters();
parms._train = frame._key;
parms._radius_scale = 10;
AggregatorModel agg = new Aggregator(parms).trainModel().get();
Frame output = agg._output._output_frame.get();
Assert.assertTrue(output.numRows() < 0.5 * frame.numRows());
boolean same = true;
for (int i = 0; i < frame.numCols(); ++i) {
if (frame.vec(i).isCategorical()) {
same = (frame.domains()[i].length == output.domains()[i].length);
if (!same) break;
}
}
frame.remove();
output.remove();
agg.remove();
Assert.assertFalse(same);
}
static Frame exec_str(String str, String id) {
Val val = Exec.exec(str);
switch (val.type()) {
case Val.FRM:
Frame fr = val.getFrame();
Key k = Key.make(id);
// Smart delete any prior top-level result
Iced i = DKV.getGet(k);
if (i instanceof Lockable) ((Lockable) i).delete();
else if (i instanceof Keyed) ((Keyed) i).remove();
else if (i != null)
throw new IllegalArgumentException("Attempting to overright an unexpected key");
DKV.put(fr = new Frame(k, fr._names, fr.vecs()));
System.out.println(fr);
checkSaneFrame();
return fr;
case Val.NUM:
System.out.println("num= " + val.getNum());
assert id == null;
checkSaneFrame();
return null;
case Val.STR:
System.out.println("str= " + val.getStr());
assert id == null;
checkSaneFrame();
return null;
default:
throw water.H2O.fail();
}
}
public GLMModelV3 make_model(int version, MakeGLMModelV3 args) {
GLMModel model = DKV.getGet(args.model.key());
if (model == null) throw new IllegalArgumentException("missing source model " + args.model);
String[] names = model._output.coefficientNames();
Map<String, Double> coefs = model.coefficients();
for (int i = 0; i < args.names.length; ++i) coefs.put(args.names[i], args.beta[i]);
double[] beta = model.beta().clone();
for (int i = 0; i < beta.length; ++i) beta[i] = coefs.get(names[i]);
GLMModel m =
new GLMModel(
args.dest != null ? args.dest.key() : Key.make(),
model._parms,
null,
new double[] {.5},
Double.NaN,
Double.NaN,
-1);
DataInfo dinfo = model.dinfo();
dinfo.setPredictorTransform(TransformType.NONE);
// GLMOutput(DataInfo dinfo, String[] column_names, String[][] domains, String[]
// coefficient_names, boolean binomial) {
m._output =
new GLMOutput(
model.dinfo(),
model._output._names,
model._output._domains,
model._output.coefficientNames(),
model._output._binomial,
beta);
DKV.put(m._key, m);
GLMModelV3 res = new GLMModelV3();
res.fillFromImpl(m);
return res;
}
@Override
protected Frame predictScoreImpl(Frame orig, Frame adaptedFr, String destination_key) {
Frame adaptFrm = new Frame(adaptedFr);
for (int i = 0; i < _parms._k; i++)
adaptFrm.add("PC" + String.valueOf(i + 1), adaptFrm.anyVec().makeZero());
new MRTask() {
@Override
public void map(Chunk chks[]) {
double tmp[] = new double[_output._names.length];
double preds[] = new double[_parms._k];
for (int row = 0; row < chks[0]._len; row++) {
double p[] = score0(chks, row, tmp, preds);
for (int c = 0; c < preds.length; c++) chks[_output._names.length + c].set(row, p[c]);
}
}
}.doAll(adaptFrm);
// Return the projection into principal component space
int x = _output._names.length, y = adaptFrm.numCols();
Frame f =
adaptFrm.extractFrame(
x, y); // this will call vec_impl() and we cannot call the delete() below just yet
f =
new Frame(
(null == destination_key ? Key.make() : Key.make(destination_key)),
f.names(),
f.vecs());
DKV.put(f);
makeMetricBuilder(null).makeModelMetrics(this, orig);
return f;
}
private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
try {
if (fs == null) return;
for (FileStatus file : fs.listStatus(p)) {
Path pfs = file.getPath();
if (file.isDir()) {
addFolder(fs, pfs, succeeded, failed);
} else {
Key k = Key.make(pfs.toString());
long size = file.getLen();
Value val = null;
if (pfs.getName().endsWith(Extensions.JSON)) {
JsonParser parser = new JsonParser();
JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
JsonElement v = json.get(Constants.VERSION);
if (v == null) throw new RuntimeException("Missing version");
JsonElement type = json.get(Constants.TYPE);
if (type == null) throw new RuntimeException("Missing type");
Class c = Class.forName(type.getAsString());
OldModel model = (OldModel) c.newInstance();
model.fromJson(json);
} else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
FSDataInputStream s = fs.open(pfs);
int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
byte[] mem = MemoryManager.malloc1(sz);
s.readFully(mem);
// Convert to a ValueArray (hope it fits in 1Meg!)
ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
val = new Value(k, ary, Value.HDFS);
} else if (size >= 2 * ValueArray.CHUNK_SZ) {
val =
new Value(
k,
new ValueArray(k, size),
Value.HDFS); // ValueArray byte wrapper over a large file
} else {
val = new Value(k, (int) size, Value.HDFS); // Plain Value
val.setdsk();
}
DKV.put(k, val);
Log.info("PersistHdfs: DKV.put(" + k + ")");
JsonObject o = new JsonObject();
o.addProperty(Constants.KEY, k.toString());
o.addProperty(Constants.FILE, pfs.toString());
o.addProperty(Constants.VALUE_SIZE, file.getLen());
succeeded.add(o);
}
}
} catch (Exception e) {
Log.err(e);
JsonObject o = new JsonObject();
o.addProperty(Constants.FILE, p.toString());
o.addProperty(Constants.ERROR, e.getMessage());
failed.add(o);
}
}
@Test
public void testCategoricalProstate() throws InterruptedException, ExecutionException {
GLRM job = null;
GLRMModel model = null;
Frame train = null;
final int[] cats = new int[] {1, 3, 4, 5}; // Categoricals: CAPSULE, RACE, DPROS, DCAPS
try {
Scope.enter();
train = parse_test_file(Key.make("prostate.hex"), "smalldata/logreg/prostate.csv");
for (int i = 0; i < cats.length; i++)
Scope.track(train.replace(cats[i], train.vec(cats[i]).toCategoricalVec())._key);
train.remove("ID").remove();
DKV.put(train._key, train);
GLRMParameters parms = new GLRMParameters();
parms._train = train._key;
parms._k = 8;
parms._gamma_x = parms._gamma_y = 0.1;
parms._regularization_x = GLRMModel.GLRMParameters.Regularizer.Quadratic;
parms._regularization_y = GLRMModel.GLRMParameters.Regularizer.Quadratic;
parms._init = GLRM.Initialization.PlusPlus;
parms._transform = DataInfo.TransformType.STANDARDIZE;
parms._recover_svd = false;
parms._max_iterations = 200;
try {
job = new GLRM(parms);
model = job.trainModel().get();
Log.info(
"Iteration "
+ model._output._iterations
+ ": Objective value = "
+ model._output._objective);
model.score(train).delete();
ModelMetricsGLRM mm = (ModelMetricsGLRM) ModelMetrics.getFromDKV(model, train);
Log.info(
"Numeric Sum of Squared Error = "
+ mm._numerr
+ "\tCategorical Misclassification Error = "
+ mm._caterr);
} catch (Throwable t) {
t.printStackTrace();
throw new RuntimeException(t);
} finally {
job.remove();
}
} catch (Throwable t) {
t.printStackTrace();
throw new RuntimeException(t);
} finally {
if (train != null) train.delete();
if (model != null) model.delete();
Scope.exit();
}
}
// Close all AppendableVec
public Futures closeAppendables(Futures fs) {
_col0 = null; // Reset cache
int len = vecs().length;
for (int i = 0; i < len; i++) {
Vec v = _vecs[i];
if (v instanceof AppendableVec)
DKV.put(_keys[i], _vecs[i] = ((AppendableVec) v).close(fs), fs);
}
return fs;
}
public Vec replace(int col, Vec nv) {
assert col < _names.length;
Vec rv = vecs()[col];
assert rv.group().equals(nv.group());
_vecs[col] = nv;
_keys[col] = nv._key;
if (DKV.get(nv._key) == null) // If not already in KV, put it there
DKV.put(nv._key, nv);
return rv;
}
// @Ignore("PUBDEV-1643")
@Test
public void testDuplicatesCarsGrid() {
Grid grid = null;
Frame fr = null;
Vec old = null;
try {
fr = parse_test_file("smalldata/junit/cars_20mpg.csv");
fr.remove("name").remove(); // Remove unique id
old = fr.remove("economy");
fr.add("economy", old); // response to last column
DKV.put(fr);
// Setup random hyperparameter search space
HashMap<String, Object[]> hyperParms =
new HashMap<String, Object[]>() {
{
put("_ntrees", new Integer[] {5, 5});
put("_max_depth", new Integer[] {2, 2});
put("_mtries", new Integer[] {-1, -1});
put("_sample_rate", new Double[] {.1, .1});
}
};
// Fire off a grid search
DRFModel.DRFParameters params = new DRFModel.DRFParameters();
params._train = fr._key;
params._response_column = "economy";
// Get the Grid for this modeling class and frame
Job<Grid> gs = GridSearch.startGridSearch(null, params, hyperParms);
grid = gs.get();
// Check that duplicate model have not been constructed
Model[] models = grid.getModels();
assertTrue("Number of returned models has to be > 0", models.length > 0);
// But all off them should be same
Key<Model> modelKey = models[0]._key;
for (Model m : models) {
assertTrue("Number of constructed models has to be equal to 1", modelKey == m._key);
}
} finally {
if (old != null) {
old.remove();
}
if (fr != null) {
fr.remove();
}
if (grid != null) {
grid.remove();
}
}
}
// Make a new NFSFileVec key which holds the filename implicitly.
// This name is used by the DVecs to load data on-demand.
public static Key make(File f) {
long size = f.length();
Key k1 = PersistNFS.decodeFile(f);
byte[] bits = new byte[1 + 1 + 4 + k1._kb.length];
bits[0] = Key.VEC;
bits[1] = 0; // Not homed
UDP.set4(bits, 2, -1); // 0xFFFFFFFF in the chunk# area
System.arraycopy(k1._kb, 0, bits, 1 + 1 + 4, k1._kb.length);
Key k = Key.make(bits);
// Insert the top-level FileVec key into the store
DKV.put(k, new NFSFileVec(k, size));
return k;
}
public static Key makeByteVec(Key k, String... data) {
byte[][] chunks = new byte[data.length][];
long[] espc = new long[data.length + 1];
for (int i = 0; i < chunks.length; ++i) {
chunks[i] = data[i].getBytes();
espc[i + 1] = espc[i] + data[i].length();
}
Futures fs = new Futures();
Key key = Vec.newKey();
ByteVec bv = new ByteVec(key, Vec.ESPC.rowLayout(key, espc));
for (int i = 0; i < chunks.length; ++i) {
Key chunkKey = bv.chunkKey(i);
DKV.put(
chunkKey,
new Value(chunkKey, chunks[i].length, chunks[i], TypeMap.C1NCHUNK, Value.ICE),
fs);
}
DKV.put(bv._key, bv, fs);
Frame fr = new Frame(k, new String[] {"makeByteVec"}, new Vec[] {bv});
DKV.put(k, fr, fs);
fs.blockForPending();
return k;
}
@Test
public void testMerge() {
Frame l = null, r = null, f = null;
try {
l = frame("name", vec(ar("Cliff", "Arno", "Tomas", "Spencer"), ari(0, 1, 2, 3)));
l.add("age", vec(ar(">dirt", "middle", "middle", "young'n"), ari(0, 1, 2, 3)));
l = new Frame(l);
DKV.put(l);
System.out.println(l);
r = frame("name", vec(ar("Arno", "Tomas", "Michael", "Cliff"), ari(0, 1, 2, 3)));
r.add("skill", vec(ar("science", "linearmath", "sparkling", "hacker"), ari(0, 1, 2, 3)));
r = new Frame(r);
DKV.put(r);
System.out.println(r);
String x = String.format("(merge %s %s #1 #0 )", l._key, r._key);
Val res = Exec.exec(x);
f = res.getFrame();
System.out.println(f);
} finally {
if (f != null) f.delete();
if (r != null) r.delete();
if (l != null) l.delete();
}
}
public Frame(String[] names, Vec[] vecs) {
// assert names==null || names.length == vecs.length : "Number of columns does not match to
// number of cols' names.";
_names = names;
_vecs = vecs;
_keys = new Key[vecs.length];
for (int i = 0; i < vecs.length; i++) {
Key k = _keys[i] = vecs[i]._key;
if (DKV.get(k) == null) // If not already in KV, put it there
DKV.put(k, vecs[i]);
}
Vec v0 = anyVec();
if (v0 == null) return;
VectorGroup grp = v0.group();
for (int i = 0; i < vecs.length; i++) assert grp.equals(vecs[i].group());
}
@Override
public Job fork() {
DKV.put(destination_key, new GLMGrid(self(), _jobs));
assert _maxParallelism >= 1;
final H2OCountedCompleter fjt =
new H2OCallback<ParallelGLMs>() {
@Override
public void callback(ParallelGLMs pgs) {
remove();
}
};
start(fjt);
H2O.submitTask(new ParallelGLMs(this, _jobs, H2O.CLOUD.size(), fjt));
return this;
}
@Override
public void map(Chunk cs) {
int idx = _chunkOffset + cs.cidx();
Key ckey = Vec.chunkKey(_v._key, idx);
if (_cmap != null) {
assert !cs.hasFloat()
: "Input chunk (" + cs.getClass() + ") has float, but is expected to be categorical";
NewChunk nc = new NewChunk(_v, idx);
// loop over rows and update ints for new domain mapping according to vecs[c].domain()
for (int r = 0; r < cs._len; ++r) {
if (cs.isNA(r)) nc.addNA();
else nc.addNum(_cmap[(int) cs.at8(r)], 0);
}
nc.close(_fs);
} else {
DKV.put(ckey, cs.deepCopy(), _fs, true);
}
}
@Override
Val apply(Env env, Env.StackHelp stk, AST asts[]) {
QuantileModel.QuantileParameters parms = new QuantileModel.QuantileParameters();
Frame fr = stk.track(asts[1].exec(env)).getFrame();
Frame fr_wkey = new Frame(fr); // Force a bogus Key for Quantiles ModelBuilder
DKV.put(fr_wkey);
parms._train = fr_wkey._key;
parms._probs = ((ASTNumList) asts[2]).expand();
for (double d : parms._probs)
if (d < 0 || d > 1)
throw new IllegalArgumentException("Probability must be between 0 and 1: " + d);
String inter = asts[3].exec(env).getStr();
parms._combine_method = QuantileModel.CombineMethod.valueOf(inter.toUpperCase());
parms._weights_column = asts[4].str().equals("_") ? null : asts[4].str();
// Compute Quantiles
QuantileModel q = new Quantile(parms).trainModel().get();
// Remove bogus Key
DKV.remove(fr_wkey._key);
// Reshape all outputs as a Frame, with probs in col 0 and the
// quantiles in cols 1 thru fr.numCols() - except the optional weights vec
int ncols = fr.numCols();
if (parms._weights_column != null) ncols--;
Vec[] vecs = new Vec[1 /*1 more for the probs themselves*/ + ncols];
String[] names = new String[vecs.length];
vecs[0] = Vec.makeCon(null, parms._probs);
names[0] = "Probs";
int w = 0;
for (int i = 0; i < vecs.length - 1; ++i) {
if (fr._names[i].equals(parms._weights_column)) w = 1;
assert (w == 0 || w == 1);
vecs[i + 1] = Vec.makeCon(null, q._output._quantiles[i]);
names[i + 1] = fr._names[w + i] + "Quantiles";
}
q.delete();
return new ValFrame(new Frame(names, vecs));
}
@Override
public Job fork() {
DKV.put(destination_key, new GLMGrid(self(), _jobs));
assert _maxParallelism >= 1;
final H2OCountedCompleter fjt = new H2O.H2OEmptyCompleter();
fjt.setPendingCount(_jobs.length - 1);
start(fjt);
for (int i = 0; i < Math.min(_jobs.length, _maxParallelism); ++i) {
_jobs[i].run(
new H2OCallback(fjt) {
@Override
public void callback(H2OCountedCompleter t) {
int nextJob = _idx.getAndIncrement();
if (nextJob < _jobs.length) {
_jobs[nextJob].run(clone());
}
}
});
}
return this;
}
/**
* TimeAveraging as part of Elastic Averaging Algorithm Cf. equation 6 of arXiv:1412.6651v5
*
* @param nodeAverageModel current average of per-node models
* @return Time-average of node-averages (consensus model, "the" model)
*/
public static DeepLearningModelInfo timeAverage(DeepLearningModelInfo nodeAverageModel) {
float pa = (float) nodeAverageModel.get_params()._elastic_averaging_moving_rate;
assert (pa > 0 && pa <= 1);
DeepLearningModelInfo elasticAverage =
DKV.getGet(nodeAverageModel.elasticAverageModelInfoKey()); // get latest version from DKV
if (elasticAverage == null || pa == 1) {
elasticAverage = nodeAverageModel.deep_clone();
} else {
nodeAverageModel.mult(pa);
elasticAverage.mult(1 - pa);
elasticAverage.add(nodeAverageModel); // ignore processed local value set here
elasticAverage.set_processed_global(nodeAverageModel.get_processed_global());
}
elasticAverage.set_processed_local(0);
DKV.put(elasticAverage.elasticAverageModelInfoKey(), elasticAverage);
// nodeAverageModel.computeStats();
// elasticAverage.computeStats();
// Log.info("Local Model :\n" + nodeAverageModel.toString());
// Log.info("Elastic Average:\n" + elasticAverage.toString());
return elasticAverage;
}
@Test
public void testCollisionOfDRFParamsChecksum() {
Frame fr = null;
try {
fr = parse_test_file("smalldata/junit/cars.csv");
fr.remove("name").remove();
Vec old = fr.remove("economy (mpg)");
fr.add("economy (mpg)", old); // response to last column
DKV.put(fr);
// {"_model_id":null,"_train":{"name":"_83da9e0754c5eb9f6b812fe17e7945e5","type":"Key"},"_valid":null,"_nfolds":0,"_keep_cross_validation_predictions":false,"_fold_assignment":"AUTO","_distribution":"AUTO","_tweedie_power":1.5,"_ignored_columns":null,"_ignore_const_cols":true,"_weights_column":null,"_offset_column":null,"_fold_column":null,"_score_each_iteration":false,"_response_column":"economy (mpg)","_balance_classes":false,"_max_after_balance_size":5.0,"_class_sampling_factors":null,"_max_hit_ratio_k":10,"_max_confusion_matrix_size":20,"_checkpoint":null,"_ntrees":9,"_max_depth":15,"_min_rows":1.0,"_nbins":20,"_nbins_cats":1024,"_r2_stopping":0.999999,"_seed":-4522296119273841674,"_nbins_top_level":1024,"_build_tree_one_node":false,"_initial_score_interval":4000,"_score_interval":4000,"_mtries":3,"_sample_rate":0.6499997,"_binomial_double_trees":false}
DRFModel.DRFParameters params1 = new DRFModel.DRFParameters();
params1._train = fr._key;
params1._response_column = "economy (mpg)";
params1._seed = -4522296119273841674L;
params1._mtries = 3;
params1._max_depth = 15;
params1._ntrees = 9;
params1._sample_rate = 0.6499997f;
// {"_model_id":null,"_train":{"name":"_83da9e0754c5eb9f6b812fe17e7945e5","type":"Key"},"_valid":null,"_nfolds":0,"_keep_cross_validation_predictions":false,"_fold_assignment":"AUTO","_distribution":"AUTO","_tweedie_power":1.5,"_ignored_columns":null,"_ignore_const_cols":true,"_weights_column":null,"_offset_column":null,"_fold_column":null,"_score_each_iteration":false,"_response_column":"economy (mpg)","_balance_classes":false,"_max_after_balance_size":5.0,"_class_sampling_factors":null,"_max_hit_ratio_k":10,"_max_confusion_matrix_size":20,"_checkpoint":null,"_ntrees":13,"_max_depth":1,"_min_rows":1.0,"_nbins":20,"_nbins_cats":1024,"_r2_stopping":0.999999,"_seed":-4522296119273841674,"_nbins_top_level":1024,"_build_tree_one_node":false,"_initial_score_interval":4000,"_score_interval":4000,"_mtries":1,"_sample_rate":0.6499997,"_binomial_double_trees":false}
DRFModel.DRFParameters params2 = new DRFModel.DRFParameters();
params2._train = fr._key;
params2._response_column = "economy (mpg)";
params2._seed = -4522296119273841674L;
params2._mtries = 1;
params2._max_depth = 1;
params2._ntrees = 13;
params2._sample_rate = 0.6499997f;
long csum1 = params1.checksum();
long csum2 = params2.checksum();
Assert.assertNotEquals("Checksums shoudl be different", csum1, csum2);
} finally {
if (fr != null) {
fr.remove();
}
}
}
// GLRM scoring is data imputation based on feature domains using reconstructed XY (see Udell
// (2015), Section 5.3)
private Frame reconstruct(
Frame orig,
Frame adaptedFr,
Key destination_key,
boolean save_imputed,
boolean reverse_transform) {
final int ncols = _output._names.length;
assert ncols == adaptedFr.numCols();
String prefix = "reconstr_";
// Need [A,X,P] where A = adaptedFr, X = loading frame, P = imputed frame
// Note: A is adapted to original training frame, P has columns shuffled so cats come before
// nums!
Frame fullFrm = new Frame(adaptedFr);
Frame loadingFrm = DKV.get(_output._representation_key).get();
fullFrm.add(loadingFrm);
String[][] adaptedDomme = adaptedFr.domains();
for (int i = 0; i < ncols; i++) {
Vec v = fullFrm.anyVec().makeZero();
v.setDomain(adaptedDomme[i]);
fullFrm.add(prefix + _output._names[i], v);
}
GLRMScore gs = new GLRMScore(ncols, _parms._k, save_imputed, reverse_transform).doAll(fullFrm);
// Return the imputed training frame
int x = ncols + _parms._k, y = fullFrm.numCols();
Frame f =
fullFrm.extractFrame(
x, y); // this will call vec_impl() and we cannot call the delete() below just yet
f = new Frame((null == destination_key ? Key.make() : destination_key), f.names(), f.vecs());
DKV.put(f);
gs._mb.makeModelMetrics(
GLRMModel.this, orig, null, null); // save error metrics based on imputed data
return f;
}
@Test
public void testCarsGrid() {
Grid<GBMModel.GBMParameters> grid = null;
Frame fr = null;
Vec old = null;
try {
fr = parse_test_file("smalldata/junit/cars.csv");
fr.remove("name").remove(); // Remove unique id
old = fr.remove("cylinders");
fr.add("cylinders", old.toCategoricalVec()); // response to last column
DKV.put(fr);
// Setup hyperparameter search space
final Double[] legalLearnRateOpts = new Double[] {0.01, 0.1, 0.3};
final Double[] illegalLearnRateOpts = new Double[] {-1.0};
HashMap<String, Object[]> hyperParms =
new HashMap<String, Object[]>() {
{
put("_ntrees", new Integer[] {1, 2});
put("_distribution", new DistributionFamily[] {DistributionFamily.multinomial});
put("_max_depth", new Integer[] {1, 2, 5});
put("_learn_rate", ArrayUtils.join(legalLearnRateOpts, illegalLearnRateOpts));
}
};
// Name of used hyper parameters
String[] hyperParamNames = hyperParms.keySet().toArray(new String[hyperParms.size()]);
Arrays.sort(hyperParamNames);
int hyperSpaceSize = ArrayUtils.crossProductSize(hyperParms);
// Fire off a grid search
GBMModel.GBMParameters params = new GBMModel.GBMParameters();
params._train = fr._key;
params._response_column = "cylinders";
// Get the Grid for this modeling class and frame
Job<Grid> gs = GridSearch.startGridSearch(null, params, hyperParms);
grid = (Grid<GBMModel.GBMParameters>) gs.get();
// Make sure number of produced models match size of specified hyper space
Assert.assertEquals(
"Size of grid (models+failures) should match to size of hyper space",
hyperSpaceSize,
grid.getModelCount() + grid.getFailureCount());
//
// Make sure that names of used parameters match
//
String[] gridHyperNames = grid.getHyperNames();
Arrays.sort(gridHyperNames);
Assert.assertArrayEquals(
"Hyper parameters names should match!", hyperParamNames, gridHyperNames);
//
// Make sure that values of used parameters match as well to the specified values
//
Key<Model>[] mKeys = grid.getModelKeys();
Map<String, Set<Object>> usedHyperParams = GridTestUtils.initMap(hyperParamNames);
for (Key<Model> mKey : mKeys) {
GBMModel gbm = (GBMModel) mKey.get();
System.out.println(
gbm._output._scored_train[gbm._output._ntrees]._mse
+ " "
+ Arrays.deepToString(
ArrayUtils.zip(grid.getHyperNames(), grid.getHyperValues(gbm._parms))));
GridTestUtils.extractParams(usedHyperParams, gbm._parms, hyperParamNames);
}
// Remove illegal options
hyperParms.put("_learn_rate", legalLearnRateOpts);
GridTestUtils.assertParamsEqual(
"Grid models parameters have to cover specified hyper space",
hyperParms,
usedHyperParams);
// Verify model failure
Map<String, Set<Object>> failedHyperParams = GridTestUtils.initMap(hyperParamNames);
;
for (Model.Parameters failedParams : grid.getFailedParameters()) {
GridTestUtils.extractParams(failedHyperParams, failedParams, hyperParamNames);
}
hyperParms.put("_learn_rate", illegalLearnRateOpts);
GridTestUtils.assertParamsEqual(
"Failed model parameters have to correspond to specified hyper space",
hyperParms,
failedHyperParams);
} finally {
if (old != null) {
old.remove();
}
if (fr != null) {
fr.remove();
}
if (grid != null) {
grid.remove();
}
}
}
// @Ignore("PUBDEV-1648")
@Test
public void testRandomCarsGrid() {
Grid grid = null;
GBMModel gbmRebuilt = null;
Frame fr = null;
Vec old = null;
try {
fr = parse_test_file("smalldata/junit/cars.csv");
fr.remove("name").remove();
old = fr.remove("economy (mpg)");
fr.add("economy (mpg)", old); // response to last column
DKV.put(fr);
// Setup random hyperparameter search space
HashMap<String, Object[]> hyperParms = new HashMap<>();
hyperParms.put("_distribution", new DistributionFamily[] {DistributionFamily.gaussian});
// Construct random grid search space
Random rng = new Random();
Integer ntreesDim = rng.nextInt(4) + 1;
Integer maxDepthDim = rng.nextInt(4) + 1;
Integer learnRateDim = rng.nextInt(4) + 1;
Integer[] ntreesArr = interval(1, 25);
ArrayList<Integer> ntreesList = new ArrayList<>(Arrays.asList(ntreesArr));
Collections.shuffle(ntreesList);
Integer[] ntreesSpace = new Integer[ntreesDim];
for (int i = 0; i < ntreesDim; i++) {
ntreesSpace[i] = ntreesList.get(i);
}
Integer[] maxDepthArr = interval(1, 10);
ArrayList<Integer> maxDepthList = new ArrayList<>(Arrays.asList(maxDepthArr));
Collections.shuffle(maxDepthList);
Integer[] maxDepthSpace = new Integer[maxDepthDim];
for (int i = 0; i < maxDepthDim; i++) {
maxDepthSpace[i] = maxDepthList.get(i);
}
Double[] learnRateArr = interval(0.01, 1.0, 0.01);
ArrayList<Double> learnRateList = new ArrayList<>(Arrays.asList(learnRateArr));
Collections.shuffle(learnRateList);
Double[] learnRateSpace = new Double[learnRateDim];
for (int i = 0; i < learnRateDim; i++) {
learnRateSpace[i] = learnRateList.get(i);
}
hyperParms.put("_ntrees", ntreesSpace);
hyperParms.put("_max_depth", maxDepthSpace);
hyperParms.put("_learn_rate", learnRateSpace);
// Fire off a grid search
GBMModel.GBMParameters params = new GBMModel.GBMParameters();
params._train = fr._key;
params._response_column = "economy (mpg)";
// Get the Grid for this modeling class and frame
Job<Grid> gs = GridSearch.startGridSearch(null, params, hyperParms);
grid = gs.get();
System.out.println("ntrees search space: " + Arrays.toString(ntreesSpace));
System.out.println("max_depth search space: " + Arrays.toString(maxDepthSpace));
System.out.println("learn_rate search space: " + Arrays.toString(learnRateSpace));
// Check that cardinality of grid
Model[] ms = grid.getModels();
Integer numModels = ms.length;
System.out.println("Grid consists of " + numModels + " models");
assertTrue(numModels == ntreesDim * maxDepthDim * learnRateDim);
// Pick a random model from the grid
HashMap<String, Object[]> randomHyperParms = new HashMap<>();
randomHyperParms.put("_distribution", new DistributionFamily[] {DistributionFamily.gaussian});
Integer ntreeVal = ntreesSpace[rng.nextInt(ntreesSpace.length)];
randomHyperParms.put("_ntrees", new Integer[] {ntreeVal});
Integer maxDepthVal = maxDepthSpace[rng.nextInt(maxDepthSpace.length)];
randomHyperParms.put("_max_depth", maxDepthSpace);
Double learnRateVal = learnRateSpace[rng.nextInt(learnRateSpace.length)];
randomHyperParms.put("_learn_rate", learnRateSpace);
// TODO: GBMModel gbmFromGrid = (GBMModel) g2.model(randomHyperParms).get();
// Rebuild it with it's parameters
params._distribution = DistributionFamily.gaussian;
params._ntrees = ntreeVal;
params._max_depth = maxDepthVal;
params._learn_rate = learnRateVal;
GBM gbm = new GBM(params);
gbmRebuilt = gbm.trainModel().get();
assertTrue(gbm.isStopped());
// Make sure the MSE metrics match
// double fromGridMSE = gbmFromGrid._output._scored_train[gbmFromGrid._output._ntrees]._mse;
double rebuiltMSE = gbmRebuilt._output._scored_train[gbmRebuilt._output._ntrees]._mse;
// System.out.println("The random grid model's MSE: " + fromGridMSE);
System.out.println("The rebuilt model's MSE: " + rebuiltMSE);
// assertEquals(fromGridMSE, rebuiltMSE);
} finally {
if (old != null) old.remove();
if (fr != null) fr.remove();
if (grid != null) grid.remove();
if (gbmRebuilt != null) gbmRebuilt.remove();
}
}
// @Ignore("PUBDEV-1648")
@Test
public void testRandomCarsGrid() {
Grid grid = null;
DRFModel drfRebuilt = null;
Frame fr = null;
try {
fr = parse_test_file("smalldata/junit/cars.csv");
fr.remove("name").remove();
Vec old = fr.remove("economy (mpg)");
fr.add("economy (mpg)", old); // response to last column
DKV.put(fr);
// Setup random hyperparameter search space
HashMap<String, Object[]> hyperParms = new HashMap<>();
// Construct random grid search space
long seed = System.nanoTime();
Random rng = new Random(seed);
// Limit to 1-3 randomly, 4 times. Average total number of models is
// 2^4, or 16. Max is 81 models.
Integer ntreesDim = rng.nextInt(3) + 1;
Integer maxDepthDim = rng.nextInt(3) + 1;
Integer mtriesDim = rng.nextInt(3) + 1;
Integer sampleRateDim = rng.nextInt(3) + 1;
Integer[] ntreesArr = interval(1, 15);
ArrayList<Integer> ntreesList = new ArrayList<>(Arrays.asList(ntreesArr));
Collections.shuffle(ntreesList);
Integer[] ntreesSpace = new Integer[ntreesDim];
for (int i = 0; i < ntreesDim; i++) {
ntreesSpace[i] = ntreesList.get(i);
}
Integer[] maxDepthArr = interval(1, 10);
ArrayList<Integer> maxDepthList = new ArrayList<>(Arrays.asList(maxDepthArr));
Collections.shuffle(maxDepthList);
Integer[] maxDepthSpace = new Integer[maxDepthDim];
for (int i = 0; i < maxDepthDim; i++) {
maxDepthSpace[i] = maxDepthList.get(i);
}
Integer[] mtriesArr = interval(1, 5);
ArrayList<Integer> mtriesList = new ArrayList<>(Arrays.asList(mtriesArr));
Collections.shuffle(mtriesList);
Integer[] mtriesSpace = new Integer[mtriesDim];
for (int i = 0; i < mtriesDim; i++) {
mtriesSpace[i] = mtriesList.get(i);
}
Double[] sampleRateArr = interval(0.01, 0.99, 0.01);
ArrayList<Double> sampleRateList = new ArrayList<>(Arrays.asList(sampleRateArr));
Collections.shuffle(sampleRateList);
Double[] sampleRateSpace = new Double[sampleRateDim];
for (int i = 0; i < sampleRateDim; i++) {
sampleRateSpace[i] = sampleRateList.get(i);
}
hyperParms.put("_ntrees", ntreesSpace);
hyperParms.put("_max_depth", maxDepthSpace);
hyperParms.put("_mtries", mtriesSpace);
hyperParms.put("_sample_rate", sampleRateSpace);
// Fire off a grid search
DRFModel.DRFParameters params = new DRFModel.DRFParameters();
params._train = fr._key;
params._response_column = "economy (mpg)";
// Get the Grid for this modeling class and frame
Job<Grid> gs = GridSearch.startGridSearch(null, params, hyperParms);
grid = gs.get();
System.out.println("Test seed: " + seed);
System.out.println("ntrees search space: " + Arrays.toString(ntreesSpace));
System.out.println("max_depth search space: " + Arrays.toString(maxDepthSpace));
System.out.println("mtries search space: " + Arrays.toString(mtriesSpace));
System.out.println("sample_rate search space: " + Arrays.toString(sampleRateSpace));
// Check that cardinality of grid
Model[] ms = grid.getModels();
int numModels = ms.length;
System.out.println("Grid consists of " + numModels + " models");
assertEquals(
"Number of models should match hyper space size",
numModels,
ntreesDim * maxDepthDim * sampleRateDim * mtriesDim + grid.getFailureCount());
// Pick a random model from the grid
HashMap<String, Object[]> randomHyperParms = new HashMap<>();
Integer ntreeVal = ntreesSpace[rng.nextInt(ntreesSpace.length)];
randomHyperParms.put("_ntrees", new Integer[] {ntreeVal});
Integer maxDepthVal = maxDepthSpace[rng.nextInt(maxDepthSpace.length)];
randomHyperParms.put("_max_depth", maxDepthSpace);
Integer mtriesVal = mtriesSpace[rng.nextInt(mtriesSpace.length)];
randomHyperParms.put("_max_depth", mtriesSpace);
Double sampleRateVal = sampleRateSpace[rng.nextInt(sampleRateSpace.length)];
randomHyperParms.put("_sample_rate", sampleRateSpace);
// TODO: DRFModel drfFromGrid = (DRFModel) g2.model(randomHyperParms).get();
// Rebuild it with it's parameters
params._ntrees = ntreeVal;
params._max_depth = maxDepthVal;
params._mtries = mtriesVal;
drfRebuilt = new DRF(params).trainModel().get();
// Make sure the MSE metrics match
// double fromGridMSE = drfFromGrid._output._scored_train[drfFromGrid._output._ntrees]._mse;
double rebuiltMSE = drfRebuilt._output._scored_train[drfRebuilt._output._ntrees]._mse;
// System.out.println("The random grid model's MSE: " + fromGridMSE);
System.out.println("The rebuilt model's MSE: " + rebuiltMSE);
// assertEquals(fromGridMSE, rebuiltMSE);
} finally {
if (fr != null) {
fr.remove();
}
if (grid != null) {
grid.remove();
}
if (drfRebuilt != null) {
drfRebuilt.remove();
}
}
}
@Test
public void testSetColumnLossCats() throws InterruptedException, ExecutionException {
GLRM job = null;
GLRMModel model = null;
Frame train = null;
final int[] cats = new int[] {1, 3, 4, 5}; // Categoricals: CAPSULE, RACE, DPROS, DCAPS
Scope.enter();
try {
train = parse_test_file(Key.make("prostate.hex"), "smalldata/logreg/prostate.csv");
for (int i = 0; i < cats.length; i++)
Scope.track(train.replace(cats[i], train.vec(cats[i]).toCategoricalVec())._key);
train.remove("ID").remove();
DKV.put(train._key, train);
GLRMParameters parms = new GLRMParameters();
parms._train = train._key;
parms._k = 12;
parms._loss = GLRMParameters.Loss.Quadratic;
parms._multi_loss = GLRMParameters.Loss.Categorical;
parms._loss_by_col =
new GLRMParameters.Loss[] {
GLRMParameters.Loss.Ordinal, GLRMParameters.Loss.Poisson, GLRMParameters.Loss.Absolute
};
parms._loss_by_col_idx = new int[] {3 /* DPROS */, 1 /* AGE */, 6 /* VOL */};
parms._init = GLRM.Initialization.PlusPlus;
parms._min_step_size = 1e-5;
parms._recover_svd = false;
parms._max_iterations = 2000;
try {
job = new GLRM(parms);
model = job.trainModel().get();
Log.info(
"Iteration "
+ model._output._iterations
+ ": Objective value = "
+ model._output._objective);
GLRMTest.checkLossbyCol(parms, model);
model.score(train).delete();
ModelMetricsGLRM mm = (ModelMetricsGLRM) ModelMetrics.getFromDKV(model, train);
Log.info(
"Numeric Sum of Squared Error = "
+ mm._numerr
+ "\tCategorical Misclassification Error = "
+ mm._caterr);
} catch (Throwable t) {
t.printStackTrace();
throw new RuntimeException(t);
} finally {
job.remove();
}
} catch (Throwable t) {
t.printStackTrace();
throw new RuntimeException(t);
} finally {
if (train != null) train.delete();
if (model != null) model.delete();
Scope.exit();
}
}
public static Frame[] shuffleSplitFrame(
Frame fr, Key[] keys, final double ratios[], final long seed) {
// Sanity check the ratios
assert keys.length == ratios.length;
double sum = ratios[0];
for (int i = 1; i < ratios.length; i++) {
sum += ratios[i];
ratios[i] = sum;
}
assert water.util.MathUtils.equalsWithinOneSmallUlp(sum, 1.0);
// Do the split, into ratios.length groupings of NewChunks
final int ncols = fr.numCols();
MRTask mr =
new MRTask() {
@Override
public void map(Chunk cs[], NewChunk ncs[]) {
Random rng = new Random(seed * cs[0].cidx());
int nrows = cs[0]._len;
for (int i = 0; i < nrows; i++) {
double r = rng.nextDouble();
int x = 0; // Pick the NewChunk split
for (; x < ratios.length - 1; x++) if (r < ratios[x]) break;
x *= ncols;
// Helper string holder
ValueString vstr = new ValueString();
// Copy row to correct set of NewChunks
for (int j = 0; j < ncols; j++) {
byte colType = cs[j].vec().get_type();
switch (colType) {
case Vec.T_BAD:
break; /* NOP */
case Vec.T_STR:
ncs[x + j].addStr(cs[j], i);
break;
case Vec.T_UUID:
ncs[x + j].addUUID(cs[j], i);
break;
case Vec.T_NUM: /* fallthrough */
case Vec.T_ENUM:
case Vec.T_TIME:
ncs[x + j].addNum(cs[j].atd(i));
break;
default:
if (colType > Vec.T_TIME && colType <= Vec.T_TIMELAST)
ncs[x + j].addNum(cs[j].atd(i));
else throw new IllegalArgumentException("Unsupported vector type: " + colType);
break;
}
}
}
}
}.doAll(ncols * ratios.length, fr);
// Build output frames
Frame frames[] = new Frame[ratios.length];
Vec[] vecs = fr.vecs();
String[] names = fr.names();
Futures fs = new Futures();
for (int i = 0; i < ratios.length; i++) {
Vec[] nvecs = new Vec[ncols];
for (int c = 0; c < ncols; c++) {
mr.appendables()[i * ncols + c].setDomain(vecs[c].domain());
nvecs[c] = mr.appendables()[i * ncols + c].close(fs);
}
frames[i] = new Frame(keys[i], fr.names(), nvecs);
DKV.put(frames[i], fs);
}
fs.blockForPending();
return frames;
}
@Test
public void testExpandCatsProstate() throws InterruptedException, ExecutionException {
double[][] prostate =
ard(
ard(0, 71, 1, 0, 0, 4.8, 14.0, 7),
ard(1, 70, 1, 1, 0, 8.4, 21.8, 5),
ard(0, 73, 1, 3, 0, 10.0, 27.4, 6),
ard(1, 68, 1, 0, 0, 6.7, 16.7, 6));
double[][] pros_expandR =
ard(
ard(1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 71, 4.8, 14.0, 7),
ard(0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 70, 8.4, 21.8, 5),
ard(0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 73, 10.0, 27.4, 6),
ard(1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 68, 6.7, 16.7, 6));
String[] pros_cols =
new String[] {"Capsule", "Age", "Race", "Dpros", "Dcaps", "PSA", "Vol", "Gleason"};
String[][] pros_domains =
new String[][] {
new String[] {"No", "Yes"},
null,
new String[] {"Other", "White", "Black"},
new String[] {"None", "UniLeft", "UniRight", "Bilobar"},
new String[] {"No", "Yes"},
null,
null,
null
};
final int[] cats = new int[] {1, 3, 4, 5}; // Categoricals: CAPSULE, RACE, DPROS, DCAPS
Frame fr = null;
try {
Scope.enter();
fr = parse_test_file(Key.make("prostate.hex"), "smalldata/logreg/prostate.csv");
for (int i = 0; i < cats.length; i++)
Scope.track(fr.replace(cats[i], fr.vec(cats[i]).toCategoricalVec())._key);
fr.remove("ID").remove();
DKV.put(fr._key, fr);
DataInfo dinfo =
new DataInfo(
Key.make(),
fr,
null,
0,
true,
DataInfo.TransformType.NONE,
DataInfo.TransformType.NONE,
false,
false,
false, /* weights */
false, /* offset */
false, /* fold */
false);
Log.info("Original matrix:\n" + colFormat(pros_cols, "%8.7s") + ArrayUtils.pprint(prostate));
double[][] pros_perm = ArrayUtils.permuteCols(prostate, dinfo._permutation);
Log.info(
"Permuted matrix:\n"
+ colFormat(pros_cols, "%8.7s", dinfo._permutation)
+ ArrayUtils.pprint(pros_perm));
double[][] pros_exp = GLRM.expandCats(pros_perm, dinfo);
Log.info(
"Expanded matrix:\n"
+ colExpFormat(pros_cols, pros_domains, "%8.7s", dinfo._permutation)
+ ArrayUtils.pprint(pros_exp));
Assert.assertArrayEquals(pros_expandR, pros_exp);
} catch (Throwable t) {
t.printStackTrace();
throw new RuntimeException(t);
} finally {
if (fr != null) fr.delete();
Scope.exit();
}
}
/**
* This method trains a stacked autoencoder
*
* @param trainData Training dataset as a JavaRDD
* @param batchSize Size of a training mini-batch
* @param layerSizes Number of neurons for each layer
* @param epochs Number of epochs to train
* @param responseColumn Name of the response column
* @param modelName Name of the model
* @return DeepLearningModel
*/
public DeepLearningModel train(
JavaRDD<LabeledPoint> trainData,
int batchSize,
int[] layerSizes,
String activationType,
int epochs,
String responseColumn,
String modelName,
MLModel mlModel,
long modelID) {
// build stacked autoencoder by training the model with training data
double trainingFraction = 1;
try {
Scope.enter();
if (trainData != null) {
int numberOfFeatures = mlModel.getFeatures().size();
List<Feature> features = mlModel.getFeatures();
String[] names = new String[numberOfFeatures + 1];
for (int i = 0; i < numberOfFeatures; i++) {
names[i] = features.get(i).getName();
}
names[numberOfFeatures] = mlModel.getResponseVariable();
Frame frame = DeeplearningModelUtils.javaRDDToFrame(names, trainData);
// H2O uses default C<x> for column header
// String classifColName = "C" + frame.numCols();
String classifColName = mlModel.getResponseVariable();
// Convert response to categorical (digits 1 to <num of columns>)
int ci = frame.find(classifColName);
Scope.track(frame.replace(ci, frame.vecs()[ci].toEnum())._key);
// Splitting train file to train, validation and test
// Using FrameSplitter (instead of SuffleSplitFrame) gives a weird exception
// barrier onExCompletion for hex.deeplearning.DeepLearning$DeepLearningDriver@78ec854
double[] ratios = new double[] {trainingFraction, 1 - trainingFraction};
@SuppressWarnings("unchecked")
Frame[] splits =
ShuffleSplitFrame.shuffleSplitFrame(
frame, generateNumKeys(frame._key, ratios.length), ratios, 123456789);
Frame trainFrame = splits[0];
Frame vframe = splits[1];
if (log.isDebugEnabled()) {
log.debug("Creating Deeplearning parameters");
}
DeepLearningParameters deeplearningParameters = new DeepLearningParameters();
// convert model name
String dlModelName = modelName.replace('.', '_').replace('-', '_');
// populate model parameters
deeplearningParameters._model_id = Key.make(dlModelName + "_dl");
deeplearningParameters._train = trainFrame._key;
deeplearningParameters._valid = vframe._key;
deeplearningParameters._response_column = classifColName; // last column is the response
// This is causin all the predictions to be 0.0
// p._autoencoder = true;
deeplearningParameters._activation = getActivationType(activationType);
deeplearningParameters._hidden = layerSizes;
deeplearningParameters._train_samples_per_iteration = batchSize;
deeplearningParameters._input_dropout_ratio = 0.2;
deeplearningParameters._l1 = 1e-5;
deeplearningParameters._max_w2 = 10;
deeplearningParameters._epochs = epochs;
// speed up training
deeplearningParameters._adaptive_rate =
true; // disable adaptive per-weight learning rate -> default
// settings for learning rate and momentum are probably
// not ideal (slow convergence)
deeplearningParameters._replicate_training_data =
true; // avoid extra communication cost upfront, got
// enough data on each node for load balancing
deeplearningParameters._overwrite_with_best_model =
true; // no need to keep the best model around
deeplearningParameters._diagnostics =
false; // no need to compute statistics during training
deeplearningParameters._classification_stop = -1;
deeplearningParameters._score_interval =
60; // score and print progress report (only) every 20 seconds
deeplearningParameters._score_training_samples =
batchSize / 10; // only score on a small sample of the
// training set -> don't want to spend
// too much time scoring (note: there
// will be at least 1 row per chunk)
DKV.put(trainFrame);
DKV.put(vframe);
deeplearning = new DeepLearning(deeplearningParameters);
if (log.isDebugEnabled()) {
log.debug("Start training deeplearning model ....");
}
try {
dlModel = deeplearning.trainModel().get();
if (log.isDebugEnabled()) {
log.debug("Successfully finished Training deeplearning model.");
}
} catch (RuntimeException ex) {
log.error("Error in training Stacked Autoencoder classifier model", ex);
}
} else {
log.error("Train file not found!");
}
} catch (RuntimeException ex) {
log.error("Failed to train the deeplearning model [id] " + modelID + ". " + ex.getMessage());
} finally {
Scope.exit();
}
return dlModel;
}
/**
* Train a Deep Learning model, assumes that all members are populated If checkpoint == null,
* then start training a new model, otherwise continue from a checkpoint
*/
public final void buildModel() {
DeepLearningModel cp = null;
if (_parms._checkpoint == null) {
cp =
new DeepLearningModel(
dest(),
_parms,
new DeepLearningModel.DeepLearningModelOutput(DeepLearning.this),
_train,
_valid,
nclasses());
cp.model_info().initializeMembers();
} else {
final DeepLearningModel previous = DKV.getGet(_parms._checkpoint);
if (previous == null) throw new IllegalArgumentException("Checkpoint not found.");
Log.info("Resuming from checkpoint.");
_job.update(0, "Resuming from checkpoint");
if (isClassifier() != previous._output.isClassifier())
throw new H2OIllegalArgumentException(
"Response type must be the same as for the checkpointed model.");
if (isSupervised() != previous._output.isSupervised())
throw new H2OIllegalArgumentException(
"Model type must be the same as for the checkpointed model.");
// check the user-given arguments for consistency
DeepLearningParameters oldP =
previous._parms; // sanitized parameters for checkpointed model
DeepLearningParameters newP = _parms; // user-given parameters for restart
DeepLearningParameters oldP2 = (DeepLearningParameters) oldP.clone();
DeepLearningParameters newP2 = (DeepLearningParameters) newP.clone();
DeepLearningParameters.Sanity.modifyParms(
oldP, oldP2, nclasses()); // sanitize the user-given parameters
DeepLearningParameters.Sanity.modifyParms(
newP, newP2, nclasses()); // sanitize the user-given parameters
DeepLearningParameters.Sanity.checkpoint(oldP2, newP2);
DataInfo dinfo;
try {
// PUBDEV-2513: Adapt _train and _valid (in-place) to match the frames that were used for
// the previous model
// This can add or remove dummy columns (can happen if the dataset is sparse and datasets
// have different non-const columns)
for (String st : previous.adaptTestForTrain(_train, true, false)) Log.warn(st);
for (String st : previous.adaptTestForTrain(_valid, true, false)) Log.warn(st);
dinfo = makeDataInfo(_train, _valid, _parms, nclasses());
DKV.put(dinfo);
cp = new DeepLearningModel(dest(), _parms, previous, false, dinfo);
cp.write_lock(_job);
if (!Arrays.equals(cp._output._names, previous._output._names)) {
throw new H2OIllegalArgumentException(
"The columns of the training data must be the same as for the checkpointed model. Check ignored columns (or disable ignore_const_cols).");
}
if (!Arrays.deepEquals(cp._output._domains, previous._output._domains)) {
throw new H2OIllegalArgumentException(
"Categorical factor levels of the training data must be the same as for the checkpointed model.");
}
if (dinfo.fullN() != previous.model_info().data_info().fullN()) {
throw new H2OIllegalArgumentException(
"Total number of predictors is different than for the checkpointed model.");
}
if (_parms._epochs <= previous.epoch_counter) {
throw new H2OIllegalArgumentException(
"Total number of epochs must be larger than the number of epochs already trained for the checkpointed model ("
+ previous.epoch_counter
+ ").");
}
// these are the mutable parameters that are to be used by the model (stored in
// model_info._parms)
final DeepLearningParameters actualNewP =
cp.model_info()
.get_params(); // actually used parameters for model building (defaults filled in,
// etc.)
assert (actualNewP != previous.model_info().get_params());
assert (actualNewP != newP);
assert (actualNewP != oldP);
DeepLearningParameters.Sanity.update(actualNewP, newP, nclasses());
Log.info(
"Continuing training after "
+ String.format("%.3f", previous.epoch_counter)
+ " epochs from the checkpointed model.");
cp.update(_job);
} catch (H2OIllegalArgumentException ex) {
if (cp != null) {
cp.unlock(_job);
cp.delete();
cp = null;
}
throw ex;
} finally {
if (cp != null) cp.unlock(_job);
}
}
trainModel(cp);
// clean up, but don't delete weights and biases if user asked for export
List<Key> keep = new ArrayList<>();
try {
if (_parms._export_weights_and_biases
&& cp._output.weights != null
&& cp._output.biases != null) {
for (Key k : Arrays.asList(cp._output.weights)) {
keep.add(k);
for (Vec vk : ((Frame) DKV.getGet(k)).vecs()) {
keep.add(vk._key);
}
}
for (Key k : Arrays.asList(cp._output.biases)) {
keep.add(k);
for (Vec vk : ((Frame) DKV.getGet(k)).vecs()) {
keep.add(vk._key);
}
}
}
} finally {
Scope.exit(keep.toArray(new Key[keep.size()]));
}
}
@Override
public void onCompletion(CountedCompleter cc) {
DKV.put(_v);
}
private static Frame register(Frame f) {
if (f._key != null) DKV.put(f._key, f);
allFrames.add(f);
return f;
}
