// @Ignore("PUBDEV-1648")
@Test
public void testRandomCarsGrid() {
Grid grid = null;
GBMModel gbmRebuilt = null;
Frame fr = null;
Vec old = null;
try {
fr = parse_test_file("smalldata/junit/cars.csv");
fr.remove("name").remove();
old = fr.remove("economy (mpg)");
fr.add("economy (mpg)", old); // response to last column
DKV.put(fr);
// Setup random hyperparameter search space
HashMap<String, Object[]> hyperParms = new HashMap<>();
hyperParms.put("_distribution", new DistributionFamily[] {DistributionFamily.gaussian});
// Construct random grid search space
Random rng = new Random();
Integer ntreesDim = rng.nextInt(4) + 1;
Integer maxDepthDim = rng.nextInt(4) + 1;
Integer learnRateDim = rng.nextInt(4) + 1;
Integer[] ntreesArr = interval(1, 25);
ArrayList<Integer> ntreesList = new ArrayList<>(Arrays.asList(ntreesArr));
Collections.shuffle(ntreesList);
Integer[] ntreesSpace = new Integer[ntreesDim];
for (int i = 0; i < ntreesDim; i++) {
ntreesSpace[i] = ntreesList.get(i);
}
Integer[] maxDepthArr = interval(1, 10);
ArrayList<Integer> maxDepthList = new ArrayList<>(Arrays.asList(maxDepthArr));
Collections.shuffle(maxDepthList);
Integer[] maxDepthSpace = new Integer[maxDepthDim];
for (int i = 0; i < maxDepthDim; i++) {
maxDepthSpace[i] = maxDepthList.get(i);
}
Double[] learnRateArr = interval(0.01, 1.0, 0.01);
ArrayList<Double> learnRateList = new ArrayList<>(Arrays.asList(learnRateArr));
Collections.shuffle(learnRateList);
Double[] learnRateSpace = new Double[learnRateDim];
for (int i = 0; i < learnRateDim; i++) {
learnRateSpace[i] = learnRateList.get(i);
}
hyperParms.put("_ntrees", ntreesSpace);
hyperParms.put("_max_depth", maxDepthSpace);
hyperParms.put("_learn_rate", learnRateSpace);
// Fire off a grid search
GBMModel.GBMParameters params = new GBMModel.GBMParameters();
params._train = fr._key;
params._response_column = "economy (mpg)";
// Get the Grid for this modeling class and frame
Job<Grid> gs = GridSearch.startGridSearch(null, params, hyperParms);
grid = gs.get();
System.out.println("ntrees search space: " + Arrays.toString(ntreesSpace));
System.out.println("max_depth search space: " + Arrays.toString(maxDepthSpace));
System.out.println("learn_rate search space: " + Arrays.toString(learnRateSpace));
// Check that cardinality of grid
Model[] ms = grid.getModels();
Integer numModels = ms.length;
System.out.println("Grid consists of " + numModels + " models");
assertTrue(numModels == ntreesDim * maxDepthDim * learnRateDim);
// Pick a random model from the grid
HashMap<String, Object[]> randomHyperParms = new HashMap<>();
randomHyperParms.put("_distribution", new DistributionFamily[] {DistributionFamily.gaussian});
Integer ntreeVal = ntreesSpace[rng.nextInt(ntreesSpace.length)];
randomHyperParms.put("_ntrees", new Integer[] {ntreeVal});
Integer maxDepthVal = maxDepthSpace[rng.nextInt(maxDepthSpace.length)];
randomHyperParms.put("_max_depth", maxDepthSpace);
Double learnRateVal = learnRateSpace[rng.nextInt(learnRateSpace.length)];
randomHyperParms.put("_learn_rate", learnRateSpace);
// TODO: GBMModel gbmFromGrid = (GBMModel) g2.model(randomHyperParms).get();
// Rebuild it with it's parameters
params._distribution = DistributionFamily.gaussian;
params._ntrees = ntreeVal;
params._max_depth = maxDepthVal;
params._learn_rate = learnRateVal;
GBM gbm = new GBM(params);
gbmRebuilt = gbm.trainModel().get();
assertTrue(gbm.isStopped());
// Make sure the MSE metrics match
// double fromGridMSE = gbmFromGrid._output._scored_train[gbmFromGrid._output._ntrees]._mse;
double rebuiltMSE = gbmRebuilt._output._scored_train[gbmRebuilt._output._ntrees]._mse;
// System.out.println("The random grid model's MSE: " + fromGridMSE);
System.out.println("The rebuilt model's MSE: " + rebuiltMSE);
// assertEquals(fromGridMSE, rebuiltMSE);
} finally {
if (old != null) old.remove();
if (fr != null) fr.remove();
if (grid != null) grid.remove();
if (gbmRebuilt != null) gbmRebuilt.remove();
}
}
