// -------------------------------------------------------------------------- // Build the next k-trees, which is trying to correct the residual error from // the prior trees. From LSE2, page 387. Step 2b ii, iii. private DTree[] buildNextKTrees(Frame fr) { // We're going to build K (nclass) trees - each focused on correcting // errors for a single class. final DTree[] ktrees = new DTree[_nclass]; // Initial set of histograms. All trees; one leaf per tree (the root // leaf); all columns DHistogram hcs[][][] = new DHistogram[_nclass][1 /*just root leaf*/][_ncols]; for (int k = 0; k < _nclass; k++) { // Initially setup as-if an empty-split had just happened if (_distribution == null || _distribution[k] != 0) { // The Boolean Optimization // This optimization assumes the 2nd tree of a 2-class system is the // inverse of the first. This is false for DRF (and true for GBM) - // DRF picks a random different set of columns for the 2nd tree. if (k == 1 && _nclass == 2) continue; ktrees[k] = new DTree(fr._names, _ncols, (char) nbins, (char) _nclass, min_rows); new GBMUndecidedNode( ktrees[k], -1, DHistogram.initialHist(fr, _ncols, nbins, hcs[k][0], false)); // The "root" node } } int[] leafs = new int[_nclass]; // Define a "working set" of leaf splits, from here to tree._len // ---- // ESL2, page 387. Step 2b ii. // One Big Loop till the ktrees are of proper depth. // Adds a layer to the trees each pass. int depth = 0; for (; depth < max_depth; depth++) { if (!Job.isRunning(self())) return null; hcs = buildLayer(fr, ktrees, leafs, hcs, false, false); // If we did not make any new splits, then the tree is split-to-death if (hcs == null) break; } // Each tree bottomed-out in a DecidedNode; go 1 more level and insert // LeafNodes to hold predictions. for (int k = 0; k < _nclass; k++) { DTree tree = ktrees[k]; if (tree == null) continue; int leaf = leafs[k] = tree.len(); for (int nid = 0; nid < leaf; nid++) { if (tree.node(nid) instanceof DecidedNode) { DecidedNode dn = tree.decided(nid); for (int i = 0; i < dn._nids.length; i++) { int cnid = dn._nids[i]; if (cnid == -1 || // Bottomed out (predictors or responses known constant) tree.node(cnid) instanceof UndecidedNode || // Or chopped off for depth (tree.node(cnid) instanceof DecidedNode && // Or not possible to split ((DecidedNode) tree.node(cnid))._split.col() == -1)) dn._nids[i] = new GBMLeafNode(tree, nid).nid(); // Mark a leaf here } // Handle the trivial non-splitting tree if (nid == 0 && dn._split.col() == -1) new GBMLeafNode(tree, -1, 0); } } } // -- k-trees are done // ---- // ESL2, page 387. Step 2b iii. Compute the gammas, and store them back // into the tree leaves. Includes learn_rate. // gamma_i_k = (nclass-1)/nclass * (sum res_i / sum (|res_i|*(1-|res_i|))) // For regression: // gamma_i_k = sum res_i / count(res_i) GammaPass gp = new GammaPass(ktrees, leafs).doAll(fr); double m1class = _nclass > 1 ? (double) (_nclass - 1) / _nclass : 1.0; // K-1/K for (int k = 0; k < _nclass; k++) { final DTree tree = ktrees[k]; if (tree == null) continue; for (int i = 0; i < tree._len - leafs[k]; i++) { double g = gp._gss[k][i] == 0 // Constant response? ? (gp._rss[k][i] == 0 ? 0 : 1000) // Cap (exponential) learn, instead of dealing with Inf : learn_rate * m1class * gp._rss[k][i] / gp._gss[k][i]; assert !Double.isNaN(g); ((LeafNode) tree.node(leafs[k] + i))._pred = g; } } // ---- // ESL2, page 387. Step 2b iv. Cache the sum of all the trees, plus the // new tree, in the 'tree' columns. Also, zap the NIDs for next pass. // Tree <== f(Tree) // Nids <== 0 new MRTask2() { @Override public void map(Chunk chks[]) { // For all tree/klasses for (int k = 0; k < _nclass; k++) { final DTree tree = ktrees[k]; if (tree == null) continue; final Chunk nids = chk_nids(chks, k); final Chunk ct = chk_tree(chks, k); for (int row = 0; row < nids._len; row++) { int nid = (int) nids.at80(row); if (nid < 0) continue; ct.set0(row, (float) (ct.at0(row) + ((LeafNode) tree.node(nid))._pred)); nids.set0(row, 0); } } } }.doAll(fr); // Collect leaves stats for (int i = 0; i < ktrees.length; i++) if (ktrees[i] != null) ktrees[i].leaves = ktrees[i].len() - leafs[i]; // DEBUG: Print the generated K trees // printGenerateTrees(ktrees); return ktrees; }
// -------------------------------------------------------------------------- // Build the next random k-trees representing tid-th tree private void buildNextKTrees(Frame fr, int mtrys, float sample_rate, Random rand, int tid) { // We're going to build K (nclass) trees - each focused on correcting // errors for a single class. final DTree[] ktrees = new DTree[_nclass]; // Initial set of histograms. All trees; one leaf per tree (the root // leaf); all columns DHistogram hcs[][][] = new DHistogram[_nclass][1 /*just root leaf*/][_ncols]; // Adjust real bins for the top-levels int adj_nbins = Math.max(_parms._nbins_top_level, _parms._nbins); // Use for all k-trees the same seed. NOTE: this is only to make a fair // view for all k-trees final double[] _distribution = _model._output._distribution; long rseed = rand.nextLong(); // Initially setup as-if an empty-split had just happened for (int k = 0; k < _nclass; k++) { if (_distribution[k] != 0) { // Ignore missing classes // The Boolean Optimization // This optimization assumes the 2nd tree of a 2-class system is the // inverse of the first (and that the same columns were picked) if (k == 1 && _nclass == 2 && _model.binomialOpt()) continue; ktrees[k] = new DRFTree( fr, _ncols, (char) _parms._nbins, (char) _parms._nbins_cats, (char) _nclass, _parms._min_rows, mtrys, rseed); new DRFUndecidedNode( ktrees[k], -1, DHistogram.initialHist( fr, _ncols, adj_nbins, _parms._nbins_cats, hcs[k][0])); // The "root" node } } // Sample - mark the lines by putting 'OUT_OF_BAG' into nid(<klass>) vector Timer t_1 = new Timer(); Sample ss[] = new Sample[_nclass]; for (int k = 0; k < _nclass; k++) if (ktrees[k] != null) ss[k] = new Sample((DRFTree) ktrees[k], sample_rate) .dfork(0, new Frame(vec_nids(fr, k), vec_resp(fr)), _parms._build_tree_one_node); for (int k = 0; k < _nclass; k++) if (ss[k] != null) ss[k].getResult(); Log.debug("Sampling took: + " + t_1); int[] leafs = new int[_nclass]; // Define a "working set" of leaf splits, from leafs[i] to tree._len for // each tree i // ---- // One Big Loop till the ktrees are of proper depth. // Adds a layer to the trees each pass. Timer t_2 = new Timer(); int depth = 0; for (; depth < _parms._max_depth; depth++) { if (!isRunning()) return; hcs = buildLayer( fr, _parms._nbins, _parms._nbins_cats, ktrees, leafs, hcs, true, _parms._build_tree_one_node); // If we did not make any new splits, then the tree is split-to-death if (hcs == null) break; } Log.debug("Tree build took: " + t_2); // Each tree bottomed-out in a DecidedNode; go 1 more level and insert // LeafNodes to hold predictions. Timer t_3 = new Timer(); for (int k = 0; k < _nclass; k++) { DTree tree = ktrees[k]; if (tree == null) continue; int leaf = leafs[k] = tree.len(); for (int nid = 0; nid < leaf; nid++) { if (tree.node(nid) instanceof DecidedNode) { DecidedNode dn = tree.decided(nid); if (dn._split._col == -1) { // No decision here, no row should have this NID now if (nid == 0) { // Handle the trivial non-splitting tree LeafNode ln = new DRFLeafNode(tree, -1, 0); ln._pred = (float) (isClassifier() ? _model._output._priorClassDist[k] : _initialPrediction); } continue; } for (int i = 0; i < dn._nids.length; i++) { int cnid = dn._nids[i]; if (cnid == -1 || // Bottomed out (predictors or responses known constant) tree.node(cnid) instanceof UndecidedNode || // Or chopped off for depth (tree.node(cnid) instanceof DecidedNode && // Or not possible to split ((DecidedNode) tree.node(cnid))._split.col() == -1)) { LeafNode ln = new DRFLeafNode(tree, nid); ln._pred = (float) dn.pred(i); // Set prediction into the leaf dn._nids[i] = ln.nid(); // Mark a leaf here } } } } } // -- k-trees are done Log.debug("Nodes propagation: " + t_3); // ---- // Move rows into the final leaf rows Timer t_4 = new Timer(); CollectPreds cp = new CollectPreds(ktrees, leafs, _model.defaultThreshold()) .doAll(fr, _parms._build_tree_one_node); if (isClassifier()) asVotes(_treeMeasuresOnOOB) .append(cp.rightVotes, cp.allRows); // Track right votes over OOB rows for this tree else /* regression */ asSSE(_treeMeasuresOnOOB).append(cp.sse, cp.allRows); Log.debug("CollectPreds done: " + t_4); // Grow the model by K-trees _model._output.addKTrees(ktrees); }