public final Vec[] vecs() { if (_vecs != null) return _vecs; // Load all Vec headers; load them all in parallel by spawning F/J tasks. final Vec[] vecs = new Vec[_keys.length]; Futures fs = new Futures(); for (int i = 0; i < _keys.length; i++) { final int ii = i; final Key k = _keys[i]; H2OCountedCompleter t = new H2OCountedCompleter() { // We need higher priority here as there is a danger of deadlock in // case of many calls from MRTask2 at once (e.g. frame with many // vectors invokes rollup tasks for all vectors in parallel). Should // probably be done in CPS style in the future @Override public byte priority() { return H2O.MIN_HI_PRIORITY; } @Override public void compute2() { vecs[ii] = DKV.get(k).get(); tryComplete(); } }; H2O.submitTask(t); fs.add(t); } fs.blockForPending(); return _vecs = vecs; }
/** Clean-up code which is executed after each {@link Job#exec()} call in any case (normal/exceptional). */ protected void cleanup() { // Clean-up global list of temporary vectors Futures fs = new Futures(); cleanupTrash(_gVecTrash, fs); if (!_lVecTrash.isEmpty()) cleanupTrash(_lVecTrash, fs); fs.blockForPending(); }
@SuppressWarnings("unused") // called through reflection by RequestServer public RemoveAllV3 remove(int version, RemoveAllV3 u) { Log.info("Removing all objects"); Futures fs = new Futures(); for (Job j : Job.jobs()) { j.cancel(); j.remove(fs); } fs.blockForPending(); // Bulk brainless key removal. Completely wipes all Keys without regard. new MRTask() { @Override public byte priority() { return H2O.GUI_PRIORITY; } @Override public void setupLocal() { H2O.raw_clear(); water.fvec.Vec.ESPC.clear(); } }.doAllNodes(); Log.info("Finished removing objects"); return u; }
/** * On-the-fly version for varimp. After generation a new tree, its tree votes are collected on * shuffled OOB rows and variable importance is recomputed. * * <p>The <a * href="http://www.stat.berkeley.edu/~breiman/RandomForests/cc_home.htm#varimp">page</a> says: * <cite> "In every tree grown in the forest, put down the oob cases and count the number of votes * cast for the correct class. Now randomly permute the values of variable m in the oob cases and * put these cases down the tree. Subtract the number of votes for the correct class in the * variable-m-permuted oob data from the number of votes for the correct class in the untouched * oob data. The average of this number over all trees in the forest is the raw importance score * for variable m." </cite> */ @Override protected VarImp doVarImpCalc( final DRFModel model, DTree[] ktrees, final int tid, final Frame fTrain, boolean scale) { // Check if we have already serialized 'ktrees'-trees in the model assert model.ntrees() - 1 == tid : "Cannot compute DRF varimp since 'ktrees' are not serialized in the model! tid=" + tid; assert _treeMeasuresOnOOB.npredictors() - 1 == tid : "Tree votes over OOB rows for this tree (var ktrees) were not found!"; // Compute tree votes over shuffled data final CompressedTree[ /*nclass*/] theTree = model.ctree(tid); // get the last tree FIXME we should pass only keys final int nclasses = model.nclasses(); Futures fs = new Futures(); for (int var = 0; var < _ncols; var++) { final int variable = var; H2OCountedCompleter task4var = classification ? new H2OCountedCompleter() { @Override public void compute2() { // Compute this tree votes over all data over given variable TreeVotes cd = TreeMeasuresCollector.collectVotes( theTree, nclasses, fTrain, _ncols, sample_rate, variable); assert cd.npredictors() == 1; asVotes(_treeMeasuresOnSOOB[variable]).append(cd); tryComplete(); } } : /* regression */ new H2OCountedCompleter() { @Override public void compute2() { // Compute this tree votes over all data over given variable TreeSSE cd = TreeMeasuresCollector.collectSSE( theTree, nclasses, fTrain, _ncols, sample_rate, variable); assert cd.npredictors() == 1; asSSE(_treeMeasuresOnSOOB[variable]).append(cd); tryComplete(); } }; H2O.submitTask(task4var); // Fork computation fs.add(task4var); } fs.blockForPending(); // Wait for results // Compute varimp for individual features (_ncols) final float[] varimp = new float[_ncols]; // output variable importance final float[] varimpSD = new float[_ncols]; // output variable importance sd for (int var = 0; var < _ncols; var++) { double[ /*2*/] imp = classification ? asVotes(_treeMeasuresOnSOOB[var]).imp(asVotes(_treeMeasuresOnOOB)) : asSSE(_treeMeasuresOnSOOB[var]).imp(asSSE(_treeMeasuresOnOOB)); varimp[var] = (float) imp[0]; varimpSD[var] = (float) imp[1]; } return new VarImp.VarImpMDA(varimp, varimpSD, model.ntrees()); }
// Will fail if locked by anybody other than 'job_key' public void delete(Key job_key, float dummy) { if (_key != null) { Log.debug(Log.Tag.Sys.LOCKS, "lock-then-delete " + _key + " by job " + job_key); new PriorWriteLock(job_key).invoke(_key); } Futures fs = new Futures(); delete_impl(fs); if (_key != null) DKV.remove(_key, fs); // Delete self also fs.blockForPending(); }
/** * Global redistribution of a Frame (balancing of chunks), done by calling process (all-to-one + * one-to-all) * * @param fr Input frame * @param seed RNG seed * @param shuffle whether to shuffle the data globally * @return Shuffled frame */ public static Frame shuffleAndBalance( final Frame fr, int splits, long seed, final boolean local, final boolean shuffle) { if ((fr.vecs()[0].nChunks() < splits || shuffle) && fr.numRows() > splits) { Vec[] vecs = fr.vecs().clone(); Log.info("Load balancing dataset, splitting it into up to " + splits + " chunks."); long[] idx = null; if (shuffle) { idx = new long[splits]; for (int r = 0; r < idx.length; ++r) idx[r] = r; Utils.shuffleArray(idx, seed); } Key keys[] = new Vec.VectorGroup().addVecs(vecs.length); final long rows_per_new_chunk = (long) (Math.ceil((double) fr.numRows() / splits)); // loop over cols (same indexing for each column) Futures fs = new Futures(); for (int col = 0; col < vecs.length; col++) { AppendableVec vec = new AppendableVec(keys[col]); // create outgoing chunks for this col NewChunk[] outCkg = new NewChunk[splits]; for (int i = 0; i < splits; ++i) outCkg[i] = new NewChunk(vec, i); // loop over all incoming chunks for (int ckg = 0; ckg < vecs[col].nChunks(); ckg++) { final Chunk inCkg = vecs[col].chunkForChunkIdx(ckg); // loop over local rows of incoming chunks (fast path) for (int row = 0; row < inCkg._len; ++row) { int outCkgIdx = (int) ((inCkg._start + row) / rows_per_new_chunk); // destination chunk idx if (shuffle) outCkgIdx = (int) (idx[outCkgIdx]); // shuffle: choose a different output chunk assert (outCkgIdx >= 0 && outCkgIdx < splits); outCkg[outCkgIdx].addNum(inCkg.at0(row)); } } for (int i = 0; i < outCkg.length; ++i) outCkg[i].close(i, fs); Vec t = vec.close(fs); t._domain = vecs[col]._domain; vecs[col] = t; } fs.blockForPending(); Log.info("Load balancing done."); return new Frame(fr.names(), vecs); } return fr; }
// Convert a chunk# into a chunk - does lazy-chunk creation. As chunks are // asked-for the first time, we make the Key and an empty backing DVec. // Touching the DVec will force the file load. @Override public Value chunkIdx(int cidx) { final long nchk = nChunks(); assert 0 <= cidx && cidx < nchk; Key dkey = chunkKey(cidx); Value val1 = DKV.get(dkey); // Check for an existing one... will fetch data as needed if (val1 != null) return val1; // Found an existing one? // Lazily create a DVec for this chunk int len = (int) (cidx < nchk - 1 ? CHUNK_SZ : (_len - chunk2StartElem(cidx))); // DVec is just the raw file data with a null-compression scheme Value val2 = new Value(dkey, len, null, TypeMap.C1NCHUNK, _be); val2.setdsk(); // It is already on disk. // If not-home, then block till the Key is everywhere. Most calls here are // from the parser loading a text file, and the parser splits the work such // that most puts here are on home - so this is a simple speed optimization: // do not make a Futures nor block on it on home. Futures fs = dkey.home() ? null : new Futures(); // Atomically insert: fails on a race, but then return the old version Value val3 = DKV.DputIfMatch(dkey, val2, null, fs); if (!dkey.home() && fs != null) fs.blockForPending(); return val3 == null ? val2 : val3; }
public static Key makeByteVec(Key k, String... data) { byte[][] chunks = new byte[data.length][]; long[] espc = new long[data.length + 1]; for (int i = 0; i < chunks.length; ++i) { chunks[i] = data[i].getBytes(); espc[i + 1] = espc[i] + data[i].length(); } Futures fs = new Futures(); Key key = Vec.newKey(); ByteVec bv = new ByteVec(key, Vec.ESPC.rowLayout(key, espc)); for (int i = 0; i < chunks.length; ++i) { Key chunkKey = bv.chunkKey(i); DKV.put( chunkKey, new Value(chunkKey, chunks[i].length, chunks[i], TypeMap.C1NCHUNK, Value.ICE), fs); } DKV.put(bv._key, bv, fs); Frame fr = new Frame(k, new String[] {"makeByteVec"}, new Vec[] {bv}); DKV.put(k, fr, fs); fs.blockForPending(); return k; }
public static Frame[] shuffleSplitFrame( Frame fr, Key[] keys, final double ratios[], final long seed) { // Sanity check the ratios assert keys.length == ratios.length; double sum = ratios[0]; for (int i = 1; i < ratios.length; i++) { sum += ratios[i]; ratios[i] = sum; } assert water.util.MathUtils.equalsWithinOneSmallUlp(sum, 1.0); // Do the split, into ratios.length groupings of NewChunks final int ncols = fr.numCols(); MRTask mr = new MRTask() { @Override public void map(Chunk cs[], NewChunk ncs[]) { Random rng = new Random(seed * cs[0].cidx()); int nrows = cs[0]._len; for (int i = 0; i < nrows; i++) { double r = rng.nextDouble(); int x = 0; // Pick the NewChunk split for (; x < ratios.length - 1; x++) if (r < ratios[x]) break; x *= ncols; // Helper string holder ValueString vstr = new ValueString(); // Copy row to correct set of NewChunks for (int j = 0; j < ncols; j++) { byte colType = cs[j].vec().get_type(); switch (colType) { case Vec.T_BAD: break; /* NOP */ case Vec.T_STR: ncs[x + j].addStr(cs[j], i); break; case Vec.T_UUID: ncs[x + j].addUUID(cs[j], i); break; case Vec.T_NUM: /* fallthrough */ case Vec.T_ENUM: case Vec.T_TIME: ncs[x + j].addNum(cs[j].atd(i)); break; default: if (colType > Vec.T_TIME && colType <= Vec.T_TIMELAST) ncs[x + j].addNum(cs[j].atd(i)); else throw new IllegalArgumentException("Unsupported vector type: " + colType); break; } } } } }.doAll(ncols * ratios.length, fr); // Build output frames Frame frames[] = new Frame[ratios.length]; Vec[] vecs = fr.vecs(); String[] names = fr.names(); Futures fs = new Futures(); for (int i = 0; i < ratios.length; i++) { Vec[] nvecs = new Vec[ncols]; for (int c = 0; c < ncols; c++) { mr.appendables()[i * ncols + c].setDomain(vecs[c].domain()); nvecs[c] = mr.appendables()[i * ncols + c].close(fs); } frames[i] = new Frame(keys[i], fr.names(), nvecs); DKV.put(frames[i], fs); } fs.blockForPending(); return frames; }
/** User call which empty local trash of vectors. */ protected final void emptyLTrash() { if (_lVecTrash.isEmpty()) return; Futures fs = new Futures(); cleanupTrash(_lVecTrash, fs); fs.blockForPending(); }
public static void remove(Key key) { Futures fs = new Futures(); remove(key, fs); // Recursively delete, gather pending deletes fs.blockForPending(); // Block until all is deleted }
// This put is a top-level user-update, and not a reflected or retried // update. i.e., The User has initiated a change against the K/V store. // This is a WEAK update: it is only strongly ordered with other updates to // the SAME key on the SAME node. public static void put(Key key, Value val) { Futures fs = new Futures(); put(key, val, fs); fs.blockForPending(); // Block for remote-put to complete }