@SuppressWarnings("unused") // called through reflection by RequestServer public RemoveAllV3 remove(int version, RemoveAllV3 u) { Log.info("Removing all objects"); Futures fs = new Futures(); for (Job j : Job.jobs()) { j.cancel(); j.remove(fs); } fs.blockForPending(); // Bulk brainless key removal. Completely wipes all Keys without regard. new MRTask() { @Override public byte priority() { return H2O.GUI_PRIORITY; } @Override public void setupLocal() { H2O.raw_clear(); water.fvec.Vec.ESPC.clear(); } }.doAllNodes(); Log.info("Finished removing objects"); return u; }
/** Clean-up code which is executed after each {@link Job#exec()} call in any case (normal/exceptional). */ protected void cleanup() { // Clean-up global list of temporary vectors Futures fs = new Futures(); cleanupTrash(_gVecTrash, fs); if (!_lVecTrash.isEmpty()) cleanupTrash(_lVecTrash, fs); fs.blockForPending(); }
public final Vec[] vecs() { if (_vecs != null) return _vecs; // Load all Vec headers; load them all in parallel by spawning F/J tasks. final Vec[] vecs = new Vec[_keys.length]; Futures fs = new Futures(); for (int i = 0; i < _keys.length; i++) { final int ii = i; final Key k = _keys[i]; H2OCountedCompleter t = new H2OCountedCompleter() { // We need higher priority here as there is a danger of deadlock in // case of many calls from MRTask2 at once (e.g. frame with many // vectors invokes rollup tasks for all vectors in parallel). Should // probably be done in CPS style in the future @Override public byte priority() { return H2O.MIN_HI_PRIORITY; } @Override public void compute2() { vecs[ii] = DKV.get(k).get(); tryComplete(); } }; H2O.submitTask(t); fs.add(t); } fs.blockForPending(); return _vecs = vecs; }
@Test public void findForJavaApiMustWork() throws Exception { LinkedList<Future<Integer>> listFutures = new LinkedList<Future<Integer>>(); for (int i = 0; i < 10; i++) { final Integer fi = i; listFutures.add( Futures.future( new Callable<Integer>() { public Integer call() { return fi; } }, system.dispatcher())); } final Integer expect = 5; Future<Option<Integer>> f = Futures.find( listFutures, new Function<Integer, Boolean>() { public Boolean apply(Integer i) { return i == 5; } }, system.dispatcher()); assertEquals(expect, Await.result(f, timeout).get()); }
@Test public void reduceForJavaApiMustWork() throws Exception { LinkedList<Future<String>> listFutures = new LinkedList<Future<String>>(); StringBuilder expected = new StringBuilder(); for (int i = 0; i < 10; i++) { expected.append("test"); listFutures.add( Futures.future( new Callable<String>() { public String call() { return "test"; } }, system.dispatcher())); } Future<String> result = Futures.reduce( listFutures, new Function2<String, String, String>() { public String apply(String r, String t) { return r + t; } }, system.dispatcher()); assertEquals(Await.result(result, timeout), expected.toString()); }
/** * On-the-fly version for varimp. After generation a new tree, its tree votes are collected on * shuffled OOB rows and variable importance is recomputed. * * <p>The <a * href="http://www.stat.berkeley.edu/~breiman/RandomForests/cc_home.htm#varimp">page</a> says: * <cite> "In every tree grown in the forest, put down the oob cases and count the number of votes * cast for the correct class. Now randomly permute the values of variable m in the oob cases and * put these cases down the tree. Subtract the number of votes for the correct class in the * variable-m-permuted oob data from the number of votes for the correct class in the untouched * oob data. The average of this number over all trees in the forest is the raw importance score * for variable m." </cite> */ @Override protected VarImp doVarImpCalc( final DRFModel model, DTree[] ktrees, final int tid, final Frame fTrain, boolean scale) { // Check if we have already serialized 'ktrees'-trees in the model assert model.ntrees() - 1 == tid : "Cannot compute DRF varimp since 'ktrees' are not serialized in the model! tid=" + tid; assert _treeMeasuresOnOOB.npredictors() - 1 == tid : "Tree votes over OOB rows for this tree (var ktrees) were not found!"; // Compute tree votes over shuffled data final CompressedTree[ /*nclass*/] theTree = model.ctree(tid); // get the last tree FIXME we should pass only keys final int nclasses = model.nclasses(); Futures fs = new Futures(); for (int var = 0; var < _ncols; var++) { final int variable = var; H2OCountedCompleter task4var = classification ? new H2OCountedCompleter() { @Override public void compute2() { // Compute this tree votes over all data over given variable TreeVotes cd = TreeMeasuresCollector.collectVotes( theTree, nclasses, fTrain, _ncols, sample_rate, variable); assert cd.npredictors() == 1; asVotes(_treeMeasuresOnSOOB[variable]).append(cd); tryComplete(); } } : /* regression */ new H2OCountedCompleter() { @Override public void compute2() { // Compute this tree votes over all data over given variable TreeSSE cd = TreeMeasuresCollector.collectSSE( theTree, nclasses, fTrain, _ncols, sample_rate, variable); assert cd.npredictors() == 1; asSSE(_treeMeasuresOnSOOB[variable]).append(cd); tryComplete(); } }; H2O.submitTask(task4var); // Fork computation fs.add(task4var); } fs.blockForPending(); // Wait for results // Compute varimp for individual features (_ncols) final float[] varimp = new float[_ncols]; // output variable importance final float[] varimpSD = new float[_ncols]; // output variable importance sd for (int var = 0; var < _ncols; var++) { double[ /*2*/] imp = classification ? asVotes(_treeMeasuresOnSOOB[var]).imp(asVotes(_treeMeasuresOnOOB)) : asSSE(_treeMeasuresOnSOOB[var]).imp(asSSE(_treeMeasuresOnOOB)); varimp[var] = (float) imp[0]; varimpSD[var] = (float) imp[1]; } return new VarImp.VarImpMDA(varimp, varimpSD, model.ntrees()); }
// Will fail if locked by anybody other than 'job_key' public void delete(Key job_key, float dummy) { if (_key != null) { Log.debug(Log.Tag.Sys.LOCKS, "lock-then-delete " + _key + " by job " + job_key); new PriorWriteLock(job_key).invoke(_key); } Futures fs = new Futures(); delete_impl(fs); if (_key != null) DKV.remove(_key, fs); // Delete self also fs.blockForPending(); }
@Override public void read( final PentoQuery query, final Distribution distribution, final PentoCallback handler, final OperationContext operationContext) { List<PentoStoreWorker> workers = readWorkerFactory.getWorkers(operationContext, distribution); for (PentoStoreWorker worker : workers) { Callable callable = worker.execute(query); ListenableFuture future = ioExecutor.submit(callable); Futures.addCallback( future, new FutureCallback() { public void onSuccess(Object response) { handler.callback(response); } public void onFailure(Throwable thrown) { logger.error(thrown.getMessage()); handler.error(thrown); } }); } }
// TODO: should this actually throw only ExecutionException? public <T> T callWithTimeout( Callable<T> callable, long timeoutDuration, TimeUnit timeoutUnit, boolean amInterruptible) throws Exception { checkNotNull(callable); checkNotNull(timeoutUnit); checkArgument(timeoutDuration > 0, "bad timeout: " + timeoutDuration); Future<T> future = executor.submit(callable); try { if (amInterruptible) { try { return future.get(timeoutDuration, timeoutUnit); } catch (InterruptedException e) { future.cancel(true); throw e; } } else { Future<T> uninterruptible = Futures.makeUninterruptible(future); return uninterruptible.get(timeoutDuration, timeoutUnit); } } catch (ExecutionException e) { throw Throwables.throwCause(e, true); } catch (TimeoutException e) { future.cancel(true); throw new UncheckedTimeoutException(e); } }
// Error cases public void testSameThreadExecutionException() { SettableFuture<String> f = SettableFuture.create(); Exception e = new IllegalArgumentException("foo not found"); MockCallback callback = new MockCallback(e); Futures.addCallback(f, callback); f.setException(e); }
@Test public void traverseForJavaApiMustWork() throws Exception { LinkedList<String> listStrings = new LinkedList<String>(); LinkedList<String> expectedStrings = new LinkedList<String>(); for (int i = 0; i < 10; i++) { expectedStrings.add("TEST"); listStrings.add("test"); } Future<Iterable<String>> result = Futures.traverse( listStrings, new Function<String, Future<String>>() { public Future<String> apply(final String r) { return Futures.future( new Callable<String>() { public String call() { return r.toUpperCase(); } }, system.dispatcher()); } }, system.dispatcher()); assertEquals(Await.result(result, timeout), expectedStrings); }
static void invalidate(H2ONode h2o, Key key, Value newval, Futures fs) { assert newval._key != null && key.home(); // Prevent the new Value from being overwritten by Yet Another PUT by // read-locking it. It's safe to read, but not to over-write, until this // invalidate completes on the *prior* value. newval.read_lock(); // block further writes until all invalidates complete fs.add(RPC.call(h2o, new TaskInvalidateKey(key, newval))); }
public void testExecutorSuccess() { CountingSameThreadExecutor ex = new CountingSameThreadExecutor(); SettableFuture<String> f = SettableFuture.create(); MockCallback callback = new MockCallback("foo"); Futures.addCallback(f, callback, ex); f.set("foo"); assertEquals(1, ex.runCount); }
@Test public void blockMustBeCallable() throws Exception { Promise<String> p = Futures.promise(); Duration d = Duration.create(1, TimeUnit.SECONDS); p.success("foo"); Await.ready(p.future(), d); assertEquals(Await.result(p.future(), d), "foo"); }
public static <T> Map<T, Exception> awaitCompletion( Map<T, ? extends Future<?>> responses, ExecutorService exec, @Nullable Long maxTime, final Logger logger, final String logPrefix) { if (responses.size() == 0) return ImmutableMap.of(); final int total = responses.size(); final CountDownLatch doneSignal = new CountDownLatch(total); final AtomicInteger complete = new AtomicInteger(0); final AtomicInteger errors = new AtomicInteger(0); final long start = System.currentTimeMillis(); final Map<T, Exception> errorMap = Maps.newHashMap(); for (final java.util.Map.Entry<T, ? extends Future<?>> future : responses.entrySet()) { Futures.makeListenable(future.getValue(), exec) .addListener( new Runnable() { @Override public void run() { try { future.getValue().get(); complete.incrementAndGet(); } catch (Exception e) { errors.incrementAndGet(); logException(logger, logPrefix, total, complete.get(), errors.get(), start, e); errorMap.put(future.getKey(), e); } doneSignal.countDown(); } @Override public String toString() { return "callGetOnFuture(" + future.getKey() + "," + future.getValue() + ")"; } }, exec); } try { if (maxTime != null) doneSignal.await(maxTime, TimeUnit.MILLISECONDS); else doneSignal.await(); if (errors.get() > 0) { String message = message(logPrefix, total, complete.get(), errors.get(), start); RuntimeException exception = new RuntimeException(message); logger.error(exception, message); } if (logger.isTraceEnabled()) { String message = message(logPrefix, total, complete.get(), errors.get(), start); logger.trace(message); } } catch (InterruptedException e) { String message = message(logPrefix, total, complete.get(), errors.get(), start); TimeoutException exception = new TimeoutException(message); logger.error(exception, message); Throwables.propagate(exception); } return errorMap; }
@Test public void mapToMustBeCallable() throws Exception { Promise<Object> p = Futures.promise(); Future<String> f = p.future().mapTo(classTag(String.class)); Duration d = Duration.create(1, TimeUnit.SECONDS); p.success("foo"); Await.ready(p.future(), d); assertEquals(Await.result(p.future(), d), "foo"); }
/** * Global redistribution of a Frame (balancing of chunks), done by calling process (all-to-one + * one-to-all) * * @param fr Input frame * @param seed RNG seed * @param shuffle whether to shuffle the data globally * @return Shuffled frame */ public static Frame shuffleAndBalance( final Frame fr, int splits, long seed, final boolean local, final boolean shuffle) { if ((fr.vecs()[0].nChunks() < splits || shuffle) && fr.numRows() > splits) { Vec[] vecs = fr.vecs().clone(); Log.info("Load balancing dataset, splitting it into up to " + splits + " chunks."); long[] idx = null; if (shuffle) { idx = new long[splits]; for (int r = 0; r < idx.length; ++r) idx[r] = r; Utils.shuffleArray(idx, seed); } Key keys[] = new Vec.VectorGroup().addVecs(vecs.length); final long rows_per_new_chunk = (long) (Math.ceil((double) fr.numRows() / splits)); // loop over cols (same indexing for each column) Futures fs = new Futures(); for (int col = 0; col < vecs.length; col++) { AppendableVec vec = new AppendableVec(keys[col]); // create outgoing chunks for this col NewChunk[] outCkg = new NewChunk[splits]; for (int i = 0; i < splits; ++i) outCkg[i] = new NewChunk(vec, i); // loop over all incoming chunks for (int ckg = 0; ckg < vecs[col].nChunks(); ckg++) { final Chunk inCkg = vecs[col].chunkForChunkIdx(ckg); // loop over local rows of incoming chunks (fast path) for (int row = 0; row < inCkg._len; ++row) { int outCkgIdx = (int) ((inCkg._start + row) / rows_per_new_chunk); // destination chunk idx if (shuffle) outCkgIdx = (int) (idx[outCkgIdx]); // shuffle: choose a different output chunk assert (outCkgIdx >= 0 && outCkgIdx < splits); outCkg[outCkgIdx].addNum(inCkg.at0(row)); } } for (int i = 0; i < outCkg.length; ++i) outCkg[i].close(i, fs); Vec t = vec.close(fs); t._domain = vecs[col]._domain; vecs[col] = t; } fs.blockForPending(); Log.info("Load balancing done."); return new Frame(fr.names(), vecs); } return fr; }
// TODO: Improve this test, perhaps with an Actor @Test public void mustSequenceAFutureList() throws Exception { LinkedList<Future<String>> listFutures = new LinkedList<Future<String>>(); LinkedList<String> listExpected = new LinkedList<String>(); for (int i = 0; i < 10; i++) { listExpected.add("test"); listFutures.add( Futures.future( new Callable<String>() { public String call() { return "test"; } }, system.dispatcher())); } Future<Iterable<String>> futureList = Futures.sequence(listFutures, system.dispatcher()); assertEquals(Await.result(futureList, timeout), listExpected); }
// Convert a chunk# into a chunk - does lazy-chunk creation. As chunks are // asked-for the first time, we make the Key and an empty backing DVec. // Touching the DVec will force the file load. @Override public Value chunkIdx(int cidx) { final long nchk = nChunks(); assert 0 <= cidx && cidx < nchk; Key dkey = chunkKey(cidx); Value val1 = DKV.get(dkey); // Check for an existing one... will fetch data as needed if (val1 != null) return val1; // Found an existing one? // Lazily create a DVec for this chunk int len = (int) (cidx < nchk - 1 ? CHUNK_SZ : (_len - chunk2StartElem(cidx))); // DVec is just the raw file data with a null-compression scheme Value val2 = new Value(dkey, len, null, TypeMap.C1NCHUNK, _be); val2.setdsk(); // It is already on disk. // If not-home, then block till the Key is everywhere. Most calls here are // from the parser loading a text file, and the parser splits the work such // that most puts here are on home - so this is a simple speed optimization: // do not make a Futures nor block on it on home. Futures fs = dkey.home() ? null : new Futures(); // Atomically insert: fails on a race, but then return the old version Value val3 = DKV.DputIfMatch(dkey, val2, null, fs); if (!dkey.home() && fs != null) fs.blockForPending(); return val3 == null ? val2 : val3; }
public void testWildcardFuture() { SettableFuture<String> settable = SettableFuture.create(); ListenableFuture<?> f = settable; FutureCallback<Object> callback = new FutureCallback<Object>() { @Override public void onSuccess(Object result) {} @Override public void onFailure(Throwable t) {} }; Futures.addCallback(f, callback); }
public static Key makeByteVec(Key k, String... data) { byte[][] chunks = new byte[data.length][]; long[] espc = new long[data.length + 1]; for (int i = 0; i < chunks.length; ++i) { chunks[i] = data[i].getBytes(); espc[i + 1] = espc[i] + data[i].length(); } Futures fs = new Futures(); Key key = Vec.newKey(); ByteVec bv = new ByteVec(key, Vec.ESPC.rowLayout(key, espc)); for (int i = 0; i < chunks.length; ++i) { Key chunkKey = bv.chunkKey(i); DKV.put( chunkKey, new Value(chunkKey, chunks[i].length, chunks[i], TypeMap.C1NCHUNK, Value.ICE), fs); } DKV.put(bv._key, bv, fs); Frame fr = new Frame(k, new String[] {"makeByteVec"}, new Vec[] {bv}); DKV.put(k, fr, fs); fs.blockForPending(); return k; }
@GwtIncompatible("Mockito") public void testOnSuccessThrowsRuntimeException() throws Exception { RuntimeException exception = new RuntimeException(); String result = "result"; SettableFuture<String> future = SettableFuture.create(); @SuppressWarnings("unchecked") // Safe for a mock FutureCallback<String> callback = Mockito.mock(FutureCallback.class); Futures.addCallback(future, callback); Mockito.doThrow(exception).when(callback).onSuccess(result); future.set(result); assertEquals(result, future.get()); Mockito.verify(callback).onSuccess(result); Mockito.verifyNoMoreInteractions(callback); }
@Test public void mustBeAbleToExecuteAnOnResultCallback() throws Throwable { final CountDownLatch latch = new CountDownLatch(1); Promise<String> cf = Futures.promise(); Future<String> f = cf.future(); f.onSuccess( new OnSuccess<String>() { public void onSuccess(String result) { if (result.equals("foo")) latch.countDown(); } }, system.dispatcher()); cf.success("foo"); assertTrue(latch.await(5000, TimeUnit.MILLISECONDS)); assertEquals(Await.result(f, timeout), "foo"); }
@Test public void mustBeAbleToForeachAFuture() throws Throwable { final CountDownLatch latch = new CountDownLatch(1); Promise<String> cf = Futures.promise(); Future<String> f = cf.future(); f.foreach( new Foreach<String>() { public void each(String future) { latch.countDown(); } }, system.dispatcher()); cf.success("foo"); assertTrue(latch.await(5000, TimeUnit.MILLISECONDS)); assertEquals(Await.result(f, timeout), "foo"); }
@Test public void mustBeAbleToExecuteAnOnExceptionCallback() throws Throwable { final CountDownLatch latch = new CountDownLatch(1); Promise<String> cf = Futures.promise(); Future<String> f = cf.future(); f.onFailure( new OnFailure() { public void onFailure(Throwable t) { if (t instanceof NullPointerException) latch.countDown(); } }, system.dispatcher()); Throwable exception = new NullPointerException(); cf.failure(exception); assertTrue(latch.await(5000, TimeUnit.MILLISECONDS)); assertEquals(f.value().get().failed().get(), exception); }
@Test public void recoverWithToMustBeCallable() throws Exception { final IllegalStateException fail = new IllegalStateException("OHNOES"); Promise<Object> p = Futures.promise(); Future<Object> f = p.future() .recoverWith( new Recover<Future<Object>>() { public Future<Object> recover(Throwable t) throws Throwable { if (t == fail) return Futures.<Object>successful("foo"); else throw t; } }, system.dispatcher()); Duration d = Duration.create(1, TimeUnit.SECONDS); p.failure(fail); assertEquals(Await.result(f, d), "foo"); }
@GwtIncompatible("Mockito") public void testOnSuccessThrowsError() throws Exception { class TestError extends Error {} TestError error = new TestError(); String result = "result"; SettableFuture<String> future = SettableFuture.create(); @SuppressWarnings("unchecked") // Safe for a mock FutureCallback<String> callback = Mockito.mock(FutureCallback.class); Futures.addCallback(future, callback); Mockito.doThrow(error).when(callback).onSuccess(result); try { future.set(result); fail("Should have thrown"); } catch (TestError e) { assertSame(error, e); } assertEquals(result, future.get()); Mockito.verify(callback).onSuccess(result); Mockito.verifyNoMoreInteractions(callback); }
public void testCancel() { SettableFuture<String> f = SettableFuture.create(); FutureCallback<String> callback = new FutureCallback<String>() { private boolean called = false; @Override public void onSuccess(String result) { fail("Was not expecting onSuccess() to be called."); } @Override public synchronized void onFailure(Throwable t) { assertFalse(called); assertThat(t).isInstanceOf(CancellationException.class); called = true; } }; Futures.addCallback(f, callback); f.cancel(true); }
@Test public void mustBeAbleToFilterAFuture() throws Throwable { final CountDownLatch latch = new CountDownLatch(1); Promise<String> cf = Futures.promise(); Future<String> f = cf.future(); Future<String> r = f.filter( Filter.filterOf( new Function<String, Boolean>() { public Boolean apply(String r) { latch.countDown(); return r.equals("foo"); } }), system.dispatcher()); cf.success("foo"); assertTrue(latch.await(5000, TimeUnit.MILLISECONDS)); assertEquals(Await.result(f, timeout), "foo"); assertEquals(Await.result(r, timeout), "foo"); }
@Test public void mustBeAbleToMapAFuture() throws Exception { Future<String> f1 = Futures.future( new Callable<String>() { public String call() { return "Hello"; } }, system.dispatcher()); Future<String> f2 = f1.map( new Mapper<String, String>() { public String apply(String s) { return s + " World"; } }, system.dispatcher()); assertEquals("Hello World", Await.result(f2, timeout)); }