public void testDelayedTasksThatFiredAtTheSameTimeAreExecutedConcurrently()
throws InterruptedException, ExecutionException {
final AppScheduledExecutorService service = new AppScheduledExecutorService(getName());
final List<LogInfo> log = Collections.synchronizedList(new ArrayList<>());
int delay = 500;
int N = 20;
List<? extends Future<?>> futures =
ContainerUtil.map(
Collections.nCopies(N, ""),
__ ->
service.schedule(
() -> {
log.add(new LogInfo(0));
TimeoutUtil.sleep(10 * 1000);
},
delay,
TimeUnit.MILLISECONDS));
for (Future<?> future : futures) {
future.get();
}
assertEquals(N, log.size());
Set<Thread> usedThreads = ContainerUtil.map2Set(log, logInfo -> logInfo.currentThread);
assertEquals(N, usedThreads.size());
service.shutdownAppScheduledExecutorService();
assertTrue(service.awaitTermination(10, TimeUnit.SECONDS));
}
@AfterClass
public static void tearDown() throws InterruptedException, ExecutionException, IOException {
DirDataImpl dirData = new DirDataImpl();
DirDataImpl.setNumber(3334);
List<String> paths = dirData.dividePath(rootPath, 30, THREADS);
List<Future<String>> list = new ArrayList<>();
try {
for (String p : paths) {
Future<String> submit =
executor.submit(
() -> {
dirData.delDir(p, 6);
return "It done!";
});
list.add(submit);
}
for (Future<String> future : list) {
future.get();
}
} finally {
executor.shutdown();
while (!executor.awaitTermination(10, TimeUnit.SECONDS)) {
LOG.info("Awaiting completion of threads.");
}
}
FileUtils.deleteDirectory(new File(rootPath));
}
@Test
public void partitionedReferencePileCollectsAllObjects()
throws InterruptedException, ExecutionException {
int backlog = 32;
PartitionedReferencePile<Slot> pile = new PartitionedReferencePile<Slot>(backlog, Slot::new);
Future[] futures = new Future[threadCnt];
for (int i = 0; i < threadCnt; i++) {
int idx = i;
futures[i] =
pool.submit(
() -> {
for (int j = 0; j < backlog * 2; j++) {
Slot s = pile.get();
s.var0 = idx;
s.var1 = j;
}
});
}
for (Future f : futures) {
f.get();
}
Collection<Slot> slots = pile.collect();
assertThat(
"All Slots allocated are available", slots, iterableWithSize(threadCnt * backlog * 2));
}
public static <T> List<T> executeAndGetResult(
Collection<? extends Callable<T>> tasks, long waitingTimeInMillis) {
ExecutorService executor = Executors.newFixedThreadPool(tasks.size());
List<T> finalResults = new ArrayList<T>(tasks.size());
List<Future<T>> temporaryResults = new ArrayList<Future<T>>(tasks.size());
try {
for (Callable<T> task : tasks) {
Future<T> future = executor.submit(task);
temporaryResults.add(future);
}
executor.awaitTermination(waitingTimeInMillis, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
throw new RuntimeException(e);
} finally {
executor.shutdown();
}
finalResults = new ArrayList<T>(temporaryResults.size());
for (Future<T> result : temporaryResults) {
try {
finalResults.add(result.get());
} catch (InterruptedException e) {
throw new RuntimeException(e);
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
}
return finalResults;
}
@Override
public Boolean call() throws Exception {
long timeoutMillis = 5000;
try {
getServersFile();
getZkRunning();
while (true) {
while (!restartQueue.isEmpty()) {
LOG.debug("Restart queue size [" + restartQueue.size() + "]");
RestartHandler handler = restartQueue.poll();
Future<ScriptContext> runner = pool.submit(handler);
ScriptContext scriptContext = runner.get(); // blocking call
if (scriptContext.getExitCode() != 0) restartQueue.add(handler);
}
try {
Thread.sleep(timeoutMillis);
} catch (InterruptedException e) {
}
}
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
pool.shutdown();
throw e;
}
}
@Override
public JDBCClient getConnection(Handler<AsyncResult<SQLConnection>> handler) {
Context ctx = vertx.getOrCreateContext();
exec.execute(
() -> {
Future<SQLConnection> res = Future.future();
try {
/*
This can block until a connection is free.
We don't want to do that while running on a worker as we can enter a deadlock situation as the worker
might have obtained a connection, and won't release it until it is run again
There is a general principle here:
*User code* should be executed on a worker and can potentially block, it's up to the *user* to deal with
deadlocks that might occur there.
If the *service code* internally blocks waiting for a resource that might be obtained by *user code*, then
this can cause deadlock, so the service should ensure it never does this, by executing such code
(e.g. getConnection) on a different thread to the worker pool.
We don't want to use the vert.x internal pool for this as the threads might end up all blocked preventing
other important operations from occurring (e.g. async file access)
*/
Connection conn = ds.getConnection();
SQLConnection sconn = new JDBCConnectionImpl(vertx, conn);
res.complete(sconn);
} catch (SQLException e) {
res.fail(e);
}
ctx.runOnContext(v -> res.setHandler(handler));
});
return this;
}
public void decodePage(
Object decodeKey,
int pageNum,
final DecodeCallback decodeCallback,
float zoom,
RectF pageSliceBounds) {
final DecodeTask decodeTask =
new DecodeTask(pageNum, decodeCallback, zoom, decodeKey, pageSliceBounds);
synchronized (decodingFutures) {
if (isRecycled) {
return;
}
final Future<?> future =
executorService.submit(
new Runnable() {
public void run() {
try {
Thread.currentThread().setPriority(Thread.NORM_PRIORITY - 1);
performDecode(decodeTask);
} catch (IOException e) {
Log.e(DECODE_SERVICE, "Decode fail", e);
}
}
});
final Future<?> removed = decodingFutures.put(decodeKey, future);
if (removed != null) {
removed.cancel(false);
}
}
}
private int[] computeClusterNumberRange(int min, int max, int hop, Optimizer optimizer)
throws ExecutionException, InterruptedException {
ExecutorService executor;
if (parallelWorkers > 1) {
executor = Executors.newFixedThreadPool(2);
} else {
executor = Executors.newFixedThreadPool(1);
}
ConcurrentMap<Integer, Double> sharedScores = new ConcurrentHashMap<>();
SearchSpaceBoundaryFinder_old upperBoundFinder =
new SearchSpaceUpperBoundaryFinder_old(min, max, hop, optimizer, sharedScores);
Future<Integer> futureUpperBound = executor.submit(upperBoundFinder);
SearchSpaceBoundaryFinder_old lowerBoundFinder =
new SearchSpaceLowerBoundaryFinder_old(min, max, hop, optimizer, sharedScores);
Future<Integer> futureLowerBound = executor.submit(lowerBoundFinder);
executor.shutdown();
int[] r = new int[2];
Integer realMin = futureLowerBound.get();
Integer realMax = futureUpperBound.get();
r[0] = realMin == null ? -1 : realMin;
r[1] = realMax == null ? -1 : realMax;
scores.putAll(lowerBoundFinder.getSplitsAndScores());
// scores.putAll(upperBoundFinder.getSplitsAndScores());
return r;
}
/* Regression test for DB-3614 */
@Test
@Category(SlowTest.class)
@Ignore("-sf- takes way too long to fail and interferes rest of build")
public void testTenTableJoinExplainDuration() throws Exception {
int size = 10;
List<String> tables = new ArrayList<String>(size);
List<String> joins = new ArrayList<String>(size - 1);
for (int i = 0; i < size; i++) {
methodWatcher.executeUpdate(format("create table tentab%s (c1 int primary key)", i));
methodWatcher.executeUpdate(format("insert into tentab%s values (1)", i));
tables.add(format("tentab%s", i));
if (i > 0) {
joins.add(format("tentab%s.c1 = tentab%s.c1", i, i - 1));
}
}
System.out.println("Tables created");
final String fromClause = Joiner.on(", ").join(tables);
final String joinCriteria = Joiner.on(" AND ").join(joins);
ExecutorService es =
Executors.newSingleThreadExecutor(
new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
Thread t = new Thread(r);
t.setDaemon(true);
return t;
}
});
try {
final CyclicBarrier startLatch = new CyclicBarrier(2);
final CountDownLatch finishLatch = new CountDownLatch(1);
Future<Void> f =
es.submit(
new Callable<Void>() {
@Override
public Void call() throws Exception {
String query =
format("EXPLAIN SELECT * FROM %s WHERE %s ", fromClause, joinCriteria);
startLatch.await();
try {
ResultSet rs = methodWatcher.executeQuery(query);
// Loose check that explain statement took a few seconds or less,
// because we're just making sure the short circuit logic in
// OptimizerImpl.checkTimeout() blocks this from taking several minutes.
Assert.assertTrue("Explain did not return result!", rs.next());
} finally {
finishLatch.countDown();
}
return null;
}
});
System.out.println("Starting wait");
startLatch.await();
f.get(1, TimeUnit.SECONDS);
System.out.println("Finished waiting");
} finally {
System.out.println("shutting down");
}
}
public NonBlockingIdentityHashMap<Long, TestKey> getMapMultithreaded()
throws InterruptedException, ExecutionException {
final int threadCount = _items.keySet().size();
final NonBlockingIdentityHashMap<Long, TestKey> map =
new NonBlockingIdentityHashMap<Long, TestKey>();
// use a barrier to open the gate for all threads at once to avoid rolling start and no actual
// concurrency
final CyclicBarrier barrier = new CyclicBarrier(threadCount);
final ExecutorService ex = Executors.newFixedThreadPool(threadCount);
final CompletionService<Integer> co = new ExecutorCompletionService<Integer>(ex);
for (Integer type : _items.keySet()) {
// A linked-list of things to insert
List<TestKey> items = _items.get(type);
TestKeyFeederThread feeder = new TestKeyFeederThread(type, items, map, barrier);
co.submit(feeder);
}
// wait for all threads to return
int itemCount = 0;
for (int retCount = 0; retCount < threadCount; retCount++) {
final Future<Integer> result = co.take();
itemCount += result.get();
}
ex.shutdown();
return map;
}
public static void main(String[] args) throws Exception {
parseArguments(args);
List<Iterator<String>> segments = Splitter.split(inputFilePath, numberOfThreads);
List<Future<Analytic>> futures = new ArrayList<Future<Analytic>>(segments.size());
ExecutorService threadPool = Executors.newFixedThreadPool(segments.size());
for (final Iterator<String> segment : segments) {
Future<Analytic> future =
threadPool.submit(
new Callable<Analytic>() {
@Override
public Analytic call() throws Exception {
return new Analyzer(segment).parse();
}
});
futures.add(future);
}
try {
ArrayList<Analytic> analytics = new ArrayList<Analytic>(futures.size());
for (Future<Analytic> future : futures) {
analytics.add(future.get());
}
Analytic result = Synthesizer.synthesize(analytics);
writeResult(result);
} finally {
threadPool.shutdown();
}
}
public static void main(String[] args) {
CustomScheduler scheduler = new CustomScheduler();
Thread t = new Thread(scheduler);
t.start();
Future<String> result =
scheduler.enqueue(
new Callable<String>() {
@Override
public String call() throws Exception {
Thread.sleep(1500);
int i = 1;
if (1 == i) {
throw new RuntimeException("test");
}
return "ok";
}
});
try {
System.out.println(result.get());
;
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}
/**
* Wait until futures are complete or the supplied timeout is reached.
*
* @param timeout Maximum time to wait for futures to complete.
* @param unit Unit of time for the timeout.
* @param futures Futures to wait for.
* @return True if all futures complete in time.
*/
public boolean awaitAll(long timeout, TimeUnit unit, Future<?>... futures) {
boolean complete;
try {
// 将timeout 转换为微秒
long nanos = unit.toNanos(timeout);
// 获取系统当前时间的微秒
long time = System.nanoTime();
for (Future<?> f : futures) {
if (nanos < 0) return false;
// 此处f的示例为Command
f.get(nanos, TimeUnit.NANOSECONDS);
long now = System.nanoTime();
nanos -= now - time;
time = now;
}
complete = true;
} catch (TimeoutException e) {
complete = false;
} catch (Exception e) {
throw new RedisCommandInterruptedException(e);
}
return complete;
}
public Integer call() {
count = 0;
try {
File[] files = directory.listFiles();
ArrayList<Future<Integer>> results = new ArrayList<Future<Integer>>();
for (File file : files)
if (file.isDirectory()) {
MatchCounter counter = new MatchCounter(file, keyword);
FutureTask<Integer> task = new FutureTask<Integer>(counter);
results.add(task);
Thread t = new Thread(task);
t.start();
} else {
if (search(file)) count++;
}
for (Future<Integer> result : results)
try {
count += result.get();
} catch (ExecutionException e) {
e.printStackTrace();
}
} catch (InterruptedException e) {
}
return count;
}
@Override
public Boolean call() throws Exception {
List<Future<Boolean>> futureList = new ArrayList<>();
List<Get> getList = new ArrayList<>(Constant.BATCH_GROUP_SIZE);
int count = 0;
for (Get get : gets) {
getList.add(get);
if (count % Constant.BATCH_GROUP_SIZE == 0) {
Future<Boolean> f =
AsyncThreadPoolFactory.ASYNC_HBASE_THREAD_POOL.submit(new GetTask(getList, result));
futureList.add(f);
getList = new ArrayList<>(Constant.BATCH_GROUP_SIZE);
}
count++;
}
Future<Boolean> f =
AsyncThreadPoolFactory.ASYNC_HBASE_THREAD_POOL.submit(new GetTask(getList, result));
futureList.add(f);
boolean isOk = false;
for (Future<Boolean> future : futureList) {
try {
isOk = future.get(500, TimeUnit.MILLISECONDS);
} catch (TimeoutException ignored) {
} catch (Exception e) {
e.printStackTrace();
} finally {
if (!isOk) {
future.cancel(Boolean.FALSE);
}
}
}
return isOk;
}
@Override
public void executeTest() throws Exception {
ODatabaseDocumentTx database =
poolFactory.get(getDatabaseURL(serverInstance.get(0)), "admin", "admin").acquire();
System.out.println("Creating Writers and Readers threads...");
final ExecutorService writerExecutors = Executors.newCachedThreadPool();
runningWriters = new CountDownLatch(serverInstance.size() * writerCount);
int serverId = 0;
int threadId = 0;
List<Callable<Void>> writerWorkers = new ArrayList<Callable<Void>>();
for (ServerRun server : serverInstance) {
for (int j = 0; j < writerCount; j++) {
Callable writer = createWriter(serverId, threadId++, getDatabaseURL(server));
writerWorkers.add(writer);
}
serverId++;
}
List<Future<Void>> futures = writerExecutors.invokeAll(writerWorkers);
System.out.println("Threads started, waiting for the end");
for (Future<Void> future : futures) {
future.get();
}
writerExecutors.shutdown();
Assert.assertTrue(writerExecutors.awaitTermination(1, TimeUnit.MINUTES));
System.out.println("All writer threads have finished, shutting down readers");
}
/**
* Somewhat multi-threaded depth-first search. Performs a DFS of the subtrees from the current
* node in parallel.
*
* @param s The tile sequence
* @param b The board to search
* @param pool The thread pool in which to submit jobs to.
* @return The board with the highest evaluation or null if no board can continue.
*/
private Board solve_pdfs(Board b, ExecutorService pool) {
List<Future<Board>> rets = new ArrayList<>(BOARD_WIDTH);
Board best = null;
int best_score = -1;
for (Direction d : directions) {
Board n = new Board(b);
if (n.move(tileSequence, d)) {
rets.add(pool.submit(new ParallelDFS(n)));
}
}
for (Future<Board> ret : rets) {
try {
Board c = ret.get();
if (c != null) {
int score = evaluate(c);
if (score > best_score) {
best = c;
best_score = score;
}
}
} catch (InterruptedException | ExecutionException e) {
System.err.println("Error: " + e.getMessage());
}
}
return best;
}
public static void main(String[] args) throws InterruptedException, ExecutionException {
List<Future<String>> futures = new ArrayList<>();
ExecutorService executor = Executors.newFixedThreadPool(5);
for (int i = 1000; i <= 1010; i++) {
futures.add(executor.submit(getTask(i)));
}
futures.add(executor.submit(getTask(10000000)));
for (Future<String> future : futures) {
System.out.println(future.get());
}
executor.shutdown();
executor.awaitTermination(10, TimeUnit.SECONDS);
/* output
500500
501501
502503
503506
504510
505515
506521
507528
508536
509545
510555
50000005000000
*/
}
public List<TravelQuote> getRankedTravelQuotes(
TravelInfo travelInfo,
Set<TravelCompany> companies,
Comparator<TravelQuote> ranking,
long time,
TimeUnit unit)
throws InterruptedException {
List<QuoteTask> tasks = new ArrayList<QuoteTask>();
for (TravelCompany company : companies) tasks.add(new QuoteTask(company, travelInfo));
List<Future<TravelQuote>> futures = exec.invokeAll(tasks, time, unit);
List<TravelQuote> quotes = new ArrayList<TravelQuote>(tasks.size());
Iterator<QuoteTask> taskIter = tasks.iterator();
for (Future<TravelQuote> f : futures) {
QuoteTask task = taskIter.next();
try {
quotes.add(f.get());
} catch (ExecutionException e) {
quotes.add(task.getFailureQuote(e.getCause()));
} catch (CancellationException e) {
quotes.add(task.getTimeoutQuote(e));
}
}
Collections.sort(quotes, ranking);
return quotes;
}
/** Execute a task that is executing something else inside. Nested Execution. */
@Test(timeout = 10000)
public void testNestedExecution() throws Exception {
Callable<String> task = new NestedExecutorTask();
ExecutorService executor = createSingleNodeExecutorService("testNestedExecution");
Future future = executor.submit(task);
future.get();
}
public static void main(String[] args) {
Callable<Integer> task =
() -> {
try {
TimeUnit.SECONDS.sleep(1);
return 123;
} catch (InterruptedException e) {
throw new IllegalStateException("task interrupted", e);
}
};
ExecutorService executor = Executors.newFixedThreadPool(1);
Future<Integer> future = executor.submit(task);
System.out.println("future done? " + future.isDone());
Integer result = -1;
try {
result = future.get();
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("future done? " + future.isDone());
System.out.println("result: " + result);
stop(executor);
}
/** Run a basic task */
@Test
public void testBasicTask() throws Exception {
Callable<String> task = new BasicTestTask();
ExecutorService executor = createSingleNodeExecutorService("testBasicTask");
Future future = executor.submit(task);
assertEquals(future.get(), BasicTestTask.RESULT);
}
@Test
public void testSubmitToKeyOwnerCallable() throws Exception {
final int k = simpleTestNodeCount;
TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(k);
final HazelcastInstance[] instances = factory.newInstances(new Config());
final AtomicInteger count = new AtomicInteger(0);
final CountDownLatch latch = new CountDownLatch(k / 2);
final ExecutionCallback callback =
new ExecutionCallback() {
public void onResponse(Object response) {
if ((Boolean) response) count.incrementAndGet();
latch.countDown();
}
public void onFailure(Throwable t) {}
};
for (int i = 0; i < k; i++) {
final HazelcastInstance instance = instances[i];
final IExecutorService service = instance.getExecutorService("testSubmitToKeyOwnerCallable");
final String script = "hazelcast.getCluster().getLocalMember().equals(member)";
final HashMap map = new HashMap();
final Member localMember = instance.getCluster().getLocalMember();
map.put("member", localMember);
int key = 0;
while (!localMember.equals(instance.getPartitionService().getPartition(++key).getOwner())) ;
if (i % 2 == 0) {
final Future f = service.submitToKeyOwner(new ScriptCallable(script, map), key);
assertTrue((Boolean) f.get(5, TimeUnit.SECONDS));
} else {
service.submitToKeyOwner(new ScriptCallable(script, map), key, callback);
}
}
assertTrue(latch.await(30, TimeUnit.SECONDS));
assertEquals(k / 2, count.get());
}
/**
* Test concurrent reader and writer (GH-458).
*
* <p><b>Test case:</b>
*
* <ol>
* <li/>start a long running reader;
* <li/>try to start a writer: it should time out;
* <li/>stop the reader;
* <li/>start the writer again: it should succeed.
* </ol>
*
* @throws Exception error during request execution
*/
@Test
@Ignore("There is no way to stop a query on the server!")
public void testReaderWriter() throws Exception {
final String readerQuery = "?query=(1%20to%20100000000000000)%5b.=1%5d";
final String writerQuery = "/test.xml";
final byte[] content = Token.token("<a/>");
final Get readerAction = new Get(readerQuery);
final Put writerAction = new Put(writerQuery, content);
final ExecutorService exec = Executors.newFixedThreadPool(2);
// start reader
exec.submit(readerAction);
Performance.sleep(TIMEOUT); // delay in order to be sure that the reader has started
// start writer
Future<HTTPResponse> writer = exec.submit(writerAction);
try {
final HTTPResponse result = writer.get(TIMEOUT, TimeUnit.MILLISECONDS);
if (result.status.isSuccess()) fail("Database modified while a reader is running");
throw new Exception(result.toString());
} catch (final TimeoutException e) {
// writer is blocked by the reader: stop it
writerAction.stop = true;
}
// stop reader
readerAction.stop = true;
// start the writer again
writer = exec.submit(writerAction);
assertEquals(HTTPCode.CREATED, writer.get().status);
}
@Test
public void testJedisClusterRunsWithMultithreaded()
throws InterruptedException, ExecutionException, IOException {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort("127.0.0.1", 7379));
final JedisCluster jc = new JedisCluster(jedisClusterNode);
jc.set("foo", "bar");
ThreadPoolExecutor executor =
new ThreadPoolExecutor(10, 100, 0, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(10));
List<Future<String>> futures = new ArrayList<Future<String>>();
for (int i = 0; i < 50; i++) {
executor.submit(
new Callable<String>() {
@Override
public String call() throws Exception {
// FIXME : invalidate slot cache from JedisCluster to test
// random connection also does work
return jc.get("foo");
}
});
}
for (Future<String> future : futures) {
String value = future.get();
assertEquals("bar", value);
}
jc.close();
}
public static <V> boolean areAllTrue(List<Future<V>> results) {
for (Future<V> result : results) {
if (!result.isDone()) {
return false;
}
}
return true;
}
public void run() {
try {
if (!src.isSet()) src.runDelayed(this);
dst.store(dstx, src.getValue());
} catch (InterruptedException e) {
dst.store(dstx, e);
}
}
@Test
public void get_shouldJoinTheCreatedThread_givenAFutureReturnedByExecute() throws Exception {
long startTime = System.nanoTime();
Future<?> result = asyncExecutor.execute(suppressCheckedExceptions(() -> Thread.sleep(250L)));
result.get();
long endTime = System.nanoTime();
assertThat(endTime - startTime, isGreaterThan(100 * 1000000L));
}
private void cancelFutures() {
for (Future future : futures) {
try {
future.cancel(true);
} catch (Exception e) {
logger.warn("Exception when cancelling future", e);
}
}
}
@Test
public void isDone_shouldReturnFalse_whenTheThreadIsFinished_givenAFutureReturnedByExecute()
throws Exception {
Future<?> result = asyncExecutor.execute(() -> {});
Thread.sleep(50L); // give other thread a chance to finish...
assertTrue(result.isDone());
}