public NonBlockingIdentityHashMap<Long, TestKey> getMapMultithreaded()
throws InterruptedException, ExecutionException {
final int threadCount = _items.keySet().size();
final NonBlockingIdentityHashMap<Long, TestKey> map =
new NonBlockingIdentityHashMap<Long, TestKey>();
// use a barrier to open the gate for all threads at once to avoid rolling start and no actual
// concurrency
final CyclicBarrier barrier = new CyclicBarrier(threadCount);
final ExecutorService ex = Executors.newFixedThreadPool(threadCount);
final CompletionService<Integer> co = new ExecutorCompletionService<Integer>(ex);
for (Integer type : _items.keySet()) {
// A linked-list of things to insert
List<TestKey> items = _items.get(type);
TestKeyFeederThread feeder = new TestKeyFeederThread(type, items, map, barrier);
co.submit(feeder);
}
// wait for all threads to return
int itemCount = 0;
for (int retCount = 0; retCount < threadCount; retCount++) {
final Future<Integer> result = co.take();
itemCount += result.get();
}
ex.shutdown();
return map;
}
public List<TravelQuote> getRankedTravelQuotes(
TravelInfo travelInfo,
Set<TravelCompany> companies,
Comparator<TravelQuote> ranking,
long time,
TimeUnit unit)
throws InterruptedException {
List<QuoteTask> tasks = new ArrayList<QuoteTask>();
for (TravelCompany company : companies) tasks.add(new QuoteTask(company, travelInfo));
List<Future<TravelQuote>> futures = exec.invokeAll(tasks, time, unit);
List<TravelQuote> quotes = new ArrayList<TravelQuote>(tasks.size());
Iterator<QuoteTask> taskIter = tasks.iterator();
for (Future<TravelQuote> f : futures) {
QuoteTask task = taskIter.next();
try {
quotes.add(f.get());
} catch (ExecutionException e) {
quotes.add(task.getFailureQuote(e.getCause()));
} catch (CancellationException e) {
quotes.add(task.getTimeoutQuote(e));
}
}
Collections.sort(quotes, ranking);
return quotes;
}
private int[] computeClusterNumberRange(int min, int max, int hop, Optimizer optimizer)
throws ExecutionException, InterruptedException {
ExecutorService executor;
if (parallelWorkers > 1) {
executor = Executors.newFixedThreadPool(2);
} else {
executor = Executors.newFixedThreadPool(1);
}
ConcurrentMap<Integer, Double> sharedScores = new ConcurrentHashMap<>();
SearchSpaceBoundaryFinder_old upperBoundFinder =
new SearchSpaceUpperBoundaryFinder_old(min, max, hop, optimizer, sharedScores);
Future<Integer> futureUpperBound = executor.submit(upperBoundFinder);
SearchSpaceBoundaryFinder_old lowerBoundFinder =
new SearchSpaceLowerBoundaryFinder_old(min, max, hop, optimizer, sharedScores);
Future<Integer> futureLowerBound = executor.submit(lowerBoundFinder);
executor.shutdown();
int[] r = new int[2];
Integer realMin = futureLowerBound.get();
Integer realMax = futureUpperBound.get();
r[0] = realMin == null ? -1 : realMin;
r[1] = realMax == null ? -1 : realMax;
scores.putAll(lowerBoundFinder.getSplitsAndScores());
// scores.putAll(upperBoundFinder.getSplitsAndScores());
return r;
}
public void testDelayedTasksThatFiredAtTheSameTimeAreExecutedConcurrently()
throws InterruptedException, ExecutionException {
final AppScheduledExecutorService service = new AppScheduledExecutorService(getName());
final List<LogInfo> log = Collections.synchronizedList(new ArrayList<>());
int delay = 500;
int N = 20;
List<? extends Future<?>> futures =
ContainerUtil.map(
Collections.nCopies(N, ""),
__ ->
service.schedule(
() -> {
log.add(new LogInfo(0));
TimeoutUtil.sleep(10 * 1000);
},
delay,
TimeUnit.MILLISECONDS));
for (Future<?> future : futures) {
future.get();
}
assertEquals(N, log.size());
Set<Thread> usedThreads = ContainerUtil.map2Set(log, logInfo -> logInfo.currentThread);
assertEquals(N, usedThreads.size());
service.shutdownAppScheduledExecutorService();
assertTrue(service.awaitTermination(10, TimeUnit.SECONDS));
}
@Override
public Boolean call() throws Exception {
long timeoutMillis = 5000;
try {
getServersFile();
getZkRunning();
while (true) {
while (!restartQueue.isEmpty()) {
LOG.debug("Restart queue size [" + restartQueue.size() + "]");
RestartHandler handler = restartQueue.poll();
Future<ScriptContext> runner = pool.submit(handler);
ScriptContext scriptContext = runner.get(); // blocking call
if (scriptContext.getExitCode() != 0) restartQueue.add(handler);
}
try {
Thread.sleep(timeoutMillis);
} catch (InterruptedException e) {
}
}
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
pool.shutdown();
throw e;
}
}
@Test
public void testSubmitToKeyOwnerCallable() throws Exception {
final int k = simpleTestNodeCount;
TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(k);
final HazelcastInstance[] instances = factory.newInstances(new Config());
final AtomicInteger count = new AtomicInteger(0);
final CountDownLatch latch = new CountDownLatch(k / 2);
final ExecutionCallback callback =
new ExecutionCallback() {
public void onResponse(Object response) {
if ((Boolean) response) count.incrementAndGet();
latch.countDown();
}
public void onFailure(Throwable t) {}
};
for (int i = 0; i < k; i++) {
final HazelcastInstance instance = instances[i];
final IExecutorService service = instance.getExecutorService("testSubmitToKeyOwnerCallable");
final String script = "hazelcast.getCluster().getLocalMember().equals(member)";
final HashMap map = new HashMap();
final Member localMember = instance.getCluster().getLocalMember();
map.put("member", localMember);
int key = 0;
while (!localMember.equals(instance.getPartitionService().getPartition(++key).getOwner())) ;
if (i % 2 == 0) {
final Future f = service.submitToKeyOwner(new ScriptCallable(script, map), key);
assertTrue((Boolean) f.get(5, TimeUnit.SECONDS));
} else {
service.submitToKeyOwner(new ScriptCallable(script, map), key, callback);
}
}
assertTrue(latch.await(30, TimeUnit.SECONDS));
assertEquals(k / 2, count.get());
}
/** Run a basic task */
@Test
public void testBasicTask() throws Exception {
Callable<String> task = new BasicTestTask();
ExecutorService executor = createSingleNodeExecutorService("testBasicTask");
Future future = executor.submit(task);
assertEquals(future.get(), BasicTestTask.RESULT);
}
/**
* Somewhat multi-threaded depth-first search. Performs a DFS of the subtrees from the current
* node in parallel.
*
* @param s The tile sequence
* @param b The board to search
* @param pool The thread pool in which to submit jobs to.
* @return The board with the highest evaluation or null if no board can continue.
*/
private Board solve_pdfs(Board b, ExecutorService pool) {
List<Future<Board>> rets = new ArrayList<>(BOARD_WIDTH);
Board best = null;
int best_score = -1;
for (Direction d : directions) {
Board n = new Board(b);
if (n.move(tileSequence, d)) {
rets.add(pool.submit(new ParallelDFS(n)));
}
}
for (Future<Board> ret : rets) {
try {
Board c = ret.get();
if (c != null) {
int score = evaluate(c);
if (score > best_score) {
best = c;
best_score = score;
}
}
} catch (InterruptedException | ExecutionException e) {
System.err.println("Error: " + e.getMessage());
}
}
return best;
}
/** Execute a task that is executing something else inside. Nested Execution. */
@Test(timeout = 10000)
public void testNestedExecution() throws Exception {
Callable<String> task = new NestedExecutorTask();
ExecutorService executor = createSingleNodeExecutorService("testNestedExecution");
Future future = executor.submit(task);
future.get();
}
public static <T> List<T> executeAndGetResult(
Collection<? extends Callable<T>> tasks, long waitingTimeInMillis) {
ExecutorService executor = Executors.newFixedThreadPool(tasks.size());
List<T> finalResults = new ArrayList<T>(tasks.size());
List<Future<T>> temporaryResults = new ArrayList<Future<T>>(tasks.size());
try {
for (Callable<T> task : tasks) {
Future<T> future = executor.submit(task);
temporaryResults.add(future);
}
executor.awaitTermination(waitingTimeInMillis, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
throw new RuntimeException(e);
} finally {
executor.shutdown();
}
finalResults = new ArrayList<T>(temporaryResults.size());
for (Future<T> result : temporaryResults) {
try {
finalResults.add(result.get());
} catch (InterruptedException e) {
throw new RuntimeException(e);
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
}
return finalResults;
}
@Test
public void testJedisClusterRunsWithMultithreaded()
throws InterruptedException, ExecutionException, IOException {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort("127.0.0.1", 7379));
final JedisCluster jc = new JedisCluster(jedisClusterNode);
jc.set("foo", "bar");
ThreadPoolExecutor executor =
new ThreadPoolExecutor(10, 100, 0, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(10));
List<Future<String>> futures = new ArrayList<Future<String>>();
for (int i = 0; i < 50; i++) {
executor.submit(
new Callable<String>() {
@Override
public String call() throws Exception {
// FIXME : invalidate slot cache from JedisCluster to test
// random connection also does work
return jc.get("foo");
}
});
}
for (Future<String> future : futures) {
String value = future.get();
assertEquals("bar", value);
}
jc.close();
}
public Integer call() {
count = 0;
try {
File[] files = directory.listFiles();
ArrayList<Future<Integer>> results = new ArrayList<Future<Integer>>();
for (File file : files)
if (file.isDirectory()) {
MatchCounter counter = new MatchCounter(file, keyword);
FutureTask<Integer> task = new FutureTask<Integer>(counter);
results.add(task);
Thread t = new Thread(task);
t.start();
} else {
if (search(file)) count++;
}
for (Future<Integer> result : results)
try {
count += result.get();
} catch (ExecutionException e) {
e.printStackTrace();
}
} catch (InterruptedException e) {
}
return count;
}
/**
* Wait until futures are complete or the supplied timeout is reached.
*
* @param timeout Maximum time to wait for futures to complete.
* @param unit Unit of time for the timeout.
* @param futures Futures to wait for.
* @return True if all futures complete in time.
*/
public boolean awaitAll(long timeout, TimeUnit unit, Future<?>... futures) {
boolean complete;
try {
// 将timeout 转换为微秒
long nanos = unit.toNanos(timeout);
// 获取系统当前时间的微秒
long time = System.nanoTime();
for (Future<?> f : futures) {
if (nanos < 0) return false;
// 此处f的示例为Command
f.get(nanos, TimeUnit.NANOSECONDS);
long now = System.nanoTime();
nanos -= now - time;
time = now;
}
complete = true;
} catch (TimeoutException e) {
complete = false;
} catch (Exception e) {
throw new RedisCommandInterruptedException(e);
}
return complete;
}
public static <V> boolean areAllTrue(List<Future<V>> results) {
for (Future<V> result : results) {
if (!result.isDone()) {
return false;
}
}
return true;
}
private void cancelFutures() {
for (Future future : futures) {
try {
future.cancel(true);
} catch (Exception e) {
logger.warn("Exception when cancelling future", e);
}
}
}
@Test
public void testGetAsync() throws Exception {
HazelcastClient hClient = getHazelcastClient();
String key = "key";
String value1 = "value1";
IMap<String, String> map = hClient.getMap("map:test:getAsync");
map.put(key, value1);
Future<String> f1 = map.getAsync(key);
assertEquals(value1, f1.get());
}
/**
* Execute database transactions in an isolated context.
*
* @param <T> return type
* @param callable transaction
* @return
*/
synchronized <T> T exec(Callable<T> callable) {
// We want to submit db jobs to an executor to isolate
// them from the current thread,
Future<T> future = _executor.submit(callable);
try {
return future.get();
} catch (Exception e) {
throw new ContextException("DbError", e);
}
}
private void executeConcurrentRandomReadAndWriteOperations()
throws InterruptedException, ExecutionException {
for (int i = 0; i < THREAD_COUNT; i++) {
futures.add(executorService.submit(new Writer()));
}
for (int i = 0; i < THREAD_COUNT; i++) {
futures.add(executorService.submit(new Reader()));
}
for (Future<Void> future : futures) future.get();
}
private void fillFilesWithContent() throws InterruptedException, ExecutionException, IOException {
for (int i = 0; i < THREAD_COUNT; i++) {
futures.add(executorService.submit(new Writer()));
}
for (Future<Void> future : futures) future.get();
futures.clear();
buffer.flushBuffer();
}
/** Test the method isDone() */
@Test
public void testIsDoneMethod() throws Exception {
Callable<String> task = new BasicTestTask();
IExecutorService executor = createSingleNodeExecutorService("isDoneMethod");
Future future = executor.submit(task);
if (future.isDone()) {
assertTrue(future.isDone());
}
assertEquals(future.get(), BasicTestTask.RESULT);
assertTrue(future.isDone());
}
@Test
public void testSubmitMultipleNode() throws ExecutionException, InterruptedException {
final int k = simpleTestNodeCount;
TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(k);
final HazelcastInstance[] instances = factory.newInstances(new Config());
for (int i = 0; i < k; i++) {
final IExecutorService service = instances[i].getExecutorService("testSubmitMultipleNode");
final String script = "hazelcast.getAtomicLong('testSubmitMultipleNode').incrementAndGet();";
final Future future = service.submit(new ScriptCallable(script, null));
assertEquals((long) (i + 1), future.get());
}
}
private void waitForComponentsToComplete() {
for (Future<BaseEvent> future : Iterables.transform(getComponents(), busyComponentFuture)) {
try {
future.get(1, TimeUnit.MINUTES);
} catch (InterruptedException | ExecutionException | TimeoutException e) {
log.error("Failed waiting for a Component to complete", e);
}
}
for (ComponentItem component : getComponents()) {
component.setBusy(false);
}
log.debug("All components completed in canvas {}", getLabel());
}
public static void load(Collection<FileDesc> files, Path root, int blocSize, Pattern pattern)
throws IOException {
root = root.toAbsolutePath().normalize();
Visitor visitor = new Visitor(root, blocSize, pattern);
Files.walkFileTree(root, visitor);
for (Future<FileDesc> future : visitor.futures()) {
try {
files.add(future.get());
} catch (Exception e) {
log.error("", e);
}
}
}
public static boolean init() {
synchronized (cudaEngines) {
System.err.println("---------Initializing Cuda----------------");
try {
extractAndLoadNativeLibs();
JCudaDriver.setExceptionsEnabled(true);
JCudaDriver.cuInit(0);
compileKernelsPtx();
// Obtain the number of devices
int deviceCountArray[] = {0};
JCudaDriver.cuDeviceGetCount(deviceCountArray);
availableDevicesNb = deviceCountArray[0];
if (availableDevicesNb == 0) return false;
availableDevicesNb = NB_OF_DEVICE_TO_USE; // TODO
initialization = Executors.newCachedThreadPool();
System.out.println("Found " + availableDevicesNb + " GPU devices");
for (int i = 0 /*-NB_OF_DEVICE_TO_USE*/; i < availableDevicesNb; i++) {
final int index = i;
Future<?> initJob =
initialization.submit(
new Runnable() {
public void run() {
System.err.println("Initializing device n°" + index);
cudaEngines.put(index, new CudaEngine(index));
}
});
initJob.get();
initialization.shutdown();
}
} catch (InterruptedException
| ExecutionException
| IOException
| CudaException
| UnsatisfiedLinkError e) {
e.printStackTrace();
System.err.println("---------Cannot initialize Cuda !!! ----------------");
return false;
}
Runtime.getRuntime()
.addShutdownHook(
new Thread() {
@Override
public void run() {
CudaEngine.stop();
}
});
System.out.println("---------Cuda Initialized----------------");
return true;
}
}
/**
* Execute the {@link Callable} tasks in parallel (per the configured size of the {@link
* WorkerPool}) and wait for them to complete.
*
* @param tasks a list of {@link Callable}s
* @return the ordered return values
*/
public <T> List<T> invokeAll(List<Callable<T>> tasks) {
try {
List<Future<T>> executorResults = executorService.invokeAll(tasks);
List<T> results = new ArrayList<T>(tasks.size());
for (Future<T> future : executorResults) {
results.add(future.get());
}
return results;
} catch (InterruptedException e) {
throw new RuntimeException(e);
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
}
@Test
public void testExecuteMultipleNode() throws InterruptedException, ExecutionException {
final int k = simpleTestNodeCount;
TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(k);
final HazelcastInstance[] instances = factory.newInstances(new Config());
for (int i = 0; i < k; i++) {
final IExecutorService service = instances[i].getExecutorService("testExecuteMultipleNode");
final String script = "hazelcast.getAtomicLong('count').incrementAndGet();";
final int rand = new Random().nextInt(100);
final Future<Integer> future = service.submit(new ScriptRunnable(script, null), rand);
assertEquals(Integer.valueOf(rand), future.get());
}
final IAtomicLong count = instances[0].getAtomicLong("count");
assertEquals(k, count.get());
}
static String outputOf(final Process p) {
try {
Future<String> outputFuture = futureOutputOf(p.getInputStream());
Future<String> errorFuture = futureOutputOf(p.getErrorStream());
final String output = outputFuture.get();
final String error = errorFuture.get();
// Check for successful process completion
equal(error, "");
equal(p.waitFor(), 0);
equal(p.exitValue(), 0);
return output;
} catch (Throwable t) {
unexpected(t);
throw new Error(t);
}
}
public void testDelayedTasksReusePooledThreadIfExecuteAtDifferentTimes()
throws InterruptedException, ExecutionException {
final AppScheduledExecutorService service = new AppScheduledExecutorService(getName());
final List<LogInfo> log = Collections.synchronizedList(new ArrayList<>());
// pre-start one thread
Future<?> future = service.submit(EmptyRunnable.getInstance());
future.get();
service.setBackendPoolCorePoolSize(1);
assertEquals(1, service.getBackendPoolExecutorSize());
int delay = 500;
ScheduledFuture<?> f1 =
service.schedule((Runnable) () -> log.add(new LogInfo(1)), delay, TimeUnit.MILLISECONDS);
ScheduledFuture<?> f2 =
service.schedule(
(Runnable) () -> log.add(new LogInfo(2)), delay + 100, TimeUnit.MILLISECONDS);
ScheduledFuture<?> f3 =
service.schedule(
(Runnable) () -> log.add(new LogInfo(3)), delay + 200, TimeUnit.MILLISECONDS);
assertEquals(1, service.getBackendPoolExecutorSize());
assertFalse(f1.isDone());
assertFalse(f2.isDone());
assertFalse(f3.isDone());
TimeoutUtil.sleep(delay + 200 + 300);
assertTrue(f1.isDone());
assertTrue(f2.isDone());
assertTrue(f3.isDone());
assertEquals(1, service.getBackendPoolExecutorSize());
assertEquals(3, log.size());
Set<Thread> usedThreads =
new HashSet<>(
Arrays.asList(
log.get(0).currentThread, log.get(1).currentThread, log.get(2).currentThread));
if (usedThreads.size() != 1) {
System.err.println(ThreadDumper.dumpThreadsToString());
}
assertEquals(usedThreads.toString(), 1, usedThreads.size()); // must be executed in same thread
service.shutdownAppScheduledExecutorService();
assertTrue(service.awaitTermination(10, TimeUnit.SECONDS));
}
public static void main(String[] args) {
ExecutorService executorService = Executors.newFixedThreadPool(4);
List<Future<String>> results = new ArrayList<>();
String[] URLs = {
"http://www.weather.gov",
"http://www.espn.com",
"http://www.marketwatch.com",
"http://www.fandango.com"
};
for (String URL : URLs) {
Callable callable =
new Callable<String>() {
@Override
public String call() throws Exception {
Thread.sleep(1000);
System.out.println("Retrieving URL: " + URL);
return SiteDownloader.get(URL);
}
};
System.out.println("Submitting task for: " + URL);
results.add(executorService.submit(callable));
}
boolean done = false;
while (!done) {
System.out.println("Still getting data...");
done = areAllTrue(results);
try {
Thread.sleep(200);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
for (Future<String> result : results) {
try {
System.out.println("Site data: " + result.get());
} catch (InterruptedException | ExecutionException e) {
e.printStackTrace();
}
}
executorService.shutdown();
}
public static void main(String[] args) {
ActiveObjectDemo d1 = new ActiveObjectDemo();
// Prevents ConcurrentModificationException:
List<Future<?>> results = new CopyOnWriteArrayList<Future<?>>();
for (float f = 0.0f; f < 1.0f; f += 0.2f) results.add(d1.calculateFloat(f, f));
for (int i = 0; i < 5; i++) results.add(d1.calculateInt(i, i));
print("All asynch calls made");
while (results.size() > 0) {
for (Future<?> f : results)
if (f.isDone()) {
try {
print(f.get());
} catch (Exception e) {
throw new RuntimeException(e);
}
results.remove(f);
}
}
d1.shutdown();
}
