public static void main(String[] args) {
BlockingQueue<Runnable> queue = new LinkedBlockingQueue<Runnable>(12);
ThreadFactory threadFactory =
new ThreadFactory() {
public Thread newThread(Runnable r) {
int currentCount = counter.getAndIncrement();
System.out.println("Creating new thread: " + currentCount);
return new Thread(r, "mythread" + currentCount);
}
};
RejectedExecutionHandler rejectedHandler =
new RejectedExecutionHandler() {
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
if (r instanceof ThreadPoolExecutorExample) {
ThreadPoolExecutorExample example = (ThreadPoolExecutorExample) r;
System.out.println("Rejecting task with id " + example.getTaskId());
}
}
};
ThreadPoolExecutor executor =
new ThreadPoolExecutor(5, 12, 1, TimeUnit.SECONDS, queue, threadFactory, rejectedHandler);
for (int i = 0; i < 100; i++) {
executor.execute(new ThreadPoolExecutorExample(i));
}
executor.shutdown();
}
@Test
public void testJedisClusterRunsWithMultithreaded()
throws InterruptedException, ExecutionException, IOException {
Set<HostAndPort> jedisClusterNode = new HashSet<HostAndPort>();
jedisClusterNode.add(new HostAndPort("127.0.0.1", 7379));
final JedisCluster jc = new JedisCluster(jedisClusterNode);
jc.set("foo", "bar");
ThreadPoolExecutor executor =
new ThreadPoolExecutor(10, 100, 0, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(10));
List<Future<String>> futures = new ArrayList<Future<String>>();
for (int i = 0; i < 50; i++) {
executor.submit(
new Callable<String>() {
@Override
public String call() throws Exception {
// FIXME : invalidate slot cache from JedisCluster to test
// random connection also does work
return jc.get("foo");
}
});
}
for (Future<String> future : futures) {
String value = future.get();
assertEquals("bar", value);
}
jc.close();
}
/** {@inheritDoc} */
@Override
protected GridConfiguration getConfiguration(String gridName) throws Exception {
GridConfiguration c = super.getConfiguration(gridName);
c.setLocalHost(HOST);
assert c.getClientConnectionConfiguration() == null;
GridClientConnectionConfiguration clientCfg = new GridClientConnectionConfiguration();
clientCfg.setRestTcpPort(REST_TCP_PORT_BASE);
GridSslContextFactory sslCtxFactory = sslContextFactory();
if (sslCtxFactory != null) {
clientCfg.setRestTcpSslEnabled(true);
clientCfg.setRestTcpSslContextFactory(sslCtxFactory);
}
c.setClientConnectionConfiguration(clientCfg);
GridTcpDiscoverySpi disco = new GridTcpDiscoverySpi();
disco.setIpFinder(IP_FINDER);
c.setDiscoverySpi(disco);
TestCommunicationSpi spi = new TestCommunicationSpi();
spi.setLocalPort(GridTestUtils.getNextCommPort(getClass()));
c.setCommunicationSpi(spi);
c.setCacheConfiguration(
cacheConfiguration(null),
cacheConfiguration(PARTITIONED_CACHE_NAME),
cacheConfiguration(REPLICATED_CACHE_NAME),
cacheConfiguration(REPLICATED_ASYNC_CACHE_NAME));
ThreadPoolExecutor exec =
new ThreadPoolExecutor(40, 40, 0, MILLISECONDS, new LinkedBlockingQueue<Runnable>());
exec.prestartAllCoreThreads();
c.setExecutorService(exec);
c.setExecutorServiceShutdown(true);
ThreadPoolExecutor sysExec =
new ThreadPoolExecutor(40, 40, 0, MILLISECONDS, new LinkedBlockingQueue<Runnable>());
sysExec.prestartAllCoreThreads();
c.setSystemExecutorService(sysExec);
c.setSystemExecutorServiceShutdown(true);
return c;
}
/** {@inheritDoc} */
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
try {
if (executor.isShutdown()) throw new RejectedExecutionException();
else executor.getQueue().put(r);
} catch (InterruptedException ignored) {
U.warn(log, "Working thread was interrupted while loading data.");
Thread.currentThread().interrupt();
}
}
/**
* Requests repairs for the given table and column families, and blocks until all repairs have
* been completed.
*/
public RepairFuture submitRepairSession(
Range<Token> range, String tablename, boolean isSequential, String... cfnames) {
RepairFuture futureTask =
new RepairSession(range, tablename, isSequential, cfnames).getFuture();
executor.execute(futureTask);
return futureTask;
}
/**
* Stops the timer, cancelling all tasks
*
* @throws InterruptedException if interrupted while waiting for thread to return
*/
public void stop() {
stopRunner();
List<Runnable> remaining_tasks = pool.shutdownNow();
for (Runnable task : remaining_tasks) {
if (task instanceof Future) {
Future future = (Future) task;
future.cancel(true);
}
}
pool.getQueue().clear();
try {
pool.awaitTermination(Global.THREADPOOL_SHUTDOWN_WAIT_TIME, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
}
}
/** Asynchronously refreshes the tokens contained in this instance. */
private void beginRefresh() {
try {
refreshExecutor.execute(
new Runnable() {
@Override
public void run() {
Log.debug("[oauth] Refreshing access token.");
Tokens tokens;
try {
tokens =
oauthClient
.getTokensForClient(
tokenEndpoint, clientId, clientSecret, "manage_project:" + projectKey)
.get();
} catch (Exception e) {
update(null, e);
return;
}
update(tokens, null);
}
});
} catch (RejectedExecutionException e) {
// another refresh is already in progress, ignore this one
}
}
@Override
public void serviceRemoved(final URI service) {
services.remove(service.toString());
for (final DiscoveryListener discoveryListener : getListeners()) {
executor.execute(new ServiceRemovedTask(discoveryListener, service));
}
}
/**
* Requeues a periodic task unless current run state precludes it. Same idea as delayedExecute
* except drops task rather than rejecting.
*
* @param task the task
*/
void reExecutePeriodic(RunnableScheduledFuture<?> task) {
if (canRunInCurrentRunState(true)) {
super.getQueue().add(task);
if (!canRunInCurrentRunState(true) && remove(task)) task.cancel(false);
else ensurePrestart();
}
}
private void receive(SendPacket sendPacket, PinpointSocket pinpointSocket) {
try {
worker.execute(new Dispatch(sendPacket.getPayload(), pinpointSocket.getRemoteAddress()));
} catch (RejectedExecutionException e) {
// cause is clear - full stack trace not necessary
logger.warn("RejectedExecutionException Caused:{}", e.getMessage());
}
}
public void addDiscoveryListener(final DiscoveryListener listener) {
// get the listener caught up
for (final URI service : services.values()) {
executor.execute(new ServiceAddedTask(listener, service));
}
listeners.add(listener);
}
private void requestResponse(RequestPacket requestPacket, PinpointSocket pinpointSocket) {
try {
worker.execute(new RequestResponseDispatch(requestPacket, pinpointSocket));
} catch (RejectedExecutionException e) {
// cause is clear - full stack trace not necessary
logger.warn("RejectedExecutionException Caused:{}", e.getMessage());
}
}
public ThreadPoolStats stats() {
List<ThreadPoolStats.Stats> stats = new ArrayList<ThreadPoolStats.Stats>();
for (ExecutorHolder holder : executors.values()) {
String name = holder.info.getName();
// no need to have info on "same" thread pool
if ("same".equals(name)) {
continue;
}
int threads = -1;
int queue = -1;
int active = -1;
long rejected = -1;
int largest = -1;
long completed = -1;
if (holder.executor instanceof ThreadPoolExecutor) {
ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) holder.executor;
threads = threadPoolExecutor.getPoolSize();
queue = threadPoolExecutor.getQueue().size();
active = threadPoolExecutor.getActiveCount();
largest = threadPoolExecutor.getLargestPoolSize();
completed = threadPoolExecutor.getCompletedTaskCount();
RejectedExecutionHandler rejectedExecutionHandler =
threadPoolExecutor.getRejectedExecutionHandler();
if (rejectedExecutionHandler instanceof XRejectedExecutionHandler) {
rejected = ((XRejectedExecutionHandler) rejectedExecutionHandler).rejected();
}
}
stats.add(
new ThreadPoolStats.Stats(name, threads, queue, active, rejected, largest, completed));
}
return new ThreadPoolStats(stats);
}
/**
* Main execution method for delayed or periodic tasks. If pool is shut down, rejects the task.
* Otherwise adds task to queue and starts a thread, if necessary, to run it. (We cannot prestart
* the thread to run the task because the task (probably) shouldn't be run yet,) If the pool is
* shut down while the task is being added, cancel and remove it if required by state and
* run-after-shutdown parameters.
*
* @param task the task
*/
private void delayedExecute(RunnableScheduledFuture<?> task) {
if (isShutdown()) reject(task);
else {
super.getQueue().add(task);
if (isShutdown() && !canRunInCurrentRunState(task.isPeriodic()) && remove(task))
task.cancel(false);
else ensurePrestart();
}
}
public void rejectedExecution(Runnable task, ThreadPoolExecutor executor) {
((DebuggableThreadPoolExecutor) executor).onInitialRejection(task);
BlockingQueue<Runnable> queue = executor.getQueue();
while (true) {
if (executor.isShutdown()) {
((DebuggableThreadPoolExecutor) executor).onFinalRejection(task);
throw new RejectedExecutionException("ThreadPoolExecutor has shut down");
}
try {
if (queue.offer(task, 1000, TimeUnit.MILLISECONDS)) {
((DebuggableThreadPoolExecutor) executor).onFinalAccept(task);
break;
}
} catch (InterruptedException e) {
throw new AssertionError(e);
}
}
}
static void oneRun(BlockingQueue<Runnable> q, int nThreads, int iters, boolean print)
throws Exception {
ThreadPoolExecutor pool =
new ThreadPoolExecutor(nThreads + 1, Integer.MAX_VALUE, 1L, TimeUnit.SECONDS, q);
CountDownLatch done = new CountDownLatch(iters);
remaining.set(nThreads - 1);
pool.prestartAllCoreThreads();
Task t = new Task(pool, done);
long start = System.nanoTime();
pool.execute(t);
done.await();
long time = System.nanoTime() - start;
if (print) System.out.println("\t: " + LoopHelpers.rightJustify(time / iters) + " ns per task");
q.clear();
Thread.sleep(100);
pool.shutdown();
Thread.sleep(100);
pool.shutdownNow();
}
@Override
public void dispose() {
myDisposed = true;
cancelAllRequests();
if (myThreadToUse == ThreadToUse.POOLED_THREAD) {
myExecutorService.shutdown();
} else if (myThreadToUse == ThreadToUse.OWN_THREAD) {
myExecutorService.shutdown();
((ThreadPoolExecutor) myExecutorService).getQueue().clear();
}
}
/** Submit exec into pool and throw exceptions */
public void execute() {
String p = project.getProperty("number.of.threads");
if (p != null) {
((ThreadPoolExecutor) threadPool).setCorePoolSize(Integer.parseInt(p));
((ThreadPoolExecutor) threadPool).setMaximumPoolSize(Integer.parseInt(p));
}
TaskRunnable tr = new TaskRunnable();
threadPool.submit(tr);
try {
synchronized (semaphore) {
while (!tr.isFinished()) semaphore.wait();
}
} catch (InterruptedException e) {
e.printStackTrace();
}
Throwable t = tr.getException();
if (t != null) {
if (t instanceof BuildException) throw (BuildException) t;
else t.printStackTrace();
}
}
synchronized Boolean destroy() throws InterruptedException, ExecutionException {
destroyed.set(true);
startupId.incrementAndGet();
unexpectedRestartTimestamp.set(-1);
NannyDestroyCallable nannyTask =
new NannyDestroyCallable(
instanceDescriptor.get(), instancePath, deployLog, healthLog, invokeScript, ubaLog);
Future<Boolean> waitForDestory = threadPoolExecutor.submit(nannyTask);
Boolean result = waitForDestory.get();
if (result) {
nannyTask.wipeoutFiles();
}
return result;
}
public void testMustNotBeAbleToShutdownGlobalPool() {
ExecutorService service = AppExecutorUtil.getAppExecutorService();
try {
service.shutdown();
fail();
} catch (Exception ignored) {
}
try {
service.shutdownNow();
fail();
} catch (Exception ignored) {
}
try {
((ThreadPoolExecutor) service).setThreadFactory(Thread::new);
fail();
} catch (Exception ignored) {
}
try {
((ThreadPoolExecutor) service).setCorePoolSize(0);
fail();
} catch (Exception ignored) {
}
}
public void run() {
done.countDown();
remaining.incrementAndGet();
int n;
while (!Thread.interrupted() && (n = remaining.get()) > 0 && done.getCount() > 0) {
if (remaining.compareAndSet(n, n - 1)) {
try {
pool.execute(this);
} catch (RuntimeException ex) {
System.out.print("*");
while (done.getCount() > 0) done.countDown();
return;
}
}
}
}
@Override
public void publish(LogRecord record) {
final Map<String, Object> map = Loggly.getMap();
String level = "UNKN";
if (record.getLevel().equals(Level.WARNING)) {
level = "WARN";
} else if (record.getLevel().equals(Level.SEVERE)) {
level = "SEVR";
} else if (record.getLevel().equals(Level.INFO)) {
level = "INFO";
} else if (record.getLevel().equals(Level.FINE)) {
level = "FINE";
} else if (record.getLevel().equals(Level.FINEST)) {
level = "FNST";
} else if (record.getLevel().equals(Level.FINER)) {
level = "FINR";
} else if (record.getLevel().equals(Level.CONFIG)) {
level = "CONF";
} else if (record.getLevel().equals(Level.OFF)) {
level = "OFF ";
} else if (record.getLevel().equals(Level.ALL)) {
level = "ALL ";
}
// and the log message itself
if (record.getParameters() != null && record.getParameters().length > 0) {
java.util.Formatter formatter = new java.util.Formatter();
formatter.format(record.getMessage(), record.getParameters());
map.put("message", formatter.toString());
} else {
map.put("message", record.getMessage());
}
// finally, other metadata
map.put("thread", Thread.currentThread().getName());
map.put("loggerName", record.getLoggerName());
map.put("level", level);
if (record.getThrown() != null) {
StringWriter sw = new StringWriter();
record.getThrown().printStackTrace(new PrintWriter(sw));
map.put("stackTrace", sw.toString());
}
pool.submit(new LogglySample(map));
}
synchronized Boolean kill() throws InterruptedException, ExecutionException {
status.set("Kill");
startupId.incrementAndGet();
unexpectedRestartTimestamp.set(-1);
NannyDestroyCallable nannyTask =
new NannyDestroyCallable(
instanceDescriptor.get(), instancePath, deployLog, healthLog, invokeScript, ubaLog);
Future<Boolean> waitForDestory = threadPoolExecutor.submit(nannyTask);
status.set("Killing");
Boolean result = waitForDestory.get();
if (result) {
status.set("Killed " + result);
} else {
status.set("Failed to kill");
}
return result;
}
public DiscoveryRegistry(final DiscoveryAgent agent) {
executor =
new ThreadPoolExecutor(
1,
10,
30,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(1),
new ThreadFactory() {
@Override
public Thread newThread(final Runnable runable) {
final Thread t = new Thread(runable, DiscoveryRegistry.class.getSimpleName());
t.setDaemon(true);
return t;
}
});
executor.setRejectedExecutionHandler(
new RejectedExecutionHandler() {
@Override
public void rejectedExecution(final Runnable r, final ThreadPoolExecutor tpe) {
if (null == r
|| null == tpe
|| tpe.isShutdown()
|| tpe.isTerminated()
|| tpe.isTerminating()) {
return;
}
try {
tpe.getQueue().offer(r, 20, TimeUnit.SECONDS);
} catch (InterruptedException e) {
// Ignore
}
}
});
SystemInstance.get().setComponent(DiscoveryRegistry.class, this);
SystemInstance.get().setComponent(DiscoveryAgent.class, this);
if (null != agent) {
addDiscoveryAgent(agent);
}
}
public void afterExecute(Runnable r, Throwable t) {
super.afterExecute(r, t);
// exceptions wrapped by FutureTask
if (r instanceof FutureTask) {
try {
((FutureTask) r).get();
} catch (InterruptedException e) {
throw new AssertionError(e);
} catch (ExecutionException e) {
logger.error("Error in executor futuretask", e);
}
}
// exceptions for non-FutureTask runnables [i.e., added via execute() instead of submit()]
if (t != null) {
logger.error("Error in ThreadPoolExecutor", t);
}
}
protected void _run() {
lock.lock();
try {
wheel_position = (wheel_position + 1) % wheel_size;
List<MyTask> list = wheel[wheel_position];
if (list.isEmpty()) return;
for (Iterator<MyTask> it = list.iterator(); it.hasNext(); ) {
MyTask tmp = it.next();
if (tmp.getAndDecrementRound() <= 0) {
try {
pool.execute(tmp);
} catch (Throwable t) {
log.error(Util.getMessage("FailureSubmittingTaskToThreadPool"), t);
}
it.remove();
}
}
} finally {
lock.unlock();
}
}
@Override
public synchronized void close() throws SecurityException {
if (pool.isShutdown()) {
return;
}
try {
// first, anything in the retry queue should be tried one last time and then we give up on it
allowRetry = false;
for (LogglySample sample : retryQueue) {
pool.submit(sample);
}
retryQueue.clear();
System.out.println(
"Shutting down Loggly handler - waiting 90 seconds for "
+ pool.getQueue().size()
+ " logs to finish");
pool.shutdown();
try {
boolean result = pool.awaitTermination(90, TimeUnit.SECONDS);
if (!result) {
System.out.println(
"Not all Loggly messages sent out - still had "
+ pool.getQueue().size()
+ " left :(");
pool.shutdownNow();
}
} catch (InterruptedException e) {
// ignore
}
} finally {
httpClient.getConnectionManager().shutdown();
System.out.println("Loggly handler shut down");
}
}
public static void execute(Runnable runnable) {
threadpool.execute(runnable);
}
@Override
public void stop() {
if (sharedHttpServers != null) {
for (HttpServer server : sharedHttpServers.values()) {
server.close();
}
sharedHttpServers = null;
}
if (sharedNetServers != null) {
for (NetServer server : sharedNetServers.values()) {
server.close();
}
sharedNetServers = null;
}
if (timer != null) {
timer.stop();
timer = null;
}
if (eventBus != null) {
eventBus.close(null);
}
if (backgroundPool != null) {
backgroundPool.shutdown();
}
if (acceptorPool != null) {
acceptorPool.shutdown();
}
try {
if (backgroundPool != null) {
backgroundPool.awaitTermination(20, TimeUnit.SECONDS);
backgroundPool = null;
}
} catch (InterruptedException ex) {
// ignore
}
try {
if (acceptorPool != null) {
acceptorPool.awaitTermination(20, TimeUnit.SECONDS);
acceptorPool = null;
}
} catch (InterruptedException ex) {
// ignore
}
// log.info("Release external resources from worker pool");
if (corePool != null) {
corePool.releaseExternalResources();
corePool = null;
}
// log.info("Release external resources: done");
setContext(null);
}
/** Shuts down internal thread pools. */
public void shutdown() {
refreshExecutor.shutdownNow();
refreshTimer.cancel();
}
