@Override
public Map<String, String> handleProbe(String... keys) {
if (keys == null || keys.length == 0) {
return null;
}
ThreadPoolExecutor exec = extract(executor);
if (exec == null) {
return null;
}
Map<String, String> map = new HashMap<>();
for (String key : keys) {
switch (key) {
case KEY:
map.put("active-thread", String.valueOf(exec.getActiveCount()));
map.put("min-thread", String.valueOf(exec.getCorePoolSize()));
map.put("max-thread", String.valueOf(exec.getMaximumPoolSize()));
map.put("current-pool-size", String.valueOf(exec.getPoolSize()));
map.put("largest-pool-size", String.valueOf(exec.getLargestPoolSize()));
map.put("keep-alive", String.valueOf(exec.getKeepAliveTime(TimeUnit.MILLISECONDS)));
map.put("queue-size", String.valueOf(exec.getQueue().size()));
break;
}
}
return map.isEmpty() ? null : map;
}
/* */ public void startDownloading(ThreadPoolExecutor executorService) {
/* 70 */ if (this.started)
throw new IllegalStateException("Cannot start download job that has already started");
/* 71 */ this.started = true;
/* */
/* 73 */ if (this.allFiles.isEmpty())
/* 74 */ System.out.println(
"Download job '" + this.name + "' skipped as there are no files to download");
/* */ else {
/* 77 */ int threads = executorService.getMaximumPoolSize();
/* 78 */ this.remainingThreads.set(threads);
/* 79 */ System.out.println(
"Download job '"
+ this.name
+ "' started ("
+ threads
+ " threads, "
+ this.allFiles.size()
+ " files)");
/* 80 */ for (int i = 0; i < threads; i++)
/* 81 */ executorService.submit(
new Runnable()
/* */ {
/* */ public void run() {
/* 84 */ DownloadJob.this.popAndDownload();
/* */ }
/* */ });
/* */ }
/* */ }
public boolean offer(Runnable o) {
// we can't do any checks
if (parent == null) return super.offer(o);
int poolSize = parent.getPoolSize();
// we are maxed out on threads, simply queue the object
if (parent.getPoolSize() == parent.getMaximumPoolSize()) return super.offer(o);
// we have idle threads, just add it to the queue
// note that we don't use getActiveCount(), see BZ 49730
AtomicInteger submittedTasksCount = StandardThreadExecutor.this.submittedTasksCount;
if (submittedTasksCount != null) {
if (submittedTasksCount.get() <= poolSize) return super.offer(o);
}
// if we have less threads than maximum force creation of a new thread
if (poolSize < parent.getMaximumPoolSize()) return false;
// if we reached here, we need to add it to the queue
return super.offer(o);
}
/* ------------------------------------------------------------ */
public boolean isLowOnThreads() {
if (_executor instanceof ThreadPoolExecutor) {
final ThreadPoolExecutor tpe = (ThreadPoolExecutor) _executor;
// getActiveCount() locks the thread pool, so execute it last
return tpe.getPoolSize() == tpe.getMaximumPoolSize()
&& tpe.getQueue().size() >= tpe.getPoolSize() - tpe.getActiveCount();
}
return false;
}
@Test
public void testScaleDown() throws Exception {
final int min = between(1, 3);
final int max = between(min + 1, 6);
final ThreadBarrier barrier = new ThreadBarrier(max + 1);
final ThreadPoolExecutor pool =
EsExecutors.newScaling(
min,
max,
between(1, 100),
TimeUnit.MILLISECONDS,
EsExecutors.daemonThreadFactory("test"));
assertThat("Min property", pool.getCorePoolSize(), equalTo(min));
assertThat("Max property", pool.getMaximumPoolSize(), equalTo(max));
for (int i = 0; i < max; ++i) {
final CountDownLatch latch = new CountDownLatch(1);
pool.execute(
new Runnable() {
@Override
public void run() {
latch.countDown();
try {
barrier.await();
barrier.await();
} catch (Throwable e) {
barrier.reset(e);
}
}
});
// wait until thread executes this task
// otherwise, a task might be queued
latch.await();
}
barrier.await();
assertThat("wrong pool size", pool.getPoolSize(), equalTo(max));
assertThat("wrong active size", pool.getActiveCount(), equalTo(max));
barrier.await();
assertBusy(
new Runnable() {
@Override
public void run() {
assertThat("wrong active count", pool.getActiveCount(), equalTo(0));
assertThat(
"idle threads didn't shrink below max. (" + pool.getPoolSize() + ")",
pool.getPoolSize(),
lessThan(max));
}
});
terminate(pool);
}
public void run() {
long t = System.currentTimeMillis();
int count = 1000000;
List<Long> maxList = new ArrayList<>(count);
for (int i = 0; i < count; i++) {
// maxList.add(executor.getActiveCount());
maxList.add(System.nanoTime() % count);
}
long max = 0;
long sum = 0;
for (Long m : maxList) {
if (m.intValue() > max) {
max = m.intValue();
}
sum += m.intValue();
}
double avg = (double) sum / (double) count;
try {
Thread.sleep((System.currentTimeMillis() - t));
} catch (Exception e) {
}
t = System.currentTimeMillis() - t;
int core = executor.getCorePoolSize();
int active = executor.getActiveCount();
int queue = executor.getQueue().size();
max = executor.getMaximumPoolSize();
System.out.println(
t
+ ", core: "
+ core
+ ", active: "
+ active
+ ", max: "
+ max
+ ", queue: "
+ queue
+ ", max: "
+ max);
}
@Override
public boolean offer(E e) {
// first try to transfer to a waiting worker thread
if (!tryTransfer(e)) {
// check if there might be spare capacity in the thread
// pool executor
int left = executor.getMaximumPoolSize() - executor.getCorePoolSize();
if (left > 0) {
// reject queuing the task to force the thread pool
// executor to add a worker if it can; combined
// with ForceQueuePolicy, this causes the thread
// pool to always scale up to max pool size and we
// only queue when there is no spare capacity
return false;
} else {
return super.offer(e);
}
} else {
return true;
}
}
static void checkPoolSizes(ThreadPoolExecutor pool, int size, int core, int max) {
equal(pool.getPoolSize(), size);
equal(pool.getCorePoolSize(), core);
equal(pool.getMaximumPoolSize(), max);
}
public boolean isLowOnThreads() {
return executor.getActiveCount() >= executor.getMaximumPoolSize();
}
@Override
public int getMaxThreadCount() {
return executor.getMaximumPoolSize();
}
public int getMaxThreads() {
return pool.getMaximumPoolSize();
}
public String[] getStats() {
return new String[] {
"STP:",
" + Effects:",
" |- ActiveThreads: " + _effectsScheduledThreadPool.getActiveCount(),
" |- getCorePoolSize: " + _effectsScheduledThreadPool.getCorePoolSize(),
" |- PoolSize: " + _effectsScheduledThreadPool.getPoolSize(),
" |- MaximumPoolSize: " + _effectsScheduledThreadPool.getMaximumPoolSize(),
" |- CompletedTasks: " + _effectsScheduledThreadPool.getCompletedTaskCount(),
" |- ScheduledTasks: "
+ (_effectsScheduledThreadPool.getTaskCount()
- _effectsScheduledThreadPool.getCompletedTaskCount()),
" | -------",
" + General:",
" |- ActiveThreads: " + _generalScheduledThreadPool.getActiveCount(),
" |- getCorePoolSize: " + _generalScheduledThreadPool.getCorePoolSize(),
" |- PoolSize: " + _generalScheduledThreadPool.getPoolSize(),
" |- MaximumPoolSize: " + _generalScheduledThreadPool.getMaximumPoolSize(),
" |- CompletedTasks: " + _generalScheduledThreadPool.getCompletedTaskCount(),
" |- ScheduledTasks: "
+ (_generalScheduledThreadPool.getTaskCount()
- _generalScheduledThreadPool.getCompletedTaskCount()),
" | -------",
" + AI:",
" |- ActiveThreads: " + _aiScheduledThreadPool.getActiveCount(),
" |- getCorePoolSize: " + _aiScheduledThreadPool.getCorePoolSize(),
" |- PoolSize: " + _aiScheduledThreadPool.getPoolSize(),
" |- MaximumPoolSize: " + _aiScheduledThreadPool.getMaximumPoolSize(),
" |- CompletedTasks: " + _aiScheduledThreadPool.getCompletedTaskCount(),
" |- ScheduledTasks: "
+ (_aiScheduledThreadPool.getTaskCount()
- _aiScheduledThreadPool.getCompletedTaskCount()),
"TP:",
" + Packets:",
" |- ActiveThreads: " + _generalPacketsThreadPool.getActiveCount(),
" |- getCorePoolSize: " + _generalPacketsThreadPool.getCorePoolSize(),
" |- MaximumPoolSize: " + _generalPacketsThreadPool.getMaximumPoolSize(),
" |- LargestPoolSize: " + _generalPacketsThreadPool.getLargestPoolSize(),
" |- PoolSize: " + _generalPacketsThreadPool.getPoolSize(),
" |- CompletedTasks: " + _generalPacketsThreadPool.getCompletedTaskCount(),
" |- QueuedTasks: " + _generalPacketsThreadPool.getQueue().size(),
" | -------",
" + I/O Packets:",
" |- ActiveThreads: " + _ioPacketsThreadPool.getActiveCount(),
" |- getCorePoolSize: " + _ioPacketsThreadPool.getCorePoolSize(),
" |- MaximumPoolSize: " + _ioPacketsThreadPool.getMaximumPoolSize(),
" |- LargestPoolSize: " + _ioPacketsThreadPool.getLargestPoolSize(),
" |- PoolSize: " + _ioPacketsThreadPool.getPoolSize(),
" |- CompletedTasks: " + _ioPacketsThreadPool.getCompletedTaskCount(),
" |- QueuedTasks: " + _ioPacketsThreadPool.getQueue().size(),
" | -------",
" + General Tasks:",
" |- ActiveThreads: " + _generalThreadPool.getActiveCount(),
" |- getCorePoolSize: " + _generalThreadPool.getCorePoolSize(),
" |- MaximumPoolSize: " + _generalThreadPool.getMaximumPoolSize(),
" |- LargestPoolSize: " + _generalThreadPool.getLargestPoolSize(),
" |- PoolSize: " + _generalThreadPool.getPoolSize(),
" |- CompletedTasks: " + _generalThreadPool.getCompletedTaskCount(),
" |- QueuedTasks: " + _generalThreadPool.getQueue().size(),
" | -------",
" + Javolution stats:",
" |- FastList: " + FastList.report(),
" |- FastMap: " + FastMap.report(),
" |- FastSet: " + FastSet.report(),
" | -------"
};
}
public void runScenario() {
MultiThreadedTaskRunner taskRunner = TestEnvironment.getMultiThreadedTaskRunner();
ThreadPoolExecutor executorService = (ThreadPoolExecutor) taskRunner.getExecutorService();
assertEquals(TestEnvironment.THREAD_COUNT, executorService.getMaximumPoolSize());
assertEquals(0, executorService.getActiveCount());
DataCleanerConfiguration configuration =
new DataCleanerConfigurationImpl()
.withEnvironment(new DataCleanerEnvironmentImpl().withTaskRunner(taskRunner));
AnalysisRunner runner = new AnalysisRunnerImpl(configuration);
Datastore ds = TestHelper.createSampleDatabaseDatastore("foobar");
try (DatastoreConnection con = ds.openConnection()) {
AnalysisJob job;
try (AnalysisJobBuilder analysisJobBuilder = new AnalysisJobBuilder(configuration)) {
analysisJobBuilder.setDatastore(ds);
Table table = con.getDataContext().getDefaultSchema().getTableByName("ORDERFACT");
assertNotNull(table);
Column statusColumn = table.getColumnByName("STATUS");
Column commentsColumn = table.getColumnByName("COMMENTS");
analysisJobBuilder.addSourceColumns(statusColumn, commentsColumn);
analysisJobBuilder
.addAnalyzer(AnalyzerMock.class)
.addInputColumns(analysisJobBuilder.getSourceColumns());
job = analysisJobBuilder.toAnalysisJob();
}
AnalysisResultFuture resultFuture = runner.run(job);
try {
Thread.sleep(550);
} catch (InterruptedException e) {
e.printStackTrace();
fail("Interrupted! " + e.getMessage());
}
resultFuture.cancel();
assertFalse(resultFuture.isSuccessful());
assertTrue(resultFuture.isCancelled());
assertTrue(resultFuture.isErrornous());
try {
Thread.sleep(400);
} catch (InterruptedException e) {
e.printStackTrace();
fail("Interrupted! " + e.getMessage());
}
assertEquals(TestEnvironment.THREAD_COUNT, executorService.getMaximumPoolSize());
long completedTaskCount = executorService.getCompletedTaskCount();
assertTrue("completedTaskCount was: " + completedTaskCount, completedTaskCount > 3);
int largestPoolSize = executorService.getLargestPoolSize();
assertTrue("largestPoolSize was: " + largestPoolSize, largestPoolSize > 5);
assertEquals(0, executorService.getActiveCount());
}
taskRunner.shutdown();
}
