@Override
public void run() {
try {
boolean running = true;
while (running) {
try {
// block on event availability
ThreadBoundEvent event = queue.take();
// add to the batch, and see if we can add more
batch.add(event);
if (maxBatchSize > 0) {
queue.drainTo(batch, maxBatchSize);
}
// check for the stop condition (and remove it)
// treat batches of 1 (the most common case) specially
if (batch.size() > 1) {
ListIterator<ThreadBoundEvent> itr = batch.listIterator();
while (itr.hasNext()) {
ThreadBoundEvent next = itr.next();
if (next.getClass().equals(ShutdownTask.class)) {
running = false;
((ShutdownTask) next).latch.countDown();
itr.remove();
}
}
eventProcessor.process(batch);
} else {
// just the one event, no need to iterate
if (event.getClass().equals(ShutdownTask.class)) {
running = false;
((ShutdownTask) event).latch.countDown();
} else {
eventProcessor.process(batch);
}
}
} catch (InterruptedException e) {
LOG.warn(
String.format(
"Consumer on queue %s interrupted.", Thread.currentThread().getName()));
// ignore
} catch (Throwable exception) {
LOG.error(
String.format(
"exception on queue %s while executing events",
Thread.currentThread().getName()),
exception);
} finally {
// reset the batch
batch.clear();
}
}
} catch (Throwable unexpectedThrowable) {
// we observed some cases where trying to log the inner exception threw an error
// don't use the logger here as that seems to be causing the problem in the first place
System.err.println("Caught and unexpected Throwable while logging");
System.err.println(
"This problem happens when jar files change at runtime, JVM might be UNSTABLE");
unexpectedThrowable.printStackTrace(System.err);
}
}
public DesignCIManifestRfcTouple call() {
String oldThreadName = Thread.currentThread().getName();
try {
Thread.currentThread().setName(getProcessingThreadName(oldThreadName, env.getCiId()));
ManifestRfcContainer manifestPlatformRfcs =
manifestRfcProcessor.processPlatform(
platRelation.getToCi(), env, nsPath, userId, availMode);
DesignCIManifestRfcTouple touple =
new DesignCIManifestRfcTouple(platRelation.getToCi().getCiId(), manifestPlatformRfcs);
return touple;
} finally {
countDownLatch.countDown();
Thread.currentThread().setName(oldThreadName);
}
}
/** {@inheritDoc} */
@Override
public void loadCache(GridBiInClosure<K, V> c, @Nullable Object... args) throws GridException {
ExecutorService exec =
new ThreadPoolExecutor(
threadsCnt,
threadsCnt,
0L,
MILLISECONDS,
new ArrayBlockingQueue<Runnable>(batchQueueSize),
new BlockingRejectedExecutionHandler());
Iterator<I> iter = inputIterator(args);
Collection<I> buf = new ArrayList<>(batchSize);
try {
while (iter.hasNext()) {
if (Thread.currentThread().isInterrupted()) {
U.warn(log, "Working thread was interrupted while loading data.");
break;
}
buf.add(iter.next());
if (buf.size() == batchSize) {
exec.submit(new Worker(c, buf, args));
buf = new ArrayList<>(batchSize);
}
}
if (!buf.isEmpty()) exec.submit(new Worker(c, buf, args));
} catch (RejectedExecutionException ignored) {
// Because of custom RejectedExecutionHandler.
assert false : "RejectedExecutionException was thrown while it shouldn't.";
} finally {
exec.shutdown();
try {
exec.awaitTermination(Long.MAX_VALUE, MILLISECONDS);
} catch (InterruptedException ignored) {
U.warn(log, "Working thread was interrupted while waiting for put operations to complete.");
Thread.currentThread().interrupt();
}
}
}
public void run() {
try {
while (true) indexFile(queue.take());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
public void run() {
try {
crawl(root);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
public void shutdown() {
try {
shutdown(0, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
/**
* Execute the {@link Callable} tasks in parallel (per the configured size of the {@link
* WorkerPool}) and wait for them to complete.
*
* @param tasks a map of {@link Callable}s with keys by which you will be able to access each
* return value
* @return the return values of each {@link Callable}s mapped by their input key
*/
public <K, V> Map<K, V> invokeAll(Map<K, Callable<V>> tasks) {
String caller =
LOGGER.isDebugEnabled() ? Thread.currentThread().getStackTrace()[2].toString() : "n/a";
LOGGER.debug("[%s] is invoking %d mapped tasks", caller, tasks.size());
List<K> orderedKeys = new ArrayList<K>(tasks.size());
List<Callable<V>> orderedTasks = new ArrayList<Callable<V>>(tasks.size());
for (Map.Entry<K, Callable<V>> entry : tasks.entrySet()) {
orderedKeys.add(entry.getKey());
orderedTasks.add(entry.getValue());
}
try {
long start = System.currentTimeMillis();
List<Future<V>> executorResults = executorService.invokeAll(orderedTasks);
long finish = System.currentTimeMillis();
LOGGER.debug("[%s] invoked %d mapped tasks in %d ms", caller, tasks.size(), finish - start);
Map<K, V> mappedResults = new LinkedHashMap<K, V>(tasks.size());
for (int i = 0; i < tasks.size(); i++) {
K key = orderedKeys.get(i);
V result = executorResults.get(i).get();
mappedResults.put(key, result);
}
return mappedResults;
} catch (InterruptedException e) {
throw new RuntimeException(e);
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
}
/**
* @param key Key to lock.
* @return Sync object that should be passed to {@link #unlock(Object, Object)} method.
*/
public <T> Object lock(T key) {
assert key != null;
boolean interrupted = false;
Sync t = new Sync();
try {
while (true) {
Sync old = locks.putIfAbsent(key, t);
if (old != null) {
while (true) {
try {
old.await();
break;
} catch (InterruptedException ignored) {
interrupted = true;
}
}
} else return t;
}
} finally {
if (interrupted) Thread.currentThread().interrupt();
}
}
/**
* @return if <0, then this pipe should be stopped. If ==0, it should not wait. If >0, if it
* should wait
*/
int execute() {
if (!lock.tryAcquire()) return 1; // currently being accessed
Thread myThread = Thread.currentThread();
int threadIndex = -1;
for (int i = 0; i < threads.length; i++) {
if (threads[i] == null) {
threads[i] = myThread;
threadIndex = i;
break;
}
if (myThread != threads[i]) continue;
threadIndex = i;
break;
}
Signal signal;
if (threadIndex < 0) {
signal = dummySignal;
} else {
SignalImpl s = signals[threadIndex];
s.threadIndex = threadIndex;
s.signaled = false;
signal = s;
}
boolean hasData;
try {
hasData = poll(signal, null);
} finally {
signal.signal();
if (threadIndex < 0) lock.release();
}
return 0;
}
public HostConnectionPool(Host host, HostDistance hostDistance, Session.Manager manager)
throws ConnectionException {
assert hostDistance != HostDistance.IGNORED;
this.host = host;
this.hostDistance = hostDistance;
this.manager = manager;
this.newConnectionTask =
new Runnable() {
@Override
public void run() {
addConnectionIfUnderMaximum();
scheduledForCreation.decrementAndGet();
}
};
// Create initial core connections
List<Connection> l =
new ArrayList<Connection>(options().getCoreConnectionsPerHost(hostDistance));
try {
for (int i = 0; i < options().getCoreConnectionsPerHost(hostDistance); i++)
l.add(manager.connectionFactory().open(host));
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
// If asked to interrupt, we can skip opening core connections, the pool will still work.
// But we ignore otherwise cause I'm not sure we can do much better currently.
}
this.connections = new CopyOnWriteArrayList<Connection>(l);
this.open = new AtomicInteger(connections.size());
logger.trace("Created connection pool to host {}", host);
}
void renderPage(CharSequence source) {
final List<ImageInfo> imageInfos = scanForImageInfo(source);
Callable<List<ImageData>> task =
new Callable<List<ImageData>>() {
public List<ImageData> call() {
List<ImageData> result = new ArrayList<ImageData>();
for (ImageInfo imageInfo : imageInfos) result.add(imageInfo.downloadImage());
return result;
}
};
Future<List<ImageData>> future = executor.submit(task);
renderText(source);
try {
List<ImageData> imageData = future.get();
for (ImageData data : imageData) renderImage(data);
} catch (InterruptedException e) {
// Re-assert the thread's interrupted status
Thread.currentThread().interrupt();
// We don't need the result, so cancel the task too
future.cancel(true);
} catch (ExecutionException e) {
throw launderThrowable(e.getCause());
}
}
public static ThreadGroup getRootThreadGroup() {
if (rootThreadGroup != null) return rootThreadGroup;
ThreadGroup tg = Thread.currentThread().getThreadGroup();
ThreadGroup ptg;
while ((ptg = tg.getParent()) != null) tg = ptg;
return tg;
}
public void stop() {
try {
flush();
} catch (IOException e) {
LOGGER.error("Error flushing", e);
}
try {
close();
} catch (IOException e) {
LOGGER.error("Error closing", e);
}
try {
backgroundFlushTask.shutdown();
// Wait a while for existing tasks to terminate
if (!backgroundFlushTask.awaitTermination(15, TimeUnit.SECONDS)) {
backgroundFlushTask.shutdownNow(); // Cancel currently executing tasks
// Wait a while for tasks to respond to being cancelled
if (!backgroundFlushTask.awaitTermination(15, TimeUnit.SECONDS)) {
LOGGER.error("Stream did not terminate");
}
}
} catch (InterruptedException ie) {
// (Re-)Cancel if current thread also interrupted
backgroundFlushTask.shutdownNow();
// Preserve interrupt status
Thread.currentThread().interrupt();
}
}
public boolean isReadAccessAllowed() {
Thread currentThread = Thread.currentThread();
return ourDispatchThread == currentThread
|| isExceptionalThreadWithReadAccess()
|| myActionsLock.isReadLockAcquired()
|| myActionsLock.isWriteLockAcquired()
|| isDispatchThread();
}
/**
* Shutdown this session instance, only waiting a definite amount of time.
*
* <p>This closes all connections used by this sessions. Note that if you want to shutdown the
* full {@code Cluster} instance this session is part of, you should use {@link Cluster#shutdown}
* instead (which will call this method for all session but also release some additional
* resources).
*
* <p>Note that this method is not thread safe in the sense that if another shutdown is perform in
* parallel, it might return {@code true} even if the instance is not yet fully shutdown.
*
* @param timeout how long to wait for the session to shutdown.
* @param unit the unit for the timeout.
* @return {@code true} if the session has been properly shutdown within the {@code timeout},
* {@code false} otherwise.
*/
public boolean shutdown(long timeout, TimeUnit unit) {
try {
return manager.shutdown(timeout, unit);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return false;
}
}
@Override
public void run() {
try {
while (!Thread.currentThread().isInterrupted()) {
onFrozenLock.lock();
try {
Message message = received.take();
if (message instanceof RequestMessage) process(((RequestMessage) message));
else if (message instanceof ResponseMessage) process(((ResponseMessage) message));
} finally {
onFrozenLock.unlock();
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
@Override
public void createQueue(int threadIndex) {
synchronized (this) {
if (queue == null) {
queue = createQueue();
BenchLogger.sysinfo(Thread.currentThread().getName() + " create queue " + threadIndex);
}
}
}
/**
* Testing threads thoroughly seems to be a lot of fun, I am not sure yet how to properly do that.
* This test tests priority order and total sum received is equal to total sum sent. It also
* simulates clients random hanging.
*/
@Test
public void testReceiveData() {
DecorateCheckOrderAndCountSumMarshaller sumAppender =
new DecorateCheckOrderAndCountSumMarshaller();
QueueWorker worker = new QueueWorker(sumAppender);
Thread workerThread = new Thread(worker);
// run 20 clients, 10.000 items will be generated per client as defined in
// src/test/properties/app.properties
for (int i = 0; i < 20; i++) {
Runnable clientHangSimulator = null;
if (i % 5 == 0) {
// simulate occasional client hang for four of the total twenty clients and check worker is
// not biased.
clientHangSimulator =
() -> {
if (ThreadLocalRandom.current().nextInt(1001) % 1000 == 0) {
try {
Thread.sleep(500L);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
};
}
executorService.execute(new DataGeneratorTask(clientHangSimulator));
}
workerThread.start();
try {
barrier.await();
System.out.println("Fired test");
Thread.sleep(1000);
executorService.shutdown();
// runs actually much faster, may need update if item count per client is drastically
// increased in app.properties
executorService.awaitTermination(2 * 60, TimeUnit.SECONDS);
System.out.println("exe service awaited");
} catch (InterruptedException | BrokenBarrierException e) {
e.printStackTrace();
}
try {
workerThread.join(Long.MAX_VALUE);
} catch (InterruptedException e) {
e.printStackTrace();
}
Assert.assertEquals(
"Sum generated does not match sum received",
sumAppender.getSum(),
AppContext.getInstance().getTotalGeneratedAmount());
Assert.assertFalse("Worker didn't exit successfully", workerThread.isAlive());
}
@Override
public void run() {
System.out.println(
System.currentTimeMillis() + ":Thread ID:" + Thread.currentThread().getId());
try {
Thread.sleep(100);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
/**
* @param log Logger.
* @param time Time.
* @param msg Message.
*/
private static void log0(@Nullable IgniteLogger log, long time, String msg) {
if (log != null) {
if (log.isDebugEnabled()) log.debug(msg);
else log.warning(msg);
} else
X.println(
String.format(
"[%s][%s]%s",
DEBUG_DATE_FMT.get().format(time), Thread.currentThread().getName(), msg));
}
synchronized void waitingCall() {
try {
while (!Thread.interrupted()) {
wait();
System.out.print(Thread.currentThread() + " ");
}
} catch (InterruptedException e) {
// OK to exit this way
}
}
private void Pause(String description) throws Exception {
Utilities.Log.Debug(
"Pausing for "
+ Settings.PAUSE_SECONDS
+ " seconds: "
+ description
+ ". Other threads may be processing these tasks.");
Thread.currentThread().sleep(Settings.PAUSE_SECONDS * 1000);
Utilities.Log.Debug("Retrying after pause: " + description + ".");
}
private static void assertIsDispatchThread(String message) {
if (ShutDownTracker.isShutdownHookRunning()) return;
final Thread currentThread = Thread.currentThread();
if (ourDispatchThread == currentThread) return;
if (EventQueue.isDispatchThread()) {
ourDispatchThread = currentThread;
}
if (ourDispatchThread == currentThread) return;
Integer safeCounter = ourEdtSafe.get();
if (safeCounter != null && safeCounter > 0) return;
LOG.error(
message,
"Current thread: " + describe(Thread.currentThread()),
"Our dispatch thread:" + describe(ourDispatchThread),
"SystemEventQueueThread: " + describe(getEventQueueThread()));
}
public void assertReadAccessAllowed() {
if (myHeadlessMode) return;
if (!isReadAccessAllowed()) {
LOG.error(
"Read access is allowed from event dispatch thread or inside read-action only (see com.intellij.openapi.application.Application.runReadAction())",
"Current thread: " + describe(Thread.currentThread()),
"Our dispatch thread:" + describe(ourDispatchThread),
"SystemEventQueueThread: " + describe(getEventQueueThread()));
}
}
public void run() {
try {
while (true) {
TimeUnit.MILLISECONDS.sleep(100);
print(Thread.currentThread() + " " + this);
}
} catch (InterruptedException e) {
print("sleep() interrupted");
}
}
protected int pick(int iThink, boolean couldCreate) {
final int id = Thread.currentThread().hashCode();
final int s = _availSafe.size();
for (int i = 0; i < s; i++) {
DBPort p = _availSafe.get(i);
if (p._lastThread == id) return i;
}
if (couldCreate) return -1;
return iThink;
}
private void shutdownExecutor(ExecutorService executor) {
if (executor == null) {
return;
}
executor.shutdown();
try {
executor.awaitTermination(10, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
/** {@inheritDoc} */
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
try {
if (executor.isShutdown()) throw new RejectedExecutionException();
else executor.getQueue().put(r);
} catch (InterruptedException ignored) {
U.warn(log, "Working thread was interrupted while loading data.");
Thread.currentThread().interrupt();
}
}
/**
* @param key Removed key.
* @param ver Removed version.
* @throws IgniteCheckedException If failed.
*/
public void onDeferredDelete(KeyCacheObject key, GridCacheVersion ver)
throws IgniteCheckedException {
try {
T2<KeyCacheObject, GridCacheVersion> evicted = rmvQueue.add(new T2<>(key, ver));
if (evicted != null) cctx.dht().removeVersionedEntry(evicted.get1(), evicted.get2());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IgniteInterruptedCheckedException(e);
}
}
private static ZooKeeper newZooKeeper() throws IOException {
System.out.printf("%s: Creating new ZooKeeper%n", Thread.currentThread().getName());
return new ZooKeeper(
hostString,
timeout,
new Watcher() {
@Override
public void process(WatchedEvent event) {
// System.out.println(event);
}
});
}
