void start() {
if (t == null) {
t = new Thread(this, "UDP.OutgoingPacketHandler thread");
t.setDaemon(true);
t.start();
}
}
/** Stops unicast and multicast receiver threads */
void stopThreads() {
Thread tmp;
// 1. Stop the multicast receiver thread
if (mcast_receiver != null) {
if (mcast_receiver.isAlive()) {
tmp = mcast_receiver;
mcast_receiver = null;
closeMulticastSocket(); // will cause the multicast thread to terminate
tmp.interrupt();
try {
tmp.join(100);
} catch (Exception e) {
}
tmp = null;
}
mcast_receiver = null;
}
// 2. Stop the unicast receiver thread
if (ucast_receiver != null) {
ucast_receiver.stop();
ucast_receiver = null;
}
// 3. Stop the in_packet_handler thread
if (incoming_packet_handler != null) {
incoming_packet_handler.stop();
}
}
protected void await(boolean throwInterrupt) throws InterruptedException {
if (!signaled.get()) {
lock.acquired = false;
sendAwaitConditionRequest(lock.name, lock.owner);
boolean interrupted = false;
while (!signaled.get()) {
parker.set(Thread.currentThread());
LockSupport.park(this);
if (Thread.interrupted()) {
// If we were interrupted and haven't received a response yet then we try to
// clean up the lock request and throw the exception
if (!signaled.get()) {
sendDeleteAwaitConditionRequest(lock.name, lock.owner);
throw new InterruptedException();
}
// In the case that we were signaled and interrupted
// we want to return the signal but still interrupt
// our thread
interrupted = true;
}
}
if (interrupted) Thread.currentThread().interrupt();
}
// We set as if this signal was no released. This way if the
// condition is reused again, but the client condition isn't lost
// we won't think we were signaled immediately
signaled.set(false);
}
/**
* @param views Guaranteed to be non-null and to have >= 2 members, or else this thread would
* not be started
*/
public synchronized void start(Map<Address, View> views) {
if (thread == null || thread.isAlive()) {
this.coords.clear();
// now remove all members which don't have us in their view, so RPCs won't block (e.g.
// FLUSH)
// https://jira.jboss.org/browse/JGRP-1061
sanitizeViews(views);
// Add all different coordinators of the views into the hashmap and sets their members:
Collection<Address> coordinators = Util.determineMergeCoords(views);
for (Address coord : coordinators) {
View view = views.get(coord);
if (view != null) this.coords.put(coord, new ArrayList<Address>(view.getMembers()));
}
// For the merge participants which are not coordinator, we simply add them, and the
// associated
// membership list consists only of themselves
Collection<Address> merge_participants = Util.determineMergeParticipants(views);
merge_participants.removeAll(coordinators);
for (Address merge_participant : merge_participants) {
Collection<Address> tmp = new ArrayList<Address>();
tmp.add(merge_participant);
coords.putIfAbsent(merge_participant, tmp);
}
thread = gms.getThreadFactory().newThread(this, "MergeTask");
thread.setDaemon(true);
thread.start();
}
}
private void startEventHandlerThread() {
if (event_queue == null) event_queue = new Queue();
if (evt_thread == null) {
evt_thread = new Thread(this, "GMS.EventHandlerThread");
evt_thread.setDaemon(true);
evt_thread.start();
}
}
public void start() {
if (thread == null) {
thread = new Thread(this, "UDP.UcastReceiverThread");
thread.setDaemon(true);
running = true;
thread.start();
}
}
synchronized void start() {
if (queue.closed()) queue.reset();
if (thread == null || !thread.isAlive()) {
thread = getThreadFactory().newThread(this, "ViewHandler");
thread.setDaemon(
false); // thread cannot terminate if we have tasks left, e.g. when we as coord leave
thread.start();
}
}
synchronized void waitUntilCompleted(long timeout, boolean resume) {
if (thread != null) {
try {
thread.join(timeout);
} catch (InterruptedException e) {
Thread.currentThread().interrupt(); // set interrupt flag again
}
thread = null; // Added after Brian noticed that ViewHandler leaks class loaders
}
if (resume) resumeForce();
}
protected void handleInterruptRequest(Address source, long requestId) {
Owner owner = new Owner(source, requestId);
Runnable runnable = removeKeyForValue(_running, owner);
Thread thread = null;
if (runnable != null) {
thread = _runnableThreads.remove(runnable);
}
if (thread != null) {
thread.interrupt();
} else if (log.isTraceEnabled())
log.trace("Message could not be interrupted due to it already returned");
}
public void stop() {
Thread tmp;
if (thread != null && thread.isAlive()) {
running = false;
tmp = thread;
thread = null;
closeSocket(); // this will cause the thread to break out of its loop
tmp.interrupt();
tmp = null;
}
thread = null;
}
protected void stopFlusher() {
flushing = false;
Thread tmp = flusher;
while (tmp != null && tmp.isAlive()) {
tmp.interrupt();
ack_promise.setResult(null);
try {
tmp.join();
} catch (InterruptedException e) {
}
}
}
public void receive(Message msg) {
if (!msg.isFlagSet(Message.OOB)) {
try {
System.out.println(Thread.currentThread() + ": waiting on latch");
latch.await(25000, TimeUnit.MILLISECONDS);
System.out.println(Thread.currentThread() + ": DONE waiting on latch");
} catch (InterruptedException e) {
e.printStackTrace();
}
}
msgs.add((Integer) msg.getObject());
}
/**
* Tests concurrent reception of multiple messages with a different conn_id
* (https://issues.jboss.org/browse/JGRP-1347)
*/
public void testMultipleConcurrentResets() throws Exception {
sendAndCheck(a, b_addr, 1, r2);
// now close connection on A unilaterally
System.out.println("==== Closing the connection on A");
removeConnection(u1, b_addr);
r2.clear();
final UNICAST unicast = (UNICAST) b.getProtocolStack().findProtocol(UNICAST.class);
int NUM = 10;
final List<Message> msgs = new ArrayList<Message>(NUM);
for (int i = 1; i <= NUM; i++) {
Message msg = new Message(b_addr, a_addr, "m" + i);
UNICAST.UnicastHeader hdr = UNICAST.UnicastHeader.createDataHeader(1, (short) 2, true);
msg.putHeader(unicast.getId(), hdr);
msgs.add(msg);
}
Thread[] threads = new Thread[NUM];
final CyclicBarrier barrier = new CyclicBarrier(NUM + 1);
for (int i = 0; i < NUM; i++) {
final int index = i;
threads[i] =
new Thread() {
public void run() {
try {
barrier.await();
unicast.up(new Event(Event.MSG, msgs.get(index)));
} catch (Exception e) {
e.printStackTrace();
}
}
};
threads[i].start();
}
barrier.await();
for (Thread thread : threads) thread.join();
List<Message> list = r2.getMessages();
System.out.println("list = " + print(list));
assert list.size() == 1
: "list must have 1 element but has " + list.size() + ": " + print(list);
}
protected synchronized void acquire(boolean throwInterrupt) throws InterruptedException {
if (!acquired) {
owner = getOwner();
sendGrantLockRequest(name, owner, 0, false);
boolean interrupted = false;
while (!acquired) {
try {
this.wait();
} catch (InterruptedException e) {
// If we haven't acquired the lock yet and were interrupted, then we have to clean up
// the lock
// request and throw the exception
if (throwInterrupt && !acquired) {
_unlock(true);
throw e;
}
// If we did get the lock then we will return with the lock and interrupt status.
// If we don't throw exceptions then we just set the interrupt flag and let it loop
// around
interrupted = true;
}
}
if (interrupted) Thread.currentThread().interrupt();
}
}
protected void flush(final Address new_coord) throws InterruptedException {
// wait until all threads currently sending messages have returned (new threads after
// flushing=true) will block
// flushing is set to true in startFlusher()
while (flushing && running) {
if (in_flight_sends.get() == 0) break;
Thread.sleep(100);
}
send_lock.lockInterruptibly();
try {
if (log.isTraceEnabled())
log.trace(local_addr + ": coord changed from " + coord + " to " + new_coord);
coord = new_coord;
is_coord = Objects.equals(local_addr, coord);
flushMessagesInForwardTable();
} finally {
if (log.isTraceEnabled()) log.trace(local_addr + ": flushing completed");
flushing = false;
ack_mode = true; // go to ack-mode after flushing
num_acks = 0;
send_cond.signalAll();
send_lock.unlock();
}
}
private void send(Address dest) throws Exception {
final CountDownLatch latch = new CountDownLatch(1);
final BlockingReceiver receiver = new BlockingReceiver(latch);
final int NUM = 10;
b.setReceiver(receiver);
a.send(dest, 1); // the only regular message
for (int i = 2; i <= NUM; i++) a.send(new Message(dest, i).setFlag(Message.Flag.OOB));
sendStableMessages(a, b);
List<Integer> list = receiver.getMsgs();
for (int i = 0; i < 20; i++) {
if (list.size() == NUM - 1) break;
sendStableMessages(a, b);
Util.sleep(500); // give the asynchronous msgs some time to be received
}
System.out.println("list = " + list);
assert list.size() == NUM - 1 : "list is " + list;
assert list.contains(2) && list.contains(10);
System.out.println("[" + Thread.currentThread().getName() + "]: releasing latch");
latch.countDown();
for (int i = 0; i < 20; i++) {
if (list.size() == NUM) break;
sendStableMessages(a, b);
Util.sleep(1000); // give the asynchronous msgs some time to be received
}
System.out.println("list = " + list);
assert list.size() == NUM : "list is " + list;
for (int i = 1; i <= NUM; i++) assert list.contains(i);
}
protected long await(long nanoSeconds) throws InterruptedException {
long target_nano = System.nanoTime() + nanoSeconds;
if (!signaled.get()) {
// We release the lock at the same time as waiting on the
// condition
lock.acquired = false;
sendAwaitConditionRequest(lock.name, lock.owner);
boolean interrupted = false;
while (!signaled.get()) {
long wait_nano = target_nano - System.nanoTime();
// If we waited max time break out
if (wait_nano > 0) {
parker.set(Thread.currentThread());
LockSupport.parkNanos(this, wait_nano);
if (Thread.interrupted()) {
// If we were interrupted and haven't received a response yet then we try to
// clean up the lock request and throw the exception
if (!signaled.get()) {
sendDeleteAwaitConditionRequest(lock.name, lock.owner);
throw new InterruptedException();
}
// In the case that we were signaled and interrupted
// we want to return the signal but still interrupt
// our thread
interrupted = true;
}
} else {
break;
}
}
if (interrupted) Thread.currentThread().interrupt();
}
// We set as if this signal was no released. This way if the
// condition is reused again, but the client condition isn't lost
// we won't think we were signaled immediately
// If we weren't signaled then delete our request
if (!signaled.getAndSet(false)) {
sendDeleteAwaitConditionRequest(lock.name, lock.owner);
}
return target_nano - System.nanoTime();
}
private void send(Address dest) throws ChannelNotConnectedException, ChannelClosedException {
final ReentrantLock lock = new ReentrantLock();
final BlockingReceiver receiver = new BlockingReceiver(lock);
final int NUM = 10;
c2.setReceiver(receiver);
System.out.println("[" + Thread.currentThread().getName() + "]: locking lock");
lock.lock();
c1.send(new Message(dest, null, 1));
for (int i = 2; i <= NUM; i++) {
Message msg = new Message(dest, null, i);
msg.setFlag(Message.OOB);
c1.send(msg);
}
sendStableMessages(c1, c2);
Util.sleep(500);
List<Integer> list = receiver.getMsgs();
for (int i = 0; i < 20; i++) {
if (list.size() == NUM - 1) break;
System.out.println("list = " + list);
Util.sleep(1000); // give the asynchronous msgs some time to be received
}
System.out.println("list = " + list);
assert list.size() == NUM - 1 : "list is " + list;
assert list.contains(2) && list.contains(10);
System.out.println("[" + Thread.currentThread().getName() + "]: unlocking lock");
lock.unlock();
for (int i = 0; i < 20; i++) {
if (list.size() == NUM) break;
System.out.println("list = " + list);
Util.sleep(1000); // give the asynchronous msgs some time to be received
}
System.out.println("list = " + list);
assert list.size() == NUM : "list is " + list;
for (int i = 1; i <= NUM; i++) assert list.contains(i);
}
/** Starts the unicast and multicast receiver threads */
void startThreads() throws Exception {
if (ucast_receiver == null) {
// start the listener thread of the ucast_recv_sock
ucast_receiver = new UcastReceiver();
ucast_receiver.start();
if (Trace.trace) {
Trace.info("UDP.startThreads()", "created unicast receiver thread");
}
}
if (ip_mcast) {
if (mcast_receiver != null) {
if (mcast_receiver.isAlive()) {
if (Trace.trace) {
Trace.info(
"UDP.createThreads()",
"did not create new multicastreceiver thread as existing "
+ "multicast receiver thread is still running");
}
} else {
mcast_receiver = null; // will be created just below...
}
}
if (mcast_receiver == null) {
mcast_receiver = new Thread(this, "UDP mcast receiver");
mcast_receiver.setPriority(Thread.MAX_PRIORITY); // needed ????
mcast_receiver.setDaemon(true);
mcast_receiver.start();
}
}
if (use_outgoing_packet_handler) {
outgoing_packet_handler.start();
}
if (use_incoming_packet_handler) {
incoming_packet_handler.start();
}
}
protected void handleTaskSubmittedRequest(
Runnable runnable, Address source, long requestId, long threadId) {
// We store in our map so that when that task is
// finished so that we can send back to the owner
// with the results
_running.put(runnable, new Owner(source, requestId));
// We give the task to the thread that is now waiting for it to be returned
// If we can't offer then we have to respond back to
// caller that we can't handle it. They must have
// gotten our address when we had a consumer, but
// they went away between then and now.
boolean received;
try {
_tasks.put(threadId, runnable);
CyclicBarrier barrier = _taskBarriers.remove(threadId);
if (received = (barrier != null)) {
// Only wait 10 milliseconds, in case if the consumer was
// stopped between when we were told it was available and now
barrier.await(10, TimeUnit.MILLISECONDS);
}
} catch (InterruptedException e) {
if (log.isDebugEnabled()) log.debug("Interrupted while handing off task");
Thread.currentThread().interrupt();
received = false;
} catch (BrokenBarrierException e) {
if (log.isDebugEnabled())
log.debug(
"Consumer " + threadId + " has been interrupted, " + "must retry to submit elsewhere");
received = false;
} catch (TimeoutException e) {
if (log.isDebugEnabled())
log.debug("Timeout waiting to hand off to barrier, consumer " + threadId + " must be slow");
// This should only happen if the consumer put the latch then got
// interrupted but hadn't yet removed the latch, should almost never
// happen
received = false;
}
if (!received) {
// Clean up the tasks request
_tasks.remove(threadId);
if (log.isDebugEnabled()) log.debug("Run rejected not able to pass off to consumer");
// If we couldn't hand off the task we have to tell the client
// and also reupdate the coordinator that our consumer is ready
sendRequest(source, Type.RUN_REJECTED, requestId, null);
_running.remove(runnable);
}
}
public ReceiverThread() {
nullifier =
new Thread(
new Runnable() {
public void run() {
// nullifies throughput display
while (running) {
Util.sleep(2000);
if ((System.currentTimeMillis() - startTimeThroughput) > 2000) {
control.throughput.setText("0 KB/sec");
}
}
}
});
nullifier.start();
}
protected synchronized boolean acquireTryLock(long timeout, boolean use_timeout)
throws InterruptedException {
if (denied) return false;
if (!acquired) {
is_trylock = true;
this.timeout = timeout;
if (owner == null) owner = getOwner();
sendGrantLockRequest(name, owner, timeout, true);
long target_time = use_timeout ? System.currentTimeMillis() + timeout : 0;
boolean interrupted = false;
while (!acquired && !denied) {
if (use_timeout) {
long wait_time = target_time - System.currentTimeMillis();
if (wait_time <= 0) break;
else {
this.timeout = wait_time;
try {
this.wait(wait_time);
} catch (InterruptedException e) {
// If we were interrupted and haven't received a response yet then we try to
// clean up the lock request and throw the exception
if (!acquired && !denied) {
_unlock(true);
throw e;
}
// In the case that we were told if we acquired or denied the lock then return that,
// but
// make sure we set the interrupt status
interrupted = true;
}
}
} else {
try {
this.wait();
} catch (InterruptedException e) {
interrupted = true;
}
}
}
if (interrupted) Thread.currentThread().interrupt();
}
if (!acquired || denied) _unlock(true);
return acquired && !denied;
}
@Override
public void await() throws InterruptedException {
InterruptedException ex = null;
try {
await(true);
} catch (InterruptedException e) {
ex = e;
throw ex;
} finally {
lock.lock();
// If we are throwing an InterruptedException
// then clear the interrupt state as well.
if (ex != null) {
Thread.interrupted();
}
}
}
/**
* Initialisation if a supplied key is defined in the properties. This supplied key must be in a
* keystore which can be generated using the keystoreGenerator file in demos. The keystore must be
* on the classpath to find it.
*
* @throws KeyStoreException
* @throws Exception
* @throws IOException
* @throws NoSuchAlgorithmException
* @throws CertificateException
* @throws UnrecoverableKeyException
*/
private void initConfiguredKey() throws Exception {
InputStream inputStream = null;
// must not use default keystore type - as does not support secret keys
KeyStore store = KeyStore.getInstance("JCEKS");
SecretKey tempKey = null;
try {
// load in keystore using this thread's classloader
inputStream =
Thread.currentThread().getContextClassLoader().getResourceAsStream(keyStoreName);
if (inputStream == null) inputStream = new FileInputStream(keyStoreName);
// we can't find a keystore here -
if (inputStream == null) {
throw new Exception(
"Unable to load keystore " + keyStoreName + " ensure file is on classpath");
}
// we have located a file lets load the keystore
try {
store.load(inputStream, storePassword.toCharArray());
// loaded keystore - get the key
tempKey = (SecretKey) store.getKey(alias, keyPassword.toCharArray());
} catch (IOException e) {
throw new Exception("Unable to load keystore " + keyStoreName + ": " + e);
} catch (NoSuchAlgorithmException e) {
throw new Exception("No Such algorithm " + keyStoreName + ": " + e);
} catch (CertificateException e) {
throw new Exception("Certificate exception " + keyStoreName + ": " + e);
}
if (tempKey == null)
throw new Exception("Unable to retrieve key '" + alias + "' from keystore " + keyStoreName);
// set the key here
setSecretKey(tempKey);
if (symAlgorithm.equals(DEFAULT_SYM_ALGO)) symAlgorithm = tempKey.getAlgorithm();
// set the fact we are using a supplied key
suppliedKey = true;
queue_down = queue_up = false;
} finally {
Util.close(inputStream);
}
}
@Override
public long awaitNanos(long nanosTimeout) throws InterruptedException {
long beforeLock;
InterruptedException ex = null;
try {
beforeLock = await(nanosTimeout) + System.nanoTime();
} catch (InterruptedException e) {
ex = e;
throw ex;
} finally {
lock.lock();
// If we are throwing an InterruptedException
// then clear the interrupt state as well.
if (ex != null) {
Thread.interrupted();
}
}
return beforeLock - System.nanoTime();
}
public void run() {
long end_time, wait_time;
List<Request> requests = new LinkedList<Request>();
while (Thread.currentThread().equals(thread) && !suspended) {
try {
boolean keepGoing = false;
end_time = System.currentTimeMillis() + max_bundling_time;
do {
Request firstRequest =
(Request)
queue.remove(INTERVAL); // throws a TimeoutException if it runs into timeout
requests.add(firstRequest);
if (!view_bundling) break;
if (queue.size() > 0) {
Request nextReq = (Request) queue.peek();
keepGoing = view_bundling && firstRequest.canBeProcessedTogether(nextReq);
} else {
wait_time = end_time - System.currentTimeMillis();
if (wait_time > 0)
queue.waitUntilClosed(
wait_time); // misnomer: waits until element has been added or q closed
keepGoing =
queue.size() > 0 && firstRequest.canBeProcessedTogether((Request) queue.peek());
}
} while (keepGoing && System.currentTimeMillis() < end_time);
try {
process(requests);
} finally {
requests.clear();
}
} catch (QueueClosedException e) {
break;
} catch (TimeoutException e) {
break;
} catch (Throwable catchall) {
Util.sleep(50);
}
}
}
/** Tests 2 channels calling partial FLUSHes and one calling FLUSH simultaneously */
@Test
public void testConcurrentFlushAndPartialFlush() throws Exception {
c1 = createChannel(true, 3);
c1.connect("testConcurrentFlushAndPartialFlush");
c2 = createChannel(c1);
c2.connect("testConcurrentFlushAndPartialFlush");
c3 = createChannel(c1);
c3.connect("testConcurrentFlushAndPartialFlush");
assertViewsReceived(c1, c2, c3);
final CountDownLatch startFlushLatch = new CountDownLatch(1);
final CountDownLatch stopFlushLatch = new CountDownLatch(1);
// 2 because either total or partial has to finish first
final CountDownLatch flushStartReceived = new CountDownLatch(2);
// 5 because we have total and partial flush
final CountDownLatch flushStopReceived = new CountDownLatch(5);
Thread t1 =
new Thread() {
public void run() {
try {
startFlushLatch.await();
boolean rc = Util.startFlush(c1);
System.out.println("t1: rc=" + rc);
} catch (InterruptedException e) {
interrupt();
}
try {
stopFlushLatch.await();
} catch (InterruptedException e) {
interrupt();
} finally {
c1.stopFlush();
}
}
};
Thread t2 =
new Thread() {
public void run() {
try {
startFlushLatch.await();
// partial, only between c2 and c3
boolean rc = Util.startFlush(c2, (Arrays.asList(c2.getAddress(), c3.getAddress())));
System.out.println("t2: partial flush rc=" + rc);
} catch (InterruptedException e) {
interrupt();
}
try {
stopFlushLatch.await();
} catch (InterruptedException e) {
interrupt();
} finally {
c2.stopFlush(Arrays.asList(c2.getAddress(), c3.getAddress()));
}
}
};
Listener l1 = new Listener("c1", c1, flushStartReceived, flushStopReceived);
Listener l2 = new Listener("c2", c2, flushStartReceived, flushStopReceived);
Listener l3 = new Listener("c3", c3, flushStartReceived, flushStopReceived);
t1.start();
t2.start();
startFlushLatch.countDown();
assertTrue(flushStartReceived.await(60, TimeUnit.SECONDS));
// at this stage both channels should have started a flush?
stopFlushLatch.countDown();
t1.join();
t2.join();
assertTrue(flushStopReceived.await(60, TimeUnit.SECONDS));
assert l1.blockReceived;
assert l1.unblockReceived;
assert l2.blockReceived;
assert l2.unblockReceived;
assert l3.blockReceived;
assert l3.unblockReceived;
}
public Object down(Event evt) {
switch (evt.getType()) {
case ExecutorEvent.TASK_SUBMIT:
Runnable runnable = evt.getArg();
// We are limited to a number of concurrent request id's
// equal to 2^63-1. This is quite large and if it
// overflows it will still be positive
long requestId = Math.abs(counter.getAndIncrement());
if (requestId == Long.MIN_VALUE) {
// TODO: need to fix this it isn't safe for concurrent modifications
counter.set(0);
requestId = Math.abs(counter.getAndIncrement());
}
// Need to make sure to put the requestId in our map before
// adding the runnable to awaiting consumer in case if
// coordinator sent a consumer found and their original task
// is no longer around
// see https://issues.jboss.org/browse/JGRP-1744
_requestId.put(runnable, requestId);
_awaitingConsumer.add(runnable);
sendToCoordinator(RUN_REQUEST, requestId, local_addr);
break;
case ExecutorEvent.CONSUMER_READY:
Thread currentThread = Thread.currentThread();
long threadId = currentThread.getId();
_consumerId.put(threadId, PRESENT);
try {
for (; ; ) {
CyclicBarrier barrier = new CyclicBarrier(2);
_taskBarriers.put(threadId, barrier);
// We only send to the coordinator that we are ready after
// making the barrier, wait for request to come and let
// us free
sendToCoordinator(Type.CONSUMER_READY, threadId, local_addr);
try {
barrier.await();
break;
} catch (BrokenBarrierException e) {
if (log.isDebugEnabled())
log.debug(
"Producer timed out before we picked up"
+ " the task, have to tell coordinator"
+ " we are still good.");
}
}
// This should always be non nullable since the latch
// was freed
runnable = _tasks.remove(threadId);
_runnableThreads.put(runnable, currentThread);
return runnable;
} catch (InterruptedException e) {
if (log.isDebugEnabled()) log.debug("Consumer " + threadId + " stopped via interrupt");
sendToCoordinator(Type.CONSUMER_UNREADY, threadId, local_addr);
Thread.currentThread().interrupt();
} finally {
// Make sure the barriers are cleaned up as well
_taskBarriers.remove(threadId);
_consumerId.remove(threadId);
}
break;
case ExecutorEvent.TASK_COMPLETE:
Object arg = evt.getArg();
Throwable throwable = null;
if (arg instanceof Object[]) {
Object[] array = (Object[]) arg;
runnable = (Runnable) array[0];
throwable = (Throwable) array[1];
} else {
runnable = (Runnable) arg;
}
Owner owner = _running.remove(runnable);
// This won't remove anything if owner doesn't come back
_runnableThreads.remove(runnable);
Object value = null;
boolean exception = false;
if (throwable != null) {
// InterruptedException is special telling us that
// we interrupted the thread while waiting but still got
// a task therefore we have to reject it.
if (throwable instanceof InterruptedException) {
if (log.isDebugEnabled())
log.debug("Run rejected due to interrupted exception returned");
sendRequest(owner.address, Type.RUN_REJECTED, owner.requestId, null);
break;
}
value = throwable;
exception = true;
} else if (runnable instanceof RunnableFuture<?>) {
RunnableFuture<?> future = (RunnableFuture<?>) runnable;
boolean interrupted = false;
boolean gotValue = false;
// We have the value, before we interrupt at least get it!
while (!gotValue) {
try {
value = future.get();
gotValue = true;
} catch (InterruptedException e) {
interrupted = true;
} catch (ExecutionException e) {
value = e.getCause();
exception = true;
gotValue = true;
}
}
if (interrupted) {
Thread.currentThread().interrupt();
}
}
if (owner != null) {
final Type type;
final Object valueToSend;
if (value == null) {
type = Type.RESULT_SUCCESS;
valueToSend = value;
}
// Both serializable values and exceptions would go in here
else if (value instanceof Serializable
|| value instanceof Externalizable
|| value instanceof Streamable) {
type = exception ? Type.RESULT_EXCEPTION : Type.RESULT_SUCCESS;
valueToSend = value;
}
// This would happen if the value wasn't serializable,
// so we have to send back to the client that the class
// wasn't serializable
else {
type = Type.RESULT_EXCEPTION;
valueToSend = new NotSerializableException(value.getClass().getName());
}
if (local_addr.equals(owner.getAddress())) {
if (log.isTraceEnabled())
log.trace(
"[redirect] <--> ["
+ local_addr
+ "] "
+ type.name()
+ " ["
+ value
+ (owner.requestId != -1 ? " request id: " + owner.requestId : "")
+ "]");
if (type == Type.RESULT_SUCCESS) {
handleValueResponse(local_addr, owner.requestId, valueToSend);
} else if (type == Type.RESULT_EXCEPTION) {
handleExceptionResponse(local_addr, owner.requestId, (Throwable) valueToSend);
}
} else {
sendRequest(owner.getAddress(), type, owner.requestId, valueToSend);
}
} else {
if (log.isTraceEnabled()) {
log.trace("Could not return result - most likely because it was interrupted");
}
}
break;
case ExecutorEvent.TASK_CANCEL:
Object[] array = evt.getArg();
runnable = (Runnable) array[0];
if (_awaitingConsumer.remove(runnable)) {
_requestId.remove(runnable);
ExecutorNotification notification = notifiers.remove(runnable);
if (notification != null) {
notification.interrupted(runnable);
}
if (log.isTraceEnabled())
log.trace("Cancelled task " + runnable + " before it was picked up");
return Boolean.TRUE;
}
// This is guaranteed to not be null so don't take cost of auto unboxing
else if (array[1] == Boolean.TRUE) {
owner = removeKeyForValue(_awaitingReturn, runnable);
if (owner != null) {
Long requestIdValue = _requestId.remove(runnable);
// We only cancel if the requestId is still available
// this means the result hasn't been returned yet and
// we still have a chance to interrupt
if (requestIdValue != null) {
if (requestIdValue != owner.getRequestId()) {
log.warn("Cancelling requestId didn't match waiting");
}
sendRequest(owner.getAddress(), Type.INTERRUPT_RUN, owner.getRequestId(), null);
}
} else {
if (log.isTraceEnabled()) log.warn("Couldn't interrupt server task: " + runnable);
}
ExecutorNotification notification = notifiers.remove(runnable);
if (notification != null) {
notification.interrupted(runnable);
}
return Boolean.TRUE;
} else {
return Boolean.FALSE;
}
case ExecutorEvent.ALL_TASK_CANCEL:
array = evt.getArg();
// This is a RunnableFuture<?> so this cast is okay
@SuppressWarnings("unchecked")
Set<Runnable> runnables = (Set<Runnable>) array[0];
Boolean booleanValue = (Boolean) array[1];
List<Runnable> notRan = new ArrayList<>();
for (Runnable cancelRunnable : runnables) {
// Removed from the consumer
if (!_awaitingConsumer.remove(cancelRunnable) && booleanValue == Boolean.TRUE) {
synchronized (_awaitingReturn) {
owner = removeKeyForValue(_awaitingReturn, cancelRunnable);
if (owner != null) {
Long requestIdValue = _requestId.remove(cancelRunnable);
if (requestIdValue != owner.getRequestId()) {
log.warn("Cancelling requestId didn't match waiting");
}
sendRequest(owner.getAddress(), Type.INTERRUPT_RUN, owner.getRequestId(), null);
}
ExecutorNotification notification = notifiers.remove(cancelRunnable);
if (notification != null) {
log.trace("Notifying listener");
notification.interrupted(cancelRunnable);
}
}
} else {
_requestId.remove(cancelRunnable);
notRan.add(cancelRunnable);
}
}
return notRan;
case Event.SET_LOCAL_ADDRESS:
local_addr = evt.getArg();
break;
case Event.VIEW_CHANGE:
handleView(evt.getArg());
break;
}
return down_prot.down(evt);
}
/** Tests 2 channels calling FLUSH simultaneously */
@Test
public void testConcurrentFlush() throws Exception {
c1 = createChannel(true, 2);
c1.connect("testConcurrentFlush");
c2 = createChannel(c1);
c2.connect("testConcurrentFlush");
assertViewsReceived(c1, c2);
final CountDownLatch startFlushLatch = new CountDownLatch(1);
final CountDownLatch stopFlushLatch = new CountDownLatch(1);
final CountDownLatch flushStartReceived = new CountDownLatch(2);
final CountDownLatch flushStopReceived = new CountDownLatch(2);
Thread t1 =
new Thread() {
public void run() {
try {
startFlushLatch.await();
boolean rc = Util.startFlush(c1);
System.out.println("t1: rc=" + rc);
} catch (InterruptedException e) {
interrupt();
}
try {
stopFlushLatch.await();
} catch (InterruptedException e) {
interrupt();
} finally {
c1.stopFlush();
}
}
};
Thread t2 =
new Thread() {
public void run() {
try {
startFlushLatch.await();
boolean rc = Util.startFlush(c2);
System.out.println("t2: rc=" + rc);
} catch (InterruptedException e) {
interrupt();
}
try {
stopFlushLatch.await();
} catch (InterruptedException e) {
interrupt();
} finally {
c2.stopFlush();
}
}
};
Listener l1 = new Listener("c1", c1, flushStartReceived, flushStopReceived);
Listener l2 = new Listener("c2", c2, flushStartReceived, flushStopReceived);
t1.start();
t2.start();
startFlushLatch.countDown();
assertTrue(flushStartReceived.await(60, TimeUnit.SECONDS));
// at this stage both channels should have started a flush
stopFlushLatch.countDown();
t1.join();
t2.join();
assertTrue(flushStopReceived.await(60, TimeUnit.SECONDS));
assert l1.blockReceived;
assert l1.unblockReceived;
assert l2.blockReceived;
assert l2.unblockReceived;
}
public synchronized void stop() {
Thread tmp = thread;
if (thread != null && thread.isAlive()) tmp.interrupt();
thread = null;
}