/** @throws Exception Exception. */
private void checkLock(final boolean fair) throws Exception {
// Check only 'false' cases here. Successful lock is tested over the grid.
final IgniteLock lock = createReentrantLock("acquire", false, fair);
lock.lock();
IgniteInternalFuture fut =
GridTestUtils.runAsync(
new Callable<Void>() {
@Override
public Void call() throws Exception {
assertNotNull(lock);
assert !lock.tryLock();
assert !lock.tryLock(10, MICROSECONDS);
return null;
}
});
fut.get();
lock.unlock();
removeReentrantLock("acquire", fair);
}
/**
* @param failoverSafe Failover safe flag.
* @throws Exception
*/
private void checkFailover(final boolean failoverSafe, final boolean fair) throws Exception {
IgniteEx g = startGrid(NODES_CNT + 1);
// For vars locality.
{
// Ensure not exists.
assert g.reentrantLock("lock", failoverSafe, fair, false) == null;
IgniteLock lock = g.reentrantLock("lock", failoverSafe, fair, true);
lock.lock();
assert lock.tryLock();
assertEquals(2, lock.getHoldCount());
}
Ignite g0 = grid(0);
final IgniteLock lock0 = g0.reentrantLock("lock", false, fair, false);
assert !lock0.tryLock();
assertEquals(0, lock0.getHoldCount());
IgniteInternalFuture<?> fut =
multithreadedAsync(
new Callable<Object>() {
@Override
public Object call() throws Exception {
try {
lock0.lock();
info("Acquired in separate thread.");
// Lock is acquired silently only in failoverSafe mode.
assertTrue(failoverSafe);
lock0.unlock();
info("Released lock in separate thread.");
} catch (IgniteException e) {
if (!failoverSafe) info("Ignored expected exception: " + e);
else throw e;
}
return null;
}
},
1);
Thread.sleep(100);
g.close();
fut.get(500);
lock0.close();
}
/** @throws Exception If any error occurs. */
public void testMultipleStartOnCoordinatorStop() throws Exception {
for (int k = 0; k < 3; k++) {
log.info("Iteration: " + k);
clientFlagGlobal = false;
final int START_NODES = 5;
final int JOIN_NODES = 10;
startGrids(START_NODES);
final CyclicBarrier barrier = new CyclicBarrier(JOIN_NODES + 1);
final AtomicInteger startIdx = new AtomicInteger(START_NODES);
IgniteInternalFuture<?> fut =
GridTestUtils.runMultiThreadedAsync(
new Callable<Object>() {
@Override
public Object call() throws Exception {
int idx = startIdx.getAndIncrement();
Thread.currentThread().setName("start-thread-" + idx);
barrier.await();
Ignite ignite = startGrid(idx);
assertFalse(ignite.configuration().isClientMode());
log.info("Started node: " + ignite.name());
return null;
}
},
JOIN_NODES,
"start-thread");
barrier.await();
U.sleep(ThreadLocalRandom.current().nextInt(10, 100));
for (int i = 0; i < START_NODES; i++) stopGrid(i);
fut.get();
stopAllGrids();
}
}
/** {@inheritDoc} */
@Override
public IgniteInternalFuture<IgniteInternalTx> rollbackAsync() {
if (log.isDebugEnabled()) log.debug("Rolling back near tx: " + this);
GridNearTxFinishFuture fut = rollbackFut.get();
if (fut != null) return fut;
if (!rollbackFut.compareAndSet(null, fut = new GridNearTxFinishFuture<>(cctx, this, false)))
return rollbackFut.get();
cctx.mvcc().addFuture(fut, fut.futureId());
IgniteInternalFuture<?> prepFut = this.prepFut.get();
if (prepFut == null || prepFut.isDone()) {
try {
// Check for errors in prepare future.
if (prepFut != null) prepFut.get();
} catch (IgniteCheckedException e) {
if (log.isDebugEnabled())
log.debug("Got optimistic tx failure [tx=" + this + ", err=" + e + ']');
}
fut.finish();
} else {
prepFut.listen(
new CI1<IgniteInternalFuture<?>>() {
@Override
public void apply(IgniteInternalFuture<?> f) {
try {
// Check for errors in prepare future.
f.get();
} catch (IgniteCheckedException e) {
if (log.isDebugEnabled())
log.debug("Got optimistic tx failure [tx=" + this + ", err=" + e + ']');
}
GridNearTxFinishFuture fut0 = rollbackFut.get();
fut0.finish();
}
});
}
return fut;
}
/**
* Rolls back local part of colocated transaction.
*
* @return Commit future.
*/
public IgniteInternalFuture<IgniteInternalTx> rollbackAsyncLocal() {
if (log.isDebugEnabled()) log.debug("Rolling back colocated tx locally: " + this);
final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, /*commit*/ false);
cctx.mvcc().addFuture(fut, fut.futureId());
IgniteInternalFuture<?> prep = prepFut.get();
if (prep == null || prep.isDone()) {
try {
if (prep != null) prep.get();
} catch (IgniteCheckedException e) {
if (log.isDebugEnabled())
log.debug(
"Failed to prepare transaction during rollback (will ignore) [tx="
+ this
+ ", msg="
+ e.getMessage()
+ ']');
}
fut.finish();
} else
prep.listen(
new CI1<IgniteInternalFuture<?>>() {
@Override
public void apply(IgniteInternalFuture<?> f) {
try {
f.get(); // Check for errors of a parent future.
} catch (IgniteCheckedException e) {
log.debug(
"Failed to prepare transaction during rollback (will ignore) [tx="
+ this
+ ", msg="
+ e.getMessage()
+ ']');
}
fut.finish();
}
});
return fut;
}
/** @throws Exception If any error occurs. */
public void testMultiThreadedClientsRestart() throws Exception {
fail("https://issues.apache.org/jira/browse/IGNITE-1123");
clientFlagGlobal = false;
info("Test timeout: " + (getTestTimeout() / (60 * 1000)) + " min.");
startGridsMultiThreaded(GRID_CNT);
clientFlagGlobal = true;
startGridsMultiThreaded(GRID_CNT, CLIENT_GRID_CNT);
final AtomicBoolean done = new AtomicBoolean();
final AtomicInteger clientIdx = new AtomicInteger(GRID_CNT);
IgniteInternalFuture<?> fut1 =
multithreadedAsync(
new Callable<Object>() {
@Override
public Object call() throws Exception {
clientFlagPerThread.set(true);
int idx = clientIdx.getAndIncrement();
while (!done.get()) {
stopGrid(idx, true);
startGrid(idx);
}
return null;
}
},
CLIENT_GRID_CNT);
Thread.sleep(getTestTimeout() - 60 * 1000);
done.set(true);
fut1.get();
}
/**
* JUnit.
*
* @throws Exception If failed.
*/
@SuppressWarnings({"TooBroadScope"})
public void testRestarts() throws Exception {
int duration = 60 * 1000;
int qryThreadNum = 10;
final long nodeLifeTime = 2 * 1000;
final int logFreq = 20;
final IgniteCache<Integer, Integer> cache = grid(0).cache(null);
assert cache != null;
for (int i = 0; i < KEY_CNT; i++) cache.put(i, i);
assertEquals(KEY_CNT, cache.localSize());
final AtomicInteger qryCnt = new AtomicInteger();
final AtomicBoolean done = new AtomicBoolean();
IgniteInternalFuture<?> fut1 =
multithreadedAsync(
new CAX() {
@Override
public void applyx() throws IgniteCheckedException {
while (!done.get()) {
Collection<Cache.Entry<Integer, Integer>> res =
cache.query(new SqlQuery(Integer.class, "_val >= 0")).getAll();
assertFalse(res.isEmpty());
int c = qryCnt.incrementAndGet();
if (c % logFreq == 0) info("Executed queries: " + c);
}
}
},
qryThreadNum);
final AtomicInteger restartCnt = new AtomicInteger();
CollectingEventListener lsnr = new CollectingEventListener();
for (int i = 0; i < GRID_CNT; i++)
grid(i).events().localListen(lsnr, EventType.EVT_CACHE_REBALANCE_STOPPED);
IgniteInternalFuture<?> fut2 =
multithreadedAsync(
new Callable<Object>() {
@SuppressWarnings({"BusyWait"})
@Override
public Object call() throws Exception {
while (!done.get()) {
int idx = GRID_CNT;
startGrid(idx);
Thread.sleep(nodeLifeTime);
stopGrid(idx);
int c = restartCnt.incrementAndGet();
if (c % logFreq == 0) info("Node restarts: " + c);
}
return true;
}
},
1);
Thread.sleep(duration);
done.set(true);
fut1.get();
fut2.get();
info("Awaiting rebalance events [restartCnt=" + restartCnt.get() + ']');
boolean success = lsnr.awaitEvents(GRID_CNT * 2 * restartCnt.get(), 15000);
for (int i = 0; i < GRID_CNT; i++)
grid(i).events().stopLocalListen(lsnr, EventType.EVT_CACHE_REBALANCE_STOPPED);
assert success;
}
/**
* Commits local part of colocated transaction.
*
* @return Commit future.
*/
public IgniteInternalFuture<IgniteInternalTx> commitAsyncLocal() {
if (log.isDebugEnabled()) log.debug("Committing colocated tx locally: " + this);
// In optimistic mode prepare was called explicitly.
if (pessimistic()) prepareAsync();
IgniteInternalFuture<?> prep = prepFut.get();
// Do not create finish future if there are no remote nodes.
if (F.isEmpty(dhtMap) && F.isEmpty(nearMap)) {
if (prep != null) return (IgniteInternalFuture<IgniteInternalTx>) (IgniteInternalFuture) prep;
return new GridFinishedFuture<IgniteInternalTx>(this);
}
final GridDhtTxFinishFuture fut = new GridDhtTxFinishFuture<>(cctx, this, /*commit*/ true);
cctx.mvcc().addFuture(fut, fut.futureId());
if (prep == null || prep.isDone()) {
assert prep != null || optimistic();
try {
if (prep != null) prep.get(); // Check for errors of a parent future.
fut.finish();
} catch (IgniteTxOptimisticCheckedException e) {
if (log.isDebugEnabled())
log.debug("Failed optimistically to prepare transaction [tx=" + this + ", e=" + e + ']');
fut.onError(e);
} catch (IgniteCheckedException e) {
U.error(log, "Failed to prepare transaction: " + this, e);
fut.onError(e);
}
} else
prep.listen(
new CI1<IgniteInternalFuture<?>>() {
@Override
public void apply(IgniteInternalFuture<?> f) {
try {
f.get(); // Check for errors of a parent future.
fut.finish();
} catch (IgniteTxOptimisticCheckedException e) {
if (log.isDebugEnabled())
log.debug(
"Failed optimistically to prepare transaction [tx="
+ this
+ ", e="
+ e
+ ']');
fut.onError(e);
} catch (IgniteCheckedException e) {
U.error(log, "Failed to prepare transaction: " + this, e);
fut.onError(e);
}
}
});
return fut;
}
/** @throws Exception If failed. */
private void testTryLockTimed(final boolean fair) throws Exception {
final IgniteLock lock0 = grid(0).reentrantLock("lock", true, fair, true);
assertEquals(0, lock0.getHoldCount());
assertFalse(lock0.hasQueuedThreads());
final int totalThreads = 2;
final Set<Thread> startedThreads = new GridConcurrentHashSet<>();
lock0.lock();
IgniteInternalFuture<?> fut =
multithreadedAsync(
new Callable<Object>() {
@Override
public Object call() throws Exception {
assertFalse(lock0.isHeldByCurrentThread());
startedThreads.add(Thread.currentThread());
boolean isInterrupted = false;
boolean locked = false;
try {
locked = lock0.tryLock(100, TimeUnit.MILLISECONDS);
} catch (IgniteInterruptedException e) {
isInterrupted = true;
} finally {
// Assert that thread was not interrupted.
assertFalse(isInterrupted);
// Assert that tryLock returned false.
assertFalse(locked);
// Assert that lock is still owned by main thread.
assertTrue(lock0.isLocked());
// Assert that this thread doesn't own the lock.
assertFalse(lock0.isHeldByCurrentThread());
// Release lock.
if (locked) lock0.unlock();
}
return null;
}
},
totalThreads);
fut.get();
lock0.unlock();
assertFalse(lock0.isLocked());
for (Thread t : startedThreads) assertFalse(lock0.hasQueuedThread(t));
lock0.close();
}
/**
* @param setPart If {@code true} sets partition for scan query.
* @throws Exception If failed.
*/
private void scanQueryReconnectInProgress(boolean setPart) throws Exception {
Ignite cln = grid(serverCount());
assertTrue(cln.cluster().localNode().isClient());
final Ignite srv = clientRouter(cln);
final IgniteCache<Integer, Person> clnCache = cln.getOrCreateCache(QUERY_CACHE);
clnCache.put(1, new Person(1, "name1", "surname1"));
clnCache.put(2, new Person(2, "name2", "surname2"));
clnCache.put(3, new Person(3, "name3", "surname3"));
final ScanQuery<Integer, Person> scanQry = new ScanQuery<>();
scanQry.setPageSize(1);
scanQry.setFilter(
new IgniteBiPredicate<Integer, Person>() {
@Override
public boolean apply(Integer integer, Person person) {
return true;
}
});
if (setPart) scanQry.setPartition(1);
blockMessage(GridCacheQueryResponse.class);
final IgniteInternalFuture<Object> fut =
GridTestUtils.runAsync(
new Callable<Object>() {
@Override
public Object call() throws Exception {
try {
QueryCursor<Cache.Entry<Integer, Person>> qryCursor = clnCache.query(scanQry);
qryCursor.getAll();
} catch (CacheException e) {
checkAndWait(e);
return true;
}
return false;
}
});
// Check that client waiting operation.
GridTestUtils.assertThrows(
log,
new Callable<Object>() {
@Override
public Object call() throws Exception {
return fut.get(200);
}
},
IgniteFutureTimeoutCheckedException.class,
null);
assertNotDone(fut);
unblockMessage();
reconnectClientNode(cln, srv, null);
assertTrue((Boolean) fut.get(2, SECONDS));
QueryCursor<Cache.Entry<Integer, Person>> qryCursor2 = clnCache.query(scanQry);
assertEquals(setPart ? 1 : 3, qryCursor2.getAll().size());
}
/** @throws Exception If failed. */
private void testLockInterruptiblyMultinode(final boolean fair) throws Exception {
if (gridCount() == 1) return;
// Initialize reentrant lock.
final IgniteLock lock0 = grid(0).reentrantLock("lock", true, fair, true);
assertEquals(0, lock0.getHoldCount());
assertFalse(lock0.hasQueuedThreads());
lock0.lock();
// Number of threads, one per node.
final int threadCount = gridCount();
final AtomicLong threadCounter = new AtomicLong(0);
IgniteInternalFuture<?> fut =
multithreadedAsync(
new Callable<Object>() {
@Override
public Object call() throws Exception {
final int localNodeId = (int) threadCounter.getAndIncrement();
final Ignite grid = grid(localNodeId);
IgniteClosure<Ignite, Void> closure =
new IgniteClosure<Ignite, Void>() {
@Override
public Void apply(Ignite ignite) {
final IgniteLock l = ignite.reentrantLock("lock", true, true, true);
final AtomicReference<Thread> thread = new AtomicReference<>();
final AtomicBoolean done = new AtomicBoolean(false);
final AtomicBoolean exceptionThrown = new AtomicBoolean(false);
final IgniteCountDownLatch latch =
ignite.countDownLatch("latch", threadCount, false, true);
IgniteInternalFuture<?> fut =
GridTestUtils.runAsync(
new Callable<Void>() {
@Override
public Void call() throws Exception {
try {
thread.set(Thread.currentThread());
l.lockInterruptibly();
} catch (IgniteInterruptedException e) {
exceptionThrown.set(true);
} finally {
done.set(true);
}
return null;
}
});
// Wait until l.lock() has been called.
while (!l.hasQueuedThreads()) {
// No-op.
}
latch.countDown();
latch.await();
thread.get().interrupt();
while (!done.get()) {
// No-op.
}
try {
fut.get();
} catch (IgniteCheckedException e) {
fail(e.getMessage());
throw new RuntimeException(e);
}
assertTrue(exceptionThrown.get());
return null;
}
};
closure.apply(grid);
return null;
}
},
threadCount);
fut.get();
lock0.unlock();
info("Checking if interrupted threads are removed from global waiting queue...");
// Check if interrupted threads are removed from global waiting queue.
boolean locked = lock0.tryLock(1000, MILLISECONDS);
info("Interrupted threads successfully removed from global waiting queue. ");
assertTrue(locked);
lock0.unlock();
assertFalse(lock0.isLocked());
lock0.close();
}
/** @throws Exception If failed. */
private void testReentrantLockMultinode1(final boolean fair) throws Exception {
if (gridCount() == 1) return;
IgniteLock lock = grid(0).reentrantLock("s1", true, fair, true);
List<IgniteInternalFuture<?>> futs = new ArrayList<>();
for (int i = 0; i < gridCount(); i++) {
final Ignite ignite = grid(i);
futs.add(
GridTestUtils.runAsync(
new Callable<Void>() {
@Override
public Void call() throws Exception {
IgniteLock lock = ignite.reentrantLock("s1", true, fair, false);
assertNotNull(lock);
IgniteCondition cond1 = lock.getOrCreateCondition("c1");
IgniteCondition cond2 = lock.getOrCreateCondition("c2");
try {
boolean wait = lock.tryLock(30_000, MILLISECONDS);
assertTrue(wait);
cond2.signal();
cond1.await();
} finally {
lock.unlock();
}
return null;
}
}));
}
boolean done = false;
while (!done) {
done = true;
for (IgniteInternalFuture<?> fut : futs) {
if (!fut.isDone()) done = false;
}
try {
lock.lock();
lock.getOrCreateCondition("c1").signal();
lock.getOrCreateCondition("c2").await(10, MILLISECONDS);
} finally {
lock.unlock();
}
}
for (IgniteInternalFuture<?> fut : futs) fut.get(30_000);
}
/**
* Tests if lock is evenly acquired among nodes when fair flag is set on. Since exact ordering of
* lock acquisitions cannot be guaranteed because it also depends on the OS thread scheduling,
* certain deviation from uniform distribution is tolerated.
*
* @throws Exception If failed.
*/
public void testFairness() throws Exception {
if (gridCount() == 1) return;
// Total number of ops.
final long opsCount = 10000;
// Allowed deviation from uniform distribution.
final double tolerance = 0.05;
// Shared counter.
final String OPS_COUNTER = "ops_counter";
// Number of threads, one per node.
final int threadCount = gridCount();
final AtomicLong threadCounter = new AtomicLong(0);
Ignite ignite = startGrid(gridCount());
// Initialize reentrant lock.
IgniteLock l = ignite.reentrantLock("lock", true, true, true);
// Initialize OPS_COUNTER.
ignite.getOrCreateCache(OPS_COUNTER).put(OPS_COUNTER, (long) 0);
final Map<Integer, Long> counts = new ConcurrentHashMap<>();
IgniteInternalFuture<?> fut =
multithreadedAsync(
new Callable<Object>() {
@Override
public Object call() throws Exception {
final int localNodeId = (int) threadCounter.getAndIncrement();
final Ignite grid = grid(localNodeId);
IgniteClosure<Ignite, Long> closure =
new IgniteClosure<Ignite, Long>() {
@Override
public Long apply(Ignite ignite) {
IgniteLock l = ignite.reentrantLock("lock", true, true, true);
long localCount = 0;
IgniteCountDownLatch latch =
ignite.countDownLatch("latch", threadCount, false, true);
latch.countDown();
latch.await();
while (true) {
l.lock();
try {
long opsCounter =
(long) ignite.getOrCreateCache(OPS_COUNTER).get(OPS_COUNTER);
if (opsCounter == opsCount) break;
ignite.getOrCreateCache(OPS_COUNTER).put(OPS_COUNTER, ++opsCounter);
localCount++;
if (localCount > 1000) {
assertTrue(localCount < (1 + tolerance) * opsCounter / threadCount);
assertTrue(localCount > (1 - tolerance) * opsCounter / threadCount);
}
if (localCount % 100 == 0) {
info(
"Node [id="
+ ignite.cluster().localNode().id()
+ "] acquired "
+ localCount
+ " times. "
+ "Total ops count: "
+ opsCounter
+ "/"
+ opsCount
+ "]");
}
} finally {
l.unlock();
}
}
return localCount;
}
};
long localCount = closure.apply(grid);
counts.put(localNodeId, localCount);
return null;
}
},
threadCount);
fut.get();
long totalSum = 0;
for (int i = 0; i < gridCount(); i++) {
totalSum += counts.get(i);
info("Node " + grid(i).localNode().id() + " acquired the lock " + counts.get(i) + " times. ");
}
assertEquals(totalSum, opsCount);
l.close();
ignite.close();
}
/** @throws Exception If failed. */
private void testHasConditionQueuedThreads(final boolean fair) throws Exception {
final IgniteLock lock0 = grid(0).reentrantLock("lock", true, fair, true);
assertEquals(0, lock0.getHoldCount());
assertFalse(lock0.hasQueuedThreads());
final int totalThreads = 5;
final Set<Thread> startedThreads = new GridConcurrentHashSet<>();
final Set<Thread> finishedThreads = new GridConcurrentHashSet<>();
IgniteInternalFuture<?> fut =
multithreadedAsync(
new Callable<Object>() {
@Override
public Object call() throws Exception {
assertFalse(lock0.isHeldByCurrentThread());
IgniteCondition cond = lock0.getOrCreateCondition("cond");
lock0.lock();
startedThreads.add(Thread.currentThread());
// Wait until every thread tries to lock.
do {
cond.await();
Thread.sleep(1000);
} while (startedThreads.size() != totalThreads);
try {
info(
"Acquired in separate thread. Number of threads waiting on condition: "
+ lock0.getWaitQueueLength(cond));
assertTrue(lock0.isHeldByCurrentThread());
assertFalse(lock0.hasQueuedThread(Thread.currentThread()));
finishedThreads.add(Thread.currentThread());
if (startedThreads.size() != finishedThreads.size()) {
assertTrue(lock0.hasWaiters(cond));
}
for (Thread t : startedThreads) {
if (!finishedThreads.contains(t)) assertTrue(lock0.hasWaiters(cond));
}
assertTrue(
lock0.getWaitQueueLength(cond)
== (startedThreads.size() - finishedThreads.size()));
} finally {
cond.signal();
lock0.unlock();
assertFalse(lock0.isHeldByCurrentThread());
}
return null;
}
},
totalThreads);
IgniteCondition cond = lock0.getOrCreateCondition("cond");
lock0.lock();
try {
// Wait until all threads are waiting on condition.
while (lock0.getWaitQueueLength(cond) != totalThreads) {
lock0.unlock();
Thread.sleep(1000);
lock0.lock();
}
// Signal once to get things started.
cond.signal();
} finally {
lock0.unlock();
}
fut.get();
assertFalse(lock0.hasQueuedThreads());
for (Thread t : startedThreads) assertFalse(lock0.hasQueuedThread(t));
lock0.close();
}
/** @throws Exception If failed. */
private void testHasQueuedThreads(final boolean fair) throws Exception {
final IgniteLock lock0 = grid(0).reentrantLock("lock", true, fair, true);
assertEquals(0, lock0.getHoldCount());
assertFalse(lock0.hasQueuedThreads());
final int totalThreads = 5;
final Set<Thread> startedThreads = new GridConcurrentHashSet<>();
final Set<Thread> finishedThreads = new GridConcurrentHashSet<>();
IgniteInternalFuture<?> fut =
multithreadedAsync(
new Callable<Object>() {
@Override
public Object call() throws Exception {
assertFalse(lock0.isHeldByCurrentThread());
startedThreads.add(Thread.currentThread());
lock0.lock();
// Wait until every thread tries to lock.
do {
Thread.sleep(1000);
} while (startedThreads.size() != totalThreads);
try {
info("Acquired in separate thread. ");
assertTrue(lock0.isHeldByCurrentThread());
assertFalse(lock0.hasQueuedThread(Thread.currentThread()));
finishedThreads.add(Thread.currentThread());
if (startedThreads.size() != finishedThreads.size()) {
assertTrue(lock0.hasQueuedThreads());
}
for (Thread t : startedThreads) {
assertTrue(lock0.hasQueuedThread(t) != finishedThreads.contains(t));
}
} finally {
lock0.unlock();
assertFalse(lock0.isHeldByCurrentThread());
}
return null;
}
},
totalThreads);
fut.get();
assertFalse(lock0.hasQueuedThreads());
for (Thread t : startedThreads) assertFalse(lock0.hasQueuedThread(t));
lock0.close();
}
/**
* Executes one test iteration.
*
* @throws Exception If failed.
*/
private void doTestFailover() throws Exception {
try {
done = false;
nodes = new AtomicReferenceArray<>(GRID_CNT);
startGridsMultiThreaded(GRID_CNT, false);
for (int i = 0; i < GRID_CNT; i++) assertTrue(nodes.compareAndSet(i, null, ignite(i)));
List<IgniteInternalFuture> futs = new ArrayList<>();
for (int i = 0; i < GRID_CNT + 1; i++) {
futs.add(
multithreadedAsync(
new Runnable() {
@Override
public void run() {
T2<Ignite, Integer> ignite;
while ((ignite = randomNode()) != null) {
IgniteCache<Object, Object> cache = ignite.get1().cache(null);
for (int i = 0; i < 100; i++)
cache.containsKey(ThreadLocalRandom.current().nextInt(100_000));
assertTrue(nodes.compareAndSet(ignite.get2(), null, ignite.get1()));
try {
Thread.sleep(ThreadLocalRandom.current().nextLong(50));
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
},
1,
"containsKey-thread-" + i));
futs.add(
multithreadedAsync(
new Runnable() {
@Override
public void run() {
T2<Ignite, Integer> ignite;
while ((ignite = randomNode()) != null) {
IgniteCache<Object, Object> cache = ignite.get1().cache(null);
for (int i = 0; i < 100; i++)
cache.put(ThreadLocalRandom.current().nextInt(100_000), UUID.randomUUID());
assertTrue(nodes.compareAndSet(ignite.get2(), null, ignite.get1()));
try {
Thread.sleep(ThreadLocalRandom.current().nextLong(50));
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
},
1,
"put-thread-" + i));
}
try {
int aliveGrids = GRID_CNT;
while (aliveGrids > 0) {
T2<Ignite, Integer> ignite = randomNode();
assert ignite != null;
Ignite ignite0 = ignite.get1();
log.info("Stop node: " + ignite0.name());
ignite0.close();
log.info("Node stop finished: " + ignite0.name());
aliveGrids--;
}
} finally {
done = true;
}
for (IgniteInternalFuture fut : futs) fut.get();
} finally {
done = true;
}
}
/** @throws Exception If failed. */
public void testReconnectQueryInProgress() throws Exception {
Ignite cln = grid(serverCount());
assertTrue(cln.cluster().localNode().isClient());
final Ignite srv = clientRouter(cln);
final IgniteCache<Integer, Person> clnCache = cln.getOrCreateCache(QUERY_CACHE);
clnCache.put(1, new Person(1, "name1", "surname1"));
clnCache.put(2, new Person(2, "name2", "surname2"));
clnCache.put(3, new Person(3, "name3", "surname3"));
blockMessage(GridQueryNextPageResponse.class);
final SqlQuery<Integer, Person> qry = new SqlQuery<>(Person.class, "_key <> 0");
qry.setPageSize(1);
final QueryCursor<Cache.Entry<Integer, Person>> cur1 = clnCache.query(qry);
final IgniteInternalFuture<Object> fut =
GridTestUtils.runAsync(
new Callable<Object>() {
@Override
public Object call() throws Exception {
try {
cur1.getAll();
} catch (CacheException e) {
checkAndWait(e);
return true;
}
return false;
}
});
// Check that client waiting operation.
GridTestUtils.assertThrows(
log,
new Callable<Object>() {
@Override
public Object call() throws Exception {
return fut.get(200);
}
},
IgniteFutureTimeoutCheckedException.class,
null);
assertNotDone(fut);
unblockMessage();
reconnectClientNode(cln, srv, null);
assertTrue((Boolean) fut.get(2, SECONDS));
QueryCursor<Cache.Entry<Integer, Person>> cur2 = clnCache.query(qry);
assertEquals(3, cur2.getAll().size());
}
/** @throws Exception If failed. */
private void testTryLock(final boolean fair) throws Exception {
final IgniteLock lock0 = grid(0).reentrantLock("lock", true, fair, true);
assertEquals(0, lock0.getHoldCount());
assertFalse(lock0.hasQueuedThreads());
final int totalThreads = 2;
final Set<Thread> startedThreads = new GridConcurrentHashSet<>();
lock0.lock();
IgniteInternalFuture<?> fut =
multithreadedAsync(
new Callable<Object>() {
@Override
public Object call() throws Exception {
assertFalse(lock0.isHeldByCurrentThread());
startedThreads.add(Thread.currentThread());
boolean isInterrupted = false;
boolean locked = false;
try {
locked = lock0.tryLock();
} catch (IgniteInterruptedException e) {
isInterrupted = true;
fail("tryLock() method is uninterruptible.");
} finally {
// Assert that thread was not interrupted.
assertFalse(isInterrupted);
// Assert that lock is locked.
assertTrue(lock0.isLocked());
// Assert that this thread does own the lock.
assertEquals(locked, lock0.isHeldByCurrentThread());
// Release lock.
if (locked) lock0.unlock();
}
return null;
}
},
totalThreads);
// Wait for all threads to attempt to acquire lock.
while (startedThreads.size() != totalThreads) {
Thread.sleep(500);
}
for (Thread t : startedThreads) t.interrupt();
fut.get();
lock0.unlock();
assertFalse(lock0.isLocked());
for (Thread t : startedThreads) assertFalse(lock0.hasQueuedThread(t));
lock0.close();
}
/**
* @param cacheMode Cache mode.
* @param sameAff If {@code false} uses different number of partitions for caches.
* @param concurrency Transaction concurrency.
* @param isolation Transaction isolation.
* @throws Exception If failed.
*/
private void crossCacheTxFailover(
CacheMode cacheMode,
boolean sameAff,
final TransactionConcurrency concurrency,
final TransactionIsolation isolation)
throws Exception {
IgniteKernal ignite0 = (IgniteKernal) ignite(0);
final AtomicBoolean stop = new AtomicBoolean();
try {
ignite0.createCache(cacheConfiguration(CACHE1, cacheMode, 256));
ignite0.createCache(cacheConfiguration(CACHE2, cacheMode, sameAff ? 256 : 128));
final AtomicInteger threadIdx = new AtomicInteger();
IgniteInternalFuture<?> fut =
GridTestUtils.runMultiThreadedAsync(
new Callable<Void>() {
@Override
public Void call() throws Exception {
int idx = threadIdx.getAndIncrement();
Ignite ignite = ignite(idx % GRID_CNT);
log.info(
"Started update thread [node="
+ ignite.name()
+ ", client="
+ ignite.configuration().isClientMode()
+ ']');
IgniteCache<TestKey, TestValue> cache1 = ignite.cache(CACHE1);
IgniteCache<TestKey, TestValue> cache2 = ignite.cache(CACHE2);
assertNotSame(cache1, cache2);
IgniteTransactions txs = ignite.transactions();
ThreadLocalRandom rnd = ThreadLocalRandom.current();
long iter = 0;
while (!stop.get()) {
boolean sameKey = rnd.nextBoolean();
try {
try (Transaction tx = txs.txStart(concurrency, isolation)) {
if (sameKey) {
TestKey key = new TestKey(rnd.nextLong(KEY_RANGE));
cacheOperation(rnd, cache1, key);
cacheOperation(rnd, cache2, key);
} else {
TestKey key1 = new TestKey(rnd.nextLong(KEY_RANGE));
TestKey key2 = new TestKey(key1.key() + 1);
cacheOperation(rnd, cache1, key1);
cacheOperation(rnd, cache2, key2);
}
tx.commit();
}
} catch (CacheException | IgniteException e) {
log.info("Update error: " + e);
}
if (iter++ % 500 == 0) log.info("Iteration: " + iter);
}
return null;
}
/**
* @param rnd Random.
* @param cache Cache.
* @param key Key.
*/
private void cacheOperation(
ThreadLocalRandom rnd, IgniteCache<TestKey, TestValue> cache, TestKey key) {
switch (rnd.nextInt(4)) {
case 0:
cache.put(key, new TestValue(rnd.nextLong()));
break;
case 1:
cache.remove(key);
break;
case 2:
cache.invoke(key, new TestEntryProcessor(rnd.nextBoolean() ? 1L : null));
break;
case 3:
cache.get(key);
break;
default:
assert false;
}
}
},
10,
"tx-thread");
long stopTime = System.currentTimeMillis() + 3 * 60_000;
long topVer = ignite0.cluster().topologyVersion();
boolean failed = false;
while (System.currentTimeMillis() < stopTime) {
log.info("Start node.");
IgniteKernal ignite = (IgniteKernal) startGrid(GRID_CNT);
assertFalse(ignite.configuration().isClientMode());
topVer++;
IgniteInternalFuture<?> affFut =
ignite
.context()
.cache()
.context()
.exchange()
.affinityReadyFuture(new AffinityTopologyVersion(topVer));
try {
if (affFut != null) affFut.get(30_000);
} catch (IgniteFutureTimeoutCheckedException e) {
log.error("Failed to wait for affinity future after start: " + topVer);
failed = true;
break;
}
Thread.sleep(500);
log.info("Stop node.");
stopGrid(GRID_CNT);
topVer++;
affFut =
ignite0
.context()
.cache()
.context()
.exchange()
.affinityReadyFuture(new AffinityTopologyVersion(topVer));
try {
if (affFut != null) affFut.get(30_000);
} catch (IgniteFutureTimeoutCheckedException e) {
log.error("Failed to wait for affinity future after stop: " + topVer);
failed = true;
break;
}
}
stop.set(true);
fut.get();
assertFalse("Test failed, see log for details.", failed);
} finally {
stop.set(true);
ignite0.destroyCache(CACHE1);
ignite0.destroyCache(CACHE2);
awaitPartitionMapExchange();
}
}
/** @throws Exception If failed. */
public void testSyncConsistent() throws Exception {
final AtomicBoolean stop = new AtomicBoolean();
final AtomicLong x = new AtomicLong();
final AtomicLong y = new AtomicLong();
final AtomicLong z = new AtomicLong();
final Random rnd = new Random();
final String oops = "Oops!";
final GridSnapshotLock<T3<Long, Long, Long>> lock =
new GridSnapshotLock<T3<Long, Long, Long>>() {
@Override
protected T3<Long, Long, Long> doSnapshot() {
if (rnd.nextBoolean()) throw new IgniteException(oops);
return new T3<>(x.get(), y.get(), z.get());
}
};
IgniteInternalFuture<?> fut1 =
multithreadedAsync(
new Callable<Object>() {
@Override
public Object call() throws Exception {
Random rnd = new Random();
while (!stop.get()) {
if (rnd.nextBoolean()) {
if (!lock.tryBeginUpdate()) continue;
} else lock.beginUpdate();
int n = 1 + rnd.nextInt(1000);
if (rnd.nextBoolean()) x.addAndGet(n);
else y.addAndGet(n);
z.addAndGet(n);
lock.endUpdate();
}
return null;
}
},
15,
"update");
IgniteInternalFuture<?> fut2 =
multithreadedAsync(
new Callable<Object>() {
@Override
public Object call() throws Exception {
while (!stop.get()) {
T3<Long, Long, Long> t;
try {
t = lock.snapshot();
} catch (IgniteException e) {
assertEquals(oops, e.getMessage());
continue;
}
assertEquals(t.get3().longValue(), t.get1() + t.get2());
}
return null;
}
},
8,
"snapshot");
Thread.sleep(20000);
stop.set(true);
fut1.get();
fut2.get();
}
/**
* Starts activity.
*
* @throws IgniteInterruptedCheckedException If interrupted.
*/
public void init() throws IgniteInterruptedCheckedException {
if (isDone()) return;
if (init.compareAndSet(false, true)) {
if (isDone()) return;
try {
// Wait for event to occur to make sure that discovery
// will return corresponding nodes.
U.await(evtLatch);
assert discoEvt != null : this;
assert !dummy && !forcePreload : this;
ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx, exchId.topologyVersion());
oldestNode.set(oldest);
startCaches();
// True if client node joined or failed.
boolean clientNodeEvt;
if (F.isEmpty(reqs)) {
int type = discoEvt.type();
assert type == EVT_NODE_JOINED || type == EVT_NODE_LEFT || type == EVT_NODE_FAILED
: discoEvt;
clientNodeEvt = CU.clientNode(discoEvt.eventNode());
} else {
assert discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT : discoEvt;
boolean clientOnlyStart = true;
for (DynamicCacheChangeRequest req : reqs) {
if (!req.clientStartOnly()) {
clientOnlyStart = false;
break;
}
}
clientNodeEvt = clientOnlyStart;
}
if (clientNodeEvt) {
ClusterNode node = discoEvt.eventNode();
// Client need to initialize affinity for local join event or for stated client caches.
if (!node.isLocal()) {
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
GridDhtPartitionTopology top = cacheCtx.topology();
top.updateTopologyVersion(exchId, this, -1, stopping(cacheCtx.cacheId()));
if (cacheCtx.affinity().affinityTopologyVersion() == AffinityTopologyVersion.NONE) {
initTopology(cacheCtx);
top.beforeExchange(this);
} else
cacheCtx.affinity().clientEventTopologyChange(discoEvt, exchId.topologyVersion());
}
if (exchId.isLeft())
cctx.mvcc().removeExplicitNodeLocks(exchId.nodeId(), exchId.topologyVersion());
onDone(exchId.topologyVersion());
skipPreload = cctx.kernalContext().clientNode();
return;
}
}
if (cctx.kernalContext().clientNode()) {
skipPreload = true;
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
GridDhtPartitionTopology top = cacheCtx.topology();
top.updateTopologyVersion(exchId, this, -1, stopping(cacheCtx.cacheId()));
}
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
initTopology(cacheCtx);
}
if (oldestNode.get() != null) {
rmtNodes =
new ConcurrentLinkedQueue<>(
CU.aliveRemoteServerNodesWithCaches(cctx, exchId.topologyVersion()));
rmtIds = Collections.unmodifiableSet(new HashSet<>(F.nodeIds(rmtNodes)));
ready.set(true);
initFut.onDone(true);
if (log.isDebugEnabled()) log.debug("Initialized future: " + this);
sendPartitions();
} else onDone(exchId.topologyVersion());
return;
}
assert oldestNode.get() != null;
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (isCacheAdded(cacheCtx.cacheId(), exchId.topologyVersion())) {
if (cacheCtx
.discovery()
.cacheAffinityNodes(cacheCtx.name(), topologyVersion())
.isEmpty())
U.quietAndWarn(log, "No server nodes found for cache client: " + cacheCtx.namex());
}
cacheCtx.preloader().onExchangeFutureAdded();
}
List<String> cachesWithoutNodes = null;
if (exchId.isLeft()) {
for (String name : cctx.cache().cacheNames()) {
if (cctx.discovery().cacheAffinityNodes(name, topologyVersion()).isEmpty()) {
if (cachesWithoutNodes == null) cachesWithoutNodes = new ArrayList<>();
cachesWithoutNodes.add(name);
// Fire event even if there is no client cache started.
if (cctx.gridEvents().isRecordable(EventType.EVT_CACHE_NODES_LEFT)) {
Event evt =
new CacheEvent(
name,
cctx.localNode(),
cctx.localNode(),
"All server nodes have left the cluster.",
EventType.EVT_CACHE_NODES_LEFT,
0,
false,
null,
null,
null,
null,
false,
null,
false,
null,
null,
null);
cctx.gridEvents().record(evt);
}
}
}
}
if (cachesWithoutNodes != null) {
StringBuilder sb =
new StringBuilder(
"All server nodes for the following caches have left the cluster: ");
for (int i = 0; i < cachesWithoutNodes.size(); i++) {
String cache = cachesWithoutNodes.get(i);
sb.append('\'').append(cache).append('\'');
if (i != cachesWithoutNodes.size() - 1) sb.append(", ");
}
U.quietAndWarn(log, sb.toString());
U.quietAndWarn(log, "Must have server nodes for caches to operate.");
}
assert discoEvt != null;
assert exchId.nodeId().equals(discoEvt.eventNode().id());
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
GridClientPartitionTopology clientTop =
cctx.exchange().clearClientTopology(cacheCtx.cacheId());
long updSeq = clientTop == null ? -1 : clientTop.lastUpdateSequence();
// Update before waiting for locks.
if (!cacheCtx.isLocal())
cacheCtx
.topology()
.updateTopologyVersion(exchId, this, updSeq, stopping(cacheCtx.cacheId()));
}
// Grab all alive remote nodes with order of equal or less than last joined node.
rmtNodes =
new ConcurrentLinkedQueue<>(
CU.aliveRemoteServerNodesWithCaches(cctx, exchId.topologyVersion()));
rmtIds = Collections.unmodifiableSet(new HashSet<>(F.nodeIds(rmtNodes)));
for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> m : singleMsgs.entrySet())
// If received any messages, process them.
onReceive(m.getKey(), m.getValue());
for (Map.Entry<UUID, GridDhtPartitionsFullMessage> m : fullMsgs.entrySet())
// If received any messages, process them.
onReceive(m.getKey(), m.getValue());
AffinityTopologyVersion topVer = exchId.topologyVersion();
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
// Must initialize topology after we get discovery event.
initTopology(cacheCtx);
cacheCtx.preloader().updateLastExchangeFuture(this);
}
IgniteInternalFuture<?> partReleaseFut = cctx.partitionReleaseFuture(topVer);
// Assign to class variable so it will be included into toString() method.
this.partReleaseFut = partReleaseFut;
if (log.isDebugEnabled()) log.debug("Before waiting for partition release future: " + this);
while (true) {
try {
partReleaseFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS);
break;
} catch (IgniteFutureTimeoutCheckedException ignored) {
// Print pending transactions and locks that might have led to hang.
dumpPendingObjects();
}
}
if (log.isDebugEnabled()) log.debug("After waiting for partition release future: " + this);
if (!F.isEmpty(reqs)) blockGateways();
if (exchId.isLeft())
cctx.mvcc().removeExplicitNodeLocks(exchId.nodeId(), exchId.topologyVersion());
IgniteInternalFuture<?> locksFut = cctx.mvcc().finishLocks(exchId.topologyVersion());
while (true) {
try {
locksFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS);
break;
} catch (IgniteFutureTimeoutCheckedException ignored) {
U.warn(
log,
"Failed to wait for locks release future. "
+ "Dumping pending objects that might be the cause: "
+ cctx.localNodeId());
U.warn(log, "Locked entries:");
Map<IgniteTxKey, Collection<GridCacheMvccCandidate>> locks =
cctx.mvcc().unfinishedLocks(exchId.topologyVersion());
for (Map.Entry<IgniteTxKey, Collection<GridCacheMvccCandidate>> e : locks.entrySet())
U.warn(log, "Locked entry [key=" + e.getKey() + ", mvcc=" + e.getValue() + ']');
}
}
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
// Notify replication manager.
GridCacheContext drCacheCtx =
cacheCtx.isNear() ? cacheCtx.near().dht().context() : cacheCtx;
if (drCacheCtx.isDrEnabled()) drCacheCtx.dr().beforeExchange(topVer, exchId.isLeft());
// Partition release future is done so we can flush the write-behind store.
cacheCtx.store().forceFlush();
// Process queued undeploys prior to sending/spreading map.
cacheCtx.preloader().unwindUndeploys();
GridDhtPartitionTopology top = cacheCtx.topology();
assert topVer.equals(top.topologyVersion())
: "Topology version is updated only in this class instances inside single ExchangeWorker thread.";
top.beforeExchange(this);
}
for (GridClientPartitionTopology top : cctx.exchange().clientTopologies()) {
top.updateTopologyVersion(exchId, this, -1, stopping(top.cacheId()));
top.beforeExchange(this);
}
} catch (IgniteInterruptedCheckedException e) {
onDone(e);
throw e;
} catch (Throwable e) {
U.error(
log,
"Failed to reinitialize local partitions (preloading will be stopped): " + exchId,
e);
onDone(e);
if (e instanceof Error) throw (Error) e;
return;
}
if (F.isEmpty(rmtIds)) {
onDone(exchId.topologyVersion());
return;
}
ready.set(true);
initFut.onDone(true);
if (log.isDebugEnabled()) log.debug("Initialized future: " + this);
// If this node is not oldest.
if (!oldestNode.get().id().equals(cctx.localNodeId())) sendPartitions();
else {
boolean allReceived = allReceived();
if (allReceived && replied.compareAndSet(false, true)) {
if (spreadPartitions()) onDone(exchId.topologyVersion());
}
}
scheduleRecheck();
} else assert false : "Skipped init future: " + this;
}
/** {@inheritDoc} */
@Override
protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
try {
IgfsDataInputStream dis = new IgfsDataInputStream(endpoint.inputStream());
byte[] hdr = new byte[IgfsMarshaller.HEADER_SIZE];
boolean first = true;
while (!Thread.currentThread().isInterrupted()) {
dis.readFully(hdr);
final long reqId = U.bytesToLong(hdr, 0);
int ordinal = U.bytesToInt(hdr, 8);
if (first) { // First message must be HANDSHAKE.
if (reqId != 0 || ordinal != IgfsIpcCommand.HANDSHAKE.ordinal()) {
if (log.isDebugEnabled())
log.debug(
"IGFS IPC handshake failed [reqId=" + reqId + ", ordinal=" + ordinal + ']');
return;
}
first = false;
}
final IgfsIpcCommand cmd = IgfsIpcCommand.valueOf(ordinal);
IgfsMessage msg = marsh.unmarshall(cmd, hdr, dis);
IgniteInternalFuture<IgfsMessage> fut = hnd.handleAsync(ses, msg, dis);
// If fut is null, no response is required.
if (fut != null) {
if (fut.isDone()) {
IgfsMessage res;
try {
res = fut.get();
} catch (IgniteCheckedException e) {
res = new IgfsControlResponse();
((IgfsControlResponse) res).error(e);
}
try {
synchronized (out) {
// Reuse header.
IgfsMarshaller.fillHeader(hdr, reqId, res.command());
marsh.marshall(res, hdr, out);
out.flush();
}
} catch (IOException | IgniteCheckedException e) {
shutdown0(e);
}
} else {
fut.listen(
new CIX1<IgniteInternalFuture<IgfsMessage>>() {
@Override
public void applyx(IgniteInternalFuture<IgfsMessage> fut) {
IgfsMessage res;
try {
res = fut.get();
} catch (IgniteCheckedException e) {
res = new IgfsControlResponse();
((IgfsControlResponse) res).error(e);
}
try {
synchronized (out) {
byte[] hdr = IgfsMarshaller.createHeader(reqId, res.command());
marsh.marshall(res, hdr, out);
out.flush();
}
} catch (IOException | IgniteCheckedException e) {
shutdown0(e);
}
}
});
}
}
}
} catch (EOFException ignored) {
// Client closed connection.
} catch (IgniteCheckedException | IOException e) {
if (!isCancelled())
U.error(log, "Failed to read data from client (will close connection)", e);
} finally {
onFinished();
}
}
