/** {@inheritDoc} */
@Override
public void printMemoryStats() {
X.println(">>> ");
X.println(">>> Grid memory stats [grid=" + gridName() + ']');
for (GridComponent comp : comps) comp.printMemoryStats();
}
/** {@inheritDoc} */
@Override
public void printMemoryStats() {
super.printMemoryStats();
X.println(">>> threadsSize: " + threads.size());
X.println(">>> futsSize: " + futs.size());
}
/** {@inheritDoc} */
@Override
public void printMemoryStats(int threshold) {
X.println(
">>> Cache partition topology stats [grid="
+ cctx.gridName()
+ ", cache="
+ cctx.name()
+ ']');
for (GridDhtLocalPartition part : locParts.values()) {
int size = part.size();
if (size >= threshold)
X.println(">>> Local partition [part=" + part.id() + ", size=" + size + ']');
}
}
/**
* Sends get atomically and handles fail.
*
* @param k Key.
*/
protected void failAtomicGet(int k) {
try {
jcache(0).get(new TestKey(String.valueOf(k)));
assert false : "p2p marshalling failed, but error response was not sent";
} catch (CacheException e) {
assert X.hasCause(e, IOException.class);
}
}
/** {@inheritDoc} */
@Override
public void printMemoryStats() {
X.println(">>> ");
X.println(">>> Mvcc manager memory stats [grid=" + cctx.gridName() + ']');
X.println(">>> rmvLocksSize: " + rmvLocks.size());
X.println(">>> dhtLocCandsSize: " + dhtLocCands.size());
X.println(">>> lockedSize: " + locked.size());
X.println(">>> futsSize: " + futs.size());
X.println(">>> near2dhtSize: " + near2dht.size());
X.println(">>> finishFutsSize: " + finishFuts.size());
}
/**
* @param log Logger.
* @param time Time.
* @param msg Message.
*/
private static void log0(@Nullable IgniteLogger log, long time, String msg) {
if (log != null) {
if (log.isDebugEnabled()) log.debug(msg);
else log.warning(msg);
} else
X.println(
String.format(
"[%s][%s]%s",
DEBUG_DATE_FMT.get().format(time), Thread.currentThread().getName(), msg));
}
/**
* Sends put atomically and handles fail.
*
* @param k Key.
*/
protected void failAtomicPut(int k) {
try {
jcache(0).put(new TestKey(String.valueOf(k)), "");
assert false : "p2p marshalling failed, but error response was not sent";
} catch (CacheException e) {
assert X.hasCause(e, IOException.class);
}
assert readCnt.get() == 0; // ensure we have read count as expected.
}
/** @return {@code True} if succeeded. */
private boolean spreadPartitions() {
try {
sendAllPartitions(rmtNodes, exchId);
return true;
} catch (IgniteCheckedException e) {
scheduleRecheck();
if (!X.hasCause(e, InterruptedException.class))
U.error(
log,
"Failed to send full partition map to nodes (will retry after timeout) [nodes="
+ F.nodeId8s(rmtNodes)
+ ", exchangeId="
+ exchId
+ ']',
e);
return false;
}
}
/** {@inheritDoc} */
@Override
public ClusterStartNodeResult call() {
JSch ssh = new JSch();
Session ses = null;
try {
if (spec.key() != null) ssh.addIdentity(spec.key().getAbsolutePath());
ses = ssh.getSession(spec.username(), spec.host(), spec.port());
if (spec.password() != null) ses.setPassword(spec.password());
ses.setConfig("StrictHostKeyChecking", "no");
ses.connect(timeout);
boolean win = isWindows(ses);
char separator = win ? '\\' : '/';
spec.fixPaths(separator);
String igniteHome = spec.igniteHome();
if (igniteHome == null) igniteHome = win ? DFLT_IGNITE_HOME_WIN : DFLT_IGNITE_HOME_LINUX;
String script = spec.script();
if (script == null) script = DFLT_SCRIPT_LINUX;
String cfg = spec.configuration();
if (cfg == null) cfg = "";
String startNodeCmd;
String scriptOutputFileName =
FILE_NAME_DATE_FORMAT.format(new Date())
+ '-'
+ UUID.randomUUID().toString().substring(0, 8)
+ ".log";
if (win)
throw new UnsupportedOperationException(
"Apache Ignite cannot be auto-started on Windows from IgniteCluster.startNodes(…) API.");
else { // Assume Unix.
int spaceIdx = script.indexOf(' ');
String scriptPath = spaceIdx > -1 ? script.substring(0, spaceIdx) : script;
String scriptArgs = spaceIdx > -1 ? script.substring(spaceIdx + 1) : "";
String rmtLogArgs = buildRemoteLogArguments(spec.username(), spec.host());
String tmpDir = env(ses, "$TMPDIR", "/tmp/");
String scriptOutputDir = tmpDir + "ignite-startNodes";
shell(ses, "mkdir " + scriptOutputDir);
// Mac os don't support ~ in double quotes. Trying get home path from remote system.
if (igniteHome.startsWith("~")) {
String homeDir = env(ses, "$HOME", "~");
igniteHome = igniteHome.replaceFirst("~", homeDir);
}
startNodeCmd =
new SB()
.
// Console output is consumed, started nodes must use Ignite file appenders for log.
a("nohup ")
.a("\"")
.a(igniteHome)
.a('/')
.a(scriptPath)
.a("\"")
.a(" ")
.a(scriptArgs)
.a(!cfg.isEmpty() ? " \"" : "")
.a(cfg)
.a(!cfg.isEmpty() ? "\"" : "")
.a(rmtLogArgs)
.a(" > ")
.a(scriptOutputDir)
.a("/")
.a(scriptOutputFileName)
.a(" 2>& 1 &")
.toString();
}
info("Starting remote node with SSH command: " + startNodeCmd, spec.logger(), log);
shell(ses, startNodeCmd);
return new ClusterStartNodeResultImpl(spec.host(), true, null);
} catch (IgniteInterruptedCheckedException e) {
return new ClusterStartNodeResultImpl(spec.host(), false, e.getMessage());
} catch (Exception e) {
return new ClusterStartNodeResultImpl(spec.host(), false, X.getFullStackTrace(e));
} finally {
if (ses != null && ses.isConnected()) ses.disconnect();
}
}
/** {@inheritDoc} */
@Override
protected void afterTest() throws Exception {
Transaction tx = jcache().unwrap(Ignite.class).transactions().tx();
if (tx != null) {
tx.close();
fail("Cache transaction remained after test completion: " + tx);
}
for (int i = 0; i < gridCount(); i++) {
info("Checking grid: " + i);
while (true) {
try {
final int fi = i;
assertTrue(
"Cache is not empty: "
+ " localSize = "
+ jcache(fi).localSize(CachePeekMode.ALL)
+ ", local entries "
+ entrySet(jcache(fi).localEntries()),
GridTestUtils.waitForCondition(
// Preloading may happen as nodes leave, so we need to wait.
new GridAbsPredicateX() {
@Override
public boolean applyx() throws IgniteCheckedException {
jcache(fi).removeAll();
if (jcache(fi).size(CachePeekMode.ALL) > 0) {
for (Cache.Entry<String, ?> k : jcache(fi).localEntries())
jcache(fi).remove(k.getKey());
}
return jcache(fi).localSize(CachePeekMode.ALL) == 0;
}
},
getTestTimeout()));
int primaryKeySize = jcache(i).localSize(CachePeekMode.PRIMARY);
int keySize = jcache(i).localSize();
int size = jcache(i).localSize();
int globalSize = jcache(i).size();
int globalPrimarySize = jcache(i).size(CachePeekMode.PRIMARY);
info(
"Size after [idx="
+ i
+ ", size="
+ size
+ ", keySize="
+ keySize
+ ", primarySize="
+ primaryKeySize
+ ", globalSize="
+ globalSize
+ ", globalPrimarySize="
+ globalPrimarySize
+ ", entrySet="
+ jcache(i).localEntries()
+ ']');
assertEquals(
"Cache is not empty [idx=" + i + ", entrySet=" + jcache(i).localEntries() + ']',
0,
jcache(i).localSize(CachePeekMode.ALL));
break;
} catch (Exception e) {
if (X.hasCause(e, ClusterTopologyCheckedException.class)) {
info("Got topology exception while tear down (will retry in 1000ms).");
U.sleep(1000);
} else throw e;
}
}
for (Cache.Entry<String, Integer> entry : jcache(i).localEntries(CachePeekMode.SWAP))
jcache(i).remove(entry.getKey());
}
assert jcache().unwrap(Ignite.class).transactions().tx() == null;
assertEquals("Cache is not empty", 0, jcache().localSize(CachePeekMode.ALL));
resetStore();
}
/** @throws Exception If failed. */
public void testCompact() throws Exception {
File file = new File(UUID.randomUUID().toString());
X.println("file: " + file.getPath());
FileSwapSpaceSpi.SwapFile f = new FileSwapSpaceSpi.SwapFile(file, 8);
Random rnd = new Random();
ArrayList<FileSwapSpaceSpi.SwapValue> arr = new ArrayList<>();
int size = 0;
for (int a = 0; a < 100; a++) {
FileSwapSpaceSpi.SwapValue[] vals = new FileSwapSpaceSpi.SwapValue[1 + rnd.nextInt(10)];
int size0 = 0;
for (int i = 0; i < vals.length; i++) {
byte[] bytes = new byte[1 + rnd.nextInt(49)];
rnd.nextBytes(bytes);
size0 += bytes.length;
vals[i] = new FileSwapSpaceSpi.SwapValue(bytes);
arr.add(vals[i]);
}
f.write(new FileSwapSpaceSpi.SwapValues(vals, size0), 1);
size += size0;
assertEquals(f.length(), size);
assertEquals(file.length(), size);
}
int i = 0;
for (FileSwapSpaceSpi.SwapValue val : arr) assertEquals(val.idx(), ++i);
i = 0;
for (int cnt = arr.size() / 2; i < cnt; i++) {
FileSwapSpaceSpi.SwapValue v = arr.remove(rnd.nextInt(arr.size()));
assertTrue(f.tryRemove(v.idx(), v));
}
int hash0 = 0;
for (FileSwapSpaceSpi.SwapValue val : arr) hash0 += Arrays.hashCode(val.readValue(f.readCh));
ArrayList<T2<ByteBuffer, ArrayDeque<FileSwapSpaceSpi.SwapValue>>> bufs = new ArrayList();
for (; ; ) {
ArrayDeque<FileSwapSpaceSpi.SwapValue> que = new ArrayDeque<>();
ByteBuffer buf = f.compact(que, 1024);
if (buf == null) break;
bufs.add(new T2(buf, que));
}
f.delete();
int hash1 = 0;
for (FileSwapSpaceSpi.SwapValue val : arr) hash1 += Arrays.hashCode(val.value(null));
assertEquals(hash0, hash1);
File file0 = new File(UUID.randomUUID().toString());
FileSwapSpaceSpi.SwapFile f0 = new FileSwapSpaceSpi.SwapFile(file0, 8);
for (T2<ByteBuffer, ArrayDeque<FileSwapSpaceSpi.SwapValue>> t : bufs)
f0.write(t.get2(), t.get1(), 1);
int hash2 = 0;
for (FileSwapSpaceSpi.SwapValue val : arr) hash2 += Arrays.hashCode(val.readValue(f0.readCh));
assertEquals(hash2, hash1);
}
/**
* Establish TCP connection to remote hadoop process and returns client.
*
* @param desc Process descriptor.
* @return Client.
* @throws IgniteCheckedException If failed.
*/
protected HadoopCommunicationClient createTcpClient(HadoopProcessDescriptor desc)
throws IgniteCheckedException {
String addr = desc.address();
int port = desc.tcpPort();
if (log.isDebugEnabled())
log.debug(
"Trying to connect to remote process [locProcDesc="
+ locProcDesc
+ ", desc="
+ desc
+ ']');
boolean conn = false;
HadoopTcpNioCommunicationClient client = null;
IgniteCheckedException errs = null;
int connectAttempts = 1;
long connTimeout0 = connTimeout;
int attempt = 1;
while (!conn) { // Reconnection on handshake timeout.
try {
SocketChannel ch = SocketChannel.open();
ch.configureBlocking(true);
ch.socket().setTcpNoDelay(tcpNoDelay);
ch.socket().setKeepAlive(true);
if (sockRcvBuf > 0) ch.socket().setReceiveBufferSize(sockRcvBuf);
if (sockSndBuf > 0) ch.socket().setSendBufferSize(sockSndBuf);
ch.socket().connect(new InetSocketAddress(addr, port), (int) connTimeout);
HandshakeFinish fin = new HandshakeFinish();
GridNioSession ses = nioSrvr.createSession(ch, F.asMap(HANDSHAKE_FINISH_META, fin)).get();
client = new HadoopTcpNioCommunicationClient(ses);
if (log.isDebugEnabled()) log.debug("Waiting for handshake finish for client: " + client);
fin.await(connTimeout0);
conn = true;
} catch (HadoopHandshakeTimeoutException e) {
if (client != null) {
client.forceClose();
client = null;
}
if (log.isDebugEnabled())
log.debug(
"Handshake timedout (will retry with increased timeout) [timeout="
+ connTimeout0
+ ", desc="
+ desc
+ ", port="
+ port
+ ", err="
+ e
+ ']');
if (attempt == reconCnt || connTimeout0 > maxConnTimeout) {
if (log.isDebugEnabled())
log.debug(
"Handshake timed out (will stop attempts to perform the handshake) "
+ "[timeout="
+ connTimeout0
+ ", maxConnTimeout="
+ maxConnTimeout
+ ", attempt="
+ attempt
+ ", reconCnt="
+ reconCnt
+ ", err="
+ e.getMessage()
+ ", addr="
+ addr
+ ']');
if (errs == null)
errs =
new IgniteCheckedException(
"Failed to connect to remote Hadoop process "
+ "(is process still running?) [desc="
+ desc
+ ", addrs="
+ addr
+ ']');
errs.addSuppressed(e);
break;
} else {
attempt++;
connTimeout0 *= 2;
// Continue loop.
}
} catch (Exception e) {
if (client != null) {
client.forceClose();
client = null;
}
if (log.isDebugEnabled())
log.debug("Client creation failed [addr=" + addr + ", port=" + port + ", err=" + e + ']');
if (X.hasCause(e, SocketTimeoutException.class))
LT.warn(
log,
null,
"Connect timed out (consider increasing 'connTimeout' "
+ "configuration property) [addr="
+ addr
+ ", port="
+ port
+ ']');
if (errs == null)
errs =
new IgniteCheckedException(
"Failed to connect to remote Hadoop process (is process still running?) "
+ "[desc="
+ desc
+ ", addrs="
+ addr
+ ']');
errs.addSuppressed(e);
// Reconnect for the second time, if connection is not established.
if (connectAttempts < 2
&& (e instanceof ConnectException || X.hasCause(e, ConnectException.class))) {
connectAttempts++;
continue;
}
break;
}
}
if (client == null) {
assert errs != null;
if (X.hasCause(errs, ConnectException.class))
LT.warn(
log,
null,
"Failed to connect to a remote Hadoop process (is process still running?). "
+ "Make sure operating system firewall is disabled on local and remote host) "
+ "[addrs="
+ addr
+ ", port="
+ port
+ ']');
throw errs;
}
if (log.isDebugEnabled()) log.debug("Created client: " + client);
return client;
}
/**
* @param desc Process descriptor.
* @param port Port.
* @return Client.
* @throws IgniteCheckedException If failed.
*/
@Nullable
protected HadoopCommunicationClient createShmemClient(HadoopProcessDescriptor desc, int port)
throws IgniteCheckedException {
int attempt = 1;
int connectAttempts = 1;
long connTimeout0 = connTimeout;
while (true) {
IpcEndpoint clientEndpoint;
try {
clientEndpoint = new IpcSharedMemoryClientEndpoint(port, (int) connTimeout, log);
} catch (IgniteCheckedException e) {
// Reconnect for the second time, if connection is not established.
if (connectAttempts < 2 && X.hasCause(e, ConnectException.class)) {
connectAttempts++;
continue;
}
throw e;
}
HadoopCommunicationClient client = null;
try {
ShmemWorker worker = new ShmemWorker(clientEndpoint, false);
shmemWorkers.add(worker);
GridNioSession ses = worker.session();
HandshakeFinish fin = new HandshakeFinish();
// We are in lock, it is safe to get session and attach
ses.addMeta(HANDSHAKE_FINISH_META, fin);
client = new HadoopTcpNioCommunicationClient(ses);
new IgniteThread(worker).start();
fin.await(connTimeout0);
} catch (HadoopHandshakeTimeoutException e) {
if (log.isDebugEnabled())
log.debug(
"Handshake timed out (will retry with increased timeout) [timeout="
+ connTimeout0
+ ", err="
+ e.getMessage()
+ ", client="
+ client
+ ']');
if (client != null) client.forceClose();
if (attempt == reconCnt || connTimeout0 > maxConnTimeout) {
if (log.isDebugEnabled())
log.debug(
"Handshake timedout (will stop attempts to perform the handshake) "
+ "[timeout="
+ connTimeout0
+ ", maxConnTimeout="
+ maxConnTimeout
+ ", attempt="
+ attempt
+ ", reconCnt="
+ reconCnt
+ ", err="
+ e.getMessage()
+ ", client="
+ client
+ ']');
throw e;
} else {
attempt++;
connTimeout0 *= 2;
continue;
}
} catch (RuntimeException | Error e) {
if (log.isDebugEnabled())
log.debug(
"Caught exception (will close client) [err="
+ e.getMessage()
+ ", client="
+ client
+ ']');
if (client != null) client.forceClose();
throw e;
}
return client;
}
}
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override
public void onUtilityCacheStarted() throws IgniteCheckedException {
IgniteCacheProxy<Object, Object> proxy = ctx.cache().jcache(CU.UTILITY_CACHE_NAME);
boolean old = proxy.context().deploy().ignoreOwnership(true);
try {
metaDataCache = (IgniteCacheProxy) proxy.withNoRetries();
} finally {
proxy.context().deploy().ignoreOwnership(old);
}
if (clientNode) {
assert !metaDataCache.context().affinityNode();
metaCacheQryId =
metaDataCache
.context()
.continuousQueries()
.executeInternalQuery(
new MetaDataEntryListener(), new MetaDataEntryFilter(), false, true);
while (true) {
ClusterNode oldestSrvNode =
CU.oldestAliveCacheServerNode(ctx.cache().context(), AffinityTopologyVersion.NONE);
if (oldestSrvNode == null) break;
GridCacheQueryManager qryMgr = metaDataCache.context().queries();
CacheQuery<Map.Entry<PortableMetadataKey, BinaryMetadata>> qry =
qryMgr.createScanQuery(new MetaDataPredicate(), null, false);
qry.keepAll(false);
qry.projection(ctx.cluster().get().forNode(oldestSrvNode));
try {
CacheQueryFuture<Map.Entry<PortableMetadataKey, BinaryMetadata>> fut = qry.execute();
Map.Entry<PortableMetadataKey, BinaryMetadata> next;
while ((next = fut.next()) != null) {
assert next.getKey() != null : next;
assert next.getValue() != null : next;
addClientCacheMetaData(next.getKey(), next.getValue());
}
} catch (IgniteCheckedException e) {
if (!ctx.discovery().alive(oldestSrvNode)
|| !ctx.discovery().pingNode(oldestSrvNode.id())) continue;
else throw e;
} catch (CacheException e) {
if (X.hasCause(e, ClusterTopologyCheckedException.class, ClusterTopologyException.class))
continue;
else throw e;
}
break;
}
}
for (Map.Entry<Integer, BinaryMetadata> e : metaBuf.entrySet())
addMeta(e.getKey(), e.getValue().wrap(portableCtx));
metaBuf.clear();
startLatch.countDown();
}
/** {@inheritDoc} */
@Override
public String execute() {
assert taskSes != null;
assert startLatch != null;
assert read1Latch != null;
assert read2Latch != null;
assert read3Latch != null;
assert read1FinishedLatch != null;
assert read2FinishedLatch != null;
assert read3FinishedLatch != null;
assert rmvLatch != null;
startLatch.countDown();
try {
startLatch.await();
} catch (InterruptedException e) {
throw new IgniteException("Thread has been interrupted.", e);
}
X.println(">>> Consumer started.");
try {
read1Latch.await();
} catch (InterruptedException e) {
throw new IgniteException("Thread has been interrupted.", e);
}
// Test that checkpoints were saved properly.
assertWithRetries(
new GridAbsClosureX() {
@Override
public void applyx() {
assert GLOBAL_VAL.equals(taskSes.loadCheckpoint(GLOBAL_KEY));
assert SES_VAL.equals(taskSes.loadCheckpoint(SES_KEY));
}
});
read1FinishedLatch.countDown();
try {
read2Latch.await();
} catch (InterruptedException e) {
throw new IgniteException("Thread has been interrupted.", e);
}
// Test that checkpoints were not overwritten.
assertWithRetries(
new GridAbsClosureX() {
@Override
public void applyx() {
assert GLOBAL_VAL.equals(taskSes.loadCheckpoint(GLOBAL_KEY));
assert SES_VAL.equals(taskSes.loadCheckpoint(SES_KEY));
}
});
read2FinishedLatch.countDown();
try {
read3Latch.await();
} catch (InterruptedException e) {
throw new IgniteException("Thread has been interrupted.", e);
}
assertWithRetries(
new GridAbsClosureX() {
@Override
public void applyx() {
assertEquals(SES_VAL_OVERWRITTEN, taskSes.loadCheckpoint(GLOBAL_KEY));
assertEquals(GLOBAL_VAL_OVERWRITTEN, taskSes.loadCheckpoint(SES_KEY));
}
});
read3FinishedLatch.countDown();
try {
rmvLatch.await();
} catch (InterruptedException e) {
throw new IgniteException("Thread has been interrupted.", e);
}
// Check checkpoints are actually removed.
assert !taskSes.removeCheckpoint(GLOBAL_KEY);
assert !taskSes.removeCheckpoint(SES_KEY);
assertWithRetries(
new GridAbsClosureX() {
@Override
public void applyx() {
assert taskSes.loadCheckpoint(GLOBAL_KEY) == null;
assert taskSes.loadCheckpoint(SES_KEY) == null;
}
});
return null;
}
/** {@inheritDoc} */
@Override
public String execute() {
assert ignite != null;
assert taskSes != null;
assert startLatch != null;
assert read1Latch != null;
assert read2Latch != null;
assert read3Latch != null;
assert read1FinishedLatch != null;
assert read2FinishedLatch != null;
assert read3FinishedLatch != null;
assert rmvLatch != null;
startLatch.countDown();
try {
startLatch.await();
} catch (InterruptedException e) {
throw new IgniteException("Thread has been interrupted.", e);
}
X.println(">>> Producer started.");
taskSes.saveCheckpoint(GLOBAL_KEY, GLOBAL_VAL, GLOBAL_SCOPE, 0);
taskSes.saveCheckpoint(SES_KEY, SES_VAL, SESSION_SCOPE, 0);
read1Latch.countDown();
try {
read1FinishedLatch.await();
} catch (InterruptedException e) {
throw new IgniteException("Thread has been interrupted.", e);
}
// No retries here as other thread should have seen checkpoint already.
assert GLOBAL_VAL.equals(taskSes.loadCheckpoint(GLOBAL_KEY));
assert SES_VAL.equals(taskSes.loadCheckpoint(SES_KEY));
taskSes.saveCheckpoint(GLOBAL_KEY, SES_VAL + "-notoverwritten", GLOBAL_SCOPE, 0, false);
taskSes.saveCheckpoint(SES_KEY, GLOBAL_VAL + "-notoverwritten", SESSION_SCOPE, 0, false);
read2Latch.countDown();
try {
read2FinishedLatch.await();
} catch (InterruptedException e) {
throw new IgniteException("Thread has been interrupted.", e);
}
assert GLOBAL_VAL.equals(taskSes.loadCheckpoint(GLOBAL_KEY));
assert SES_VAL.equals(taskSes.loadCheckpoint(SES_KEY));
// Swap values.
taskSes.saveCheckpoint(GLOBAL_KEY, SES_VAL_OVERWRITTEN, GLOBAL_SCOPE, 0, true);
taskSes.saveCheckpoint(SES_KEY, GLOBAL_VAL_OVERWRITTEN, SESSION_SCOPE, 0, true);
read3Latch.countDown();
try {
read3FinishedLatch.await();
} catch (InterruptedException e) {
throw new IgniteException("Thread has been interrupted.", e);
}
assert SES_VAL_OVERWRITTEN.equals(taskSes.loadCheckpoint(GLOBAL_KEY));
assert GLOBAL_VAL_OVERWRITTEN.equals(taskSes.loadCheckpoint(SES_KEY));
// Remove checkpoints.
assert taskSes.removeCheckpoint(GLOBAL_KEY);
assert taskSes.removeCheckpoint(SES_KEY);
// Check checkpoints are actually removed.
assert !taskSes.removeCheckpoint(GLOBAL_KEY);
assert !taskSes.removeCheckpoint(SES_KEY);
rmvLatch.countDown();
assertWithRetries(
new GridAbsClosureX() {
@Override
public void applyx() {
assert taskSes.loadCheckpoint(GLOBAL_KEY) == null;
assert taskSes.loadCheckpoint(SES_KEY) == null;
}
});
return null;
}