/**
* Sends get atomically and handles fail.
*
* @param k Key.
*/
protected void failAtomicGet(int k) {
try {
jcache(0).get(new TestKey(String.valueOf(k)));
assert false : "p2p marshalling failed, but error response was not sent";
} catch (CacheException e) {
assert X.hasCause(e, IOException.class);
}
}
/**
* Sends put atomically and handles fail.
*
* @param k Key.
*/
protected void failAtomicPut(int k) {
try {
jcache(0).put(new TestKey(String.valueOf(k)), "");
assert false : "p2p marshalling failed, but error response was not sent";
} catch (CacheException e) {
assert X.hasCause(e, IOException.class);
}
assert readCnt.get() == 0; // ensure we have read count as expected.
}
/** @return {@code True} if succeeded. */
private boolean spreadPartitions() {
try {
sendAllPartitions(rmtNodes, exchId);
return true;
} catch (IgniteCheckedException e) {
scheduleRecheck();
if (!X.hasCause(e, InterruptedException.class))
U.error(
log,
"Failed to send full partition map to nodes (will retry after timeout) [nodes="
+ F.nodeId8s(rmtNodes)
+ ", exchangeId="
+ exchId
+ ']',
e);
return false;
}
}
/** {@inheritDoc} */
@Override
protected void afterTest() throws Exception {
Transaction tx = jcache().unwrap(Ignite.class).transactions().tx();
if (tx != null) {
tx.close();
fail("Cache transaction remained after test completion: " + tx);
}
for (int i = 0; i < gridCount(); i++) {
info("Checking grid: " + i);
while (true) {
try {
final int fi = i;
assertTrue(
"Cache is not empty: "
+ " localSize = "
+ jcache(fi).localSize(CachePeekMode.ALL)
+ ", local entries "
+ entrySet(jcache(fi).localEntries()),
GridTestUtils.waitForCondition(
// Preloading may happen as nodes leave, so we need to wait.
new GridAbsPredicateX() {
@Override
public boolean applyx() throws IgniteCheckedException {
jcache(fi).removeAll();
if (jcache(fi).size(CachePeekMode.ALL) > 0) {
for (Cache.Entry<String, ?> k : jcache(fi).localEntries())
jcache(fi).remove(k.getKey());
}
return jcache(fi).localSize(CachePeekMode.ALL) == 0;
}
},
getTestTimeout()));
int primaryKeySize = jcache(i).localSize(CachePeekMode.PRIMARY);
int keySize = jcache(i).localSize();
int size = jcache(i).localSize();
int globalSize = jcache(i).size();
int globalPrimarySize = jcache(i).size(CachePeekMode.PRIMARY);
info(
"Size after [idx="
+ i
+ ", size="
+ size
+ ", keySize="
+ keySize
+ ", primarySize="
+ primaryKeySize
+ ", globalSize="
+ globalSize
+ ", globalPrimarySize="
+ globalPrimarySize
+ ", entrySet="
+ jcache(i).localEntries()
+ ']');
assertEquals(
"Cache is not empty [idx=" + i + ", entrySet=" + jcache(i).localEntries() + ']',
0,
jcache(i).localSize(CachePeekMode.ALL));
break;
} catch (Exception e) {
if (X.hasCause(e, ClusterTopologyCheckedException.class)) {
info("Got topology exception while tear down (will retry in 1000ms).");
U.sleep(1000);
} else throw e;
}
}
for (Cache.Entry<String, Integer> entry : jcache(i).localEntries(CachePeekMode.SWAP))
jcache(i).remove(entry.getKey());
}
assert jcache().unwrap(Ignite.class).transactions().tx() == null;
assertEquals("Cache is not empty", 0, jcache().localSize(CachePeekMode.ALL));
resetStore();
}
/**
* Establish TCP connection to remote hadoop process and returns client.
*
* @param desc Process descriptor.
* @return Client.
* @throws IgniteCheckedException If failed.
*/
protected HadoopCommunicationClient createTcpClient(HadoopProcessDescriptor desc)
throws IgniteCheckedException {
String addr = desc.address();
int port = desc.tcpPort();
if (log.isDebugEnabled())
log.debug(
"Trying to connect to remote process [locProcDesc="
+ locProcDesc
+ ", desc="
+ desc
+ ']');
boolean conn = false;
HadoopTcpNioCommunicationClient client = null;
IgniteCheckedException errs = null;
int connectAttempts = 1;
long connTimeout0 = connTimeout;
int attempt = 1;
while (!conn) { // Reconnection on handshake timeout.
try {
SocketChannel ch = SocketChannel.open();
ch.configureBlocking(true);
ch.socket().setTcpNoDelay(tcpNoDelay);
ch.socket().setKeepAlive(true);
if (sockRcvBuf > 0) ch.socket().setReceiveBufferSize(sockRcvBuf);
if (sockSndBuf > 0) ch.socket().setSendBufferSize(sockSndBuf);
ch.socket().connect(new InetSocketAddress(addr, port), (int) connTimeout);
HandshakeFinish fin = new HandshakeFinish();
GridNioSession ses = nioSrvr.createSession(ch, F.asMap(HANDSHAKE_FINISH_META, fin)).get();
client = new HadoopTcpNioCommunicationClient(ses);
if (log.isDebugEnabled()) log.debug("Waiting for handshake finish for client: " + client);
fin.await(connTimeout0);
conn = true;
} catch (HadoopHandshakeTimeoutException e) {
if (client != null) {
client.forceClose();
client = null;
}
if (log.isDebugEnabled())
log.debug(
"Handshake timedout (will retry with increased timeout) [timeout="
+ connTimeout0
+ ", desc="
+ desc
+ ", port="
+ port
+ ", err="
+ e
+ ']');
if (attempt == reconCnt || connTimeout0 > maxConnTimeout) {
if (log.isDebugEnabled())
log.debug(
"Handshake timed out (will stop attempts to perform the handshake) "
+ "[timeout="
+ connTimeout0
+ ", maxConnTimeout="
+ maxConnTimeout
+ ", attempt="
+ attempt
+ ", reconCnt="
+ reconCnt
+ ", err="
+ e.getMessage()
+ ", addr="
+ addr
+ ']');
if (errs == null)
errs =
new IgniteCheckedException(
"Failed to connect to remote Hadoop process "
+ "(is process still running?) [desc="
+ desc
+ ", addrs="
+ addr
+ ']');
errs.addSuppressed(e);
break;
} else {
attempt++;
connTimeout0 *= 2;
// Continue loop.
}
} catch (Exception e) {
if (client != null) {
client.forceClose();
client = null;
}
if (log.isDebugEnabled())
log.debug("Client creation failed [addr=" + addr + ", port=" + port + ", err=" + e + ']');
if (X.hasCause(e, SocketTimeoutException.class))
LT.warn(
log,
null,
"Connect timed out (consider increasing 'connTimeout' "
+ "configuration property) [addr="
+ addr
+ ", port="
+ port
+ ']');
if (errs == null)
errs =
new IgniteCheckedException(
"Failed to connect to remote Hadoop process (is process still running?) "
+ "[desc="
+ desc
+ ", addrs="
+ addr
+ ']');
errs.addSuppressed(e);
// Reconnect for the second time, if connection is not established.
if (connectAttempts < 2
&& (e instanceof ConnectException || X.hasCause(e, ConnectException.class))) {
connectAttempts++;
continue;
}
break;
}
}
if (client == null) {
assert errs != null;
if (X.hasCause(errs, ConnectException.class))
LT.warn(
log,
null,
"Failed to connect to a remote Hadoop process (is process still running?). "
+ "Make sure operating system firewall is disabled on local and remote host) "
+ "[addrs="
+ addr
+ ", port="
+ port
+ ']');
throw errs;
}
if (log.isDebugEnabled()) log.debug("Created client: " + client);
return client;
}
/**
* @param desc Process descriptor.
* @param port Port.
* @return Client.
* @throws IgniteCheckedException If failed.
*/
@Nullable
protected HadoopCommunicationClient createShmemClient(HadoopProcessDescriptor desc, int port)
throws IgniteCheckedException {
int attempt = 1;
int connectAttempts = 1;
long connTimeout0 = connTimeout;
while (true) {
IpcEndpoint clientEndpoint;
try {
clientEndpoint = new IpcSharedMemoryClientEndpoint(port, (int) connTimeout, log);
} catch (IgniteCheckedException e) {
// Reconnect for the second time, if connection is not established.
if (connectAttempts < 2 && X.hasCause(e, ConnectException.class)) {
connectAttempts++;
continue;
}
throw e;
}
HadoopCommunicationClient client = null;
try {
ShmemWorker worker = new ShmemWorker(clientEndpoint, false);
shmemWorkers.add(worker);
GridNioSession ses = worker.session();
HandshakeFinish fin = new HandshakeFinish();
// We are in lock, it is safe to get session and attach
ses.addMeta(HANDSHAKE_FINISH_META, fin);
client = new HadoopTcpNioCommunicationClient(ses);
new IgniteThread(worker).start();
fin.await(connTimeout0);
} catch (HadoopHandshakeTimeoutException e) {
if (log.isDebugEnabled())
log.debug(
"Handshake timed out (will retry with increased timeout) [timeout="
+ connTimeout0
+ ", err="
+ e.getMessage()
+ ", client="
+ client
+ ']');
if (client != null) client.forceClose();
if (attempt == reconCnt || connTimeout0 > maxConnTimeout) {
if (log.isDebugEnabled())
log.debug(
"Handshake timedout (will stop attempts to perform the handshake) "
+ "[timeout="
+ connTimeout0
+ ", maxConnTimeout="
+ maxConnTimeout
+ ", attempt="
+ attempt
+ ", reconCnt="
+ reconCnt
+ ", err="
+ e.getMessage()
+ ", client="
+ client
+ ']');
throw e;
} else {
attempt++;
connTimeout0 *= 2;
continue;
}
} catch (RuntimeException | Error e) {
if (log.isDebugEnabled())
log.debug(
"Caught exception (will close client) [err="
+ e.getMessage()
+ ", client="
+ client
+ ']');
if (client != null) client.forceClose();
throw e;
}
return client;
}
}
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
@Override
public void onUtilityCacheStarted() throws IgniteCheckedException {
IgniteCacheProxy<Object, Object> proxy = ctx.cache().jcache(CU.UTILITY_CACHE_NAME);
boolean old = proxy.context().deploy().ignoreOwnership(true);
try {
metaDataCache = (IgniteCacheProxy) proxy.withNoRetries();
} finally {
proxy.context().deploy().ignoreOwnership(old);
}
if (clientNode) {
assert !metaDataCache.context().affinityNode();
metaCacheQryId =
metaDataCache
.context()
.continuousQueries()
.executeInternalQuery(
new MetaDataEntryListener(), new MetaDataEntryFilter(), false, true);
while (true) {
ClusterNode oldestSrvNode =
CU.oldestAliveCacheServerNode(ctx.cache().context(), AffinityTopologyVersion.NONE);
if (oldestSrvNode == null) break;
GridCacheQueryManager qryMgr = metaDataCache.context().queries();
CacheQuery<Map.Entry<PortableMetadataKey, BinaryMetadata>> qry =
qryMgr.createScanQuery(new MetaDataPredicate(), null, false);
qry.keepAll(false);
qry.projection(ctx.cluster().get().forNode(oldestSrvNode));
try {
CacheQueryFuture<Map.Entry<PortableMetadataKey, BinaryMetadata>> fut = qry.execute();
Map.Entry<PortableMetadataKey, BinaryMetadata> next;
while ((next = fut.next()) != null) {
assert next.getKey() != null : next;
assert next.getValue() != null : next;
addClientCacheMetaData(next.getKey(), next.getValue());
}
} catch (IgniteCheckedException e) {
if (!ctx.discovery().alive(oldestSrvNode)
|| !ctx.discovery().pingNode(oldestSrvNode.id())) continue;
else throw e;
} catch (CacheException e) {
if (X.hasCause(e, ClusterTopologyCheckedException.class, ClusterTopologyException.class))
continue;
else throw e;
}
break;
}
}
for (Map.Entry<Integer, BinaryMetadata> e : metaBuf.entrySet())
addMeta(e.getKey(), e.getValue().wrap(portableCtx));
metaBuf.clear();
startLatch.countDown();
}
