/** {@inheritDoc} */
@Override
protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
if (log.isDebugEnabled()) log.debug("GC worker started.");
File workTokDir = tokDir.getParentFile();
assert workTokDir != null;
boolean lastRunNeeded = true;
while (true) {
try {
// Sleep only if not cancelled.
if (lastRunNeeded) Thread.sleep(GC_FREQ);
} catch (InterruptedException ignored) {
// No-op.
}
if (log.isDebugEnabled()) log.debug("Starting GC iteration.");
cleanupResources(workTokDir);
// Process spaces created by this endpoint.
if (log.isDebugEnabled()) log.debug("Processing local spaces.");
for (IpcSharedMemoryClientEndpoint e : endpoints) {
if (log.isDebugEnabled()) log.debug("Processing endpoint: " + e);
if (!e.checkOtherPartyAlive()) {
endpoints.remove(e);
if (log.isDebugEnabled()) log.debug("Removed endpoint: " + e);
}
}
if (isCancelled()) {
if (lastRunNeeded) {
lastRunNeeded = false;
// Clear interrupted status.
Thread.interrupted();
} else {
Thread.currentThread().interrupt();
break;
}
}
}
}
/** @param workTokDir Token directory (common for multiple nodes). */
private void cleanupResources(File workTokDir) {
RandomAccessFile lockFile = null;
FileLock lock = null;
try {
lockFile = new RandomAccessFile(new File(workTokDir, LOCK_FILE_NAME), "rw");
lock = lockFile.getChannel().lock();
if (lock != null) processTokenDirectory(workTokDir);
else if (log.isDebugEnabled())
log.debug(
"Token directory is being processed concurrently: " + workTokDir.getAbsolutePath());
} catch (OverlappingFileLockException ignored) {
if (log.isDebugEnabled())
log.debug(
"Token directory is being processed concurrently: " + workTokDir.getAbsolutePath());
} catch (FileLockInterruptionException ignored) {
Thread.currentThread().interrupt();
} catch (IOException e) {
U.error(log, "Failed to process directory: " + workTokDir.getAbsolutePath(), e);
} finally {
U.releaseQuiet(lock);
U.closeQuiet(lockFile);
}
}
/**
* @param log Logger.
* @param time Time.
* @param msg Message.
*/
private static void log0(@Nullable IgniteLogger log, long time, String msg) {
if (log != null) {
if (log.isDebugEnabled()) log.debug(msg);
else log.warning(msg);
} else
X.println(
String.format(
"[%s][%s]%s",
DEBUG_DATE_FMT.get().format(time), Thread.currentThread().getName(), msg));
}
/**
* @param key Removed key.
* @param ver Removed version.
* @throws IgniteCheckedException If failed.
*/
public void onDeferredDelete(KeyCacheObject key, GridCacheVersion ver)
throws IgniteCheckedException {
try {
T2<KeyCacheObject, GridCacheVersion> evicted = rmvQueue.add(new T2<>(key, ver));
if (evicted != null) cctx.dht().removeVersionedEntry(evicted.get1(), evicted.get2());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IgniteInterruptedCheckedException(e);
}
}
/** {@inheritDoc} */
@Override
public void close() {
closed = true;
U.closeQuiet(srvSock);
if (gcWorker != null) {
U.cancel(gcWorker);
// This method may be called from already interrupted thread.
// Need to ensure cleaning on close.
boolean interrupted = Thread.interrupted();
try {
U.join(gcWorker);
} catch (IgniteInterruptedCheckedException e) {
U.warn(log, "Interrupted when stopping GC worker.", e);
} finally {
if (interrupted) Thread.currentThread().interrupt();
}
}
}
/**
* Processes cache query request.
*
* @param sndId Sender node id.
* @param req Query request.
*/
@SuppressWarnings("unchecked")
@Override
void processQueryRequest(UUID sndId, GridCacheQueryRequest req) {
if (req.cancel()) {
cancelIds.add(new CancelMessageId(req.id(), sndId));
if (req.fields()) removeFieldsQueryResult(sndId, req.id());
else removeQueryResult(sndId, req.id());
} else {
if (!cancelIds.contains(new CancelMessageId(req.id(), sndId))) {
if (!F.eq(req.cacheName(), cctx.name())) {
GridCacheQueryResponse res =
new GridCacheQueryResponse(
cctx.cacheId(),
req.id(),
new IgniteCheckedException(
"Received request for incorrect cache [expected="
+ cctx.name()
+ ", actual="
+ req.cacheName()));
sendQueryResponse(sndId, res, 0);
} else {
threads.put(req.id(), Thread.currentThread());
try {
GridCacheQueryInfo info = distributedQueryInfo(sndId, req);
if (info == null) return;
if (req.fields()) runFieldsQuery(info);
else runQuery(info);
} catch (Throwable e) {
U.error(log(), "Failed to run query.", e);
sendQueryResponse(
sndId, new GridCacheQueryResponse(cctx.cacheId(), req.id(), e.getCause()), 0);
if (e instanceof Error) throw (Error) e;
} finally {
threads.remove(req.id());
}
}
}
}
}
/**
* Checks if address can be reached using one argument InetAddress.isReachable() version or ping
* command if failed.
*
* @param addr Address to check.
* @param reachTimeout Timeout for the check.
* @return {@code True} if address is reachable.
*/
public static boolean reachableByPing(InetAddress addr, int reachTimeout) {
try {
if (addr.isReachable(reachTimeout)) return true;
String cmd = String.format("ping -%s 1 %s", U.isWindows() ? "n" : "c", addr.getHostAddress());
Process myProc = Runtime.getRuntime().exec(cmd);
myProc.waitFor();
return myProc.exitValue() == 0;
} catch (IOException ignore) {
return false;
} catch (InterruptedException ignored) {
Thread.currentThread().interrupt();
return false;
}
}
/**
* JUnit.
*
* @throws Exception If failed.
*/
@SuppressWarnings({"TooBroadScope"})
public void testRestarts() throws Exception {
int duration = 60 * 1000;
int qryThreadNum = 10;
final long nodeLifeTime = 2 * 1000;
final int logFreq = 20;
final IgniteCache<Integer, Integer> cache = grid(0).cache(null);
assert cache != null;
for (int i = 0; i < KEY_CNT; i++) cache.put(i, i);
assertEquals(KEY_CNT, cache.localSize());
final AtomicInteger qryCnt = new AtomicInteger();
final AtomicBoolean done = new AtomicBoolean();
IgniteInternalFuture<?> fut1 =
multithreadedAsync(
new CAX() {
@Override
public void applyx() throws IgniteCheckedException {
while (!done.get()) {
Collection<Cache.Entry<Integer, Integer>> res =
cache.query(new SqlQuery(Integer.class, "_val >= 0")).getAll();
assertFalse(res.isEmpty());
int c = qryCnt.incrementAndGet();
if (c % logFreq == 0) info("Executed queries: " + c);
}
}
},
qryThreadNum);
final AtomicInteger restartCnt = new AtomicInteger();
CollectingEventListener lsnr = new CollectingEventListener();
for (int i = 0; i < GRID_CNT; i++)
grid(i).events().localListen(lsnr, EventType.EVT_CACHE_REBALANCE_STOPPED);
IgniteInternalFuture<?> fut2 =
multithreadedAsync(
new Callable<Object>() {
@SuppressWarnings({"BusyWait"})
@Override
public Object call() throws Exception {
while (!done.get()) {
int idx = GRID_CNT;
startGrid(idx);
Thread.sleep(nodeLifeTime);
stopGrid(idx);
int c = restartCnt.incrementAndGet();
if (c % logFreq == 0) info("Node restarts: " + c);
}
return true;
}
},
1);
Thread.sleep(duration);
done.set(true);
fut1.get();
fut2.get();
info("Awaiting rebalance events [restartCnt=" + restartCnt.get() + ']');
boolean success = lsnr.awaitEvents(GRID_CNT * 2 * restartCnt.get(), 15000);
for (int i = 0; i < GRID_CNT; i++)
grid(i).events().stopLocalListen(lsnr, EventType.EVT_CACHE_REBALANCE_STOPPED);
assert success;
}
/**
* Sends cache query response.
*
* @param nodeId Node to send response.
* @param res Cache query response.
* @param timeout Message timeout.
* @return {@code true} if response was sent, {@code false} otherwise.
*/
private boolean sendQueryResponse(UUID nodeId, GridCacheQueryResponse res, long timeout) {
ClusterNode node = cctx.node(nodeId);
if (node == null) return false;
int attempt = 1;
IgniteCheckedException err = null;
while (!Thread.currentThread().isInterrupted()) {
try {
if (log.isDebugEnabled()) log.debug("Send query response: " + res);
Object topic = topic(nodeId, res.requestId());
cctx.io()
.sendOrderedMessage(
node, topic, res, cctx.ioPolicy(), timeout > 0 ? timeout : Long.MAX_VALUE);
return true;
} catch (ClusterTopologyCheckedException ignored) {
if (log.isDebugEnabled())
log.debug(
"Failed to send query response since node left grid [nodeId="
+ nodeId
+ ", res="
+ res
+ "]");
return false;
} catch (IgniteCheckedException e) {
if (err == null) err = e;
if (Thread.currentThread().isInterrupted()) break;
if (attempt < RESEND_ATTEMPTS) {
if (log.isDebugEnabled())
log.debug(
"Failed to send queries response (will try again) [nodeId="
+ nodeId
+ ", res="
+ res
+ ", attempt="
+ attempt
+ ", err="
+ e
+ "]");
if (!Thread.currentThread().isInterrupted())
try {
U.sleep(RESEND_FREQ);
} catch (IgniteInterruptedCheckedException e1) {
U.error(
log,
"Waiting for queries response resending was interrupted (response will not be sent) "
+ "[nodeId="
+ nodeId
+ ", response="
+ res
+ "]",
e1);
return false;
}
} else {
U.error(
log,
"Failed to sender cache response [nodeId=" + nodeId + ", response=" + res + "]",
err);
return false;
}
}
attempt++;
}
return false;
}
/** {@inheritDoc} */
@SuppressWarnings("ErrorNotRethrown")
@Override
public IpcEndpoint accept() throws IgniteCheckedException {
while (!Thread.currentThread().isInterrupted()) {
Socket sock = null;
boolean accepted = false;
try {
sock = srvSock.accept();
accepted = true;
InputStream inputStream = sock.getInputStream();
ObjectInputStream in = new ObjectInputStream(inputStream);
ObjectOutputStream out = new ObjectOutputStream(sock.getOutputStream());
IpcSharedMemorySpace inSpace = null;
IpcSharedMemorySpace outSpace = null;
boolean err = true;
try {
IpcSharedMemoryInitRequest req = (IpcSharedMemoryInitRequest) in.readObject();
if (log.isDebugEnabled()) log.debug("Processing request: " + req);
IgnitePair<String> p = inOutToken(req.pid(), size);
String file1 = p.get1();
String file2 = p.get2();
assert file1 != null;
assert file2 != null;
// Create tokens.
new File(file1).createNewFile();
new File(file2).createNewFile();
if (log.isDebugEnabled()) log.debug("Created token files: " + p);
inSpace = new IpcSharedMemorySpace(file1, req.pid(), pid, size, true, log);
outSpace = new IpcSharedMemorySpace(file2, pid, req.pid(), size, false, log);
IpcSharedMemoryClientEndpoint ret =
new IpcSharedMemoryClientEndpoint(inSpace, outSpace, log);
out.writeObject(
new IpcSharedMemoryInitResponse(
file2, outSpace.sharedMemoryId(), file1, inSpace.sharedMemoryId(), pid, size));
err = !in.readBoolean();
endpoints.add(ret);
return ret;
} catch (UnsatisfiedLinkError e) {
throw IpcSharedMemoryUtils.linkError(e);
} catch (IOException e) {
if (log.isDebugEnabled())
log.debug(
"Failed to process incoming connection "
+ "(was connection closed by another party):"
+ e.getMessage());
} catch (ClassNotFoundException e) {
U.error(log, "Failed to process incoming connection.", e);
} catch (ClassCastException e) {
String msg =
"Failed to process incoming connection (most probably, shared memory "
+ "rest endpoint has been configured by mistake).";
LT.warn(log, null, msg);
sendErrorResponse(out, e);
} catch (IpcOutOfSystemResourcesException e) {
if (!omitOutOfResourcesWarn) LT.warn(log, null, OUT_OF_RESOURCES_MSG);
sendErrorResponse(out, e);
} catch (IgniteCheckedException e) {
LT.error(log, e, "Failed to process incoming shared memory connection.");
sendErrorResponse(out, e);
} finally {
// Exception has been thrown, need to free system resources.
if (err) {
if (inSpace != null) inSpace.forceClose();
// Safety.
if (outSpace != null) outSpace.forceClose();
}
}
} catch (IOException e) {
if (!Thread.currentThread().isInterrupted() && !accepted)
throw new IgniteCheckedException("Failed to accept incoming connection.", e);
if (!closed)
LT.error(
log, null, "Failed to process incoming shared memory connection: " + e.getMessage());
} finally {
U.closeQuiet(sock);
}
} // while
throw new IgniteInterruptedCheckedException("Socket accept was interrupted.");
}