/** {@inheritDoc} */
    @Override
    protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
      if (log.isDebugEnabled()) log.debug("GC worker started.");

      File workTokDir = tokDir.getParentFile();

      assert workTokDir != null;

      boolean lastRunNeeded = true;

      while (true) {
        try {
          // Sleep only if not cancelled.
          if (lastRunNeeded) Thread.sleep(GC_FREQ);
        } catch (InterruptedException ignored) {
          // No-op.
        }

        if (log.isDebugEnabled()) log.debug("Starting GC iteration.");

        cleanupResources(workTokDir);

        // Process spaces created by this endpoint.
        if (log.isDebugEnabled()) log.debug("Processing local spaces.");

        for (IpcSharedMemoryClientEndpoint e : endpoints) {
          if (log.isDebugEnabled()) log.debug("Processing endpoint: " + e);

          if (!e.checkOtherPartyAlive()) {
            endpoints.remove(e);

            if (log.isDebugEnabled()) log.debug("Removed endpoint: " + e);
          }
        }

        if (isCancelled()) {
          if (lastRunNeeded) {
            lastRunNeeded = false;

            // Clear interrupted status.
            Thread.interrupted();
          } else {
            Thread.currentThread().interrupt();

            break;
          }
        }
      }
    }
    /** @param workTokDir Token directory (common for multiple nodes). */
    private void cleanupResources(File workTokDir) {
      RandomAccessFile lockFile = null;

      FileLock lock = null;

      try {
        lockFile = new RandomAccessFile(new File(workTokDir, LOCK_FILE_NAME), "rw");

        lock = lockFile.getChannel().lock();

        if (lock != null) processTokenDirectory(workTokDir);
        else if (log.isDebugEnabled())
          log.debug(
              "Token directory is being processed concurrently: " + workTokDir.getAbsolutePath());
      } catch (OverlappingFileLockException ignored) {
        if (log.isDebugEnabled())
          log.debug(
              "Token directory is being processed concurrently: " + workTokDir.getAbsolutePath());
      } catch (FileLockInterruptionException ignored) {
        Thread.currentThread().interrupt();
      } catch (IOException e) {
        U.error(log, "Failed to process directory: " + workTokDir.getAbsolutePath(), e);
      } finally {
        U.releaseQuiet(lock);
        U.closeQuiet(lockFile);
      }
    }
 /**
  * @param log Logger.
  * @param time Time.
  * @param msg Message.
  */
 private static void log0(@Nullable IgniteLogger log, long time, String msg) {
   if (log != null) {
     if (log.isDebugEnabled()) log.debug(msg);
     else log.warning(msg);
   } else
     X.println(
         String.format(
             "[%s][%s]%s",
             DEBUG_DATE_FMT.get().format(time), Thread.currentThread().getName(), msg));
 }
  /**
   * @param key Removed key.
   * @param ver Removed version.
   * @throws IgniteCheckedException If failed.
   */
  public void onDeferredDelete(KeyCacheObject key, GridCacheVersion ver)
      throws IgniteCheckedException {
    try {
      T2<KeyCacheObject, GridCacheVersion> evicted = rmvQueue.add(new T2<>(key, ver));

      if (evicted != null) cctx.dht().removeVersionedEntry(evicted.get1(), evicted.get2());
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();

      throw new IgniteInterruptedCheckedException(e);
    }
  }
  /** {@inheritDoc} */
  @Override
  public void close() {
    closed = true;

    U.closeQuiet(srvSock);

    if (gcWorker != null) {
      U.cancel(gcWorker);

      // This method may be called from already interrupted thread.
      // Need to ensure cleaning on close.
      boolean interrupted = Thread.interrupted();

      try {
        U.join(gcWorker);
      } catch (IgniteInterruptedCheckedException e) {
        U.warn(log, "Interrupted when stopping GC worker.", e);
      } finally {
        if (interrupted) Thread.currentThread().interrupt();
      }
    }
  }
  /**
   * Processes cache query request.
   *
   * @param sndId Sender node id.
   * @param req Query request.
   */
  @SuppressWarnings("unchecked")
  @Override
  void processQueryRequest(UUID sndId, GridCacheQueryRequest req) {
    if (req.cancel()) {
      cancelIds.add(new CancelMessageId(req.id(), sndId));

      if (req.fields()) removeFieldsQueryResult(sndId, req.id());
      else removeQueryResult(sndId, req.id());
    } else {
      if (!cancelIds.contains(new CancelMessageId(req.id(), sndId))) {
        if (!F.eq(req.cacheName(), cctx.name())) {
          GridCacheQueryResponse res =
              new GridCacheQueryResponse(
                  cctx.cacheId(),
                  req.id(),
                  new IgniteCheckedException(
                      "Received request for incorrect cache [expected="
                          + cctx.name()
                          + ", actual="
                          + req.cacheName()));

          sendQueryResponse(sndId, res, 0);
        } else {
          threads.put(req.id(), Thread.currentThread());

          try {
            GridCacheQueryInfo info = distributedQueryInfo(sndId, req);

            if (info == null) return;

            if (req.fields()) runFieldsQuery(info);
            else runQuery(info);
          } catch (Throwable e) {
            U.error(log(), "Failed to run query.", e);

            sendQueryResponse(
                sndId, new GridCacheQueryResponse(cctx.cacheId(), req.id(), e.getCause()), 0);

            if (e instanceof Error) throw (Error) e;
          } finally {
            threads.remove(req.id());
          }
        }
      }
    }
  }
  /**
   * Checks if address can be reached using one argument InetAddress.isReachable() version or ping
   * command if failed.
   *
   * @param addr Address to check.
   * @param reachTimeout Timeout for the check.
   * @return {@code True} if address is reachable.
   */
  public static boolean reachableByPing(InetAddress addr, int reachTimeout) {
    try {
      if (addr.isReachable(reachTimeout)) return true;

      String cmd = String.format("ping -%s 1 %s", U.isWindows() ? "n" : "c", addr.getHostAddress());

      Process myProc = Runtime.getRuntime().exec(cmd);

      myProc.waitFor();

      return myProc.exitValue() == 0;
    } catch (IOException ignore) {
      return false;
    } catch (InterruptedException ignored) {
      Thread.currentThread().interrupt();

      return false;
    }
  }
  /**
   * JUnit.
   *
   * @throws Exception If failed.
   */
  @SuppressWarnings({"TooBroadScope"})
  public void testRestarts() throws Exception {
    int duration = 60 * 1000;
    int qryThreadNum = 10;
    final long nodeLifeTime = 2 * 1000;
    final int logFreq = 20;

    final IgniteCache<Integer, Integer> cache = grid(0).cache(null);

    assert cache != null;

    for (int i = 0; i < KEY_CNT; i++) cache.put(i, i);

    assertEquals(KEY_CNT, cache.localSize());

    final AtomicInteger qryCnt = new AtomicInteger();

    final AtomicBoolean done = new AtomicBoolean();

    IgniteInternalFuture<?> fut1 =
        multithreadedAsync(
            new CAX() {
              @Override
              public void applyx() throws IgniteCheckedException {
                while (!done.get()) {
                  Collection<Cache.Entry<Integer, Integer>> res =
                      cache.query(new SqlQuery(Integer.class, "_val >= 0")).getAll();

                  assertFalse(res.isEmpty());

                  int c = qryCnt.incrementAndGet();

                  if (c % logFreq == 0) info("Executed queries: " + c);
                }
              }
            },
            qryThreadNum);

    final AtomicInteger restartCnt = new AtomicInteger();

    CollectingEventListener lsnr = new CollectingEventListener();

    for (int i = 0; i < GRID_CNT; i++)
      grid(i).events().localListen(lsnr, EventType.EVT_CACHE_REBALANCE_STOPPED);

    IgniteInternalFuture<?> fut2 =
        multithreadedAsync(
            new Callable<Object>() {
              @SuppressWarnings({"BusyWait"})
              @Override
              public Object call() throws Exception {
                while (!done.get()) {
                  int idx = GRID_CNT;

                  startGrid(idx);

                  Thread.sleep(nodeLifeTime);

                  stopGrid(idx);

                  int c = restartCnt.incrementAndGet();

                  if (c % logFreq == 0) info("Node restarts: " + c);
                }

                return true;
              }
            },
            1);

    Thread.sleep(duration);

    done.set(true);

    fut1.get();
    fut2.get();

    info("Awaiting rebalance events [restartCnt=" + restartCnt.get() + ']');

    boolean success = lsnr.awaitEvents(GRID_CNT * 2 * restartCnt.get(), 15000);

    for (int i = 0; i < GRID_CNT; i++)
      grid(i).events().stopLocalListen(lsnr, EventType.EVT_CACHE_REBALANCE_STOPPED);

    assert success;
  }
  /**
   * Sends cache query response.
   *
   * @param nodeId Node to send response.
   * @param res Cache query response.
   * @param timeout Message timeout.
   * @return {@code true} if response was sent, {@code false} otherwise.
   */
  private boolean sendQueryResponse(UUID nodeId, GridCacheQueryResponse res, long timeout) {
    ClusterNode node = cctx.node(nodeId);

    if (node == null) return false;

    int attempt = 1;

    IgniteCheckedException err = null;

    while (!Thread.currentThread().isInterrupted()) {
      try {
        if (log.isDebugEnabled()) log.debug("Send query response: " + res);

        Object topic = topic(nodeId, res.requestId());

        cctx.io()
            .sendOrderedMessage(
                node, topic, res, cctx.ioPolicy(), timeout > 0 ? timeout : Long.MAX_VALUE);

        return true;
      } catch (ClusterTopologyCheckedException ignored) {
        if (log.isDebugEnabled())
          log.debug(
              "Failed to send query response since node left grid [nodeId="
                  + nodeId
                  + ", res="
                  + res
                  + "]");

        return false;
      } catch (IgniteCheckedException e) {
        if (err == null) err = e;

        if (Thread.currentThread().isInterrupted()) break;

        if (attempt < RESEND_ATTEMPTS) {
          if (log.isDebugEnabled())
            log.debug(
                "Failed to send queries response (will try again) [nodeId="
                    + nodeId
                    + ", res="
                    + res
                    + ", attempt="
                    + attempt
                    + ", err="
                    + e
                    + "]");

          if (!Thread.currentThread().isInterrupted())
            try {
              U.sleep(RESEND_FREQ);
            } catch (IgniteInterruptedCheckedException e1) {
              U.error(
                  log,
                  "Waiting for queries response resending was interrupted (response will not be sent) "
                      + "[nodeId="
                      + nodeId
                      + ", response="
                      + res
                      + "]",
                  e1);

              return false;
            }
        } else {
          U.error(
              log,
              "Failed to sender cache response [nodeId=" + nodeId + ", response=" + res + "]",
              err);

          return false;
        }
      }

      attempt++;
    }

    return false;
  }
  /** {@inheritDoc} */
  @SuppressWarnings("ErrorNotRethrown")
  @Override
  public IpcEndpoint accept() throws IgniteCheckedException {
    while (!Thread.currentThread().isInterrupted()) {
      Socket sock = null;

      boolean accepted = false;

      try {
        sock = srvSock.accept();

        accepted = true;

        InputStream inputStream = sock.getInputStream();
        ObjectInputStream in = new ObjectInputStream(inputStream);

        ObjectOutputStream out = new ObjectOutputStream(sock.getOutputStream());

        IpcSharedMemorySpace inSpace = null;

        IpcSharedMemorySpace outSpace = null;

        boolean err = true;

        try {
          IpcSharedMemoryInitRequest req = (IpcSharedMemoryInitRequest) in.readObject();

          if (log.isDebugEnabled()) log.debug("Processing request: " + req);

          IgnitePair<String> p = inOutToken(req.pid(), size);

          String file1 = p.get1();
          String file2 = p.get2();

          assert file1 != null;
          assert file2 != null;

          // Create tokens.
          new File(file1).createNewFile();
          new File(file2).createNewFile();

          if (log.isDebugEnabled()) log.debug("Created token files: " + p);

          inSpace = new IpcSharedMemorySpace(file1, req.pid(), pid, size, true, log);

          outSpace = new IpcSharedMemorySpace(file2, pid, req.pid(), size, false, log);

          IpcSharedMemoryClientEndpoint ret =
              new IpcSharedMemoryClientEndpoint(inSpace, outSpace, log);

          out.writeObject(
              new IpcSharedMemoryInitResponse(
                  file2, outSpace.sharedMemoryId(), file1, inSpace.sharedMemoryId(), pid, size));

          err = !in.readBoolean();

          endpoints.add(ret);

          return ret;
        } catch (UnsatisfiedLinkError e) {
          throw IpcSharedMemoryUtils.linkError(e);
        } catch (IOException e) {
          if (log.isDebugEnabled())
            log.debug(
                "Failed to process incoming connection "
                    + "(was connection closed by another party):"
                    + e.getMessage());
        } catch (ClassNotFoundException e) {
          U.error(log, "Failed to process incoming connection.", e);
        } catch (ClassCastException e) {
          String msg =
              "Failed to process incoming connection (most probably, shared memory "
                  + "rest endpoint has been configured by mistake).";

          LT.warn(log, null, msg);

          sendErrorResponse(out, e);
        } catch (IpcOutOfSystemResourcesException e) {
          if (!omitOutOfResourcesWarn) LT.warn(log, null, OUT_OF_RESOURCES_MSG);

          sendErrorResponse(out, e);
        } catch (IgniteCheckedException e) {
          LT.error(log, e, "Failed to process incoming shared memory connection.");

          sendErrorResponse(out, e);
        } finally {
          // Exception has been thrown, need to free system resources.
          if (err) {
            if (inSpace != null) inSpace.forceClose();

            // Safety.
            if (outSpace != null) outSpace.forceClose();
          }
        }
      } catch (IOException e) {
        if (!Thread.currentThread().isInterrupted() && !accepted)
          throw new IgniteCheckedException("Failed to accept incoming connection.", e);

        if (!closed)
          LT.error(
              log, null, "Failed to process incoming shared memory connection: " + e.getMessage());
      } finally {
        U.closeQuiet(sock);
      }
    } // while

    throw new IgniteInterruptedCheckedException("Socket accept was interrupted.");
  }