/**
   * Sends message to Hadoop process.
   *
   * @param desc
   * @param msg
   * @throws IgniteCheckedException
   */
  public void sendMessage(HadoopProcessDescriptor desc, HadoopMessage msg)
      throws IgniteCheckedException {
    assert desc != null;
    assert msg != null;

    if (log.isTraceEnabled())
      log.trace("Sending message to Hadoop process [desc=" + desc + ", msg=" + msg + ']');

    HadoopCommunicationClient client = null;

    boolean closeOnRelease = true;

    try {
      client = reserveClient(desc);

      client.sendMessage(desc, msg);

      closeOnRelease = false;
    } finally {
      if (client != null) {
        if (closeOnRelease) {
          client.forceClose();

          clients.remove(desc.processId(), client);
        } else client.release();
      }
    }
  }
  /**
   * Returns existing or just created client to node.
   *
   * @param desc Node to which client should be open.
   * @return The existing or just created client.
   * @throws IgniteCheckedException Thrown if any exception occurs.
   */
  private HadoopCommunicationClient reserveClient(HadoopProcessDescriptor desc)
      throws IgniteCheckedException {
    assert desc != null;

    UUID procId = desc.processId();

    while (true) {
      HadoopCommunicationClient client = clients.get(procId);

      if (client == null) {
        if (log.isDebugEnabled())
          log.debug(
              "Did not find client for remote process [locProcDesc="
                  + locProcDesc
                  + ", desc="
                  + desc
                  + ']');

        // Do not allow concurrent connects.
        Object sync = locks.lock(procId);

        try {
          client = clients.get(procId);

          if (client == null) {
            HadoopCommunicationClient old = clients.put(procId, client = createNioClient(desc));

            assert old == null;
          }
        } finally {
          locks.unlock(procId, sync);
        }

        assert client != null;
      }

      if (client.reserve()) return client;
      else
        // Client has just been closed by idle worker. Help it and try again.
        clients.remove(procId, client);
    }
  }
        /** {@inheritDoc} */
        @Override
        public void onDisconnected(GridNioSession ses, @Nullable Exception e) {
          if (log.isDebugEnabled()) log.debug("Closed connection for session: " + ses);

          if (e != null) U.error(log, "Session disconnected due to exception: " + ses, e);

          HadoopProcessDescriptor desc = ses.meta(PROCESS_META);

          if (desc != null) {
            HadoopCommunicationClient rmv = clients.remove(desc.processId());

            if (rmv != null) rmv.forceClose();
          }

          HadoopMessageListener lsnr0 = lsnr;

          if (lsnr0 != null)
            // Notify listener about connection close.
            lsnr0.onConnectionLost(desc);
        }
  /**
   * Creates new shared memory communication server.
   *
   * @return Server.
   * @throws IgniteCheckedException If failed.
   */
  @Nullable
  private IpcSharedMemoryServerEndpoint resetShmemServer() throws IgniteCheckedException {
    if (boundTcpShmemPort >= 0)
      throw new IgniteCheckedException(
          "Shared memory server was already created on port " + boundTcpShmemPort);

    if (shmemPort == -1 || U.isWindows()) return null;

    IgniteCheckedException lastEx = null;

    // If configured TCP port is busy, find first available in range.
    for (int port = shmemPort; port < shmemPort + locPortRange; port++) {
      try {
        IpcSharedMemoryServerEndpoint srv =
            new IpcSharedMemoryServerEndpoint(
                log.getLogger(IpcSharedMemoryServerEndpoint.class),
                locProcDesc.processId(),
                gridName);

        srv.setPort(port);

        srv.omitOutOfResourcesWarning(true);

        srv.start();

        boundTcpShmemPort = port;

        // Ack Port the TCP server was bound to.
        if (log.isInfoEnabled())
          log.info(
              "Successfully bound shared memory communication to TCP port [port="
                  + boundTcpShmemPort
                  + ", locHost="
                  + locHost
                  + ']');

        return srv;
      } catch (IgniteCheckedException e) {
        lastEx = e;

        if (log.isDebugEnabled())
          log.debug(
              "Failed to bind to local port (will try next port within range) [port="
                  + port
                  + ", locHost="
                  + locHost
                  + ']');
      }
    }

    // If free port wasn't found.
    throw new IgniteCheckedException(
        "Failed to bind shared memory communication to any port within range [startPort="
            + locPort
            + ", portRange="
            + locPortRange
            + ", locHost="
            + locHost
            + ']',
        lastEx);
  }
    /** {@inheritDoc} */
    @Override
    public void onMessageReceived(GridNioSession ses, Object msg) throws IgniteCheckedException {
      HadoopProcessDescriptor desc = ses.meta(PROCESS_META);

      UUID rmtProcId = desc == null ? null : desc.processId();

      if (rmtProcId == null) {
        if (!(msg instanceof ProcessHandshakeMessage)) {
          log.warning(
              "Invalid handshake message received, will close connection [ses="
                  + ses
                  + ", msg="
                  + msg
                  + ']');

          ses.close();

          return;
        }

        ProcessHandshakeMessage nId = (ProcessHandshakeMessage) msg;

        if (log.isDebugEnabled())
          log.debug("Received handshake message [ses=" + ses + ", msg=" + msg + ']');

        ses.addMeta(PROCESS_META, nId.processDescriptor());

        if (!ses.accepted())
          // Send handshake reply.
          ses.send(locIdMsg);
        else {
          //
          rmtProcId = nId.processDescriptor().processId();

          if (log.isDebugEnabled()) log.debug("Finished handshake with remote client: " + ses);

          Object sync = locks.tryLock(rmtProcId);

          if (sync != null) {
            try {
              if (clients.get(rmtProcId) == null) {
                if (log.isDebugEnabled())
                  log.debug("Will reuse session for descriptor: " + rmtProcId);

                // Handshake finished flag is true.
                clients.put(rmtProcId, new HadoopTcpNioCommunicationClient(ses));
              } else {
                if (log.isDebugEnabled())
                  log.debug(
                      "Will not reuse client as another already exists [locProcDesc="
                          + locProcDesc
                          + ", desc="
                          + desc
                          + ']');
              }
            } finally {
              locks.unlock(rmtProcId, sync);
            }
          } else {
            if (log.isDebugEnabled())
              log.debug(
                  "Concurrent connection is being established, will not reuse client session ["
                      + "locProcDesc="
                      + locProcDesc
                      + ", desc="
                      + desc
                      + ']');
          }
        }

        if (log.isDebugEnabled())
          log.debug(
              "Handshake is finished for session [ses="
                  + ses
                  + ", locProcDesc="
                  + locProcDesc
                  + ']');

        HandshakeFinish to = ses.meta(HANDSHAKE_FINISH_META);

        if (to != null) to.finish();

        // Notify session opened (both parties).
        proceedSessionOpened(ses);
      } else {
        if (msgQueueLimit > 0) {
          GridNioMessageTracker tracker = ses.meta(TRACKER_META);

          if (tracker == null) {
            GridNioMessageTracker old =
                ses.addMeta(TRACKER_META, tracker = new GridNioMessageTracker(ses, msgQueueLimit));

            assert old == null;
          }

          tracker.onMessageReceived();
        }

        proceedMessageReceived(ses, msg);
      }
    }