/** @param e Error. */
  void onError(Throwable e) {
    tx.commitError(e);

    if (err.compareAndSet(null, e)) {
      boolean marked = tx.setRollbackOnly();

      if (e instanceof GridCacheTxRollbackException) {
        if (marked) {
          try {
            tx.rollback();
          } catch (GridException ex) {
            U.error(log, "Failed to automatically rollback transaction: " + tx, ex);
          }
        }
      } else if (tx.implicit()
          && tx.isSystemInvalidate()) { // Finish implicit transaction on heuristic error.
        try {
          tx.close();
        } catch (GridException ex) {
          U.error(log, "Failed to invalidate transaction: " + tx, ex);
        }
      }

      onComplete();
    }
  }
Example #2
0
  /**
   * Message received callback.
   *
   * @param src Sender node ID.
   * @param msg Received message.
   * @return {@code True}.
   */
  public boolean onMessageReceived(UUID src, GridHadoopMessage msg) {
    if (msg instanceof GridHadoopShuffleMessage) {
      GridHadoopShuffleMessage m = (GridHadoopShuffleMessage) msg;

      try {
        job(m.jobId()).onShuffleMessage(m);
      } catch (GridException e) {
        U.error(log, "Message handling failed.", e);
      }

      try {
        // Reply with ack.
        send0(src, new GridHadoopShuffleAck(m.id(), m.jobId()));
      } catch (GridException e) {
        U.error(
            log,
            "Failed to reply back to shuffle message sender [snd=" + src + ", msg=" + msg + ']',
            e);
      }
    } else if (msg instanceof GridHadoopShuffleAck) {
      GridHadoopShuffleAck m = (GridHadoopShuffleAck) msg;

      try {
        job(m.jobId()).onShuffleAck(m);
      } catch (GridException e) {
        U.error(log, "Message handling failed.", e);
      }
    } else
      throw new IllegalStateException(
          "Unknown message type received to Hadoop shuffle [src=" + src + ", msg=" + msg + ']');

    return true;
  }
  /**
   * @param nodeId Sender node ID.
   * @param msg Response to prepare request.
   */
  private void processPrepareResponse(UUID nodeId, GridDistributedTxPrepareResponse<K, V> msg) {
    assert nodeId != null;
    assert msg != null;

    GridReplicatedTxLocal<K, V> tx = ctx.tm().tx(msg.version());

    if (tx == null) {
      if (log.isDebugEnabled())
        log.debug(
            "Received prepare response for non-existing transaction [senderNodeId="
                + nodeId
                + ", res="
                + msg
                + ']');

      return;
    }

    GridReplicatedTxPrepareFuture<K, V> future = (GridReplicatedTxPrepareFuture<K, V>) tx.future();

    if (future != null) future.onResult(nodeId, msg);
    else
      U.error(
          log,
          "Received prepare response for transaction with no future [res="
              + msg
              + ", tx="
              + tx
              + ']');
  }
    /** @param workTokDir Token directory (common for multiple nodes). */
    private void cleanupResources(File workTokDir) {
      RandomAccessFile lockFile = null;

      FileLock lock = null;

      try {
        lockFile = new RandomAccessFile(new File(workTokDir, LOCK_FILE_NAME), "rw");

        lock = lockFile.getChannel().lock();

        if (lock != null) processTokenDirectory(workTokDir);
        else if (log.isDebugEnabled())
          log.debug(
              "Token directory is being processed concurrently: " + workTokDir.getAbsolutePath());
      } catch (OverlappingFileLockException ignored) {
        if (log.isDebugEnabled())
          log.debug(
              "Token directory is being processed concurrently: " + workTokDir.getAbsolutePath());
      } catch (FileLockInterruptionException ignored) {
        Thread.currentThread().interrupt();
      } catch (IOException e) {
        U.error(log, "Failed to process directory: " + workTokDir.getAbsolutePath(), e);
      } finally {
        U.releaseQuiet(lock);
        U.closeQuiet(lockFile);
      }
    }
  /** Initializes future. */
  @SuppressWarnings({"unchecked"})
  void finish() {
    if (mappings != null) {
      finish(mappings.values());

      markInitialized();

      if (!isSync()) {
        boolean complete = true;

        for (GridFuture<?> f : pending())
          // Mini-future in non-sync mode gets done when message gets sent.
          if (isMini(f) && !f.isDone()) complete = false;

        if (complete) onComplete();
      }
    } else {
      assert !commit;

      try {
        tx.rollback();
      } catch (GridException e) {
        U.error(log, "Failed to rollback empty transaction: " + tx, e);
      }

      markInitialized();
    }
  }
  /** Clears swap entries for evicted partition. */
  private void clearSwap() {
    assert state() == EVICTED;
    assert !GridQueryProcessor.isEnabled(cctx.config())
        : "Indexing needs to have unswapped values.";

    try {
      GridCloseableIterator<Map.Entry<byte[], GridCacheSwapEntry>> it = cctx.swap().iterator(id);

      boolean isLocStore = cctx.store().isLocal();

      if (it != null) {
        // We can safely remove these values because no entries will be created for evicted
        // partition.
        while (it.hasNext()) {
          Map.Entry<byte[], GridCacheSwapEntry> entry = it.next();

          byte[] keyBytes = entry.getKey();

          KeyCacheObject key = cctx.toCacheKeyObject(keyBytes);

          cctx.swap().remove(key);

          if (isLocStore) cctx.store().remove(null, key.value(cctx.cacheObjectContext(), false));
        }
      }
    } catch (IgniteCheckedException e) {
      U.error(log, "Failed to clear swap for evicted partition: " + this, e);
    }
  }
  private void recheck() {
    // If this is the oldest node.
    if (oldestNode.get().id().equals(cctx.localNodeId())) {
      Collection<UUID> remaining = remaining();

      if (!remaining.isEmpty()) {
        try {
          cctx.io()
              .safeSend(
                  cctx.discovery().nodes(remaining),
                  new GridDhtPartitionsSingleRequest(exchId),
                  SYSTEM_POOL,
                  null);
        } catch (IgniteCheckedException e) {
          U.error(
              log,
              "Failed to request partitions from nodes [exchangeId="
                  + exchId
                  + ", nodes="
                  + remaining
                  + ']',
              e);
        }
      }
      // Resend full partition map because last attempt failed.
      else {
        if (spreadPartitions()) onDone(exchId.topologyVersion());
      }
    } else sendPartitions();

    // Schedule another send.
    scheduleRecheck();
  }
  private void sendPartitions() {
    ClusterNode oldestNode = this.oldestNode.get();

    try {
      sendLocalPartitions(oldestNode, exchId);
    } catch (ClusterTopologyCheckedException ignore) {
      if (log.isDebugEnabled())
        log.debug(
            "Oldest node left during partition exchange [nodeId="
                + oldestNode.id()
                + ", exchId="
                + exchId
                + ']');
    } catch (IgniteCheckedException e) {
      scheduleRecheck();

      U.error(
          log,
          "Failed to send local partitions to oldest node (will retry after timeout) [oldestNodeId="
              + oldestNode.id()
              + ", exchId="
              + exchId
              + ']',
          e);
    }
  }
  /** {@inheritDoc} */
  @Override
  public boolean onDone(GridCacheTx tx, Throwable err) {
    if ((initialized() || err != null) && super.onDone(tx, err)) {
      if (error() instanceof GridCacheTxHeuristicException) {
        long topVer = this.tx.topologyVersion();

        for (GridCacheTxEntry<K, V> e : this.tx.writeMap().values()) {
          try {
            if (e.op() != NOOP && !cctx.affinity().localNode(e.key(), topVer)) {
              GridCacheEntryEx<K, V> cacheEntry = cctx.cache().peekEx(e.key());

              if (cacheEntry != null) cacheEntry.invalidate(null, this.tx.xidVersion());
            }
          } catch (Throwable t) {
            U.error(log, "Failed to invalidate entry.", t);

            if (t instanceof Error) throw (Error) t;
          }
        }
      }

      // Don't forget to clean up.
      cctx.mvcc().removeFuture(this);

      return true;
    }

    return false;
  }
 /**
  * @param out Output stream.
  * @param err Error cause.
  */
 private void sendErrorResponse(ObjectOutput out, Exception err) {
   try {
     out.writeObject(new IpcSharedMemoryInitResponse(err));
   } catch (IOException e) {
     U.error(log, "Failed to send error response to client.", e);
   }
 }
  /**
   * Processes unlock request.
   *
   * @param nodeId Sender node ID.
   * @param req Unlock request.
   */
  @SuppressWarnings({"unchecked"})
  private void processUnlockRequest(UUID nodeId, GridDistributedUnlockRequest req) {
    assert nodeId != null;

    try {
      ClassLoader ldr = ctx.deploy().globalLoader();
      List<byte[]> keys = req.keyBytes();

      for (byte[] keyBytes : keys) {
        K key = (K) U.unmarshal(ctx.marshaller(), new ByteArrayInputStream(keyBytes), ldr);

        while (true) {
          boolean created = false;

          GridDistributedCacheEntry<K, V> entry = peekexx(key);

          if (entry == null) {
            entry = entryexx(key);

            created = true;
          }

          try {
            entry.doneRemote(
                req.version(), req.version(), req.committedVersions(), req.rolledbackVersions());

            // Note that we don't reorder completed versions here,
            // as there is no point to reorder relative to the version
            // we are about to remove.
            if (entry.removeLock(req.version())) {
              if (log.isDebugEnabled())
                log.debug("Removed lock [lockId=" + req.version() + ", key=" + key + ']');

              if (created && entry.markObsolete(req.version())) removeIfObsolete(entry.key());
            } else if (log.isDebugEnabled())
              log.debug(
                  "Received unlock request for unknown candidate "
                      + "(added to cancelled locks set): "
                      + req);

            break;
          } catch (GridCacheEntryRemovedException ignored) {
            if (log.isDebugEnabled())
              log.debug(
                  "Received remove lock request for removed entry (will retry) [entry="
                      + entry
                      + ", req="
                      + req
                      + ']');
          }
        }
      }
    } catch (GridException e) {
      U.error(log, "Failed to unmarshal unlock key (unlock will not be performed): " + req, e);
    }
  }
  /**
   * Removes locks regardless of whether they are owned or not for given version and keys.
   *
   * @param ver Lock version.
   * @param keys Keys.
   */
  @SuppressWarnings({"unchecked"})
  public void removeLocks(GridCacheVersion ver, Collection<? extends K> keys) {
    if (keys.isEmpty()) return;

    Collection<GridRichNode> nodes = ctx.remoteNodes(keys);

    try {
      // Send request to remove from remote nodes.
      GridDistributedUnlockRequest<K, V> req = new GridDistributedUnlockRequest<K, V>(keys.size());

      req.version(ver);

      for (K key : keys) {
        while (true) {
          GridDistributedCacheEntry<K, V> entry = peekexx(key);

          try {
            if (entry != null) {
              GridCacheMvccCandidate<K> cand = entry.candidate(ver);

              if (cand != null) {
                // Remove candidate from local node first.
                if (entry.removeLock(cand.version())) {
                  // If there is only local node in this lock's topology,
                  // then there is no reason to distribute the request.
                  if (nodes.isEmpty()) continue;

                  req.addKey(entry.key(), entry.getOrMarshalKeyBytes(), ctx);
                }
              }
            }

            break;
          } catch (GridCacheEntryRemovedException ignored) {
            if (log.isDebugEnabled())
              log.debug(
                  "Attempted to remove lock from removed entry (will retry) [rmvVer="
                      + ver
                      + ", entry="
                      + entry
                      + ']');
          }
        }
      }

      if (nodes.isEmpty()) return;

      req.completedVersions(ctx.tm().committedVersions(ver), ctx.tm().rolledbackVersions(ver));

      if (!req.keyBytes().isEmpty())
        // We don't wait for reply to this message.
        ctx.io().safeSend(nodes, req, null);
    } catch (GridException ex) {
      U.error(log, "Failed to unlock the lock for keys: " + keys, ex);
    }
  }
  /** Perform cleanup of the trash directory. */
  private void delete() {
    IgfsFileInfo info = null;

    try {
      info = meta.info(TRASH_ID);
    } catch (ClusterTopologyServerNotFoundException e) {
      LT.warn(log, e, "Server nodes not found.");
    } catch (IgniteCheckedException e) {
      U.error(log, "Cannot obtain trash directory info.", e);
    }

    if (info != null) {
      for (Map.Entry<String, IgfsListingEntry> entry : info.listing().entrySet()) {
        IgniteUuid fileId = entry.getValue().fileId();

        if (log.isDebugEnabled())
          log.debug(
              "Deleting IGFS trash entry [name=" + entry.getKey() + ", fileId=" + fileId + ']');

        try {
          if (!cancelled) {
            if (delete(entry.getKey(), fileId)) {
              if (log.isDebugEnabled())
                log.debug(
                    "Sending delete confirmation message [name="
                        + entry.getKey()
                        + ", fileId="
                        + fileId
                        + ']');

              sendDeleteMessage(new IgfsDeleteMessage(fileId));
            }
          } else break;
        } catch (IgniteInterruptedCheckedException ignored) {
          // Ignore this exception while stopping.
        } catch (IgniteCheckedException e) {
          U.error(log, "Failed to delete entry from the trash directory: " + entry.getKey(), e);

          sendDeleteMessage(new IgfsDeleteMessage(fileId, e));
        }
      }
    }
  }
Example #14
0
  /** @param jobId Job id. */
  public void jobFinished(GridHadoopJobId jobId) {
    GridHadoopShuffleJob job = jobs.remove(jobId);

    if (job != null) {
      try {
        job.close();
      } catch (GridException e) {
        U.error(log, "Failed to close job: " + jobId, e);
      }
    }
  }
  /**
   * Checks entry for empty value.
   *
   * @param entry Entry to check.
   * @return {@code True} if entry is empty.
   */
  private boolean empty(GridCacheEntry<K, V> entry) {
    try {
      return entry.peek(F.asList(GLOBAL)) == null;
    } catch (GridException e) {
      U.error(null, e.getMessage(), e);

      assert false : "Should never happen: " + e;

      return false;
    }
  }
Example #16
0
  /**
   * Stops shuffle.
   *
   * @param cancel If should cancel all ongoing activities.
   */
  @Override
  public void stop(boolean cancel) {
    for (GridHadoopShuffleJob job : jobs.values()) {
      try {
        job.close();
      } catch (GridException e) {
        U.error(log, "Failed to close job.", e);
      }
    }

    jobs.clear();
  }
  /** {@inheritDoc} */
  @Override
  void onCancelAtStop() {
    super.onCancelAtStop();

    for (GridCacheQueryFutureAdapter fut : futs.values())
      try {
        fut.cancel();
      } catch (IgniteCheckedException e) {
        U.error(log, "Failed to cancel running query future: " + fut, e);
      }

    U.interrupt(threads.values());
  }
  /**
   * Notifies single listener.
   *
   * @param lsnr Listener.
   */
  private void notifyListener(IgniteInClosure<? super IgniteInternalFuture<R>> lsnr) {
    assert lsnr != null;

    try {
      lsnr.apply(this);
    } catch (IllegalStateException e) {
      U.error(
          null,
          "Failed to notify listener (is grid stopped?) [fut="
              + this
              + ", lsnr="
              + lsnr
              + ", err="
              + e.getMessage()
              + ']',
          e);
    } catch (RuntimeException | Error e) {
      U.error(null, "Failed to notify listener: " + lsnr, e);

      throw e;
    }
  }
Example #19
0
  /**
   * Notifies single listener.
   *
   * @param lsnr Listener.
   */
  private void notifyListener(GridInClosure<? super GridFuture<R>> lsnr) {
    assert lsnr != null;

    try {
      lsnr.apply(this);
    } catch (IllegalStateException ignore) {
      U.warn(
          null,
          "Failed to notify listener (grid is stopped) [grid="
              + ctx.gridName()
              + ", lsnr="
              + lsnr
              + ']');
    } catch (RuntimeException e) {
      U.error(log, "Failed to notify listener: " + lsnr, e);

      throw e;
    } catch (Error e) {
      U.error(log, "Failed to notify listener: " + lsnr, e);

      throw e;
    }
  }
Example #20
0
    void cancelAll() {
      GridException err =
          new GridException("Data loader has been cancelled: " + GridDataLoaderImpl.this);

      for (GridFuture<?> f : locFuts) {
        try {
          f.cancel();
        } catch (GridException e) {
          U.error(log, "Failed to cancel mini-future.", e);
        }
      }

      for (GridFutureAdapter<?> f : reqs.values()) f.onDone(err);
    }
  /**
   * @param nodeId Node ID.
   * @param retryCnt Number of retries.
   */
  private void sendAllPartitions(final UUID nodeId, final int retryCnt) {
    ClusterNode n = cctx.node(nodeId);

    try {
      if (n != null) sendAllPartitions(F.asList(n), exchId);
    } catch (IgniteCheckedException e) {
      if (e instanceof ClusterTopologyCheckedException || !cctx.discovery().alive(n)) {
        log.debug(
            "Failed to send full partition map to node, node left grid "
                + "[rmtNode="
                + nodeId
                + ", exchangeId="
                + exchId
                + ']');

        return;
      }

      if (retryCnt > 0) {
        long timeout = cctx.gridConfig().getNetworkSendRetryDelay();

        LT.error(
            log,
            e,
            "Failed to send full partition map to node (will retry after timeout) "
                + "[node="
                + nodeId
                + ", exchangeId="
                + exchId
                + ", timeout="
                + timeout
                + ']');

        cctx.time()
            .addTimeoutObject(
                new GridTimeoutObjectAdapter(timeout) {
                  @Override
                  public void onTimeout() {
                    sendAllPartitions(nodeId, retryCnt - 1);
                  }
                });
      } else
        U.error(
            log,
            "Failed to send full partition map [node=" + n + ", exchangeId=" + exchId + ']',
            e);
    }
  }
  /**
   * Processes cache query request.
   *
   * @param sndId Sender node id.
   * @param req Query request.
   */
  @SuppressWarnings("unchecked")
  @Override
  void processQueryRequest(UUID sndId, GridCacheQueryRequest req) {
    if (req.cancel()) {
      cancelIds.add(new CancelMessageId(req.id(), sndId));

      if (req.fields()) removeFieldsQueryResult(sndId, req.id());
      else removeQueryResult(sndId, req.id());
    } else {
      if (!cancelIds.contains(new CancelMessageId(req.id(), sndId))) {
        if (!F.eq(req.cacheName(), cctx.name())) {
          GridCacheQueryResponse res =
              new GridCacheQueryResponse(
                  cctx.cacheId(),
                  req.id(),
                  new IgniteCheckedException(
                      "Received request for incorrect cache [expected="
                          + cctx.name()
                          + ", actual="
                          + req.cacheName()));

          sendQueryResponse(sndId, res, 0);
        } else {
          threads.put(req.id(), Thread.currentThread());

          try {
            GridCacheQueryInfo info = distributedQueryInfo(sndId, req);

            if (info == null) return;

            if (req.fields()) runFieldsQuery(info);
            else runQuery(info);
          } catch (Throwable e) {
            U.error(log(), "Failed to run query.", e);

            sendQueryResponse(
                sndId, new GridCacheQueryResponse(cctx.cacheId(), req.id(), e.getCause()), 0);

            if (e instanceof Error) throw (Error) e;
          } finally {
            threads.remove(req.id());
          }
        }
      }
    }
  }
  /** @param e Error. */
  void onError(Throwable e) {
    tx.commitError(e);

    if (err.compareAndSet(null, e)) {
      boolean marked = tx.setRollbackOnly();

      if (e instanceof GridCacheTxRollbackException)
        if (marked) {
          try {
            tx.rollback();
          } catch (GridException ex) {
            U.error(log, "Failed to automatically rollback transaction: " + tx, ex);
          }
        }

      onComplete();
    }
  }
  /** @return {@code True} if succeeded. */
  private boolean spreadPartitions() {
    try {
      sendAllPartitions(rmtNodes, exchId);

      return true;
    } catch (IgniteCheckedException e) {
      scheduleRecheck();

      if (!X.hasCause(e, InterruptedException.class))
        U.error(
            log,
            "Failed to send full partition map to nodes (will retry after timeout) [nodes="
                + F.nodeId8s(rmtNodes)
                + ", exchangeId="
                + exchId
                + ']',
            e);

      return false;
    }
  }
  /**
   * Starts activity.
   *
   * @throws IgniteInterruptedCheckedException If interrupted.
   */
  public void init() throws IgniteInterruptedCheckedException {
    if (isDone()) return;

    if (init.compareAndSet(false, true)) {
      if (isDone()) return;

      try {
        // Wait for event to occur to make sure that discovery
        // will return corresponding nodes.
        U.await(evtLatch);

        assert discoEvt != null : this;
        assert !dummy && !forcePreload : this;

        ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx, exchId.topologyVersion());

        oldestNode.set(oldest);

        startCaches();

        // True if client node joined or failed.
        boolean clientNodeEvt;

        if (F.isEmpty(reqs)) {
          int type = discoEvt.type();

          assert type == EVT_NODE_JOINED || type == EVT_NODE_LEFT || type == EVT_NODE_FAILED
              : discoEvt;

          clientNodeEvt = CU.clientNode(discoEvt.eventNode());
        } else {
          assert discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT : discoEvt;

          boolean clientOnlyStart = true;

          for (DynamicCacheChangeRequest req : reqs) {
            if (!req.clientStartOnly()) {
              clientOnlyStart = false;

              break;
            }
          }

          clientNodeEvt = clientOnlyStart;
        }

        if (clientNodeEvt) {
          ClusterNode node = discoEvt.eventNode();

          // Client need to initialize affinity for local join event or for stated client caches.
          if (!node.isLocal()) {
            for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
              if (cacheCtx.isLocal()) continue;

              GridDhtPartitionTopology top = cacheCtx.topology();

              top.updateTopologyVersion(exchId, this, -1, stopping(cacheCtx.cacheId()));

              if (cacheCtx.affinity().affinityTopologyVersion() == AffinityTopologyVersion.NONE) {
                initTopology(cacheCtx);

                top.beforeExchange(this);
              } else
                cacheCtx.affinity().clientEventTopologyChange(discoEvt, exchId.topologyVersion());
            }

            if (exchId.isLeft())
              cctx.mvcc().removeExplicitNodeLocks(exchId.nodeId(), exchId.topologyVersion());

            onDone(exchId.topologyVersion());

            skipPreload = cctx.kernalContext().clientNode();

            return;
          }
        }

        if (cctx.kernalContext().clientNode()) {
          skipPreload = true;

          for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
            if (cacheCtx.isLocal()) continue;

            GridDhtPartitionTopology top = cacheCtx.topology();

            top.updateTopologyVersion(exchId, this, -1, stopping(cacheCtx.cacheId()));
          }

          for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
            if (cacheCtx.isLocal()) continue;

            initTopology(cacheCtx);
          }

          if (oldestNode.get() != null) {
            rmtNodes =
                new ConcurrentLinkedQueue<>(
                    CU.aliveRemoteServerNodesWithCaches(cctx, exchId.topologyVersion()));

            rmtIds = Collections.unmodifiableSet(new HashSet<>(F.nodeIds(rmtNodes)));

            ready.set(true);

            initFut.onDone(true);

            if (log.isDebugEnabled()) log.debug("Initialized future: " + this);

            sendPartitions();
          } else onDone(exchId.topologyVersion());

          return;
        }

        assert oldestNode.get() != null;

        for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
          if (isCacheAdded(cacheCtx.cacheId(), exchId.topologyVersion())) {
            if (cacheCtx
                .discovery()
                .cacheAffinityNodes(cacheCtx.name(), topologyVersion())
                .isEmpty())
              U.quietAndWarn(log, "No server nodes found for cache client: " + cacheCtx.namex());
          }

          cacheCtx.preloader().onExchangeFutureAdded();
        }

        List<String> cachesWithoutNodes = null;

        if (exchId.isLeft()) {
          for (String name : cctx.cache().cacheNames()) {
            if (cctx.discovery().cacheAffinityNodes(name, topologyVersion()).isEmpty()) {
              if (cachesWithoutNodes == null) cachesWithoutNodes = new ArrayList<>();

              cachesWithoutNodes.add(name);

              // Fire event even if there is no client cache started.
              if (cctx.gridEvents().isRecordable(EventType.EVT_CACHE_NODES_LEFT)) {
                Event evt =
                    new CacheEvent(
                        name,
                        cctx.localNode(),
                        cctx.localNode(),
                        "All server nodes have left the cluster.",
                        EventType.EVT_CACHE_NODES_LEFT,
                        0,
                        false,
                        null,
                        null,
                        null,
                        null,
                        false,
                        null,
                        false,
                        null,
                        null,
                        null);

                cctx.gridEvents().record(evt);
              }
            }
          }
        }

        if (cachesWithoutNodes != null) {
          StringBuilder sb =
              new StringBuilder(
                  "All server nodes for the following caches have left the cluster: ");

          for (int i = 0; i < cachesWithoutNodes.size(); i++) {
            String cache = cachesWithoutNodes.get(i);

            sb.append('\'').append(cache).append('\'');

            if (i != cachesWithoutNodes.size() - 1) sb.append(", ");
          }

          U.quietAndWarn(log, sb.toString());

          U.quietAndWarn(log, "Must have server nodes for caches to operate.");
        }

        assert discoEvt != null;

        assert exchId.nodeId().equals(discoEvt.eventNode().id());

        for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
          GridClientPartitionTopology clientTop =
              cctx.exchange().clearClientTopology(cacheCtx.cacheId());

          long updSeq = clientTop == null ? -1 : clientTop.lastUpdateSequence();

          // Update before waiting for locks.
          if (!cacheCtx.isLocal())
            cacheCtx
                .topology()
                .updateTopologyVersion(exchId, this, updSeq, stopping(cacheCtx.cacheId()));
        }

        // Grab all alive remote nodes with order of equal or less than last joined node.
        rmtNodes =
            new ConcurrentLinkedQueue<>(
                CU.aliveRemoteServerNodesWithCaches(cctx, exchId.topologyVersion()));

        rmtIds = Collections.unmodifiableSet(new HashSet<>(F.nodeIds(rmtNodes)));

        for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> m : singleMsgs.entrySet())
          // If received any messages, process them.
          onReceive(m.getKey(), m.getValue());

        for (Map.Entry<UUID, GridDhtPartitionsFullMessage> m : fullMsgs.entrySet())
          // If received any messages, process them.
          onReceive(m.getKey(), m.getValue());

        AffinityTopologyVersion topVer = exchId.topologyVersion();

        for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
          if (cacheCtx.isLocal()) continue;

          // Must initialize topology after we get discovery event.
          initTopology(cacheCtx);

          cacheCtx.preloader().updateLastExchangeFuture(this);
        }

        IgniteInternalFuture<?> partReleaseFut = cctx.partitionReleaseFuture(topVer);

        // Assign to class variable so it will be included into toString() method.
        this.partReleaseFut = partReleaseFut;

        if (log.isDebugEnabled()) log.debug("Before waiting for partition release future: " + this);

        while (true) {
          try {
            partReleaseFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS);

            break;
          } catch (IgniteFutureTimeoutCheckedException ignored) {
            // Print pending transactions and locks that might have led to hang.
            dumpPendingObjects();
          }
        }

        if (log.isDebugEnabled()) log.debug("After waiting for partition release future: " + this);

        if (!F.isEmpty(reqs)) blockGateways();

        if (exchId.isLeft())
          cctx.mvcc().removeExplicitNodeLocks(exchId.nodeId(), exchId.topologyVersion());

        IgniteInternalFuture<?> locksFut = cctx.mvcc().finishLocks(exchId.topologyVersion());

        while (true) {
          try {
            locksFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS);

            break;
          } catch (IgniteFutureTimeoutCheckedException ignored) {
            U.warn(
                log,
                "Failed to wait for locks release future. "
                    + "Dumping pending objects that might be the cause: "
                    + cctx.localNodeId());

            U.warn(log, "Locked entries:");

            Map<IgniteTxKey, Collection<GridCacheMvccCandidate>> locks =
                cctx.mvcc().unfinishedLocks(exchId.topologyVersion());

            for (Map.Entry<IgniteTxKey, Collection<GridCacheMvccCandidate>> e : locks.entrySet())
              U.warn(log, "Locked entry [key=" + e.getKey() + ", mvcc=" + e.getValue() + ']');
          }
        }

        for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
          if (cacheCtx.isLocal()) continue;

          // Notify replication manager.
          GridCacheContext drCacheCtx =
              cacheCtx.isNear() ? cacheCtx.near().dht().context() : cacheCtx;

          if (drCacheCtx.isDrEnabled()) drCacheCtx.dr().beforeExchange(topVer, exchId.isLeft());

          // Partition release future is done so we can flush the write-behind store.
          cacheCtx.store().forceFlush();

          // Process queued undeploys prior to sending/spreading map.
          cacheCtx.preloader().unwindUndeploys();

          GridDhtPartitionTopology top = cacheCtx.topology();

          assert topVer.equals(top.topologyVersion())
              : "Topology version is updated only in this class instances inside single ExchangeWorker thread.";

          top.beforeExchange(this);
        }

        for (GridClientPartitionTopology top : cctx.exchange().clientTopologies()) {
          top.updateTopologyVersion(exchId, this, -1, stopping(top.cacheId()));

          top.beforeExchange(this);
        }
      } catch (IgniteInterruptedCheckedException e) {
        onDone(e);

        throw e;
      } catch (Throwable e) {
        U.error(
            log,
            "Failed to reinitialize local partitions (preloading will be stopped): " + exchId,
            e);

        onDone(e);

        if (e instanceof Error) throw (Error) e;

        return;
      }

      if (F.isEmpty(rmtIds)) {
        onDone(exchId.topologyVersion());

        return;
      }

      ready.set(true);

      initFut.onDone(true);

      if (log.isDebugEnabled()) log.debug("Initialized future: " + this);

      // If this node is not oldest.
      if (!oldestNode.get().id().equals(cctx.localNodeId())) sendPartitions();
      else {
        boolean allReceived = allReceived();

        if (allReceived && replied.compareAndSet(false, true)) {
          if (spreadPartitions()) onDone(exchId.topologyVersion());
        }
      }

      scheduleRecheck();
    } else assert false : "Skipped init future: " + this;
  }
Example #26
0
  /** {@inheritDoc} */
  @Override
  public GridFuture<GridCacheTxEx<K, V>> prepareAsync() {
    GridNearTxPrepareFuture<K, V> fut = prepFut.get();

    if (fut == null) {
      // Future must be created before any exception can be thrown.
      if (!prepFut.compareAndSet(null, fut = new GridNearTxPrepareFuture<K, V>(cctx, this)))
        return prepFut.get();
    } else
      // Prepare was called explicitly.
      return fut;

    if (!state(PREPARING)) {
      if (setRollbackOnly()) {
        if (timedOut())
          fut.onError(
              new GridCacheTxTimeoutException(
                  "Transaction timed out and was rolled back: " + this));
        else
          fut.onError(
              new GridException(
                  "Invalid transaction state for prepare [state="
                      + state()
                      + ", tx="
                      + this
                      + ']'));
      } else
        fut.onError(
            new GridCacheTxRollbackException(
                "Invalid transaction state for prepare [state=" + state() + ", tx=" + this + ']'));

      return fut;
    }

    // For pessimistic mode we don't distribute prepare request.
    if (pessimistic()) {
      try {
        userPrepare();

        if (!state(PREPARED)) {
          setRollbackOnly();

          fut.onError(
              new GridException(
                  "Invalid transaction state for commit [state=" + state() + ", tx=" + this + ']'));

          return fut;
        }

        fut.complete();

        return fut;
      } catch (GridException e) {
        fut.onError(e);

        return fut;
      }
    }

    try {
      cctx.topology().readLock();

      try {
        topologyVersion(cctx.topology().topologyVersion());

        userPrepare();
      } finally {
        cctx.topology().readUnlock();
      }

      // This will attempt to locally commit
      // EVENTUALLY CONSISTENT transactions.
      fut.onPreparedEC();

      // Make sure to add future before calling prepare.
      cctx.mvcc().addFuture(fut);

      fut.prepare();
    } catch (GridCacheTxTimeoutException e) {
      fut.onError(e);
    } catch (GridCacheTxOptimisticException e) {
      fut.onError(e);
    } catch (GridException e) {
      setRollbackOnly();

      String msg = "Failed to prepare transaction (will attempt rollback): " + this;

      log.error(msg, e);

      try {
        rollback();
      } catch (GridException e1) {
        U.error(log, "Failed to rollback transaction: " + this, e1);
      }

      fut.onError(new GridCacheTxRollbackException(msg, e));
    }

    return fut;
  }
  /** {@inheritDoc} */
  @Override
  public void unlockAll(
      Collection<? extends K> keys, GridPredicate<? super GridCacheEntry<K, V>>[] filter) {
    if (keys.isEmpty()) return;

    try {
      GridCacheVersion ver = null;

      Collection<GridRichNode> affNodes = null;

      int keyCnt = -1;

      Map<GridRichNode, GridNearUnlockRequest<K, V>> map = null;

      Collection<K> locKeys = new LinkedList<K>();

      GridCacheVersion obsoleteVer = ctx.versions().next();

      for (K key : keys) {
        while (true) {
          GridDistributedCacheEntry<K, V> entry = peekExx(key);

          if (entry == null || !ctx.isAll(entry.wrap(false), filter)) break; // While.

          try {
            GridCacheMvccCandidate<K> cand =
                entry.candidate(ctx.nodeId(), Thread.currentThread().getId());

            if (cand != null) {
              ver = cand.version();

              if (affNodes == null) {
                affNodes = CU.allNodes(ctx, cand.topologyVersion());

                keyCnt = (int) Math.ceil((double) keys.size() / affNodes.size());

                map = new HashMap<GridRichNode, GridNearUnlockRequest<K, V>>(affNodes.size());
              }

              // Send request to remove from remote nodes.
              GridRichNode primary = CU.primary0(ctx.affinity(key, affNodes));

              GridNearUnlockRequest<K, V> req = map.get(primary);

              if (req == null) {
                map.put(primary, req = new GridNearUnlockRequest<K, V>(keyCnt));

                req.version(ver);
              }

              // Remove candidate from local node first.
              GridCacheMvccCandidate<K> rmv = entry.removeLock();

              if (rmv != null) {
                if (!rmv.reentry()) {
                  if (ver != null && !ver.equals(rmv.version()))
                    throw new GridException(
                        "Failed to unlock (if keys were locked separately, "
                            + "then they need to be unlocked separately): "
                            + keys);

                  if (!primary.isLocal()) {
                    assert req != null;

                    req.addKey(entry.key(), entry.getOrMarshalKeyBytes(), ctx);
                  } else locKeys.add(key);

                  if (log.isDebugEnabled()) log.debug("Removed lock (will distribute): " + rmv);
                } else if (log.isDebugEnabled())
                  log.debug(
                      "Current thread still owns lock (or there are no other nodes)"
                          + " [lock="
                          + rmv
                          + ", curThreadId="
                          + Thread.currentThread().getId()
                          + ']');
              }

              // Try to evict near entry if it's dht-mapped locally.
              evictNearEntry(entry, obsoleteVer);
            }

            break;
          } catch (GridCacheEntryRemovedException ignore) {
            if (log.isDebugEnabled())
              log.debug("Attempted to unlock removed entry (will retry): " + entry);
          }
        }
      }

      if (ver == null) return;

      for (Map.Entry<GridRichNode, GridNearUnlockRequest<K, V>> mapping : map.entrySet()) {
        GridRichNode n = mapping.getKey();

        GridDistributedUnlockRequest<K, V> req = mapping.getValue();

        if (n.isLocal()) dht.removeLocks(ctx.nodeId(), req.version(), locKeys, true);
        else if (!req.keyBytes().isEmpty())
          // We don't wait for reply to this message.
          ctx.io().send(n, req);
      }
    } catch (GridException ex) {
      U.error(log, "Failed to unlock the lock for keys: " + keys, ex);
    }
  }
Example #28
0
    /**
     * @param entries Entries to submit.
     * @param curFut Current future.
     * @throws GridInterruptedException If interrupted.
     */
    private void submit(final List<Map.Entry<K, V>> entries, final GridFutureAdapter<Object> curFut)
        throws GridInterruptedException {
      assert entries != null;
      assert !entries.isEmpty();
      assert curFut != null;

      incrementActiveTasks();

      GridFuture<Object> fut;
      if (isLocNode) {
        fut =
            ctx.closure()
                .callLocalSafe(
                    new GridDataLoadUpdateJob<>(ctx, log, cacheName, entries, false, updater),
                    false);

        locFuts.add(fut);

        fut.listenAsync(
            new GridInClosure<GridFuture<Object>>() {
              @Override
              public void apply(GridFuture<Object> t) {
                try {
                  boolean rmv = locFuts.remove(t);

                  assert rmv;

                  curFut.onDone(t.get());
                } catch (GridException e) {
                  curFut.onDone(e);
                }
              }
            });
      } else {
        byte[] entriesBytes;

        try {
          entriesBytes = ctx.config().getMarshaller().marshal(entries);

          if (updaterBytes == null) {
            assert updater != null;

            updaterBytes = ctx.config().getMarshaller().marshal(updater);
          }

          if (topicBytes == null) topicBytes = ctx.config().getMarshaller().marshal(topic);
        } catch (GridException e) {
          U.error(log, "Failed to marshal (request will not be sent).", e);

          return;
        }

        GridDeployment dep = null;
        GridPeerDeployAware jobPda0 = null;

        if (ctx.deploy().enabled()) {
          try {
            jobPda0 = jobPda;

            assert jobPda0 != null;

            dep = ctx.deploy().deploy(jobPda0.deployClass(), jobPda0.classLoader());
          } catch (GridException e) {
            U.error(
                log,
                "Failed to deploy class (request will not be sent): " + jobPda0.deployClass(),
                e);

            return;
          }

          if (dep == null)
            U.warn(log, "Failed to deploy class (request will be sent): " + jobPda0.deployClass());
        }

        long reqId = idGen.incrementAndGet();

        fut = curFut;

        reqs.put(reqId, (GridFutureAdapter<Object>) fut);

        GridDataLoadRequest<Object, Object> req =
            new GridDataLoadRequest<>(
                reqId,
                topicBytes,
                cacheName,
                updaterBytes,
                entriesBytes,
                true,
                dep != null ? dep.deployMode() : null,
                dep != null ? jobPda0.deployClass().getName() : null,
                dep != null ? dep.userVersion() : null,
                dep != null ? dep.participants() : null,
                dep != null ? dep.classLoaderId() : null,
                dep == null);

        try {
          ctx.io().send(node, TOPIC_DATALOAD, req, PUBLIC_POOL);

          if (log.isDebugEnabled())
            log.debug("Sent request to node [nodeId=" + node.id() + ", req=" + req + ']');
        } catch (GridException e) {
          if (ctx.discovery().alive(node) && ctx.discovery().pingNode(node.id()))
            ((GridFutureAdapter<Object>) fut).onDone(e);
          else
            ((GridFutureAdapter<Object>) fut)
                .onDone(
                    new GridTopologyException(
                        "Failed to send " + "request (node has left): " + node.id()));
        }
      }
    }
  /**
   * Process HTTP request.
   *
   * @param act Action.
   * @param req Http request.
   * @param res Http response.
   */
  private void processRequest(String act, HttpServletRequest req, HttpServletResponse res) {
    res.setContentType("application/json");
    res.setCharacterEncoding("UTF-8");

    GridRestCommand cmd = command(req);

    if (cmd == null) {
      res.setStatus(HttpServletResponse.SC_BAD_REQUEST);

      return;
    }

    if (!authChecker.apply(req.getHeader("X-Signature"))) {
      res.setStatus(HttpServletResponse.SC_UNAUTHORIZED);

      return;
    }

    GridRestResponse cmdRes;

    Map<String, Object> params = parameters(req);

    try {
      GridRestRequest cmdReq = createRequest(cmd, params, req);

      if (log.isDebugEnabled()) log.debug("Initialized command request: " + cmdReq);

      cmdRes = hnd.handle(cmdReq);

      if (cmdRes == null)
        throw new IllegalStateException("Received null result from handler: " + hnd);

      byte[] sesTok = cmdRes.sessionTokenBytes();

      if (sesTok != null) cmdRes.setSessionToken(U.byteArray2HexString(sesTok));

      res.setStatus(HttpServletResponse.SC_OK);
    } catch (Exception e) {
      res.setStatus(HttpServletResponse.SC_OK);

      U.error(log, "Failed to process HTTP request [action=" + act + ", req=" + req + ']', e);

      cmdRes = new GridRestResponse(STATUS_FAILED, e.getMessage());
    } catch (Throwable e) {
      U.error(log, "Failed to process HTTP request [action=" + act + ", req=" + req + ']', e);

      throw e;
    }

    JsonConfig cfg = new GridJettyJsonConfig();

    // Workaround for not needed transformation of string into JSON object.
    if (cmdRes.getResponse() instanceof String)
      cfg.registerJsonValueProcessor(cmdRes.getClass(), "response", SKIP_STR_VAL_PROC);

    if (cmdRes.getResponse() instanceof GridClientTaskResultBean
        && ((GridClientTaskResultBean) cmdRes.getResponse()).getResult() instanceof String)
      cfg.registerJsonValueProcessor(cmdRes.getResponse().getClass(), "result", SKIP_STR_VAL_PROC);

    JSON json;

    try {
      json = JSONSerializer.toJSON(cmdRes, cfg);
    } catch (JSONException e) {
      U.error(log, "Failed to convert response to JSON: " + cmdRes, e);

      json = JSONSerializer.toJSON(new GridRestResponse(STATUS_FAILED, e.getMessage()), cfg);
    }

    try {
      if (log.isDebugEnabled())
        log.debug("Parsed command response into JSON object: " + json.toString(2));

      res.getWriter().write(json.toString());

      if (log.isDebugEnabled())
        log.debug(
            "Processed HTTP request [action=" + act + ", jsonRes=" + cmdRes + ", req=" + req + ']');
    } catch (IOException e) {
      U.error(log, "Failed to send HTTP response: " + json.toString(2), e);
    }
  }
  /**
   * Removes locks regardless of whether they are owned or not for given version and keys.
   *
   * @param ver Lock version.
   * @param keys Keys.
   */
  @SuppressWarnings({"unchecked"})
  public void removeLocks(GridCacheVersion ver, Collection<? extends K> keys) {
    if (keys.isEmpty()) return;

    try {
      Collection<GridRichNode> affNodes = null;

      int keyCnt = -1;

      Map<GridNode, GridNearUnlockRequest<K, V>> map = null;

      for (K key : keys) {
        // Send request to remove from remote nodes.
        GridNearUnlockRequest<K, V> req = null;

        while (true) {
          GridDistributedCacheEntry<K, V> entry = peekExx(key);

          try {
            if (entry != null) {
              GridCacheMvccCandidate<K> cand = entry.candidate(ver);

              if (cand != null) {
                if (affNodes == null) {
                  affNodes = CU.allNodes(ctx, cand.topologyVersion());

                  keyCnt = (int) Math.ceil((double) keys.size() / affNodes.size());

                  map = new HashMap<GridNode, GridNearUnlockRequest<K, V>>(affNodes.size());
                }

                GridRichNode primary = CU.primary0(ctx.affinity(key, affNodes));

                if (!primary.isLocal()) {
                  req = map.get(primary);

                  if (req == null) {
                    map.put(primary, req = new GridNearUnlockRequest<K, V>(keyCnt));

                    req.version(ver);
                  }
                }

                // Remove candidate from local node first.
                if (entry.removeLock(cand.version())) {
                  if (primary.isLocal()) {
                    dht.removeLocks(primary.id(), ver, F.asList(key), true);

                    assert req == null;

                    continue;
                  }

                  req.addKey(entry.key(), entry.getOrMarshalKeyBytes(), ctx);
                }
              }
            }

            break;
          } catch (GridCacheEntryRemovedException ignored) {
            if (log.isDebugEnabled())
              log.debug(
                  "Attempted to remove lock from removed entry (will retry) [rmvVer="
                      + ver
                      + ", entry="
                      + entry
                      + ']');
          }
        }
      }

      if (map == null || map.isEmpty()) return;

      Collection<GridCacheVersion> committed = ctx.tm().committedVersions(ver);
      Collection<GridCacheVersion> rolledback = ctx.tm().rolledbackVersions(ver);

      for (Map.Entry<GridNode, GridNearUnlockRequest<K, V>> mapping : map.entrySet()) {
        GridNode n = mapping.getKey();

        GridDistributedUnlockRequest<K, V> req = mapping.getValue();

        if (!req.keyBytes().isEmpty()) {
          req.completedVersions(committed, rolledback);

          // We don't wait for reply to this message.
          ctx.io().send(n, req);
        }
      }
    } catch (GridException ex) {
      U.error(log, "Failed to unlock the lock for keys: " + keys, ex);
    }
  }