/** @param e Error. */
  void onError(Throwable e) {
    tx.commitError(e);

    if (err.compareAndSet(null, e)) {
      boolean marked = tx.setRollbackOnly();

      if (e instanceof GridCacheTxRollbackException) {
        if (marked) {
          try {
            tx.rollback();
          } catch (GridException ex) {
            U.error(log, "Failed to automatically rollback transaction: " + tx, ex);
          }
        }
      } else if (tx.implicit()
          && tx.isSystemInvalidate()) { // Finish implicit transaction on heuristic error.
        try {
          tx.close();
        } catch (GridException ex) {
          U.error(log, "Failed to invalidate transaction: " + tx, ex);
        }
      }

      onComplete();
    }
  }
  /**
   * Processes unlock request.
   *
   * @param nodeId Sender node ID.
   * @param req Unlock request.
   */
  @SuppressWarnings({"unchecked"})
  private void processUnlockRequest(UUID nodeId, GridDistributedUnlockRequest req) {
    assert nodeId != null;

    try {
      ClassLoader ldr = ctx.deploy().globalLoader();
      List<byte[]> keys = req.keyBytes();

      for (byte[] keyBytes : keys) {
        K key = (K) U.unmarshal(ctx.marshaller(), new ByteArrayInputStream(keyBytes), ldr);

        while (true) {
          boolean created = false;

          GridDistributedCacheEntry<K, V> entry = peekexx(key);

          if (entry == null) {
            entry = entryexx(key);

            created = true;
          }

          try {
            entry.doneRemote(
                req.version(), req.version(), req.committedVersions(), req.rolledbackVersions());

            // Note that we don't reorder completed versions here,
            // as there is no point to reorder relative to the version
            // we are about to remove.
            if (entry.removeLock(req.version())) {
              if (log.isDebugEnabled())
                log.debug("Removed lock [lockId=" + req.version() + ", key=" + key + ']');

              if (created && entry.markObsolete(req.version())) removeIfObsolete(entry.key());
            } else if (log.isDebugEnabled())
              log.debug(
                  "Received unlock request for unknown candidate "
                      + "(added to cancelled locks set): "
                      + req);

            break;
          } catch (GridCacheEntryRemovedException ignored) {
            if (log.isDebugEnabled())
              log.debug(
                  "Received remove lock request for removed entry (will retry) [entry="
                      + entry
                      + ", req="
                      + req
                      + ']');
          }
        }
      }
    } catch (GridException e) {
      U.error(log, "Failed to unmarshal unlock key (unlock will not be performed): " + req, e);
    }
  }
  /** Initializes future. */
  @SuppressWarnings({"unchecked"})
  void finish() {
    if (mappings != null) {
      finish(mappings.values());

      markInitialized();

      if (!isSync()) {
        boolean complete = true;

        for (GridFuture<?> f : pending())
          // Mini-future in non-sync mode gets done when message gets sent.
          if (isMini(f) && !f.isDone()) complete = false;

        if (complete) onComplete();
      }
    } else {
      assert !commit;

      try {
        tx.rollback();
      } catch (GridException e) {
        U.error(log, "Failed to rollback empty transaction: " + tx, e);
      }

      markInitialized();
    }
  }
  /** {@inheritDoc} */
  @Override
  public boolean onDone(GridCacheTx tx, Throwable err) {
    if ((initialized() || err != null) && super.onDone(tx, err)) {
      if (error() instanceof GridCacheTxHeuristicException) {
        long topVer = this.tx.topologyVersion();

        for (GridCacheTxEntry<K, V> e : this.tx.writeMap().values()) {
          try {
            if (e.op() != NOOP && !cctx.affinity().localNode(e.key(), topVer)) {
              GridCacheEntryEx<K, V> cacheEntry = cctx.cache().peekEx(e.key());

              if (cacheEntry != null) cacheEntry.invalidate(null, this.tx.xidVersion());
            }
          } catch (Throwable t) {
            U.error(log, "Failed to invalidate entry.", t);

            if (t instanceof Error) throw (Error) t;
          }
        }
      }

      // Don't forget to clean up.
      cctx.mvcc().removeFuture(this);

      return true;
    }

    return false;
  }
  /**
   * @param nodeId Sender node ID.
   * @param msg Response to prepare request.
   */
  private void processPrepareResponse(UUID nodeId, GridDistributedTxPrepareResponse<K, V> msg) {
    assert nodeId != null;
    assert msg != null;

    GridReplicatedTxLocal<K, V> tx = ctx.tm().tx(msg.version());

    if (tx == null) {
      if (log.isDebugEnabled())
        log.debug(
            "Received prepare response for non-existing transaction [senderNodeId="
                + nodeId
                + ", res="
                + msg
                + ']');

      return;
    }

    GridReplicatedTxPrepareFuture<K, V> future = (GridReplicatedTxPrepareFuture<K, V>) tx.future();

    if (future != null) future.onResult(nodeId, msg);
    else
      U.error(
          log,
          "Received prepare response for transaction with no future [res="
              + msg
              + ", tx="
              + tx
              + ']');
  }
  /** {@inheritDoc} */
  @Override
  public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
    super.readExternal(in);

    topVer = in.readLong();
    implicitTx = in.readBoolean();
    implicitSingleTx = in.readBoolean();
    syncCommit = in.readBoolean();
    syncRollback = in.readBoolean();
    filterBytes = (byte[][]) in.readObject();

    dhtVers = U.readArray(in, CU.versionArrayFactory());

    miniId = U.readGridUuid(in);

    assert miniId != null;
  }
  /**
   * @param nodeId Sender node ID.
   * @param msg Finish transaction response.
   */
  private void processFinishResponse(UUID nodeId, GridDistributedTxFinishResponse<K, V> msg) {
    GridReplicatedTxCommitFuture<K, V> fut =
        (GridReplicatedTxCommitFuture<K, V>)
            ctx.mvcc().<GridCacheTx>future(msg.xid().id(), msg.futureId());

    if (fut != null) fut.onResult(nodeId);
    else U.warn(log, "Received finish response for unknown transaction: " + msg);
  }
  /** {@inheritDoc} */
  @Override
  public void writeExternal(ObjectOutput out) throws IOException {
    super.writeExternal(out);

    out.writeLong(topVer);
    out.writeBoolean(implicitTx);
    out.writeBoolean(implicitSingleTx);
    out.writeBoolean(syncCommit);
    out.writeBoolean(syncRollback);
    out.writeObject(filterBytes);

    U.writeArray(out, dhtVers);

    assert miniId != null;

    U.writeGridUuid(out, miniId);
  }
Ejemplo n.º 9
0
  /**
   * @param nodeId Sender.
   * @param res Result.
   */
  void onResult(UUID nodeId, GridNearLockResponse<K, V> res) {
    if (!isDone()) {
      if (log.isDebugEnabled())
        log.debug(
            "Received lock response from node [nodeId="
                + nodeId
                + ", res="
                + res
                + ", fut="
                + this
                + ']');

      for (GridFuture<Boolean> fut : pending()) {
        if (isMini(fut)) {
          MiniFuture mini = (MiniFuture) fut;

          if (mini.futureId().equals(res.miniId())) {
            assert mini.node().id().equals(nodeId);

            if (log.isDebugEnabled())
              log.debug("Found mini future for response [mini=" + mini + ", res=" + res + ']');

            mini.onResult(res);

            if (log.isDebugEnabled())
              log.debug(
                  "Future after processed lock response [fut="
                      + this
                      + ", mini="
                      + mini
                      + ", res="
                      + res
                      + ']');

            return;
          }
        }
      }

      U.warn(
          log,
          "Failed to find mini future for response (perhaps due to stale message) [res="
              + res
              + ", fut="
              + this
              + ']');
    } else if (log.isDebugEnabled())
      log.debug(
          "Ignoring lock response from node (future is done) [nodeId="
              + nodeId
              + ", res="
              + res
              + ", fut="
              + this
              + ']');
  }
  /**
   * Removes locks regardless of whether they are owned or not for given version and keys.
   *
   * @param ver Lock version.
   * @param keys Keys.
   */
  @SuppressWarnings({"unchecked"})
  public void removeLocks(GridCacheVersion ver, Collection<? extends K> keys) {
    if (keys.isEmpty()) return;

    Collection<GridRichNode> nodes = ctx.remoteNodes(keys);

    try {
      // Send request to remove from remote nodes.
      GridDistributedUnlockRequest<K, V> req = new GridDistributedUnlockRequest<K, V>(keys.size());

      req.version(ver);

      for (K key : keys) {
        while (true) {
          GridDistributedCacheEntry<K, V> entry = peekexx(key);

          try {
            if (entry != null) {
              GridCacheMvccCandidate<K> cand = entry.candidate(ver);

              if (cand != null) {
                // Remove candidate from local node first.
                if (entry.removeLock(cand.version())) {
                  // If there is only local node in this lock's topology,
                  // then there is no reason to distribute the request.
                  if (nodes.isEmpty()) continue;

                  req.addKey(entry.key(), entry.getOrMarshalKeyBytes(), ctx);
                }
              }
            }

            break;
          } catch (GridCacheEntryRemovedException ignored) {
            if (log.isDebugEnabled())
              log.debug(
                  "Attempted to remove lock from removed entry (will retry) [rmvVer="
                      + ver
                      + ", entry="
                      + entry
                      + ']');
          }
        }
      }

      if (nodes.isEmpty()) return;

      req.completedVersions(ctx.tm().committedVersions(ver), ctx.tm().rolledbackVersions(ver));

      if (!req.keyBytes().isEmpty())
        // We don't wait for reply to this message.
        ctx.io().safeSend(nodes, req, null);
    } catch (GridException ex) {
      U.error(log, "Failed to unlock the lock for keys: " + keys, ex);
    }
  }
  /**
   * @param cctx Context.
   * @param tx Transaction.
   * @param commit Commit flag.
   */
  public GridNearTxFinishFuture(
      GridCacheContext<K, V> cctx, GridNearTxLocal<K, V> tx, boolean commit) {
    super(cctx.kernalContext(), F.<GridCacheTx>identityReducer(tx));

    assert cctx != null;

    this.cctx = cctx;
    this.tx = tx;
    this.commit = commit;

    mappings = tx.mappings();

    futId = GridUuid.randomUuid();

    log = U.logger(ctx, logRef, GridNearTxFinishFuture.class);
  }
  /**
   * Processes lock response.
   *
   * @param nodeId Sender node ID.
   * @param res Lock response.
   */
  private void processLockResponse(UUID nodeId, GridDistributedLockResponse<K, V> res) {
    GridReplicatedLockFuture<K, V> fut = futurex(res.lockId(), res.futureId());

    if (fut == null) {
      U.warn(log, "Received lock response for non-existing future (will ignore): " + res);
    } else {
      fut.onResult(nodeId, res);

      if (fut.isDone()) {
        ctx.mvcc().removeFuture(fut);

        if (log.isDebugEnabled())
          log.debug("Received all replies for future (future was removed): " + fut);
      }
    }
  }
Ejemplo n.º 13
0
    /** @param e Error. */
    void onResult(Throwable e) {
      if (rcvRes.compareAndSet(false, true)) {
        if (log.isDebugEnabled())
          log.debug("Failed to get future result [fut=" + this + ", err=" + e + ']');

        // Fail.
        onDone(e);
      } else
        U.warn(
            log,
            "Received error after another result has been processed [fut="
                + GridNearLockFuture.this
                + ", mini="
                + this
                + ']',
            e);
    }
  /** @param e Error. */
  void onError(Throwable e) {
    tx.commitError(e);

    if (err.compareAndSet(null, e)) {
      boolean marked = tx.setRollbackOnly();

      if (e instanceof GridCacheTxRollbackException)
        if (marked) {
          try {
            tx.rollback();
          } catch (GridException ex) {
            U.error(log, "Failed to automatically rollback transaction: " + tx, ex);
          }
        }

      onComplete();
    }
  }
Ejemplo n.º 15
0
  /**
   * @param cctx Registry.
   * @param keys Keys to lock.
   * @param tx Transaction.
   * @param read Read flag.
   * @param retval Flag to return value or not.
   * @param timeout Lock acquisition timeout.
   * @param filter Filter.
   */
  public GridNearLockFuture(
      GridCacheContext<K, V> cctx,
      Collection<? extends K> keys,
      @Nullable GridNearTxLocal<K, V> tx,
      boolean read,
      boolean retval,
      long timeout,
      GridPredicate<GridCacheEntry<K, V>>[] filter) {
    super(cctx.kernalContext(), CU.boolReducer());
    assert cctx != null;
    assert keys != null;

    this.cctx = cctx;
    this.keys = keys;
    this.tx = tx;
    this.read = read;
    this.retval = retval;
    this.timeout = timeout;
    this.filter = filter;

    threadId = tx == null ? Thread.currentThread().getId() : tx.threadId();

    lockVer = tx != null ? tx.xidVersion() : cctx.versions().next();

    futId = GridUuid.randomUuid();

    entries = new ArrayList<>(keys.size());

    log = U.logger(ctx, logRef, GridNearLockFuture.class);

    if (timeout > 0) {
      timeoutObj = new LockTimeoutObject();

      cctx.time().addTimeoutObject(timeoutObj);
    }

    valMap = new ConcurrentHashMap8<>(keys.size(), 1f);
  }
Ejemplo n.º 16
0
  /** {@inheritDoc} */
  @SuppressWarnings({"CatchGenericClass", "ThrowableInstanceNeverThrown"})
  @Override
  public void finish(boolean commit) throws GridException {
    if (log.isDebugEnabled())
      log.debug("Finishing near local tx [tx=" + this + ", commit=" + commit + "]");

    if (commit) {
      if (!state(COMMITTING)) {
        GridCacheTxState state = state();

        if (state != COMMITTING && state != COMMITTED)
          throw new GridException(
              "Invalid transaction state for commit [state=" + state() + ", tx=" + this + ']');
        else {
          if (log.isDebugEnabled())
            log.debug(
                "Invalid transaction state for commit (another thread is committing): " + this);

          return;
        }
      }
    } else {
      if (!state(ROLLING_BACK)) {
        if (log.isDebugEnabled())
          log.debug(
              "Invalid transaction state for rollback [state=" + state() + ", tx=" + this + ']');

        return;
      }
    }

    GridException err = null;

    // Commit to DB first. This way if there is a failure, transaction
    // won't be committed.
    try {
      if (commit && !isRollbackOnly()) userCommit();
      else userRollback();
    } catch (GridException e) {
      err = e;

      commit = false;

      // If heuristic error.
      if (!isRollbackOnly()) {
        invalidate = true;

        U.warn(
            log,
            "Set transaction invalidation flag to true due to error [tx="
                + this
                + ", err="
                + err
                + ']');
      }
    }

    if (err != null) {
      state(UNKNOWN);

      throw err;
    } else {
      if (!state(commit ? COMMITTED : ROLLED_BACK)) {
        state(UNKNOWN);

        throw new GridException("Invalid transaction state for commit or rollback: " + this);
      }
    }
  }
  /**
   * Removes locks regardless of whether they are owned or not for given version and keys.
   *
   * @param ver Lock version.
   * @param keys Keys.
   */
  @SuppressWarnings({"unchecked"})
  public void removeLocks(GridCacheVersion ver, Collection<? extends K> keys) {
    if (keys.isEmpty()) return;

    try {
      Collection<GridRichNode> affNodes = null;

      int keyCnt = -1;

      Map<GridNode, GridNearUnlockRequest<K, V>> map = null;

      for (K key : keys) {
        // Send request to remove from remote nodes.
        GridNearUnlockRequest<K, V> req = null;

        while (true) {
          GridDistributedCacheEntry<K, V> entry = peekExx(key);

          try {
            if (entry != null) {
              GridCacheMvccCandidate<K> cand = entry.candidate(ver);

              if (cand != null) {
                if (affNodes == null) {
                  affNodes = CU.allNodes(ctx, cand.topologyVersion());

                  keyCnt = (int) Math.ceil((double) keys.size() / affNodes.size());

                  map = new HashMap<GridNode, GridNearUnlockRequest<K, V>>(affNodes.size());
                }

                GridRichNode primary = CU.primary0(ctx.affinity(key, affNodes));

                if (!primary.isLocal()) {
                  req = map.get(primary);

                  if (req == null) {
                    map.put(primary, req = new GridNearUnlockRequest<K, V>(keyCnt));

                    req.version(ver);
                  }
                }

                // Remove candidate from local node first.
                if (entry.removeLock(cand.version())) {
                  if (primary.isLocal()) {
                    dht.removeLocks(primary.id(), ver, F.asList(key), true);

                    assert req == null;

                    continue;
                  }

                  req.addKey(entry.key(), entry.getOrMarshalKeyBytes(), ctx);
                }
              }
            }

            break;
          } catch (GridCacheEntryRemovedException ignored) {
            if (log.isDebugEnabled())
              log.debug(
                  "Attempted to remove lock from removed entry (will retry) [rmvVer="
                      + ver
                      + ", entry="
                      + entry
                      + ']');
          }
        }
      }

      if (map == null || map.isEmpty()) return;

      Collection<GridCacheVersion> committed = ctx.tm().committedVersions(ver);
      Collection<GridCacheVersion> rolledback = ctx.tm().rolledbackVersions(ver);

      for (Map.Entry<GridNode, GridNearUnlockRequest<K, V>> mapping : map.entrySet()) {
        GridNode n = mapping.getKey();

        GridDistributedUnlockRequest<K, V> req = mapping.getValue();

        if (!req.keyBytes().isEmpty()) {
          req.completedVersions(committed, rolledback);

          // We don't wait for reply to this message.
          ctx.io().send(n, req);
        }
      }
    } catch (GridException ex) {
      U.error(log, "Failed to unlock the lock for keys: " + keys, ex);
    }
  }
  /** {@inheritDoc} */
  @Override
  public void unlockAll(
      Collection<? extends K> keys, GridPredicate<? super GridCacheEntry<K, V>>[] filter) {
    if (keys.isEmpty()) return;

    try {
      GridCacheVersion ver = null;

      Collection<GridRichNode> affNodes = null;

      int keyCnt = -1;

      Map<GridRichNode, GridNearUnlockRequest<K, V>> map = null;

      Collection<K> locKeys = new LinkedList<K>();

      GridCacheVersion obsoleteVer = ctx.versions().next();

      for (K key : keys) {
        while (true) {
          GridDistributedCacheEntry<K, V> entry = peekExx(key);

          if (entry == null || !ctx.isAll(entry.wrap(false), filter)) break; // While.

          try {
            GridCacheMvccCandidate<K> cand =
                entry.candidate(ctx.nodeId(), Thread.currentThread().getId());

            if (cand != null) {
              ver = cand.version();

              if (affNodes == null) {
                affNodes = CU.allNodes(ctx, cand.topologyVersion());

                keyCnt = (int) Math.ceil((double) keys.size() / affNodes.size());

                map = new HashMap<GridRichNode, GridNearUnlockRequest<K, V>>(affNodes.size());
              }

              // Send request to remove from remote nodes.
              GridRichNode primary = CU.primary0(ctx.affinity(key, affNodes));

              GridNearUnlockRequest<K, V> req = map.get(primary);

              if (req == null) {
                map.put(primary, req = new GridNearUnlockRequest<K, V>(keyCnt));

                req.version(ver);
              }

              // Remove candidate from local node first.
              GridCacheMvccCandidate<K> rmv = entry.removeLock();

              if (rmv != null) {
                if (!rmv.reentry()) {
                  if (ver != null && !ver.equals(rmv.version()))
                    throw new GridException(
                        "Failed to unlock (if keys were locked separately, "
                            + "then they need to be unlocked separately): "
                            + keys);

                  if (!primary.isLocal()) {
                    assert req != null;

                    req.addKey(entry.key(), entry.getOrMarshalKeyBytes(), ctx);
                  } else locKeys.add(key);

                  if (log.isDebugEnabled()) log.debug("Removed lock (will distribute): " + rmv);
                } else if (log.isDebugEnabled())
                  log.debug(
                      "Current thread still owns lock (or there are no other nodes)"
                          + " [lock="
                          + rmv
                          + ", curThreadId="
                          + Thread.currentThread().getId()
                          + ']');
              }

              // Try to evict near entry if it's dht-mapped locally.
              evictNearEntry(entry, obsoleteVer);
            }

            break;
          } catch (GridCacheEntryRemovedException ignore) {
            if (log.isDebugEnabled())
              log.debug("Attempted to unlock removed entry (will retry): " + entry);
          }
        }
      }

      if (ver == null) return;

      for (Map.Entry<GridRichNode, GridNearUnlockRequest<K, V>> mapping : map.entrySet()) {
        GridRichNode n = mapping.getKey();

        GridDistributedUnlockRequest<K, V> req = mapping.getValue();

        if (n.isLocal()) dht.removeLocks(ctx.nodeId(), req.version(), locKeys, true);
        else if (!req.keyBytes().isEmpty())
          // We don't wait for reply to this message.
          ctx.io().send(n, req);
      }
    } catch (GridException ex) {
      U.error(log, "Failed to unlock the lock for keys: " + keys, ex);
    }
  }
  /**
   * @param nodeId Primary node ID.
   * @param req Request.
   * @return Remote transaction.
   * @throws GridException If failed.
   * @throws GridDistributedLockCancelledException If lock has been cancelled.
   */
  @SuppressWarnings({"RedundantTypeArguments"})
  @Nullable
  public GridNearTxRemote<K, V> startRemoteTxForFinish(
      UUID nodeId, GridDhtTxFinishRequest<K, V> req)
      throws GridException, GridDistributedLockCancelledException {
    GridNearTxRemote<K, V> tx = null;

    ClassLoader ldr = ctx.deploy().globalLoader();

    if (ldr != null) {
      for (GridCacheTxEntry<K, V> txEntry : req.nearWrites()) {
        GridDistributedCacheEntry<K, V> entry = null;

        while (true) {
          try {
            entry = peekExx(txEntry.key());

            if (entry != null) {
              entry.keyBytes(txEntry.keyBytes());

              // Handle implicit locks for pessimistic transactions.
              tx = ctx.tm().tx(req.version());

              if (tx != null) {
                if (tx.local()) return null;

                if (tx.markFinalizing()) tx.addWrite(txEntry.key(), txEntry.keyBytes());
                else return null;
              } else {
                tx =
                    new GridNearTxRemote<K, V>(
                        nodeId,
                        req.nearNodeId(),
                        req.threadId(),
                        req.version(),
                        null,
                        PESSIMISTIC,
                        req.isolation(),
                        req.isInvalidate(),
                        0,
                        txEntry.key(),
                        txEntry.keyBytes(),
                        txEntry.value(),
                        txEntry.valueBytes(),
                        ctx);

                if (tx.empty()) return tx;

                tx = ctx.tm().onCreated(tx);

                if (tx == null || !ctx.tm().onStarted(tx))
                  throw new GridCacheTxRollbackException(
                      "Failed to acquire lock "
                          + "(transaction has been completed): "
                          + req.version());

                if (!tx.markFinalizing()) return null;
              }

              // Add remote candidate before reordering.
              if (txEntry.explicitVersion() == null)
                entry.addRemote(
                    req.nearNodeId(),
                    nodeId,
                    req.threadId(),
                    req.version(),
                    0,
                    tx.ec(),
                    /*tx*/ true,
                    tx.implicitSingle());

              // Remote candidates for ordered lock queuing.
              entry.addRemoteCandidates(
                  Collections.<GridCacheMvccCandidate<K>>emptyList(),
                  req.version(),
                  req.committedVersions(),
                  req.rolledbackVersions());
            }

            // Double-check in case if sender node left the grid.
            if (ctx.discovery().node(req.nearNodeId()) == null) {
              if (log.isDebugEnabled())
                log.debug("Node requesting lock left grid (lock request will be ignored): " + req);

              if (tx != null) tx.rollback();

              return null;
            }

            // Entry is legit.
            break;
          } catch (GridCacheEntryRemovedException ignored) {
            assert entry.obsoleteVersion() != null
                : "Obsolete flag not set on removed entry: " + entry;

            if (log.isDebugEnabled())
              log.debug("Received entry removed exception (will retry on renewed entry): " + entry);

            if (tx != null) {
              tx.clearEntry(entry.key());

              if (log.isDebugEnabled())
                log.debug(
                    "Cleared removed entry from remote transaction (will retry) [entry="
                        + entry
                        + ", tx="
                        + tx
                        + ']');
            }
          }
        }
      }
    } else {
      String err = "Failed to acquire deployment class loader for message: " + req;

      U.warn(log, err);

      throw new GridException(err);
    }

    return tx;
  }
  /**
   * @param nodeId Primary node ID.
   * @param req Request.
   * @return Remote transaction.
   * @throws GridException If failed.
   * @throws GridDistributedLockCancelledException If lock has been cancelled.
   */
  @SuppressWarnings({"RedundantTypeArguments"})
  @Nullable
  public GridNearTxRemote<K, V> startRemoteTx(UUID nodeId, GridDhtLockRequest<K, V> req)
      throws GridException, GridDistributedLockCancelledException {
    List<byte[]> nearKeyBytes = req.nearKeyBytes();

    GridNearTxRemote<K, V> tx = null;

    ClassLoader ldr = ctx.deploy().globalLoader();

    if (ldr != null) {
      for (int i = 0; i < nearKeyBytes.size(); i++) {
        byte[] bytes = nearKeyBytes.get(i);

        if (bytes == null) continue;

        K key = req.nearKeys().get(i);

        Collection<GridCacheMvccCandidate<K>> cands = req.candidatesByIndex(i);

        if (log.isDebugEnabled()) log.debug("Unmarshalled key: " + key);

        GridNearCacheEntry<K, V> entry = null;

        while (true) {
          try {
            entry = peekExx(key);

            if (entry != null) {
              entry.keyBytes(bytes);

              // Handle implicit locks for pessimistic transactions.
              if (req.inTx()) {
                tx = ctx.tm().tx(req.version());

                if (tx != null) tx.addWrite(key, bytes, null /*Value.*/, null /*Value bytes.*/);
                else {
                  tx =
                      new GridNearTxRemote<K, V>(
                          nodeId,
                          req.nearNodeId(),
                          req.threadId(),
                          req.version(),
                          null,
                          PESSIMISTIC,
                          req.isolation(),
                          req.isInvalidate(),
                          req.timeout(),
                          key,
                          bytes,
                          null, // Value.
                          null, // Value bytes.
                          ctx);

                  if (tx.empty()) return tx;

                  tx = ctx.tm().onCreated(tx);

                  if (tx == null || !ctx.tm().onStarted(tx))
                    throw new GridCacheTxRollbackException(
                        "Failed to acquire lock "
                            + "(transaction has been completed): "
                            + req.version());
                }
              }

              // Add remote candidate before reordering.
              entry.addRemote(
                  req.nodeId(),
                  nodeId,
                  req.threadId(),
                  req.version(),
                  req.timeout(),
                  tx != null && tx.ec(),
                  tx != null,
                  tx != null && tx.implicitSingle());

              // Remote candidates for ordered lock queuing.
              entry.addRemoteCandidates(
                  cands, req.version(), req.committedVersions(), req.rolledbackVersions());

              entry.orderOwned(req.version(), req.owned(entry.key()));
            }

            // Double-check in case if sender node left the grid.
            if (ctx.discovery().node(req.nodeId()) == null) {
              if (log.isDebugEnabled())
                log.debug("Node requesting lock left grid (lock request will be ignored): " + req);

              if (tx != null) tx.rollback();

              return null;
            }

            // Entry is legit.
            break;
          } catch (GridCacheEntryRemovedException ignored) {
            assert entry.obsoleteVersion() != null
                : "Obsolete flag not set on removed entry: " + entry;

            if (log.isDebugEnabled())
              log.debug("Received entry removed exception (will retry on renewed entry): " + entry);

            if (tx != null) {
              tx.clearEntry(entry.key());

              if (log.isDebugEnabled())
                log.debug(
                    "Cleared removed entry from remote transaction (will retry) [entry="
                        + entry
                        + ", tx="
                        + tx
                        + ']');
            }
          }
        }
      }
    } else {
      String err = "Failed to acquire deployment class loader for message: " + req;

      U.warn(log, err);

      throw new GridException(err);
    }

    return tx;
  }
  /** {@inheritDoc} */
  @SuppressWarnings({"unchecked"})
  @Override
  public void unlockAll(
      Collection<? extends K> keys, GridPredicate<? super GridCacheEntry<K, V>>[] filter) {
    if (keys == null || keys.isEmpty()) return;

    Collection<? extends GridNode> nodes = ctx.remoteNodes(keys);

    try {
      GridDistributedUnlockRequest<K, V> req = new GridDistributedUnlockRequest<K, V>(keys.size());

      for (K key : keys) {
        GridDistributedCacheEntry<K, V> entry = entryexx(key);

        if (!ctx.isAll(entry.wrap(false), filter)) continue;

        // Unlock local lock first.
        GridCacheMvccCandidate<K> rmv = entry.removeLock();

        if (rmv != null && !nodes.isEmpty()) {
          if (!rmv.reentry()) {
            req.addKey(entry.key(), entry.getOrMarshalKeyBytes(), ctx);

            // We are assuming that lock ID is the same for all keys.
            req.version(rmv.version());

            if (log.isDebugEnabled()) log.debug("Removed lock (will distribute): " + rmv);
          } else {
            if (log.isDebugEnabled())
              log.debug(
                  "Locally unlocked lock reentry without distributing to other nodes [removed="
                      + rmv
                      + ", entry="
                      + entry
                      + ']');
          }
        } else {
          if (log.isDebugEnabled())
            log.debug(
                "Current thread still owns lock (or there are no other nodes) [lock="
                    + rmv
                    + ", curThreadId="
                    + Thread.currentThread().getId()
                    + ']');
        }
      }

      // Don't proceed of no keys to unlock.
      if (req.keyBytes().isEmpty()) {
        if (log.isDebugEnabled())
          log.debug("No keys to unlock locally (was it reentry unlock?): " + keys);

        return;
      }

      // We don't wait for reply to this message. Receiving side will have
      // to make sure that unlock requests don't come before lock requests.
      ctx.io().safeSend(nodes, req, null);
    } catch (GridException e) {
      U.error(log, "Failed to unlock keys: " + keys, e);
    }
  }
Ejemplo n.º 22
0
  /**
   * @param nodeId Reader to add.
   * @param msgId Message ID.
   * @return Future for all relevant transactions that were active at the time of adding reader, or
   *     {@code null} if reader was added
   * @throws GridCacheEntryRemovedException If entry was removed.
   */
  @Nullable
  public GridFuture<Boolean> addReader(UUID nodeId, long msgId)
      throws GridCacheEntryRemovedException {
    // Don't add local node as reader.
    if (cctx.nodeId().equals(nodeId)) return null;

    GridNode node = cctx.discovery().node(nodeId);

    // If remote node has no near cache, don't add it.
    if (node == null || !U.hasNearCache(node, cctx.dht().near().name())) return null;

    // If remote node is (primary?) or back up, don't add it as a reader.
    if (U.nodeIds(cctx.affinity(partition(), CU.allNodes(cctx))).contains(nodeId)) return null;

    boolean ret = false;

    GridCacheMultiTxFuture<K, V> txFut;

    Collection<GridCacheMvccCandidate<K>> cands = null;

    synchronized (mux) {
      checkObsolete();

      txFut = this.txFut;

      ReaderId reader = readerId(nodeId);

      if (reader == null) {
        reader = new ReaderId(nodeId, msgId);

        readers = new LinkedList<ReaderId>(readers);

        readers.add(reader);

        // Seal.
        readers = Collections.unmodifiableList(readers);

        txFut = this.txFut = new GridCacheMultiTxFuture<K, V>(cctx);

        cands = localCandidates();

        ret = true;
      } else {
        long id = reader.messageId();

        if (id < msgId) reader.messageId(msgId);
      }
    }

    if (ret) {
      assert txFut != null;

      if (!F.isEmpty(cands)) {
        for (GridCacheMvccCandidate<K> c : cands) {
          GridCacheTxEx<K, V> tx = cctx.tm().<GridCacheTxEx<K, V>>tx(c.version());

          if (tx != null) {
            assert tx.local();

            txFut.addTx(tx);
          }
        }
      }

      txFut.init();

      if (!txFut.isDone()) {
        txFut.listenAsync(
            new CI1<GridFuture<?>>() {
              @Override
              public void apply(GridFuture<?> f) {
                synchronized (mux) {
                  // Release memory.
                  GridDhtCacheEntry.this.txFut = null;
                }
              }
            });
      } else
        // Release memory.
        txFut = this.txFut = null;
    }

    return txFut;
  }
  /**
   * @param nodeId Sender node ID.
   * @param req Finish transaction message.
   */
  @SuppressWarnings({"CatchGenericClass"})
  private void processFinishRequest(UUID nodeId, GridDistributedTxFinishRequest<K, V> req) {
    assert nodeId != null;
    assert req != null;

    GridReplicatedTxRemote<K, V> tx = ctx.tm().tx(req.version());

    try {
      ClassLoader ldr = ctx.deploy().globalLoader();

      if (req.commit()) {
        // If lock was acquired explicitly.
        if (tx == null) {
          // Create transaction and add entries.
          tx =
              ctx.tm()
                  .onCreated(
                      new GridReplicatedTxRemote<K, V>(
                          ldr,
                          nodeId,
                          req.threadId(),
                          req.version(),
                          req.commitVersion(),
                          PESSIMISTIC,
                          READ_COMMITTED,
                          req.isInvalidate(),
                          /*timeout */ 0,
                          /*read entries*/ null,
                          req.writes(),
                          ctx));

          if (tx == null || !ctx.tm().onStarted(tx))
            throw new GridCacheTxRollbackException(
                "Attempt to start a completed " + "transaction: " + req);
        } else {
          boolean set = tx.commitVersion(req.commitVersion());

          assert set;
        }

        Collection<GridCacheTxEntry<K, V>> writeEntries = req.writes();

        if (!F.isEmpty(writeEntries)) {
          // In OPTIMISTIC mode, we get the values at PREPARE stage.
          assert tx.concurrency() == PESSIMISTIC;

          for (GridCacheTxEntry<K, V> entry : writeEntries) {
            // Unmarshal write entries.
            entry.unmarshal(ctx, ldr);

            if (log.isDebugEnabled())
              log.debug(
                  "Unmarshalled transaction entry from pessimistic transaction [key="
                      + entry.key()
                      + ", value="
                      + entry.value()
                      + ", tx="
                      + tx
                      + ']');

            if (!tx.setWriteValue(entry))
              U.warn(
                  log,
                  "Received entry to commit that was not present in transaction [entry="
                      + entry
                      + ", tx="
                      + tx
                      + ']');
          }
        }

        // Add completed versions.
        tx.doneRemote(req.baseVersion(), req.committedVersions(), req.rolledbackVersions());

        if (tx.pessimistic()) tx.prepare();

        tx.commit();
      } else if (tx != null) {
        tx.doneRemote(req.baseVersion(), req.committedVersions(), req.rolledbackVersions());

        tx.rollback();
      }

      if (req.replyRequired()) {
        GridCacheMessage<K, V> res =
            new GridDistributedTxFinishResponse<K, V>(req.version(), req.futureId());

        try {
          ctx.io().send(nodeId, res);
        } catch (Throwable e) {
          // Double-check.
          if (ctx.discovery().node(nodeId) == null) {
            if (log.isDebugEnabled())
              log.debug(
                  "Node left while sending finish response [nodeId="
                      + nodeId
                      + ", res="
                      + res
                      + ']');
          } else
            U.error(
                log,
                "Failed to send finish response to node [nodeId=" + nodeId + ", res=" + res + ']',
                e);
        }
      }
    } catch (GridCacheTxRollbackException e) {
      if (log.isDebugEnabled())
        log.debug("Attempted to start a completed transaction (will ignore): " + e);
    } catch (Throwable e) {
      U.error(
          log,
          "Failed completing transaction [commit=" + req.commit() + ", tx=" + CU.txString(tx) + ']',
          e);

      if (tx != null) tx.rollback();
    }
  }
Ejemplo n.º 24
0
  /** {@inheritDoc} */
  @Override
  public void writeExternal(ObjectOutput out) throws IOException {
    super.writeExternal(out);

    U.writeCollection(out, nearKeyBytes);
  }
  /**
   * @param nodeId Sender node ID.
   * @param msg Prepare request.
   */
  @SuppressWarnings({"InstanceofCatchParameter"})
  private void processPrepareRequest(UUID nodeId, GridDistributedTxPrepareRequest<K, V> msg) {
    assert nodeId != null;
    assert msg != null;

    GridReplicatedTxRemote<K, V> tx = null;

    GridDistributedTxPrepareResponse<K, V> res;

    try {
      tx =
          new GridReplicatedTxRemote<K, V>(
              ctx.deploy().globalLoader(),
              nodeId,
              msg.threadId(),
              msg.version(),
              msg.commitVersion(),
              msg.concurrency(),
              msg.isolation(),
              msg.isInvalidate(),
              msg.timeout(),
              msg.reads(),
              msg.writes(),
              ctx);

      tx = ctx.tm().onCreated(tx);

      if (tx == null || !ctx.tm().onStarted(tx))
        throw new GridCacheTxRollbackException("Attempt to start a completed transaction: " + tx);

      // Prepare prior to reordering, so the pending locks added
      // in prepare phase will get properly ordered as well.
      tx.prepare();

      // Add remote candidates and reorder completed and uncompleted versions.
      tx.addRemoteCandidates(
          msg.candidatesByKey(), msg.committedVersions(), msg.rolledbackVersions());

      if (msg.concurrency() == EVENTUALLY_CONSISTENT) {
        if (log.isDebugEnabled()) log.debug("Committing transaction during remote prepare: " + tx);

        tx.commit();

        if (log.isDebugEnabled()) log.debug("Committed transaction during remote prepare: " + tx);

        // Don't send response.
        return;
      }

      res = new GridDistributedTxPrepareResponse<K, V>(msg.version());

      Map<K, Collection<GridCacheMvccCandidate<K>>> cands = tx.localCandidates();

      // Add local candidates (completed version must be set below).
      res.candidates(cands);
    } catch (GridException e) {
      if (e instanceof GridCacheTxRollbackException) {
        if (log.isDebugEnabled())
          log.debug("Transaction was rolled back before prepare completed: " + tx);
      } else if (e instanceof GridCacheTxOptimisticException) {
        if (log.isDebugEnabled())
          log.debug("Optimistic failure for remote transaction (will rollback): " + tx);
      } else {
        U.error(log, "Failed to process prepare request: " + msg, e);
      }

      if (tx != null)
        // Automatically rollback remote transactions.
        tx.rollback();

      // Don't send response.
      if (msg.concurrency() == EVENTUALLY_CONSISTENT) return;

      res = new GridDistributedTxPrepareResponse<K, V>(msg.version());

      res.error(e);
    }

    // Add completed versions.
    res.completedVersions(
        ctx.tm().committedVersions(msg.version()), ctx.tm().rolledbackVersions(msg.version()));

    assert msg.concurrency() != EVENTUALLY_CONSISTENT;

    GridNode node = ctx.discovery().node(nodeId);

    if (node != null) {
      try {
        // Reply back to sender.
        ctx.io().send(node, res);
      } catch (GridException e) {
        U.error(
            log,
            "Failed to send tx response to node (did the node leave grid?) [node="
                + node.id()
                + ", msg="
                + res
                + ']',
            e);

        if (tx != null) tx.rollback();
      }
    }
  }
Ejemplo n.º 26
0
  /** {@inheritDoc} */
  @Override
  public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
    super.readExternal(in);

    nearKeyBytes = U.readList(in);
  }
  /**
   * Processes lock request.
   *
   * @param nodeId Sender node ID.
   * @param msg Lock request.
   */
  @SuppressWarnings({"unchecked", "ThrowableInstanceNeverThrown"})
  private void processLockRequest(UUID nodeId, GridDistributedLockRequest<K, V> msg) {
    assert !nodeId.equals(locNodeId);

    List<byte[]> keys = msg.keyBytes();

    int cnt = keys.size();

    GridReplicatedTxRemote<K, V> tx = null;

    GridDistributedLockResponse res;

    ClassLoader ldr = null;

    try {
      ldr = ctx.deploy().globalLoader();

      if (ldr != null) {
        res = new GridDistributedLockResponse(msg.version(), msg.futureId(), cnt);

        for (int i = 0; i < keys.size(); i++) {
          byte[] bytes = keys.get(i);
          K key = msg.keys().get(i);

          Collection<GridCacheMvccCandidate<K>> cands = msg.candidatesByIndex(i);

          if (bytes == null) continue;

          if (log.isDebugEnabled()) log.debug("Unmarshalled key: " + key);

          GridDistributedCacheEntry<K, V> entry = null;

          while (true) {
            try {
              entry = entryexx(key);

              // Handle implicit locks for pessimistic transactions.
              if (msg.inTx()) {
                tx = ctx.tm().tx(msg.version());

                if (tx != null) {
                  if (msg.txRead()) tx.addRead(key, bytes);
                  else tx.addWrite(key, bytes);
                } else {
                  tx =
                      new GridReplicatedTxRemote<K, V>(
                          nodeId,
                          msg.threadId(),
                          msg.version(),
                          null,
                          PESSIMISTIC,
                          msg.isolation(),
                          msg.isInvalidate(),
                          msg.timeout(),
                          key,
                          bytes,
                          msg.txRead(),
                          ctx);

                  tx = ctx.tm().onCreated(tx);

                  if (tx == null || !ctx.tm().onStarted(tx))
                    throw new GridCacheTxRollbackException(
                        "Failed to acquire lock "
                            + "(transaction has been completed): "
                            + msg.version());
                }
              }

              // Add remote candidate before reordering.
              entry.addRemote(
                  msg.nodeId(),
                  null,
                  msg.threadId(),
                  msg.version(),
                  msg.timeout(),
                  tx != null && tx.ec(),
                  tx != null,
                  tx != null && tx.implicitSingle());

              // Remote candidates for ordered lock queuing.
              entry.addRemoteCandidates(
                  cands, msg.version(), msg.committedVersions(), msg.rolledbackVersions());

              // Double-check in case if sender node left the grid.
              if (ctx.discovery().node(msg.nodeId()) == null) {
                if (log.isDebugEnabled())
                  log.debug(
                      "Node requesting lock left grid (lock request will be ignored): " + msg);

                if (tx != null) tx.rollback();

                return;
              }

              res.setCandidates(
                  i,
                  entry.localCandidates(),
                  ctx.tm().committedVersions(msg.version()),
                  ctx.tm().rolledbackVersions(msg.version()));

              res.addValueBytes(
                  entry.rawGet(), msg.returnValue(i) ? entry.valueBytes(null) : null, ctx);

              // Entry is legit.
              break;
            } catch (GridCacheEntryRemovedException ignored) {
              assert entry.obsoleteVersion() != null
                  : "Obsolete flag not set on removed entry: " + entry;

              if (log.isDebugEnabled())
                log.debug(
                    "Received entry removed exception (will retry on renewed entry): " + entry);

              if (tx != null) {
                tx.clearEntry(entry.key());

                if (log.isDebugEnabled())
                  log.debug(
                      "Cleared removed entry from remote transaction (will retry) [entry="
                          + entry
                          + ", tx="
                          + tx
                          + ']');
              }
            }
          }
        }
      } else {
        String err = "Failed to acquire deployment class for message: " + msg;

        U.warn(log, err);

        res =
            new GridDistributedLockResponse(msg.version(), msg.futureId(), new GridException(err));
      }
    } catch (GridCacheTxRollbackException e) {
      if (log.isDebugEnabled())
        log.debug("Received lock request for completed transaction (will ignore): " + e);

      res = new GridDistributedLockResponse(msg.version(), msg.futureId(), e);
    } catch (GridException e) {
      String err = "Failed to unmarshal at least one of the keys for lock request message: " + msg;

      log.error(err, e);

      res =
          new GridDistributedLockResponse(msg.version(), msg.futureId(), new GridException(err, e));

      if (tx != null) tx.rollback();
    } catch (GridDistributedLockCancelledException ignored) {
      // Received lock request for cancelled lock.
      if (log.isDebugEnabled())
        log.debug("Received lock request for canceled lock (will ignore): " + msg);

      if (tx != null) tx.rollback();

      // Don't send response back.
      return;
    }

    GridNode node = ctx.discovery().node(msg.nodeId());

    boolean releaseAll = false;

    if (node != null) {
      try {
        // Reply back to sender.
        ctx.io().send(node, res);
      } catch (GridException e) {
        U.error(log, "Failed to send message to node (did the node leave grid?): " + node.id(), e);

        releaseAll = ldr != null;
      }
    }
    // If sender left grid, release all locks acquired so far.
    else releaseAll = ldr != null;

    // Release all locks because sender node left grid.
    if (releaseAll) {
      for (K key : msg.keys()) {
        while (true) {
          GridDistributedCacheEntry<K, V> entry = peekexx(key);

          try {
            if (entry != null) entry.removeExplicitNodeLocks(msg.nodeId());

            break;
          } catch (GridCacheEntryRemovedException ignore) {
            if (log.isDebugEnabled())
              log.debug(
                  "Attempted to remove lock on removed entity during failure "
                      + "of replicated lock request handling (will retry): "
                      + entry);
          }
        }
      }

      U.warn(
          log, "Sender node left grid in the midst of lock acquisition (locks will be released).");
    }
  }
Ejemplo n.º 28
0
  /** {@inheritDoc} */
  @Override
  public GridFuture<GridCacheTxEx<K, V>> prepareAsync() {
    GridNearTxPrepareFuture<K, V> fut = prepFut.get();

    if (fut == null) {
      // Future must be created before any exception can be thrown.
      if (!prepFut.compareAndSet(null, fut = new GridNearTxPrepareFuture<K, V>(cctx, this)))
        return prepFut.get();
    } else
      // Prepare was called explicitly.
      return fut;

    if (!state(PREPARING)) {
      if (setRollbackOnly()) {
        if (timedOut())
          fut.onError(
              new GridCacheTxTimeoutException(
                  "Transaction timed out and was rolled back: " + this));
        else
          fut.onError(
              new GridException(
                  "Invalid transaction state for prepare [state="
                      + state()
                      + ", tx="
                      + this
                      + ']'));
      } else
        fut.onError(
            new GridCacheTxRollbackException(
                "Invalid transaction state for prepare [state=" + state() + ", tx=" + this + ']'));

      return fut;
    }

    // For pessimistic mode we don't distribute prepare request.
    if (pessimistic()) {
      try {
        userPrepare();

        if (!state(PREPARED)) {
          setRollbackOnly();

          fut.onError(
              new GridException(
                  "Invalid transaction state for commit [state=" + state() + ", tx=" + this + ']'));

          return fut;
        }

        fut.complete();

        return fut;
      } catch (GridException e) {
        fut.onError(e);

        return fut;
      }
    }

    try {
      cctx.topology().readLock();

      try {
        topologyVersion(cctx.topology().topologyVersion());

        userPrepare();
      } finally {
        cctx.topology().readUnlock();
      }

      // This will attempt to locally commit
      // EVENTUALLY CONSISTENT transactions.
      fut.onPreparedEC();

      // Make sure to add future before calling prepare.
      cctx.mvcc().addFuture(fut);

      fut.prepare();
    } catch (GridCacheTxTimeoutException e) {
      fut.onError(e);
    } catch (GridCacheTxOptimisticException e) {
      fut.onError(e);
    } catch (GridException e) {
      setRollbackOnly();

      String msg = "Failed to prepare transaction (will attempt rollback): " + this;

      log.error(msg, e);

      try {
        rollback();
      } catch (GridException e1) {
        U.error(log, "Failed to rollback transaction: " + this, e1);
      }

      fut.onError(new GridCacheTxRollbackException(msg, e));
    }

    return fut;
  }
Ejemplo n.º 29
0
  /** {@inheritDoc} */
  @Override
  public void rollback() throws GridException {
    GridNearTxPrepareFuture<K, V> prepFut = this.prepFut.get();

    GridNearTxFinishFuture<K, V> fut = rollbackFut.get();

    if (fut == null
        && !rollbackFut.compareAndSet(
            null, fut = new GridNearTxFinishFuture<K, V>(cctx, this, false))) {
      rollbackFut.get();

      return;
    }

    try {
      cctx.mvcc().addFuture(fut);

      if (prepFut == null) {
        finish(false);

        fut.finish();
      } else {
        prepFut.listenAsync(
            new CI1<GridFuture<GridCacheTxEx<K, V>>>() {
              @Override
              public void apply(GridFuture<GridCacheTxEx<K, V>> f) {
                try {
                  // Check for errors in prepare future.
                  f.get();
                } catch (GridException e) {
                  if (log.isDebugEnabled())
                    log.debug("Got optimistic tx failure [tx=" + this + ", err=" + e + ']');
                }

                try {
                  finish(false);

                  rollbackFut.get().finish();
                } catch (GridException e) {
                  U.error(log, "Failed to gracefully rollback transaction: " + this, e);

                  rollbackFut.get().onError(e);
                }
              }
            });
      }

      // TODO: Rollback Async?
      fut.get();
    } catch (Error e) {
      U.addLastCause(e, commitErr.get());

      throw e;
    } catch (RuntimeException e) {
      U.addLastCause(e, commitErr.get());

      throw e;
    } catch (GridException e) {
      U.addLastCause(e, commitErr.get());

      throw e;
    } finally {
      cctx.tm().txContextReset();
      cctx.near().dht().context().tm().txContextReset();
    }
  }
  /** {@inheritDoc} */
  @SuppressWarnings({"unchecked", "ThrowableInstanceNeverThrown"})
  @Override
  protected GridFuture<Boolean> lockAllAsync(
      Collection<? extends K> keys,
      long timeout,
      GridCacheTxLocalEx<K, V> tx,
      boolean isInvalidate,
      boolean isRead,
      boolean retval,
      GridCacheTxIsolation isolation,
      GridPredicate<? super GridCacheEntry<K, V>>[] filter) {
    if (keys.isEmpty()) return new GridFinishedFuture<Boolean>(ctx.kernalContext(), true);

    Collection<GridRichNode> nodes = ctx.remoteNodes(keys);

    final GridReplicatedLockFuture<K, V> fut =
        new GridReplicatedLockFuture<K, V>(ctx, keys, tx, this, nodes, timeout, filter);

    GridDistributedLockRequest<K, V> req =
        new GridDistributedLockRequest<K, V>(
            locNodeId,
            Thread.currentThread().getId(),
            fut.futureId(),
            fut.version(),
            tx != null,
            isRead,
            isolation,
            isInvalidate,
            timeout,
            keys.size());

    try {
      // Must add future before redying locks.
      if (!ctx.mvcc().addFuture(fut))
        throw new IllegalStateException("Duplicate future ID: " + fut);

      boolean distribute = false;

      for (K key : keys) {
        while (true) {
          GridDistributedCacheEntry<K, V> entry = null;

          try {
            entry = entryexx(key);

            if (!ctx.isAll(entry.wrap(false), filter)) {
              if (log.isDebugEnabled())
                log.debug("Entry being locked did not pass filter (will not lock): " + entry);

              fut.onDone(false);

              return fut;
            }

            // Removed exception may be thrown here.
            GridCacheMvccCandidate<K> cand = fut.addEntry(entry);

            if (cand != null) {
              req.addKeyBytes(
                  key,
                  cand.reentry() ? null : entry.getOrMarshalKeyBytes(),
                  retval,
                  entry.localCandidates(fut.version()),
                  ctx);

              req.completedVersions(
                  ctx.tm().committedVersions(fut.version()),
                  ctx.tm().rolledbackVersions(fut.version()));

              distribute = !cand.reentry();
            } else if (fut.isDone()) return fut;

            break;
          } catch (GridCacheEntryRemovedException ignored) {
            if (log.isDebugEnabled())
              log.debug("Got removed entry in lockAsync(..) method (will retry): " + entry);
          }
        }
      }

      // If nothing to distribute at this point,
      // then all locks are reentries.
      if (!distribute) fut.complete(true);

      if (nodes.isEmpty()) fut.readyLocks();

      // No reason to send request if all locks are locally re-entered,
      // or if timeout is negative and local locks could not be acquired.
      if (fut.isDone()) return fut;

      try {
        ctx.io()
            .safeSend(
                fut.nodes(),
                req,
                new P1<GridNode>() {
                  @Override
                  public boolean apply(GridNode node) {
                    fut.onNodeLeft(node.id());

                    return !fut.isDone();
                  }
                });
      } catch (GridException e) {
        U.error(
            log,
            "Failed to send lock request to node [nodes="
                + U.toShortString(nodes)
                + ", req="
                + req
                + ']',
            e);

        fut.onError(e);
      }

      return fut;
    } catch (GridException e) {
      Throwable err = new GridException("Failed to acquire asynchronous lock for keys: " + keys, e);

      // Clean-up.
      fut.onError(err);

      ctx.mvcc().removeFuture(fut);

      return fut;
    }
  }