@Override protected boolean isProperWriter(InvocationContext ctx, FlagAffectedCommand command, Object key) { if (command.hasFlag(Flag.SKIP_OWNERSHIP_CHECK)) return true; if (loaderConfig.shared()) { if (!dm.getPrimaryLocation(key).equals(address)) { log.tracef( "Skipping cache store since the cache loader is shared " + "and the caller is not the first owner of the key %s", key); return false; } } else { if (isUsingLockDelegation && !command.hasFlag(Flag.CACHE_MODE_LOCAL)) { if (ctx.isOriginLocal() && !dm.getPrimaryLocation(key).equals(address)) { // The command will be forwarded back to the originator, and the value will be stored then // (while holding the lock on the primary owner). log.tracef( "Skipping cache store on the originator because it is not the primary owner " + "of key %s", key); return false; } } if (!dm.getWriteConsistentHash().isKeyLocalToNode(address, key)) { log.tracef("Skipping cache store since the key is not local: %s", key); return false; } } return true; }
@Override public boolean hasAffinity(K key) { DistributionManager dist = this.cache.getAdvancedCache().getDistributionManager(); if (dist != null) { DataLocality locality = dist.getLocality(key); return locality.isLocal() || locality.isUncertain(); } return true; }
@Override public Object visitPrepareCommand(TxInvocationContext ctx, PrepareCommand command) throws Throwable { Object retVal = invokeNextInterceptor(ctx, command); boolean sync = isSynchronous(ctx); if (shouldInvokeRemoteTxCommand(ctx)) { int newCacheViewId = -1; stateTransferLock.waitForStateTransferToEnd(ctx, command, newCacheViewId); if (command.isOnePhaseCommit()) flushL1Caches(ctx); // if we are one-phase, don't block on this future. Collection<Address> recipients = dm.getAffectedNodes(ctx.getAffectedKeys()); prepareOnAffectedNodes(ctx, command, recipients, sync); ((LocalTxInvocationContext) ctx).remoteLocksAcquired(recipients); } else if (isL1CacheEnabled && command.isOnePhaseCommit() && !ctx.isOriginLocal() && !ctx.getLockedKeys().isEmpty()) { // We fall into this block if we are a remote node, happen to be the primary data owner and // have locked keys. // it is still our responsibility to invalidate L1 caches in the cluster. flushL1Caches(ctx); } return retVal; }
private void sendCommitCommand( TxInvocationContext ctx, CommitCommand command, Collection<Address> preparedOn) throws TimeoutException, InterruptedException { // we only send the commit command to the nodes that Collection<Address> recipients = dm.getAffectedNodes(ctx.getAffectedKeys()); // By default, use the configured commit sync settings boolean syncCommitPhase = configuration.isSyncCommitPhase(); for (Address a : preparedOn) { if (!recipients.contains(a)) { // However if we have prepared on some nodes and are now committing on different nodes, make // sure we // force sync commit so we can respond to prepare resend requests. syncCommitPhase = true; } } Map<Address, Response> responses = rpcManager.invokeRemotely(recipients, command, syncCommitPhase, true); if (!responses.isEmpty()) { List<Address> resendTo = new LinkedList<Address>(); for (Map.Entry<Address, Response> r : responses.entrySet()) { if (needToResendPrepare(r.getValue())) resendTo.add(r.getKey()); } if (!resendTo.isEmpty()) { log.debugf( "Need to resend prepares for %s to %s", command.getGlobalTransaction(), resendTo); PrepareCommand pc = buildPrepareCommandForResend(ctx, command); rpcManager.invokeRemotely(resendTo, pc, true, true); } } }
@Override public Object visitPutMapCommand(InvocationContext ctx, PutMapCommand command) throws Throwable { Object returnValue = invokeNextInterceptor(ctx, command); if (!isStoreEnabled(command) || ctx.isInTxScope()) return returnValue; Map<Object, Object> map = command.getMap(); int count = 0; for (Object key : map.keySet()) { // In non-tx mode, a node may receive the same forwarded PutMapCommand many times - but each // time // it must write only the keys locked on the primary owner that forwarded the command if (isUsingLockDelegation && command.isForwarded() && !dm.getPrimaryLocation(key).equals(ctx.getOrigin())) continue; if (isProperWriter(ctx, command, key)) { InternalCacheEntry se = getStoredEntry(key, ctx); store.store(se); log.tracef("Stored entry %s under key %s", se, key); count++; } } if (getStatisticsEnabled()) cacheStores.getAndAdd(count); return returnValue; }
@Override public Response getResponse(CacheRpcCommand command, Object returnValue) { if (command.getCommandId() == ClusteredGetCommand.COMMAND_ID) { if (returnValue == null) return null; ClusteredGetCommand clusteredGet = (ClusteredGetCommand) command; if (distributionManager.isAffectedByRehash(clusteredGet.getKey())) return UnsureResponse.INSTANCE; return SuccessfulResponse.create(returnValue); } else if (command instanceof SingleRpcCommand) { SingleRpcCommand src = (SingleRpcCommand) command; ReplicableCommand c = src.getCommand(); byte commandId = c.getCommandId(); if (c instanceof WriteCommand) { if (returnValue == null) return null; // check if this is successful. WriteCommand wc = (WriteCommand) c; return handleWriteCommand(wc, returnValue); } else if (commandId == MapCombineCommand.COMMAND_ID || commandId == ReduceCommand.COMMAND_ID || commandId == DistributedExecuteCommand.COMMAND_ID) { // Even null values should be wrapped in this case. return SuccessfulResponse.create(returnValue); } else if (c.isReturnValueExpected()) { if (returnValue == null) return null; return SuccessfulResponse.create(returnValue); } } else if (command.isReturnValueExpected()) { return SuccessfulResponse.create(returnValue); } return null; // no unnecessary response values! }
@Override public List<Address> getOwners(Collection<Object> affectedKeys) { if (affectedKeys.isEmpty()) { return InfinispanCollections.emptyList(); } return Immutables.immutableListConvert(dm.locateAll(affectedKeys)); }
/** * Method that skips invocation if: - The store is a shared one and node storing the key is not * the 1st owner of the key or, - This is an L1 put operation. */ @Override protected boolean skipKey(Object key) { if (loaderConfig.shared()) { if (!dm.getPrimaryLocation(key).equals(address)) { log.trace( "Skipping cache store since the cache loader is shared " + "and the caller is not the first owner of the key"); return true; } } else { List<Address> addresses = dm.locate(key); if (isL1Put(addresses)) { log.trace("Skipping cache store since this is an L1 put"); return true; } } return false; }
protected <KIn> Set<KIn> filterLocalPrimaryOwner(Set<KIn> nodeLocalKeys, DistributionManager dm) { Set<KIn> selectedKeys = new HashSet<KIn>(); for (KIn key : nodeLocalKeys) { Address primaryLocation = dm != null ? dm.getPrimaryLocation(key) : cdl.getAddress(); if (primaryLocation != null && primaryLocation.equals(cdl.getAddress())) { selectedKeys.add(key); } } return selectedKeys; }
private Object realRemoteGet( InvocationContext ctx, Object key, boolean storeInL1, boolean isWrite) throws Throwable { if (trace) log.tracef("Doing a remote get for key %s", key); boolean acquireRemoteLock = false; if (ctx.isInTxScope()) { TxInvocationContext txContext = (TxInvocationContext) ctx; acquireRemoteLock = isWrite && isPessimisticCache && !txContext.getAffectedKeys().contains(key); } // attempt a remote lookup InternalCacheEntry ice = dm.retrieveFromRemoteSource(key, ctx, acquireRemoteLock); if (acquireRemoteLock) { ((TxInvocationContext) ctx).addAffectedKey(key); } if (ice != null) { if (storeInL1) { if (isL1CacheEnabled) { if (trace) log.tracef("Caching remotely retrieved entry for key %s in L1", key); // This should be fail-safe try { long lifespan = ice.getLifespan() < 0 ? configuration.getL1Lifespan() : Math.min(ice.getLifespan(), configuration.getL1Lifespan()); PutKeyValueCommand put = cf.buildPutKeyValueCommand( ice.getKey(), ice.getValue(), lifespan, -1, ctx.getFlags()); lockAndWrap(ctx, key, ice); invokeNextInterceptor(ctx, put); } catch (Exception e) { // Couldn't store in L1 for some reason. But don't fail the transaction! log.infof("Unable to store entry %s in L1 cache", key); log.debug("Inability to store in L1 caused by", e); } } else { CacheEntry ce = ctx.lookupEntry(key); if (ce == null || ce.isNull() || ce.isLockPlaceholder() || ce.getValue() == null) { if (ce != null && ce.isChanged()) { ce.setValue(ice.getValue()); } else { if (isWrite) lockAndWrap(ctx, key, ice); else ctx.putLookedUpEntry(key, ice); } } } } else { if (trace) log.tracef("Not caching remotely retrieved entry for key %s in L1", key); } return ice.getValue(); } return null; }
@Override public Object visitLockControlCommand(TxInvocationContext ctx, LockControlCommand command) throws Throwable { if (ctx.isOriginLocal()) { int newCacheViewId = -1; stateTransferLock.waitForStateTransferToEnd(ctx, command, newCacheViewId); final Collection<Address> affectedNodes = dm.getAffectedNodes(command.getKeys()); ((LocalTxInvocationContext) ctx).remoteLocksAcquired(affectedNodes); rpcManager.invokeRemotely(affectedNodes, command, true, true); } return invokeNextInterceptor(ctx, command); }
@Override public Object visitRollbackCommand(TxInvocationContext ctx, RollbackCommand command) throws Throwable { if (shouldInvokeRemoteTxCommand(ctx)) { rpcManager.invokeRemotely( dm.getAffectedNodes(ctx.getAffectedKeys()), command, configuration.isSyncRollbackPhase(), true); } return invokeNextInterceptor(ctx, command); }
@Override public <T> Map<Address, List<T>> mapKeysToNodes( DistributionManager dm, String taskId, Collection<T> keysToMap, boolean useIntermediateCompositeKey) { Map<Address, List<T>> addressToKey = new HashMap<Address, List<T>>(); for (T key : keysToMap) { Address ownerOfKey = null; if (useIntermediateCompositeKey) { ownerOfKey = dm.getPrimaryLocation(new IntermediateCompositeKey<T>(taskId, key)); } else { ownerOfKey = dm.getPrimaryLocation(key); } List<T> keysAtNode = addressToKey.get(ownerOfKey); if (keysAtNode == null) { keysAtNode = new ArrayList<T>(); addressToKey.put(ownerOfKey, keysAtNode); } keysAtNode.add(key); } return addressToKey; }
@ManagedAttribute( description = "Number of replicas for each key", displayName = "Replication Degree") public double getReplicationDegree() { if (distributionManager != null) { // distributed mode return distributionManager.getConsistentHash().getNumOwners(); } else if (rpcManager != null) { // replicated or other clustered mode return this.rpcManager.getTransport().getMembers().size(); } // local mode return 1; }
/** * This method retrieves an entry from a remote cache and optionally stores it in L1 (if L1 is * enabled). * * <p>This method only works if a) this is a locally originating invocation and b) the entry in * question is not local to the current cache instance and c) the entry is not in L1. If either of * a, b or c does not hold true, this method returns a null and doesn't do anything. * * @param ctx invocation context * @param key key to retrieve * @return value of a remote get, or null * @throws Throwable if there are problems */ private Object remoteGetAndStoreInL1(InvocationContext ctx, Object key, boolean isWrite) throws Throwable { DataLocality locality = dm.getLocality(key); if (ctx.isOriginLocal() && !locality.isLocal() && isNotInL1(key)) { return realRemoteGet(ctx, key, true, isWrite); } else { // maybe we are still rehashing as a joiner? ISPN-258 if (locality.isUncertain()) { if (trace) log.tracef( "Key %s is mapped to local node %s, but a rehash is in progress so may need to look elsewhere", key, rpcManager.getAddress()); // try a remote lookup all the same return realRemoteGet(ctx, key, false, isWrite); } else { if (trace) log.tracef( "Not doing a remote get for key %s since entry is mapped to current node (%s), or is in L1. Owners are %s", key, rpcManager.getAddress(), dm.locate(key)); } } return null; }
/** * Encodes the cases for an asyncGet operation in which it makes sense to actually perform the * operation in sync. * * @param flags * @param key * @return true if we skip the thread (performing it in sync) */ private boolean asyncSkipsThread(EnumSet<Flag> flags, K key) { boolean isSkipLoader = isSkipLoader(flags); if (!isSkipLoader) { // if we can't skip the cacheloader, we really want a thread for async. return false; } CacheMode cacheMode = config.getCacheMode(); if (!cacheMode.isDistributed()) { // in these cluster modes we won't RPC for a get, so no need to fork a thread. return true; } else if (flags != null && (flags.contains(Flag.SKIP_REMOTE_LOOKUP) || flags.contains(Flag.CACHE_MODE_LOCAL))) { // with these flags we won't RPC either return true; } // finally, we will skip the thread if the key maps to the local node return distributionManager.getLocality(key).isLocal(); }
<R> R performOperation( Function<? super S2, ? extends R> function, ResultsAccumulator<R> remoteResults, Predicate<? super R> earlyTerminatePredicate) { ConsistentHash ch = dm.getConsistentHash(); TerminalOperation<R> op = new SingleRunOperation( intermediateOperations, supplierForSegments(ch, segmentsToFilter, null), function); Object id = csm.remoteStreamOperation( getParallelDistribution(), parallel, ch, segmentsToFilter, keysToFilter, Collections.emptyMap(), includeLoader, op, remoteResults, earlyTerminatePredicate); try { R localValue = op.performOperation(); remoteResults.onCompletion(null, Collections.emptySet(), localValue); if (id != null) { try { if ((earlyTerminatePredicate == null || !earlyTerminatePredicate.test(localValue)) && !csm.awaitCompletion(id, timeout, timeoutUnit)) { throw new TimeoutException(); } } catch (InterruptedException e) { throw new CacheException(e); } } log.tracef("Finished operation for id %s", id); return remoteResults.currentValue; } finally { csm.forgetOperation(id); } }
/** * If we are within one transaction we won't do any replication as replication would only be * performed at commit time. If the operation didn't originate locally we won't do any replication * either. */ private Object handleWriteCommand( InvocationContext ctx, WriteCommand command, RecipientGenerator recipientGenerator, boolean skipRemoteGet, boolean skipL1Invalidation) throws Throwable { // see if we need to load values from remote srcs first if (ctx.isOriginLocal() && !skipRemoteGet) remoteGetBeforeWrite(ctx, command.isConditional(), recipientGenerator); boolean sync = isSynchronous(ctx); // if this is local mode then skip distributing if (isLocalModeForced(ctx)) { return invokeNextInterceptor(ctx, command); } // FIRST pass this call up the chain. Only if it succeeds (no exceptions) locally do we attempt // to distribute. Object returnValue = invokeNextInterceptor(ctx, command); if (command.isSuccessful()) { if (!ctx.isInTxScope()) { NotifyingNotifiableFuture<Object> futureToReturn = null; Future<?> invalidationFuture = null; if (ctx.isOriginLocal()) { int newCacheViewId = -1; stateTransferLock.waitForStateTransferToEnd(ctx, command, newCacheViewId); List<Address> rec = recipientGenerator.generateRecipients(); int numCallRecipients = rec == null ? 0 : rec.size(); if (trace) log.tracef("Invoking command %s on hosts %s", command, rec); boolean useFuture = ctx.isUseFutureReturnType(); if (isL1CacheEnabled && !skipL1Invalidation) // Handle the case where the put is local. If in unicast mode and this is not a data // owner, nothing happens. If in multicast mode, we this node will send the multicast if (rpcManager.getTransport().getMembers().size() > numCallRecipients) { // Command was successful, we have a number of receipients and L1 should be flushed, // so request any L1 invalidations from this node if (trace) log.tracef( "Put occuring on node, requesting L1 cache invalidation for keys %s. Other data owners are %s", command.getAffectedKeys(), dm.getAffectedNodes(command.getAffectedKeys())); if (useFuture) { futureToReturn = l1Manager.flushCache( recipientGenerator.getKeys(), returnValue, ctx.getOrigin(), !(command instanceof RemoveCommand)); } else { invalidationFuture = l1Manager.flushCacheWithSimpleFuture( recipientGenerator.getKeys(), returnValue, ctx.getOrigin(), !(command instanceof RemoveCommand)); } } else { if (trace) log.tracef("Not performing invalidation! numCallRecipients=%s", numCallRecipients); } if (!isSingleOwnerAndLocal(recipientGenerator)) { if (useFuture) { if (futureToReturn == null) futureToReturn = new NotifyingFutureImpl(returnValue); rpcManager.invokeRemotelyInFuture(rec, command, futureToReturn); return futureToReturn; } else { rpcManager.invokeRemotely(rec, command, sync); } } else if (useFuture && futureToReturn != null) { return futureToReturn; } if (invalidationFuture != null && sync) { invalidationFuture.get(); // wait for the inval command to complete if (trace) log.tracef("Finished invalidating keys %s ", recipientGenerator.getKeys()); } } else { // Piggyback remote puts and cause L1 invalidations if (isL1CacheEnabled && !skipL1Invalidation) { // Command was successful and L1 should be flushed, so request any L1 invalidations from // this node if (trace) log.tracef( "Put occuring on node, requesting cache invalidation for keys %s. Origin of command is remote", command.getAffectedKeys()); // If this is a remove command, then don't pass in the origin - since the entru would be // removed from the origin's L1 cache. invalidationFuture = l1Manager.flushCacheWithSimpleFuture( recipientGenerator.getKeys(), returnValue, ctx.getOrigin(), !(command instanceof RemoveCommand)); if (sync) { invalidationFuture.get(); // wait for the inval command to complete if (trace) log.tracef("Finished invalidating keys %s ", recipientGenerator.getKeys()); } } } } } return returnValue; }
@Override public boolean localNodeIsOwner(Object key) { return dm.getLocality(key).isLocal(); }
<R> R performOperationRehashAware( Function<? super S2, ? extends R> function, boolean retryOnRehash, ResultsAccumulator<R> remoteResults, Predicate<? super R> earlyTerminatePredicate) { Set<Integer> segmentsToProcess = segmentsToFilter; TerminalOperation<R> op; do { ConsistentHash ch = dm.getReadConsistentHash(); if (retryOnRehash) { op = new SegmentRetryingOperation( intermediateOperations, supplierForSegments(ch, segmentsToProcess, null), function); } else { op = new SingleRunOperation( intermediateOperations, supplierForSegments(ch, segmentsToProcess, null), function); } Object id = csm.remoteStreamOperationRehashAware( getParallelDistribution(), parallel, ch, segmentsToProcess, keysToFilter, Collections.emptyMap(), includeLoader, op, remoteResults, earlyTerminatePredicate); try { R localValue; boolean localRun = ch.getMembers().contains(localAddress); if (localRun) { localValue = op.performOperation(); // TODO: we can do this more efficiently - since we drop all results locally if (dm.getReadConsistentHash().equals(ch)) { Set<Integer> ourSegments = ch.getPrimarySegmentsForOwner(localAddress); if (segmentsToProcess != null) { ourSegments.retainAll(segmentsToProcess); } remoteResults.onCompletion(null, ourSegments, localValue); } else { if (segmentsToProcess != null) { Set<Integer> ourSegments = ch.getPrimarySegmentsForOwner(localAddress); ourSegments.retainAll(segmentsToProcess); remoteResults.onSegmentsLost(ourSegments); } else { remoteResults.onSegmentsLost(ch.getPrimarySegmentsForOwner(localAddress)); } } } else { // This isn't actually used because localRun short circuits first localValue = null; } if (id != null) { try { if ((!localRun || earlyTerminatePredicate == null || !earlyTerminatePredicate.test(localValue)) && !csm.awaitCompletion(id, timeout, timeoutUnit)) { throw new TimeoutException(); } } catch (InterruptedException e) { throw new CacheException(e); } } if (!remoteResults.lostSegments.isEmpty()) { segmentsToProcess = new HashSet<>(remoteResults.lostSegments); remoteResults.lostSegments.clear(); log.tracef("Found %s lost segments for identifier %s", segmentsToProcess, id); } else { // If we didn't lose any segments we don't need to process anymore if (segmentsToProcess != null) { segmentsToProcess = null; } log.tracef("Finished rehash aware operation for id %s", id); } } finally { csm.forgetOperation(id); } } while (segmentsToProcess != null && !segmentsToProcess.isEmpty()); return remoteResults.currentValue; }
@Override public Address getPrimaryOwner(Object key) { return dm.getPrimaryLocation(key); }
@Override public boolean localNodeIsPrimaryOwner(Object key) { final Address address = rpcManager.getAddress(); return dm.getPrimaryLocation(key).equals(address); }
void performRehashKeyTrackingOperation( Function< Supplier<Stream<CacheEntry>>, KeyTrackingTerminalOperation<Object, ? extends T, Object>> function) { final AtomicBoolean complete = new AtomicBoolean(); ConsistentHash segmentInfoCH = dm.getReadConsistentHash(); KeyTrackingConsumer<Object, Object> results = new KeyTrackingConsumer<>(segmentInfoCH, (c) -> {}, c -> c, null, keyEquivalence); Set<Integer> segmentsToProcess = segmentsToFilter == null ? new ReplicatedConsistentHash.RangeSet(segmentInfoCH.getNumSegments()) : segmentsToFilter; do { ConsistentHash ch = dm.getReadConsistentHash(); boolean localRun = ch.getMembers().contains(localAddress); Set<Integer> segments; Set<Object> excludedKeys; if (localRun) { segments = ch.getPrimarySegmentsForOwner(localAddress); segments.retainAll(segmentsToProcess); excludedKeys = segments .stream() .flatMap(s -> results.referenceArray.get(s).stream()) .collect(Collectors.toSet()); } else { // This null is okay as it is only referenced if it was a localRun segments = null; excludedKeys = Collections.emptySet(); } KeyTrackingTerminalOperation<Object, ? extends T, Object> op = function.apply(supplierForSegments(ch, segmentsToProcess, excludedKeys)); op.handleInjection(registry); Object id = csm.remoteStreamOperationRehashAware( getParallelDistribution(), parallel, ch, segmentsToProcess, keysToFilter, new AtomicReferenceArrayToMap<>(results.referenceArray), includeLoader, op, results); try { if (localRun) { Collection<CacheEntry<Object, Object>> localValue = op.performOperationRehashAware(results); // TODO: we can do this more efficiently - this hampers performance during rehash if (dm.getReadConsistentHash().equals(ch)) { log.tracef("Found local values %s for id %s", localValue.size(), id); results.onCompletion(null, segments, localValue); } else { Set<Integer> ourSegments = ch.getPrimarySegmentsForOwner(localAddress); ourSegments.retainAll(segmentsToProcess); log.tracef( "CH changed - making %s segments suspect for identifier %s", ourSegments, id); results.onSegmentsLost(ourSegments); // We keep track of those keys so we don't fire them again results.onIntermediateResult(null, localValue); } } if (id != null) { try { if (!csm.awaitCompletion(id, timeout, timeoutUnit)) { throw new TimeoutException(); } } catch (InterruptedException e) { throw new CacheException(e); } } if (!results.lostSegments.isEmpty()) { segmentsToProcess = new HashSet<>(results.lostSegments); results.lostSegments.clear(); log.tracef("Found %s lost segments for identifier %s", segmentsToProcess, id); } else { log.tracef("Finished rehash aware operation for id %s", id); complete.set(true); } } finally { csm.forgetOperation(id); } } while (!complete.get()); }
private static boolean isOwner(Cache<?, ?> cache, Object key) { DistributionManager dm = cache.getAdvancedCache().getDistributionManager(); return dm == null || dm.locate(key).contains(addressOf(cache)); }
private boolean getMightGoRemote(InvocationContext ctx, Object key) { return ctx.isOriginLocal() && configuration.getCacheMode().isDistributed() && !ctx.hasFlag(Flag.SKIP_REMOTE_LOOKUP) && !distManager.getLocality(key).isLocal(); }
private boolean isRemote(Object key) { return distributionManager != null && !distributionManager.getLocality(key).isLocal(); }
@Override public List<Address> getOwners(Object key) { return Immutables.immutableListConvert(dm.locate(key)); }