/** @param e Error. */ void onError(Throwable e) { tx.commitError(e); if (err.compareAndSet(null, e)) { boolean marked = tx.setRollbackOnly(); if (e instanceof GridCacheTxRollbackException) { if (marked) { try { tx.rollback(); } catch (GridException ex) { U.error(log, "Failed to automatically rollback transaction: " + tx, ex); } } } else if (tx.implicit() && tx.isSystemInvalidate()) { // Finish implicit transaction on heuristic error. try { tx.close(); } catch (GridException ex) { U.error(log, "Failed to invalidate transaction: " + tx, ex); } } onComplete(); } }
/** * Message received callback. * * @param src Sender node ID. * @param msg Received message. * @return {@code True}. */ public boolean onMessageReceived(UUID src, GridHadoopMessage msg) { if (msg instanceof GridHadoopShuffleMessage) { GridHadoopShuffleMessage m = (GridHadoopShuffleMessage) msg; try { job(m.jobId()).onShuffleMessage(m); } catch (GridException e) { U.error(log, "Message handling failed.", e); } try { // Reply with ack. send0(src, new GridHadoopShuffleAck(m.id(), m.jobId())); } catch (GridException e) { U.error( log, "Failed to reply back to shuffle message sender [snd=" + src + ", msg=" + msg + ']', e); } } else if (msg instanceof GridHadoopShuffleAck) { GridHadoopShuffleAck m = (GridHadoopShuffleAck) msg; try { job(m.jobId()).onShuffleAck(m); } catch (GridException e) { U.error(log, "Message handling failed.", e); } } else throw new IllegalStateException( "Unknown message type received to Hadoop shuffle [src=" + src + ", msg=" + msg + ']'); return true; }
/** * @param nodeId Sender node ID. * @param msg Response to prepare request. */ private void processPrepareResponse(UUID nodeId, GridDistributedTxPrepareResponse<K, V> msg) { assert nodeId != null; assert msg != null; GridReplicatedTxLocal<K, V> tx = ctx.tm().tx(msg.version()); if (tx == null) { if (log.isDebugEnabled()) log.debug( "Received prepare response for non-existing transaction [senderNodeId=" + nodeId + ", res=" + msg + ']'); return; } GridReplicatedTxPrepareFuture<K, V> future = (GridReplicatedTxPrepareFuture<K, V>) tx.future(); if (future != null) future.onResult(nodeId, msg); else U.error( log, "Received prepare response for transaction with no future [res=" + msg + ", tx=" + tx + ']'); }
/** @param workTokDir Token directory (common for multiple nodes). */ private void cleanupResources(File workTokDir) { RandomAccessFile lockFile = null; FileLock lock = null; try { lockFile = new RandomAccessFile(new File(workTokDir, LOCK_FILE_NAME), "rw"); lock = lockFile.getChannel().lock(); if (lock != null) processTokenDirectory(workTokDir); else if (log.isDebugEnabled()) log.debug( "Token directory is being processed concurrently: " + workTokDir.getAbsolutePath()); } catch (OverlappingFileLockException ignored) { if (log.isDebugEnabled()) log.debug( "Token directory is being processed concurrently: " + workTokDir.getAbsolutePath()); } catch (FileLockInterruptionException ignored) { Thread.currentThread().interrupt(); } catch (IOException e) { U.error(log, "Failed to process directory: " + workTokDir.getAbsolutePath(), e); } finally { U.releaseQuiet(lock); U.closeQuiet(lockFile); } }
/** Initializes future. */ @SuppressWarnings({"unchecked"}) void finish() { if (mappings != null) { finish(mappings.values()); markInitialized(); if (!isSync()) { boolean complete = true; for (GridFuture<?> f : pending()) // Mini-future in non-sync mode gets done when message gets sent. if (isMini(f) && !f.isDone()) complete = false; if (complete) onComplete(); } } else { assert !commit; try { tx.rollback(); } catch (GridException e) { U.error(log, "Failed to rollback empty transaction: " + tx, e); } markInitialized(); } }
/** Clears swap entries for evicted partition. */ private void clearSwap() { assert state() == EVICTED; assert !GridQueryProcessor.isEnabled(cctx.config()) : "Indexing needs to have unswapped values."; try { GridCloseableIterator<Map.Entry<byte[], GridCacheSwapEntry>> it = cctx.swap().iterator(id); boolean isLocStore = cctx.store().isLocal(); if (it != null) { // We can safely remove these values because no entries will be created for evicted // partition. while (it.hasNext()) { Map.Entry<byte[], GridCacheSwapEntry> entry = it.next(); byte[] keyBytes = entry.getKey(); KeyCacheObject key = cctx.toCacheKeyObject(keyBytes); cctx.swap().remove(key); if (isLocStore) cctx.store().remove(null, key.value(cctx.cacheObjectContext(), false)); } } } catch (IgniteCheckedException e) { U.error(log, "Failed to clear swap for evicted partition: " + this, e); } }
private void recheck() { // If this is the oldest node. if (oldestNode.get().id().equals(cctx.localNodeId())) { Collection<UUID> remaining = remaining(); if (!remaining.isEmpty()) { try { cctx.io() .safeSend( cctx.discovery().nodes(remaining), new GridDhtPartitionsSingleRequest(exchId), SYSTEM_POOL, null); } catch (IgniteCheckedException e) { U.error( log, "Failed to request partitions from nodes [exchangeId=" + exchId + ", nodes=" + remaining + ']', e); } } // Resend full partition map because last attempt failed. else { if (spreadPartitions()) onDone(exchId.topologyVersion()); } } else sendPartitions(); // Schedule another send. scheduleRecheck(); }
private void sendPartitions() { ClusterNode oldestNode = this.oldestNode.get(); try { sendLocalPartitions(oldestNode, exchId); } catch (ClusterTopologyCheckedException ignore) { if (log.isDebugEnabled()) log.debug( "Oldest node left during partition exchange [nodeId=" + oldestNode.id() + ", exchId=" + exchId + ']'); } catch (IgniteCheckedException e) { scheduleRecheck(); U.error( log, "Failed to send local partitions to oldest node (will retry after timeout) [oldestNodeId=" + oldestNode.id() + ", exchId=" + exchId + ']', e); } }
/** {@inheritDoc} */ @Override public boolean onDone(GridCacheTx tx, Throwable err) { if ((initialized() || err != null) && super.onDone(tx, err)) { if (error() instanceof GridCacheTxHeuristicException) { long topVer = this.tx.topologyVersion(); for (GridCacheTxEntry<K, V> e : this.tx.writeMap().values()) { try { if (e.op() != NOOP && !cctx.affinity().localNode(e.key(), topVer)) { GridCacheEntryEx<K, V> cacheEntry = cctx.cache().peekEx(e.key()); if (cacheEntry != null) cacheEntry.invalidate(null, this.tx.xidVersion()); } } catch (Throwable t) { U.error(log, "Failed to invalidate entry.", t); if (t instanceof Error) throw (Error) t; } } } // Don't forget to clean up. cctx.mvcc().removeFuture(this); return true; } return false; }
/** * @param out Output stream. * @param err Error cause. */ private void sendErrorResponse(ObjectOutput out, Exception err) { try { out.writeObject(new IpcSharedMemoryInitResponse(err)); } catch (IOException e) { U.error(log, "Failed to send error response to client.", e); } }
/** * Processes unlock request. * * @param nodeId Sender node ID. * @param req Unlock request. */ @SuppressWarnings({"unchecked"}) private void processUnlockRequest(UUID nodeId, GridDistributedUnlockRequest req) { assert nodeId != null; try { ClassLoader ldr = ctx.deploy().globalLoader(); List<byte[]> keys = req.keyBytes(); for (byte[] keyBytes : keys) { K key = (K) U.unmarshal(ctx.marshaller(), new ByteArrayInputStream(keyBytes), ldr); while (true) { boolean created = false; GridDistributedCacheEntry<K, V> entry = peekexx(key); if (entry == null) { entry = entryexx(key); created = true; } try { entry.doneRemote( req.version(), req.version(), req.committedVersions(), req.rolledbackVersions()); // Note that we don't reorder completed versions here, // as there is no point to reorder relative to the version // we are about to remove. if (entry.removeLock(req.version())) { if (log.isDebugEnabled()) log.debug("Removed lock [lockId=" + req.version() + ", key=" + key + ']'); if (created && entry.markObsolete(req.version())) removeIfObsolete(entry.key()); } else if (log.isDebugEnabled()) log.debug( "Received unlock request for unknown candidate " + "(added to cancelled locks set): " + req); break; } catch (GridCacheEntryRemovedException ignored) { if (log.isDebugEnabled()) log.debug( "Received remove lock request for removed entry (will retry) [entry=" + entry + ", req=" + req + ']'); } } } } catch (GridException e) { U.error(log, "Failed to unmarshal unlock key (unlock will not be performed): " + req, e); } }
/** * Removes locks regardless of whether they are owned or not for given version and keys. * * @param ver Lock version. * @param keys Keys. */ @SuppressWarnings({"unchecked"}) public void removeLocks(GridCacheVersion ver, Collection<? extends K> keys) { if (keys.isEmpty()) return; Collection<GridRichNode> nodes = ctx.remoteNodes(keys); try { // Send request to remove from remote nodes. GridDistributedUnlockRequest<K, V> req = new GridDistributedUnlockRequest<K, V>(keys.size()); req.version(ver); for (K key : keys) { while (true) { GridDistributedCacheEntry<K, V> entry = peekexx(key); try { if (entry != null) { GridCacheMvccCandidate<K> cand = entry.candidate(ver); if (cand != null) { // Remove candidate from local node first. if (entry.removeLock(cand.version())) { // If there is only local node in this lock's topology, // then there is no reason to distribute the request. if (nodes.isEmpty()) continue; req.addKey(entry.key(), entry.getOrMarshalKeyBytes(), ctx); } } } break; } catch (GridCacheEntryRemovedException ignored) { if (log.isDebugEnabled()) log.debug( "Attempted to remove lock from removed entry (will retry) [rmvVer=" + ver + ", entry=" + entry + ']'); } } } if (nodes.isEmpty()) return; req.completedVersions(ctx.tm().committedVersions(ver), ctx.tm().rolledbackVersions(ver)); if (!req.keyBytes().isEmpty()) // We don't wait for reply to this message. ctx.io().safeSend(nodes, req, null); } catch (GridException ex) { U.error(log, "Failed to unlock the lock for keys: " + keys, ex); } }
/** Perform cleanup of the trash directory. */ private void delete() { IgfsFileInfo info = null; try { info = meta.info(TRASH_ID); } catch (ClusterTopologyServerNotFoundException e) { LT.warn(log, e, "Server nodes not found."); } catch (IgniteCheckedException e) { U.error(log, "Cannot obtain trash directory info.", e); } if (info != null) { for (Map.Entry<String, IgfsListingEntry> entry : info.listing().entrySet()) { IgniteUuid fileId = entry.getValue().fileId(); if (log.isDebugEnabled()) log.debug( "Deleting IGFS trash entry [name=" + entry.getKey() + ", fileId=" + fileId + ']'); try { if (!cancelled) { if (delete(entry.getKey(), fileId)) { if (log.isDebugEnabled()) log.debug( "Sending delete confirmation message [name=" + entry.getKey() + ", fileId=" + fileId + ']'); sendDeleteMessage(new IgfsDeleteMessage(fileId)); } } else break; } catch (IgniteInterruptedCheckedException ignored) { // Ignore this exception while stopping. } catch (IgniteCheckedException e) { U.error(log, "Failed to delete entry from the trash directory: " + entry.getKey(), e); sendDeleteMessage(new IgfsDeleteMessage(fileId, e)); } } } }
/** @param jobId Job id. */ public void jobFinished(GridHadoopJobId jobId) { GridHadoopShuffleJob job = jobs.remove(jobId); if (job != null) { try { job.close(); } catch (GridException e) { U.error(log, "Failed to close job: " + jobId, e); } } }
/** * Checks entry for empty value. * * @param entry Entry to check. * @return {@code True} if entry is empty. */ private boolean empty(GridCacheEntry<K, V> entry) { try { return entry.peek(F.asList(GLOBAL)) == null; } catch (GridException e) { U.error(null, e.getMessage(), e); assert false : "Should never happen: " + e; return false; } }
/** * Stops shuffle. * * @param cancel If should cancel all ongoing activities. */ @Override public void stop(boolean cancel) { for (GridHadoopShuffleJob job : jobs.values()) { try { job.close(); } catch (GridException e) { U.error(log, "Failed to close job.", e); } } jobs.clear(); }
/** {@inheritDoc} */ @Override void onCancelAtStop() { super.onCancelAtStop(); for (GridCacheQueryFutureAdapter fut : futs.values()) try { fut.cancel(); } catch (IgniteCheckedException e) { U.error(log, "Failed to cancel running query future: " + fut, e); } U.interrupt(threads.values()); }
/** * Notifies single listener. * * @param lsnr Listener. */ private void notifyListener(IgniteInClosure<? super IgniteInternalFuture<R>> lsnr) { assert lsnr != null; try { lsnr.apply(this); } catch (IllegalStateException e) { U.error( null, "Failed to notify listener (is grid stopped?) [fut=" + this + ", lsnr=" + lsnr + ", err=" + e.getMessage() + ']', e); } catch (RuntimeException | Error e) { U.error(null, "Failed to notify listener: " + lsnr, e); throw e; } }
/** * Notifies single listener. * * @param lsnr Listener. */ private void notifyListener(GridInClosure<? super GridFuture<R>> lsnr) { assert lsnr != null; try { lsnr.apply(this); } catch (IllegalStateException ignore) { U.warn( null, "Failed to notify listener (grid is stopped) [grid=" + ctx.gridName() + ", lsnr=" + lsnr + ']'); } catch (RuntimeException e) { U.error(log, "Failed to notify listener: " + lsnr, e); throw e; } catch (Error e) { U.error(log, "Failed to notify listener: " + lsnr, e); throw e; } }
void cancelAll() { GridException err = new GridException("Data loader has been cancelled: " + GridDataLoaderImpl.this); for (GridFuture<?> f : locFuts) { try { f.cancel(); } catch (GridException e) { U.error(log, "Failed to cancel mini-future.", e); } } for (GridFutureAdapter<?> f : reqs.values()) f.onDone(err); }
/** * @param nodeId Node ID. * @param retryCnt Number of retries. */ private void sendAllPartitions(final UUID nodeId, final int retryCnt) { ClusterNode n = cctx.node(nodeId); try { if (n != null) sendAllPartitions(F.asList(n), exchId); } catch (IgniteCheckedException e) { if (e instanceof ClusterTopologyCheckedException || !cctx.discovery().alive(n)) { log.debug( "Failed to send full partition map to node, node left grid " + "[rmtNode=" + nodeId + ", exchangeId=" + exchId + ']'); return; } if (retryCnt > 0) { long timeout = cctx.gridConfig().getNetworkSendRetryDelay(); LT.error( log, e, "Failed to send full partition map to node (will retry after timeout) " + "[node=" + nodeId + ", exchangeId=" + exchId + ", timeout=" + timeout + ']'); cctx.time() .addTimeoutObject( new GridTimeoutObjectAdapter(timeout) { @Override public void onTimeout() { sendAllPartitions(nodeId, retryCnt - 1); } }); } else U.error( log, "Failed to send full partition map [node=" + n + ", exchangeId=" + exchId + ']', e); } }
/** * Processes cache query request. * * @param sndId Sender node id. * @param req Query request. */ @SuppressWarnings("unchecked") @Override void processQueryRequest(UUID sndId, GridCacheQueryRequest req) { if (req.cancel()) { cancelIds.add(new CancelMessageId(req.id(), sndId)); if (req.fields()) removeFieldsQueryResult(sndId, req.id()); else removeQueryResult(sndId, req.id()); } else { if (!cancelIds.contains(new CancelMessageId(req.id(), sndId))) { if (!F.eq(req.cacheName(), cctx.name())) { GridCacheQueryResponse res = new GridCacheQueryResponse( cctx.cacheId(), req.id(), new IgniteCheckedException( "Received request for incorrect cache [expected=" + cctx.name() + ", actual=" + req.cacheName())); sendQueryResponse(sndId, res, 0); } else { threads.put(req.id(), Thread.currentThread()); try { GridCacheQueryInfo info = distributedQueryInfo(sndId, req); if (info == null) return; if (req.fields()) runFieldsQuery(info); else runQuery(info); } catch (Throwable e) { U.error(log(), "Failed to run query.", e); sendQueryResponse( sndId, new GridCacheQueryResponse(cctx.cacheId(), req.id(), e.getCause()), 0); if (e instanceof Error) throw (Error) e; } finally { threads.remove(req.id()); } } } } }
/** @param e Error. */ void onError(Throwable e) { tx.commitError(e); if (err.compareAndSet(null, e)) { boolean marked = tx.setRollbackOnly(); if (e instanceof GridCacheTxRollbackException) if (marked) { try { tx.rollback(); } catch (GridException ex) { U.error(log, "Failed to automatically rollback transaction: " + tx, ex); } } onComplete(); } }
/** @return {@code True} if succeeded. */ private boolean spreadPartitions() { try { sendAllPartitions(rmtNodes, exchId); return true; } catch (IgniteCheckedException e) { scheduleRecheck(); if (!X.hasCause(e, InterruptedException.class)) U.error( log, "Failed to send full partition map to nodes (will retry after timeout) [nodes=" + F.nodeId8s(rmtNodes) + ", exchangeId=" + exchId + ']', e); return false; } }
/** * Starts activity. * * @throws IgniteInterruptedCheckedException If interrupted. */ public void init() throws IgniteInterruptedCheckedException { if (isDone()) return; if (init.compareAndSet(false, true)) { if (isDone()) return; try { // Wait for event to occur to make sure that discovery // will return corresponding nodes. U.await(evtLatch); assert discoEvt != null : this; assert !dummy && !forcePreload : this; ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx, exchId.topologyVersion()); oldestNode.set(oldest); startCaches(); // True if client node joined or failed. boolean clientNodeEvt; if (F.isEmpty(reqs)) { int type = discoEvt.type(); assert type == EVT_NODE_JOINED || type == EVT_NODE_LEFT || type == EVT_NODE_FAILED : discoEvt; clientNodeEvt = CU.clientNode(discoEvt.eventNode()); } else { assert discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT : discoEvt; boolean clientOnlyStart = true; for (DynamicCacheChangeRequest req : reqs) { if (!req.clientStartOnly()) { clientOnlyStart = false; break; } } clientNodeEvt = clientOnlyStart; } if (clientNodeEvt) { ClusterNode node = discoEvt.eventNode(); // Client need to initialize affinity for local join event or for stated client caches. if (!node.isLocal()) { for (GridCacheContext cacheCtx : cctx.cacheContexts()) { if (cacheCtx.isLocal()) continue; GridDhtPartitionTopology top = cacheCtx.topology(); top.updateTopologyVersion(exchId, this, -1, stopping(cacheCtx.cacheId())); if (cacheCtx.affinity().affinityTopologyVersion() == AffinityTopologyVersion.NONE) { initTopology(cacheCtx); top.beforeExchange(this); } else cacheCtx.affinity().clientEventTopologyChange(discoEvt, exchId.topologyVersion()); } if (exchId.isLeft()) cctx.mvcc().removeExplicitNodeLocks(exchId.nodeId(), exchId.topologyVersion()); onDone(exchId.topologyVersion()); skipPreload = cctx.kernalContext().clientNode(); return; } } if (cctx.kernalContext().clientNode()) { skipPreload = true; for (GridCacheContext cacheCtx : cctx.cacheContexts()) { if (cacheCtx.isLocal()) continue; GridDhtPartitionTopology top = cacheCtx.topology(); top.updateTopologyVersion(exchId, this, -1, stopping(cacheCtx.cacheId())); } for (GridCacheContext cacheCtx : cctx.cacheContexts()) { if (cacheCtx.isLocal()) continue; initTopology(cacheCtx); } if (oldestNode.get() != null) { rmtNodes = new ConcurrentLinkedQueue<>( CU.aliveRemoteServerNodesWithCaches(cctx, exchId.topologyVersion())); rmtIds = Collections.unmodifiableSet(new HashSet<>(F.nodeIds(rmtNodes))); ready.set(true); initFut.onDone(true); if (log.isDebugEnabled()) log.debug("Initialized future: " + this); sendPartitions(); } else onDone(exchId.topologyVersion()); return; } assert oldestNode.get() != null; for (GridCacheContext cacheCtx : cctx.cacheContexts()) { if (isCacheAdded(cacheCtx.cacheId(), exchId.topologyVersion())) { if (cacheCtx .discovery() .cacheAffinityNodes(cacheCtx.name(), topologyVersion()) .isEmpty()) U.quietAndWarn(log, "No server nodes found for cache client: " + cacheCtx.namex()); } cacheCtx.preloader().onExchangeFutureAdded(); } List<String> cachesWithoutNodes = null; if (exchId.isLeft()) { for (String name : cctx.cache().cacheNames()) { if (cctx.discovery().cacheAffinityNodes(name, topologyVersion()).isEmpty()) { if (cachesWithoutNodes == null) cachesWithoutNodes = new ArrayList<>(); cachesWithoutNodes.add(name); // Fire event even if there is no client cache started. if (cctx.gridEvents().isRecordable(EventType.EVT_CACHE_NODES_LEFT)) { Event evt = new CacheEvent( name, cctx.localNode(), cctx.localNode(), "All server nodes have left the cluster.", EventType.EVT_CACHE_NODES_LEFT, 0, false, null, null, null, null, false, null, false, null, null, null); cctx.gridEvents().record(evt); } } } } if (cachesWithoutNodes != null) { StringBuilder sb = new StringBuilder( "All server nodes for the following caches have left the cluster: "); for (int i = 0; i < cachesWithoutNodes.size(); i++) { String cache = cachesWithoutNodes.get(i); sb.append('\'').append(cache).append('\''); if (i != cachesWithoutNodes.size() - 1) sb.append(", "); } U.quietAndWarn(log, sb.toString()); U.quietAndWarn(log, "Must have server nodes for caches to operate."); } assert discoEvt != null; assert exchId.nodeId().equals(discoEvt.eventNode().id()); for (GridCacheContext cacheCtx : cctx.cacheContexts()) { GridClientPartitionTopology clientTop = cctx.exchange().clearClientTopology(cacheCtx.cacheId()); long updSeq = clientTop == null ? -1 : clientTop.lastUpdateSequence(); // Update before waiting for locks. if (!cacheCtx.isLocal()) cacheCtx .topology() .updateTopologyVersion(exchId, this, updSeq, stopping(cacheCtx.cacheId())); } // Grab all alive remote nodes with order of equal or less than last joined node. rmtNodes = new ConcurrentLinkedQueue<>( CU.aliveRemoteServerNodesWithCaches(cctx, exchId.topologyVersion())); rmtIds = Collections.unmodifiableSet(new HashSet<>(F.nodeIds(rmtNodes))); for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> m : singleMsgs.entrySet()) // If received any messages, process them. onReceive(m.getKey(), m.getValue()); for (Map.Entry<UUID, GridDhtPartitionsFullMessage> m : fullMsgs.entrySet()) // If received any messages, process them. onReceive(m.getKey(), m.getValue()); AffinityTopologyVersion topVer = exchId.topologyVersion(); for (GridCacheContext cacheCtx : cctx.cacheContexts()) { if (cacheCtx.isLocal()) continue; // Must initialize topology after we get discovery event. initTopology(cacheCtx); cacheCtx.preloader().updateLastExchangeFuture(this); } IgniteInternalFuture<?> partReleaseFut = cctx.partitionReleaseFuture(topVer); // Assign to class variable so it will be included into toString() method. this.partReleaseFut = partReleaseFut; if (log.isDebugEnabled()) log.debug("Before waiting for partition release future: " + this); while (true) { try { partReleaseFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS); break; } catch (IgniteFutureTimeoutCheckedException ignored) { // Print pending transactions and locks that might have led to hang. dumpPendingObjects(); } } if (log.isDebugEnabled()) log.debug("After waiting for partition release future: " + this); if (!F.isEmpty(reqs)) blockGateways(); if (exchId.isLeft()) cctx.mvcc().removeExplicitNodeLocks(exchId.nodeId(), exchId.topologyVersion()); IgniteInternalFuture<?> locksFut = cctx.mvcc().finishLocks(exchId.topologyVersion()); while (true) { try { locksFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS); break; } catch (IgniteFutureTimeoutCheckedException ignored) { U.warn( log, "Failed to wait for locks release future. " + "Dumping pending objects that might be the cause: " + cctx.localNodeId()); U.warn(log, "Locked entries:"); Map<IgniteTxKey, Collection<GridCacheMvccCandidate>> locks = cctx.mvcc().unfinishedLocks(exchId.topologyVersion()); for (Map.Entry<IgniteTxKey, Collection<GridCacheMvccCandidate>> e : locks.entrySet()) U.warn(log, "Locked entry [key=" + e.getKey() + ", mvcc=" + e.getValue() + ']'); } } for (GridCacheContext cacheCtx : cctx.cacheContexts()) { if (cacheCtx.isLocal()) continue; // Notify replication manager. GridCacheContext drCacheCtx = cacheCtx.isNear() ? cacheCtx.near().dht().context() : cacheCtx; if (drCacheCtx.isDrEnabled()) drCacheCtx.dr().beforeExchange(topVer, exchId.isLeft()); // Partition release future is done so we can flush the write-behind store. cacheCtx.store().forceFlush(); // Process queued undeploys prior to sending/spreading map. cacheCtx.preloader().unwindUndeploys(); GridDhtPartitionTopology top = cacheCtx.topology(); assert topVer.equals(top.topologyVersion()) : "Topology version is updated only in this class instances inside single ExchangeWorker thread."; top.beforeExchange(this); } for (GridClientPartitionTopology top : cctx.exchange().clientTopologies()) { top.updateTopologyVersion(exchId, this, -1, stopping(top.cacheId())); top.beforeExchange(this); } } catch (IgniteInterruptedCheckedException e) { onDone(e); throw e; } catch (Throwable e) { U.error( log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e); onDone(e); if (e instanceof Error) throw (Error) e; return; } if (F.isEmpty(rmtIds)) { onDone(exchId.topologyVersion()); return; } ready.set(true); initFut.onDone(true); if (log.isDebugEnabled()) log.debug("Initialized future: " + this); // If this node is not oldest. if (!oldestNode.get().id().equals(cctx.localNodeId())) sendPartitions(); else { boolean allReceived = allReceived(); if (allReceived && replied.compareAndSet(false, true)) { if (spreadPartitions()) onDone(exchId.topologyVersion()); } } scheduleRecheck(); } else assert false : "Skipped init future: " + this; }
/** {@inheritDoc} */ @Override public GridFuture<GridCacheTxEx<K, V>> prepareAsync() { GridNearTxPrepareFuture<K, V> fut = prepFut.get(); if (fut == null) { // Future must be created before any exception can be thrown. if (!prepFut.compareAndSet(null, fut = new GridNearTxPrepareFuture<K, V>(cctx, this))) return prepFut.get(); } else // Prepare was called explicitly. return fut; if (!state(PREPARING)) { if (setRollbackOnly()) { if (timedOut()) fut.onError( new GridCacheTxTimeoutException( "Transaction timed out and was rolled back: " + this)); else fut.onError( new GridException( "Invalid transaction state for prepare [state=" + state() + ", tx=" + this + ']')); } else fut.onError( new GridCacheTxRollbackException( "Invalid transaction state for prepare [state=" + state() + ", tx=" + this + ']')); return fut; } // For pessimistic mode we don't distribute prepare request. if (pessimistic()) { try { userPrepare(); if (!state(PREPARED)) { setRollbackOnly(); fut.onError( new GridException( "Invalid transaction state for commit [state=" + state() + ", tx=" + this + ']')); return fut; } fut.complete(); return fut; } catch (GridException e) { fut.onError(e); return fut; } } try { cctx.topology().readLock(); try { topologyVersion(cctx.topology().topologyVersion()); userPrepare(); } finally { cctx.topology().readUnlock(); } // This will attempt to locally commit // EVENTUALLY CONSISTENT transactions. fut.onPreparedEC(); // Make sure to add future before calling prepare. cctx.mvcc().addFuture(fut); fut.prepare(); } catch (GridCacheTxTimeoutException e) { fut.onError(e); } catch (GridCacheTxOptimisticException e) { fut.onError(e); } catch (GridException e) { setRollbackOnly(); String msg = "Failed to prepare transaction (will attempt rollback): " + this; log.error(msg, e); try { rollback(); } catch (GridException e1) { U.error(log, "Failed to rollback transaction: " + this, e1); } fut.onError(new GridCacheTxRollbackException(msg, e)); } return fut; }
/** {@inheritDoc} */ @Override public void unlockAll( Collection<? extends K> keys, GridPredicate<? super GridCacheEntry<K, V>>[] filter) { if (keys.isEmpty()) return; try { GridCacheVersion ver = null; Collection<GridRichNode> affNodes = null; int keyCnt = -1; Map<GridRichNode, GridNearUnlockRequest<K, V>> map = null; Collection<K> locKeys = new LinkedList<K>(); GridCacheVersion obsoleteVer = ctx.versions().next(); for (K key : keys) { while (true) { GridDistributedCacheEntry<K, V> entry = peekExx(key); if (entry == null || !ctx.isAll(entry.wrap(false), filter)) break; // While. try { GridCacheMvccCandidate<K> cand = entry.candidate(ctx.nodeId(), Thread.currentThread().getId()); if (cand != null) { ver = cand.version(); if (affNodes == null) { affNodes = CU.allNodes(ctx, cand.topologyVersion()); keyCnt = (int) Math.ceil((double) keys.size() / affNodes.size()); map = new HashMap<GridRichNode, GridNearUnlockRequest<K, V>>(affNodes.size()); } // Send request to remove from remote nodes. GridRichNode primary = CU.primary0(ctx.affinity(key, affNodes)); GridNearUnlockRequest<K, V> req = map.get(primary); if (req == null) { map.put(primary, req = new GridNearUnlockRequest<K, V>(keyCnt)); req.version(ver); } // Remove candidate from local node first. GridCacheMvccCandidate<K> rmv = entry.removeLock(); if (rmv != null) { if (!rmv.reentry()) { if (ver != null && !ver.equals(rmv.version())) throw new GridException( "Failed to unlock (if keys were locked separately, " + "then they need to be unlocked separately): " + keys); if (!primary.isLocal()) { assert req != null; req.addKey(entry.key(), entry.getOrMarshalKeyBytes(), ctx); } else locKeys.add(key); if (log.isDebugEnabled()) log.debug("Removed lock (will distribute): " + rmv); } else if (log.isDebugEnabled()) log.debug( "Current thread still owns lock (or there are no other nodes)" + " [lock=" + rmv + ", curThreadId=" + Thread.currentThread().getId() + ']'); } // Try to evict near entry if it's dht-mapped locally. evictNearEntry(entry, obsoleteVer); } break; } catch (GridCacheEntryRemovedException ignore) { if (log.isDebugEnabled()) log.debug("Attempted to unlock removed entry (will retry): " + entry); } } } if (ver == null) return; for (Map.Entry<GridRichNode, GridNearUnlockRequest<K, V>> mapping : map.entrySet()) { GridRichNode n = mapping.getKey(); GridDistributedUnlockRequest<K, V> req = mapping.getValue(); if (n.isLocal()) dht.removeLocks(ctx.nodeId(), req.version(), locKeys, true); else if (!req.keyBytes().isEmpty()) // We don't wait for reply to this message. ctx.io().send(n, req); } } catch (GridException ex) { U.error(log, "Failed to unlock the lock for keys: " + keys, ex); } }
/** * @param entries Entries to submit. * @param curFut Current future. * @throws GridInterruptedException If interrupted. */ private void submit(final List<Map.Entry<K, V>> entries, final GridFutureAdapter<Object> curFut) throws GridInterruptedException { assert entries != null; assert !entries.isEmpty(); assert curFut != null; incrementActiveTasks(); GridFuture<Object> fut; if (isLocNode) { fut = ctx.closure() .callLocalSafe( new GridDataLoadUpdateJob<>(ctx, log, cacheName, entries, false, updater), false); locFuts.add(fut); fut.listenAsync( new GridInClosure<GridFuture<Object>>() { @Override public void apply(GridFuture<Object> t) { try { boolean rmv = locFuts.remove(t); assert rmv; curFut.onDone(t.get()); } catch (GridException e) { curFut.onDone(e); } } }); } else { byte[] entriesBytes; try { entriesBytes = ctx.config().getMarshaller().marshal(entries); if (updaterBytes == null) { assert updater != null; updaterBytes = ctx.config().getMarshaller().marshal(updater); } if (topicBytes == null) topicBytes = ctx.config().getMarshaller().marshal(topic); } catch (GridException e) { U.error(log, "Failed to marshal (request will not be sent).", e); return; } GridDeployment dep = null; GridPeerDeployAware jobPda0 = null; if (ctx.deploy().enabled()) { try { jobPda0 = jobPda; assert jobPda0 != null; dep = ctx.deploy().deploy(jobPda0.deployClass(), jobPda0.classLoader()); } catch (GridException e) { U.error( log, "Failed to deploy class (request will not be sent): " + jobPda0.deployClass(), e); return; } if (dep == null) U.warn(log, "Failed to deploy class (request will be sent): " + jobPda0.deployClass()); } long reqId = idGen.incrementAndGet(); fut = curFut; reqs.put(reqId, (GridFutureAdapter<Object>) fut); GridDataLoadRequest<Object, Object> req = new GridDataLoadRequest<>( reqId, topicBytes, cacheName, updaterBytes, entriesBytes, true, dep != null ? dep.deployMode() : null, dep != null ? jobPda0.deployClass().getName() : null, dep != null ? dep.userVersion() : null, dep != null ? dep.participants() : null, dep != null ? dep.classLoaderId() : null, dep == null); try { ctx.io().send(node, TOPIC_DATALOAD, req, PUBLIC_POOL); if (log.isDebugEnabled()) log.debug("Sent request to node [nodeId=" + node.id() + ", req=" + req + ']'); } catch (GridException e) { if (ctx.discovery().alive(node) && ctx.discovery().pingNode(node.id())) ((GridFutureAdapter<Object>) fut).onDone(e); else ((GridFutureAdapter<Object>) fut) .onDone( new GridTopologyException( "Failed to send " + "request (node has left): " + node.id())); } } }
/** * Process HTTP request. * * @param act Action. * @param req Http request. * @param res Http response. */ private void processRequest(String act, HttpServletRequest req, HttpServletResponse res) { res.setContentType("application/json"); res.setCharacterEncoding("UTF-8"); GridRestCommand cmd = command(req); if (cmd == null) { res.setStatus(HttpServletResponse.SC_BAD_REQUEST); return; } if (!authChecker.apply(req.getHeader("X-Signature"))) { res.setStatus(HttpServletResponse.SC_UNAUTHORIZED); return; } GridRestResponse cmdRes; Map<String, Object> params = parameters(req); try { GridRestRequest cmdReq = createRequest(cmd, params, req); if (log.isDebugEnabled()) log.debug("Initialized command request: " + cmdReq); cmdRes = hnd.handle(cmdReq); if (cmdRes == null) throw new IllegalStateException("Received null result from handler: " + hnd); byte[] sesTok = cmdRes.sessionTokenBytes(); if (sesTok != null) cmdRes.setSessionToken(U.byteArray2HexString(sesTok)); res.setStatus(HttpServletResponse.SC_OK); } catch (Exception e) { res.setStatus(HttpServletResponse.SC_OK); U.error(log, "Failed to process HTTP request [action=" + act + ", req=" + req + ']', e); cmdRes = new GridRestResponse(STATUS_FAILED, e.getMessage()); } catch (Throwable e) { U.error(log, "Failed to process HTTP request [action=" + act + ", req=" + req + ']', e); throw e; } JsonConfig cfg = new GridJettyJsonConfig(); // Workaround for not needed transformation of string into JSON object. if (cmdRes.getResponse() instanceof String) cfg.registerJsonValueProcessor(cmdRes.getClass(), "response", SKIP_STR_VAL_PROC); if (cmdRes.getResponse() instanceof GridClientTaskResultBean && ((GridClientTaskResultBean) cmdRes.getResponse()).getResult() instanceof String) cfg.registerJsonValueProcessor(cmdRes.getResponse().getClass(), "result", SKIP_STR_VAL_PROC); JSON json; try { json = JSONSerializer.toJSON(cmdRes, cfg); } catch (JSONException e) { U.error(log, "Failed to convert response to JSON: " + cmdRes, e); json = JSONSerializer.toJSON(new GridRestResponse(STATUS_FAILED, e.getMessage()), cfg); } try { if (log.isDebugEnabled()) log.debug("Parsed command response into JSON object: " + json.toString(2)); res.getWriter().write(json.toString()); if (log.isDebugEnabled()) log.debug( "Processed HTTP request [action=" + act + ", jsonRes=" + cmdRes + ", req=" + req + ']'); } catch (IOException e) { U.error(log, "Failed to send HTTP response: " + json.toString(2), e); } }
/** * Removes locks regardless of whether they are owned or not for given version and keys. * * @param ver Lock version. * @param keys Keys. */ @SuppressWarnings({"unchecked"}) public void removeLocks(GridCacheVersion ver, Collection<? extends K> keys) { if (keys.isEmpty()) return; try { Collection<GridRichNode> affNodes = null; int keyCnt = -1; Map<GridNode, GridNearUnlockRequest<K, V>> map = null; for (K key : keys) { // Send request to remove from remote nodes. GridNearUnlockRequest<K, V> req = null; while (true) { GridDistributedCacheEntry<K, V> entry = peekExx(key); try { if (entry != null) { GridCacheMvccCandidate<K> cand = entry.candidate(ver); if (cand != null) { if (affNodes == null) { affNodes = CU.allNodes(ctx, cand.topologyVersion()); keyCnt = (int) Math.ceil((double) keys.size() / affNodes.size()); map = new HashMap<GridNode, GridNearUnlockRequest<K, V>>(affNodes.size()); } GridRichNode primary = CU.primary0(ctx.affinity(key, affNodes)); if (!primary.isLocal()) { req = map.get(primary); if (req == null) { map.put(primary, req = new GridNearUnlockRequest<K, V>(keyCnt)); req.version(ver); } } // Remove candidate from local node first. if (entry.removeLock(cand.version())) { if (primary.isLocal()) { dht.removeLocks(primary.id(), ver, F.asList(key), true); assert req == null; continue; } req.addKey(entry.key(), entry.getOrMarshalKeyBytes(), ctx); } } } break; } catch (GridCacheEntryRemovedException ignored) { if (log.isDebugEnabled()) log.debug( "Attempted to remove lock from removed entry (will retry) [rmvVer=" + ver + ", entry=" + entry + ']'); } } } if (map == null || map.isEmpty()) return; Collection<GridCacheVersion> committed = ctx.tm().committedVersions(ver); Collection<GridCacheVersion> rolledback = ctx.tm().rolledbackVersions(ver); for (Map.Entry<GridNode, GridNearUnlockRequest<K, V>> mapping : map.entrySet()) { GridNode n = mapping.getKey(); GridDistributedUnlockRequest<K, V> req = mapping.getValue(); if (!req.keyBytes().isEmpty()) { req.completedVersions(committed, rolledback); // We don't wait for reply to this message. ctx.io().send(n, req); } } } catch (GridException ex) { U.error(log, "Failed to unlock the lock for keys: " + keys, ex); } }