/**
* Message received callback.
*
* @param src Sender node ID.
* @param msg Received message.
* @return {@code True}.
*/
public boolean onMessageReceived(UUID src, GridHadoopMessage msg) {
if (msg instanceof GridHadoopShuffleMessage) {
GridHadoopShuffleMessage m = (GridHadoopShuffleMessage) msg;
try {
job(m.jobId()).onShuffleMessage(m);
} catch (GridException e) {
U.error(log, "Message handling failed.", e);
}
try {
// Reply with ack.
send0(src, new GridHadoopShuffleAck(m.id(), m.jobId()));
} catch (GridException e) {
U.error(
log,
"Failed to reply back to shuffle message sender [snd=" + src + ", msg=" + msg + ']',
e);
}
} else if (msg instanceof GridHadoopShuffleAck) {
GridHadoopShuffleAck m = (GridHadoopShuffleAck) msg;
try {
job(m.jobId()).onShuffleAck(m);
} catch (GridException e) {
U.error(log, "Message handling failed.", e);
}
} else
throw new IllegalStateException(
"Unknown message type received to Hadoop shuffle [src=" + src + ", msg=" + msg + ']');
return true;
}
/** @param e Error. */
void onError(Throwable e) {
tx.commitError(e);
if (err.compareAndSet(null, e)) {
boolean marked = tx.setRollbackOnly();
if (e instanceof GridCacheTxRollbackException) {
if (marked) {
try {
tx.rollback();
} catch (GridException ex) {
U.error(log, "Failed to automatically rollback transaction: " + tx, ex);
}
}
} else if (tx.implicit()
&& tx.isSystemInvalidate()) { // Finish implicit transaction on heuristic error.
try {
tx.close();
} catch (GridException ex) {
U.error(log, "Failed to invalidate transaction: " + tx, ex);
}
}
onComplete();
}
}
/** Initializes future. */
@SuppressWarnings({"unchecked"})
void finish() {
if (mappings != null) {
finish(mappings.values());
markInitialized();
if (!isSync()) {
boolean complete = true;
for (GridFuture<?> f : pending())
// Mini-future in non-sync mode gets done when message gets sent.
if (isMini(f) && !f.isDone()) complete = false;
if (complete) onComplete();
}
} else {
assert !commit;
try {
tx.rollback();
} catch (GridException e) {
U.error(log, "Failed to rollback empty transaction: " + tx, e);
}
markInitialized();
}
}
/** {@inheritDoc} */
@Override
public boolean onDone(GridCacheTx tx, Throwable err) {
if ((initialized() || err != null) && super.onDone(tx, err)) {
if (error() instanceof GridCacheTxHeuristicException) {
long topVer = this.tx.topologyVersion();
for (GridCacheTxEntry<K, V> e : this.tx.writeMap().values()) {
try {
if (e.op() != NOOP && !cctx.affinity().localNode(e.key(), topVer)) {
GridCacheEntryEx<K, V> cacheEntry = cctx.cache().peekEx(e.key());
if (cacheEntry != null) cacheEntry.invalidate(null, this.tx.xidVersion());
}
} catch (Throwable t) {
U.error(log, "Failed to invalidate entry.", t);
if (t instanceof Error) throw (Error) t;
}
}
}
// Don't forget to clean up.
cctx.mvcc().removeFuture(this);
return true;
}
return false;
}
/** Clears swap entries for evicted partition. */
private void clearSwap() {
assert state() == EVICTED;
assert !GridQueryProcessor.isEnabled(cctx.config())
: "Indexing needs to have unswapped values.";
try {
GridCloseableIterator<Map.Entry<byte[], GridCacheSwapEntry>> it = cctx.swap().iterator(id);
boolean isLocStore = cctx.store().isLocal();
if (it != null) {
// We can safely remove these values because no entries will be created for evicted
// partition.
while (it.hasNext()) {
Map.Entry<byte[], GridCacheSwapEntry> entry = it.next();
byte[] keyBytes = entry.getKey();
KeyCacheObject key = cctx.toCacheKeyObject(keyBytes);
cctx.swap().remove(key);
if (isLocStore) cctx.store().remove(null, key.value(cctx.cacheObjectContext(), false));
}
}
} catch (IgniteCheckedException e) {
U.error(log, "Failed to clear swap for evicted partition: " + this, e);
}
}
private void recheck() {
// If this is the oldest node.
if (oldestNode.get().id().equals(cctx.localNodeId())) {
Collection<UUID> remaining = remaining();
if (!remaining.isEmpty()) {
try {
cctx.io()
.safeSend(
cctx.discovery().nodes(remaining),
new GridDhtPartitionsSingleRequest(exchId),
SYSTEM_POOL,
null);
} catch (IgniteCheckedException e) {
U.error(
log,
"Failed to request partitions from nodes [exchangeId="
+ exchId
+ ", nodes="
+ remaining
+ ']',
e);
}
}
// Resend full partition map because last attempt failed.
else {
if (spreadPartitions()) onDone(exchId.topologyVersion());
}
} else sendPartitions();
// Schedule another send.
scheduleRecheck();
}
private void sendPartitions() {
ClusterNode oldestNode = this.oldestNode.get();
try {
sendLocalPartitions(oldestNode, exchId);
} catch (ClusterTopologyCheckedException ignore) {
if (log.isDebugEnabled())
log.debug(
"Oldest node left during partition exchange [nodeId="
+ oldestNode.id()
+ ", exchId="
+ exchId
+ ']');
} catch (IgniteCheckedException e) {
scheduleRecheck();
U.error(
log,
"Failed to send local partitions to oldest node (will retry after timeout) [oldestNodeId="
+ oldestNode.id()
+ ", exchId="
+ exchId
+ ']',
e);
}
}
/** Perform cleanup of the trash directory. */
private void delete() {
IgfsFileInfo info = null;
try {
info = meta.info(TRASH_ID);
} catch (ClusterTopologyServerNotFoundException e) {
LT.warn(log, e, "Server nodes not found.");
} catch (IgniteCheckedException e) {
U.error(log, "Cannot obtain trash directory info.", e);
}
if (info != null) {
for (Map.Entry<String, IgfsListingEntry> entry : info.listing().entrySet()) {
IgniteUuid fileId = entry.getValue().fileId();
if (log.isDebugEnabled())
log.debug(
"Deleting IGFS trash entry [name=" + entry.getKey() + ", fileId=" + fileId + ']');
try {
if (!cancelled) {
if (delete(entry.getKey(), fileId)) {
if (log.isDebugEnabled())
log.debug(
"Sending delete confirmation message [name="
+ entry.getKey()
+ ", fileId="
+ fileId
+ ']');
sendDeleteMessage(new IgfsDeleteMessage(fileId));
}
} else break;
} catch (IgniteInterruptedCheckedException ignored) {
// Ignore this exception while stopping.
} catch (IgniteCheckedException e) {
U.error(log, "Failed to delete entry from the trash directory: " + entry.getKey(), e);
sendDeleteMessage(new IgfsDeleteMessage(fileId, e));
}
}
}
}
/** @param jobId Job id. */
public void jobFinished(GridHadoopJobId jobId) {
GridHadoopShuffleJob job = jobs.remove(jobId);
if (job != null) {
try {
job.close();
} catch (GridException e) {
U.error(log, "Failed to close job: " + jobId, e);
}
}
}
/**
* Stops shuffle.
*
* @param cancel If should cancel all ongoing activities.
*/
@Override
public void stop(boolean cancel) {
for (GridHadoopShuffleJob job : jobs.values()) {
try {
job.close();
} catch (GridException e) {
U.error(log, "Failed to close job.", e);
}
}
jobs.clear();
}
/** {@inheritDoc} */
@Override
void onCancelAtStop() {
super.onCancelAtStop();
for (GridCacheQueryFutureAdapter fut : futs.values())
try {
fut.cancel();
} catch (IgniteCheckedException e) {
U.error(log, "Failed to cancel running query future: " + fut, e);
}
U.interrupt(threads.values());
}
/**
* Notifies single listener.
*
* @param lsnr Listener.
*/
private void notifyListener(IgniteInClosure<? super IgniteInternalFuture<R>> lsnr) {
assert lsnr != null;
try {
lsnr.apply(this);
} catch (IllegalStateException e) {
U.error(
null,
"Failed to notify listener (is grid stopped?) [fut="
+ this
+ ", lsnr="
+ lsnr
+ ", err="
+ e.getMessage()
+ ']',
e);
} catch (RuntimeException | Error e) {
U.error(null, "Failed to notify listener: " + lsnr, e);
throw e;
}
}
/**
* Notifies single listener.
*
* @param lsnr Listener.
*/
private void notifyListener(GridInClosure<? super GridFuture<R>> lsnr) {
assert lsnr != null;
try {
lsnr.apply(this);
} catch (IllegalStateException ignore) {
U.warn(
null,
"Failed to notify listener (grid is stopped) [grid="
+ ctx.gridName()
+ ", lsnr="
+ lsnr
+ ']');
} catch (RuntimeException e) {
U.error(log, "Failed to notify listener: " + lsnr, e);
throw e;
} catch (Error e) {
U.error(log, "Failed to notify listener: " + lsnr, e);
throw e;
}
}
void cancelAll() {
GridException err =
new GridException("Data loader has been cancelled: " + GridDataLoaderImpl.this);
for (GridFuture<?> f : locFuts) {
try {
f.cancel();
} catch (GridException e) {
U.error(log, "Failed to cancel mini-future.", e);
}
}
for (GridFutureAdapter<?> f : reqs.values()) f.onDone(err);
}
/**
* @param nodeId Node ID.
* @param retryCnt Number of retries.
*/
private void sendAllPartitions(final UUID nodeId, final int retryCnt) {
ClusterNode n = cctx.node(nodeId);
try {
if (n != null) sendAllPartitions(F.asList(n), exchId);
} catch (IgniteCheckedException e) {
if (e instanceof ClusterTopologyCheckedException || !cctx.discovery().alive(n)) {
log.debug(
"Failed to send full partition map to node, node left grid "
+ "[rmtNode="
+ nodeId
+ ", exchangeId="
+ exchId
+ ']');
return;
}
if (retryCnt > 0) {
long timeout = cctx.gridConfig().getNetworkSendRetryDelay();
LT.error(
log,
e,
"Failed to send full partition map to node (will retry after timeout) "
+ "[node="
+ nodeId
+ ", exchangeId="
+ exchId
+ ", timeout="
+ timeout
+ ']');
cctx.time()
.addTimeoutObject(
new GridTimeoutObjectAdapter(timeout) {
@Override
public void onTimeout() {
sendAllPartitions(nodeId, retryCnt - 1);
}
});
} else
U.error(
log,
"Failed to send full partition map [node=" + n + ", exchangeId=" + exchId + ']',
e);
}
}
/**
* Processes cache query request.
*
* @param sndId Sender node id.
* @param req Query request.
*/
@SuppressWarnings("unchecked")
@Override
void processQueryRequest(UUID sndId, GridCacheQueryRequest req) {
if (req.cancel()) {
cancelIds.add(new CancelMessageId(req.id(), sndId));
if (req.fields()) removeFieldsQueryResult(sndId, req.id());
else removeQueryResult(sndId, req.id());
} else {
if (!cancelIds.contains(new CancelMessageId(req.id(), sndId))) {
if (!F.eq(req.cacheName(), cctx.name())) {
GridCacheQueryResponse res =
new GridCacheQueryResponse(
cctx.cacheId(),
req.id(),
new IgniteCheckedException(
"Received request for incorrect cache [expected="
+ cctx.name()
+ ", actual="
+ req.cacheName()));
sendQueryResponse(sndId, res, 0);
} else {
threads.put(req.id(), Thread.currentThread());
try {
GridCacheQueryInfo info = distributedQueryInfo(sndId, req);
if (info == null) return;
if (req.fields()) runFieldsQuery(info);
else runQuery(info);
} catch (Throwable e) {
U.error(log(), "Failed to run query.", e);
sendQueryResponse(
sndId, new GridCacheQueryResponse(cctx.cacheId(), req.id(), e.getCause()), 0);
if (e instanceof Error) throw (Error) e;
} finally {
threads.remove(req.id());
}
}
}
}
}
/** @param e Error. */
void onError(Throwable e) {
tx.commitError(e);
if (err.compareAndSet(null, e)) {
boolean marked = tx.setRollbackOnly();
if (e instanceof GridCacheTxRollbackException)
if (marked) {
try {
tx.rollback();
} catch (GridException ex) {
U.error(log, "Failed to automatically rollback transaction: " + tx, ex);
}
}
onComplete();
}
}
/** @return {@code True} if succeeded. */
private boolean spreadPartitions() {
try {
sendAllPartitions(rmtNodes, exchId);
return true;
} catch (IgniteCheckedException e) {
scheduleRecheck();
if (!X.hasCause(e, InterruptedException.class))
U.error(
log,
"Failed to send full partition map to nodes (will retry after timeout) [nodes="
+ F.nodeId8s(rmtNodes)
+ ", exchangeId="
+ exchId
+ ']',
e);
return false;
}
}
/**
* Starts activity.
*
* @throws IgniteInterruptedCheckedException If interrupted.
*/
public void init() throws IgniteInterruptedCheckedException {
if (isDone()) return;
if (init.compareAndSet(false, true)) {
if (isDone()) return;
try {
// Wait for event to occur to make sure that discovery
// will return corresponding nodes.
U.await(evtLatch);
assert discoEvt != null : this;
assert !dummy && !forcePreload : this;
ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx, exchId.topologyVersion());
oldestNode.set(oldest);
startCaches();
// True if client node joined or failed.
boolean clientNodeEvt;
if (F.isEmpty(reqs)) {
int type = discoEvt.type();
assert type == EVT_NODE_JOINED || type == EVT_NODE_LEFT || type == EVT_NODE_FAILED
: discoEvt;
clientNodeEvt = CU.clientNode(discoEvt.eventNode());
} else {
assert discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT : discoEvt;
boolean clientOnlyStart = true;
for (DynamicCacheChangeRequest req : reqs) {
if (!req.clientStartOnly()) {
clientOnlyStart = false;
break;
}
}
clientNodeEvt = clientOnlyStart;
}
if (clientNodeEvt) {
ClusterNode node = discoEvt.eventNode();
// Client need to initialize affinity for local join event or for stated client caches.
if (!node.isLocal()) {
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
GridDhtPartitionTopology top = cacheCtx.topology();
top.updateTopologyVersion(exchId, this, -1, stopping(cacheCtx.cacheId()));
if (cacheCtx.affinity().affinityTopologyVersion() == AffinityTopologyVersion.NONE) {
initTopology(cacheCtx);
top.beforeExchange(this);
} else
cacheCtx.affinity().clientEventTopologyChange(discoEvt, exchId.topologyVersion());
}
if (exchId.isLeft())
cctx.mvcc().removeExplicitNodeLocks(exchId.nodeId(), exchId.topologyVersion());
onDone(exchId.topologyVersion());
skipPreload = cctx.kernalContext().clientNode();
return;
}
}
if (cctx.kernalContext().clientNode()) {
skipPreload = true;
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
GridDhtPartitionTopology top = cacheCtx.topology();
top.updateTopologyVersion(exchId, this, -1, stopping(cacheCtx.cacheId()));
}
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
initTopology(cacheCtx);
}
if (oldestNode.get() != null) {
rmtNodes =
new ConcurrentLinkedQueue<>(
CU.aliveRemoteServerNodesWithCaches(cctx, exchId.topologyVersion()));
rmtIds = Collections.unmodifiableSet(new HashSet<>(F.nodeIds(rmtNodes)));
ready.set(true);
initFut.onDone(true);
if (log.isDebugEnabled()) log.debug("Initialized future: " + this);
sendPartitions();
} else onDone(exchId.topologyVersion());
return;
}
assert oldestNode.get() != null;
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (isCacheAdded(cacheCtx.cacheId(), exchId.topologyVersion())) {
if (cacheCtx
.discovery()
.cacheAffinityNodes(cacheCtx.name(), topologyVersion())
.isEmpty())
U.quietAndWarn(log, "No server nodes found for cache client: " + cacheCtx.namex());
}
cacheCtx.preloader().onExchangeFutureAdded();
}
List<String> cachesWithoutNodes = null;
if (exchId.isLeft()) {
for (String name : cctx.cache().cacheNames()) {
if (cctx.discovery().cacheAffinityNodes(name, topologyVersion()).isEmpty()) {
if (cachesWithoutNodes == null) cachesWithoutNodes = new ArrayList<>();
cachesWithoutNodes.add(name);
// Fire event even if there is no client cache started.
if (cctx.gridEvents().isRecordable(EventType.EVT_CACHE_NODES_LEFT)) {
Event evt =
new CacheEvent(
name,
cctx.localNode(),
cctx.localNode(),
"All server nodes have left the cluster.",
EventType.EVT_CACHE_NODES_LEFT,
0,
false,
null,
null,
null,
null,
false,
null,
false,
null,
null,
null);
cctx.gridEvents().record(evt);
}
}
}
}
if (cachesWithoutNodes != null) {
StringBuilder sb =
new StringBuilder(
"All server nodes for the following caches have left the cluster: ");
for (int i = 0; i < cachesWithoutNodes.size(); i++) {
String cache = cachesWithoutNodes.get(i);
sb.append('\'').append(cache).append('\'');
if (i != cachesWithoutNodes.size() - 1) sb.append(", ");
}
U.quietAndWarn(log, sb.toString());
U.quietAndWarn(log, "Must have server nodes for caches to operate.");
}
assert discoEvt != null;
assert exchId.nodeId().equals(discoEvt.eventNode().id());
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
GridClientPartitionTopology clientTop =
cctx.exchange().clearClientTopology(cacheCtx.cacheId());
long updSeq = clientTop == null ? -1 : clientTop.lastUpdateSequence();
// Update before waiting for locks.
if (!cacheCtx.isLocal())
cacheCtx
.topology()
.updateTopologyVersion(exchId, this, updSeq, stopping(cacheCtx.cacheId()));
}
// Grab all alive remote nodes with order of equal or less than last joined node.
rmtNodes =
new ConcurrentLinkedQueue<>(
CU.aliveRemoteServerNodesWithCaches(cctx, exchId.topologyVersion()));
rmtIds = Collections.unmodifiableSet(new HashSet<>(F.nodeIds(rmtNodes)));
for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> m : singleMsgs.entrySet())
// If received any messages, process them.
onReceive(m.getKey(), m.getValue());
for (Map.Entry<UUID, GridDhtPartitionsFullMessage> m : fullMsgs.entrySet())
// If received any messages, process them.
onReceive(m.getKey(), m.getValue());
AffinityTopologyVersion topVer = exchId.topologyVersion();
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
// Must initialize topology after we get discovery event.
initTopology(cacheCtx);
cacheCtx.preloader().updateLastExchangeFuture(this);
}
IgniteInternalFuture<?> partReleaseFut = cctx.partitionReleaseFuture(topVer);
// Assign to class variable so it will be included into toString() method.
this.partReleaseFut = partReleaseFut;
if (log.isDebugEnabled()) log.debug("Before waiting for partition release future: " + this);
while (true) {
try {
partReleaseFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS);
break;
} catch (IgniteFutureTimeoutCheckedException ignored) {
// Print pending transactions and locks that might have led to hang.
dumpPendingObjects();
}
}
if (log.isDebugEnabled()) log.debug("After waiting for partition release future: " + this);
if (!F.isEmpty(reqs)) blockGateways();
if (exchId.isLeft())
cctx.mvcc().removeExplicitNodeLocks(exchId.nodeId(), exchId.topologyVersion());
IgniteInternalFuture<?> locksFut = cctx.mvcc().finishLocks(exchId.topologyVersion());
while (true) {
try {
locksFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS);
break;
} catch (IgniteFutureTimeoutCheckedException ignored) {
U.warn(
log,
"Failed to wait for locks release future. "
+ "Dumping pending objects that might be the cause: "
+ cctx.localNodeId());
U.warn(log, "Locked entries:");
Map<IgniteTxKey, Collection<GridCacheMvccCandidate>> locks =
cctx.mvcc().unfinishedLocks(exchId.topologyVersion());
for (Map.Entry<IgniteTxKey, Collection<GridCacheMvccCandidate>> e : locks.entrySet())
U.warn(log, "Locked entry [key=" + e.getKey() + ", mvcc=" + e.getValue() + ']');
}
}
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
// Notify replication manager.
GridCacheContext drCacheCtx =
cacheCtx.isNear() ? cacheCtx.near().dht().context() : cacheCtx;
if (drCacheCtx.isDrEnabled()) drCacheCtx.dr().beforeExchange(topVer, exchId.isLeft());
// Partition release future is done so we can flush the write-behind store.
cacheCtx.store().forceFlush();
// Process queued undeploys prior to sending/spreading map.
cacheCtx.preloader().unwindUndeploys();
GridDhtPartitionTopology top = cacheCtx.topology();
assert topVer.equals(top.topologyVersion())
: "Topology version is updated only in this class instances inside single ExchangeWorker thread.";
top.beforeExchange(this);
}
for (GridClientPartitionTopology top : cctx.exchange().clientTopologies()) {
top.updateTopologyVersion(exchId, this, -1, stopping(top.cacheId()));
top.beforeExchange(this);
}
} catch (IgniteInterruptedCheckedException e) {
onDone(e);
throw e;
} catch (Throwable e) {
U.error(
log,
"Failed to reinitialize local partitions (preloading will be stopped): " + exchId,
e);
onDone(e);
if (e instanceof Error) throw (Error) e;
return;
}
if (F.isEmpty(rmtIds)) {
onDone(exchId.topologyVersion());
return;
}
ready.set(true);
initFut.onDone(true);
if (log.isDebugEnabled()) log.debug("Initialized future: " + this);
// If this node is not oldest.
if (!oldestNode.get().id().equals(cctx.localNodeId())) sendPartitions();
else {
boolean allReceived = allReceived();
if (allReceived && replied.compareAndSet(false, true)) {
if (spreadPartitions()) onDone(exchId.topologyVersion());
}
}
scheduleRecheck();
} else assert false : "Skipped init future: " + this;
}
/**
* Sends cache query response.
*
* @param nodeId Node to send response.
* @param res Cache query response.
* @param timeout Message timeout.
* @return {@code true} if response was sent, {@code false} otherwise.
*/
private boolean sendQueryResponse(UUID nodeId, GridCacheQueryResponse res, long timeout) {
ClusterNode node = cctx.node(nodeId);
if (node == null) return false;
int attempt = 1;
IgniteCheckedException err = null;
while (!Thread.currentThread().isInterrupted()) {
try {
if (log.isDebugEnabled()) log.debug("Send query response: " + res);
Object topic = topic(nodeId, res.requestId());
cctx.io()
.sendOrderedMessage(
node, topic, res, cctx.ioPolicy(), timeout > 0 ? timeout : Long.MAX_VALUE);
return true;
} catch (ClusterTopologyCheckedException ignored) {
if (log.isDebugEnabled())
log.debug(
"Failed to send query response since node left grid [nodeId="
+ nodeId
+ ", res="
+ res
+ "]");
return false;
} catch (IgniteCheckedException e) {
if (err == null) err = e;
if (Thread.currentThread().isInterrupted()) break;
if (attempt < RESEND_ATTEMPTS) {
if (log.isDebugEnabled())
log.debug(
"Failed to send queries response (will try again) [nodeId="
+ nodeId
+ ", res="
+ res
+ ", attempt="
+ attempt
+ ", err="
+ e
+ "]");
if (!Thread.currentThread().isInterrupted())
try {
U.sleep(RESEND_FREQ);
} catch (IgniteInterruptedCheckedException e1) {
U.error(
log,
"Waiting for queries response resending was interrupted (response will not be sent) "
+ "[nodeId="
+ nodeId
+ ", response="
+ res
+ "]",
e1);
return false;
}
} else {
U.error(
log,
"Failed to sender cache response [nodeId=" + nodeId + ", response=" + res + "]",
err);
return false;
}
}
attempt++;
}
return false;
}
/** {@inheritDoc} */
@Override
public GridFuture<GridCacheTxEx<K, V>> prepareAsync() {
GridNearTxPrepareFuture<K, V> fut = prepFut.get();
if (fut == null) {
// Future must be created before any exception can be thrown.
if (!prepFut.compareAndSet(null, fut = new GridNearTxPrepareFuture<K, V>(cctx, this)))
return prepFut.get();
} else
// Prepare was called explicitly.
return fut;
if (!state(PREPARING)) {
if (setRollbackOnly()) {
if (timedOut())
fut.onError(
new GridCacheTxTimeoutException(
"Transaction timed out and was rolled back: " + this));
else
fut.onError(
new GridException(
"Invalid transaction state for prepare [state="
+ state()
+ ", tx="
+ this
+ ']'));
} else
fut.onError(
new GridCacheTxRollbackException(
"Invalid transaction state for prepare [state=" + state() + ", tx=" + this + ']'));
return fut;
}
// For pessimistic mode we don't distribute prepare request.
if (pessimistic()) {
try {
userPrepare();
if (!state(PREPARED)) {
setRollbackOnly();
fut.onError(
new GridException(
"Invalid transaction state for commit [state=" + state() + ", tx=" + this + ']'));
return fut;
}
fut.complete();
return fut;
} catch (GridException e) {
fut.onError(e);
return fut;
}
}
try {
cctx.topology().readLock();
try {
topologyVersion(cctx.topology().topologyVersion());
userPrepare();
} finally {
cctx.topology().readUnlock();
}
// This will attempt to locally commit
// EVENTUALLY CONSISTENT transactions.
fut.onPreparedEC();
// Make sure to add future before calling prepare.
cctx.mvcc().addFuture(fut);
fut.prepare();
} catch (GridCacheTxTimeoutException e) {
fut.onError(e);
} catch (GridCacheTxOptimisticException e) {
fut.onError(e);
} catch (GridException e) {
setRollbackOnly();
String msg = "Failed to prepare transaction (will attempt rollback): " + this;
log.error(msg, e);
try {
rollback();
} catch (GridException e1) {
U.error(log, "Failed to rollback transaction: " + this, e1);
}
fut.onError(new GridCacheTxRollbackException(msg, e));
}
return fut;
}
/**
* @param entries Entries to submit.
* @param curFut Current future.
* @throws GridInterruptedException If interrupted.
*/
private void submit(final List<Map.Entry<K, V>> entries, final GridFutureAdapter<Object> curFut)
throws GridInterruptedException {
assert entries != null;
assert !entries.isEmpty();
assert curFut != null;
incrementActiveTasks();
GridFuture<Object> fut;
if (isLocNode) {
fut =
ctx.closure()
.callLocalSafe(
new GridDataLoadUpdateJob<>(ctx, log, cacheName, entries, false, updater),
false);
locFuts.add(fut);
fut.listenAsync(
new GridInClosure<GridFuture<Object>>() {
@Override
public void apply(GridFuture<Object> t) {
try {
boolean rmv = locFuts.remove(t);
assert rmv;
curFut.onDone(t.get());
} catch (GridException e) {
curFut.onDone(e);
}
}
});
} else {
byte[] entriesBytes;
try {
entriesBytes = ctx.config().getMarshaller().marshal(entries);
if (updaterBytes == null) {
assert updater != null;
updaterBytes = ctx.config().getMarshaller().marshal(updater);
}
if (topicBytes == null) topicBytes = ctx.config().getMarshaller().marshal(topic);
} catch (GridException e) {
U.error(log, "Failed to marshal (request will not be sent).", e);
return;
}
GridDeployment dep = null;
GridPeerDeployAware jobPda0 = null;
if (ctx.deploy().enabled()) {
try {
jobPda0 = jobPda;
assert jobPda0 != null;
dep = ctx.deploy().deploy(jobPda0.deployClass(), jobPda0.classLoader());
} catch (GridException e) {
U.error(
log,
"Failed to deploy class (request will not be sent): " + jobPda0.deployClass(),
e);
return;
}
if (dep == null)
U.warn(log, "Failed to deploy class (request will be sent): " + jobPda0.deployClass());
}
long reqId = idGen.incrementAndGet();
fut = curFut;
reqs.put(reqId, (GridFutureAdapter<Object>) fut);
GridDataLoadRequest<Object, Object> req =
new GridDataLoadRequest<>(
reqId,
topicBytes,
cacheName,
updaterBytes,
entriesBytes,
true,
dep != null ? dep.deployMode() : null,
dep != null ? jobPda0.deployClass().getName() : null,
dep != null ? dep.userVersion() : null,
dep != null ? dep.participants() : null,
dep != null ? dep.classLoaderId() : null,
dep == null);
try {
ctx.io().send(node, TOPIC_DATALOAD, req, PUBLIC_POOL);
if (log.isDebugEnabled())
log.debug("Sent request to node [nodeId=" + node.id() + ", req=" + req + ']');
} catch (GridException e) {
if (ctx.discovery().alive(node) && ctx.discovery().pingNode(node.id()))
((GridFutureAdapter<Object>) fut).onDone(e);
else
((GridFutureAdapter<Object>) fut)
.onDone(
new GridTopologyException(
"Failed to send " + "request (node has left): " + node.id()));
}
}
}
/** Clears values for this partition. */
private void clearAll() {
GridCacheVersion clearVer = cctx.versions().next();
boolean swap = cctx.isSwapOrOffheapEnabled();
boolean rec = cctx.events().isRecordable(EVT_CACHE_REBALANCE_OBJECT_UNLOADED);
Iterator<GridDhtCacheEntry> it = map.values().iterator();
GridCloseableIterator<Map.Entry<byte[], GridCacheSwapEntry>> swapIt = null;
if (swap
&& GridQueryProcessor.isEnabled(cctx.config())) { // Indexing needs to unswap cache values.
Iterator<GridDhtCacheEntry> unswapIt = null;
try {
swapIt = cctx.swap().iterator(id);
unswapIt = unswapIterator(swapIt);
} catch (Exception e) {
U.error(log, "Failed to clear swap for evicted partition: " + this, e);
}
if (unswapIt != null) it = F.concat(it, unswapIt);
}
try {
while (it.hasNext()) {
GridDhtCacheEntry cached = it.next();
try {
if (cached.clearInternal(clearVer, swap)) {
map.remove(cached.key(), cached);
if (!cached.isInternal()) {
mapPubSize.decrement();
if (rec)
cctx.events()
.addEvent(
cached.partition(),
cached.key(),
cctx.localNodeId(),
(IgniteUuid) null,
null,
EVT_CACHE_REBALANCE_OBJECT_UNLOADED,
null,
false,
cached.rawGet(),
cached.hasValue(),
null,
null,
null);
}
}
} catch (IgniteCheckedException e) {
U.error(log, "Failed to clear cache entry for evicted partition: " + cached, e);
}
}
} finally {
U.close(swapIt, log);
}
}
/**
* @param gridName Grid instance name. Can be {@code null}.
* @param exec Executor service.
* @param parentLog Parent logger.
*/
static void runBackgroundCheck(String gridName, Executor exec, GridLogger parentLog) {
assert exec != null;
assert parentLog != null;
final GridLogger log = parentLog.getLogger(GridDiagnostic.class);
try {
exec.execute(
new GridWorker(gridName, "grid-diagnostic-1", log) {
@Override
public void body() {
try {
InetAddress localHost = U.getLocalHost();
if (!localHost.isReachable(REACH_TIMEOUT)) {
U.warn(
log,
"Default local host is unreachable. This may lead to delays on "
+ "grid network operations. Check your OS network setting to correct it.",
"Default local host is unreachable.");
}
} catch (IOException ignore) {
U.warn(
log,
"Failed to perform network diagnostics. It is usually caused by serious "
+ "network configuration problem. Check your OS network setting to correct it.",
"Failed to perform network diagnostics.");
}
}
});
exec.execute(
new GridWorker(gridName, "grid-diagnostic-2", log) {
@Override
public void body() {
try {
InetAddress localHost = U.getLocalHost();
if (localHost.isLoopbackAddress()) {
U.warn(
log,
"Default local host is a loopback address. This can be a sign of "
+ "potential network configuration problem.",
"Default local host is a loopback address.");
}
} catch (IOException ignore) {
U.warn(
log,
"Failed to perform network diagnostics. It is usually caused by serious "
+ "network configuration problem. Check your OS network setting to correct it.",
"Failed to perform network diagnostics.");
}
}
});
exec.execute(
new GridWorker(gridName, "grid-diagnostic-3", log) {
@Override
public void body() {
String jdkStrLow = U.jdkString().toLowerCase();
if (jdkStrLow.contains("jrockit") && jdkStrLow.contains("1.5.")) {
U.warn(
log,
"BEA JRockit VM ver. 1.5.x has shown problems with NIO functionality in our "
+ "tests that were not reproducible in other VMs. We recommend using Sun VM. Should you "
+ "have further questions please contact us at [email protected]",
"BEA JRockit VM ver. 1.5.x is not supported.");
}
}
});
exec.execute(
new GridWorker(gridName, "grid-diagnostic-4", log) {
@Override
public void body() {
// Sufficiently tested OS.
if (!U.isSufficientlyTestedOs()) {
U.warn(
log,
"This operating system has been tested less rigorously: "
+ U.osString()
+ ". Our team will appreciate the feedback if you experience any problems running "
+ "gridgain in this environment. You can always send your feedback to [email protected]",
"This OS is tested less rigorously: " + U.osString());
}
}
});
exec.execute(
new GridWorker(gridName, "grid-diagnostic-5", log) {
@Override
public void body() {
// Fix for GG-1075.
if (U.allLocalMACs() == null)
U.warn(
log,
"No live network interfaces detected. If IP-multicast discovery is used - "
+ "make sure to add 127.0.0.1 as a local address.",
"No live network interfaces. Add 127.0.0.1 as a local address.");
}
});
exec.execute(
new GridWorker(gridName, "grid-diagnostic-6", log) {
@Override
public void body() {
if (System.getProperty("com.sun.management.jmxremote") != null) {
String portStr = System.getProperty("com.sun.management.jmxremote.port");
if (portStr != null)
try {
Integer.parseInt(portStr);
return;
} catch (NumberFormatException ignore) {
}
U.warn(
log,
"JMX remote management is enabled but JMX port is either not set or invalid. "
+ "Check system property 'com.sun.management.jmxremote.port' to make sure it specifies "
+ "valid TCP/IP port.",
"JMX remote port is invalid - JMX management is off.");
}
}
});
} catch (RejectedExecutionException e) {
U.error(
log,
"Failed to start background network diagnostics check due to thread pool execution "
+ "rejection. In most cases it indicates a severe configuration problem with GridGain.",
"Failed to start background network diagnostics.",
e);
}
}