/** @param workTokDir Token directory (common for multiple nodes). */
private void cleanupResources(File workTokDir) {
RandomAccessFile lockFile = null;
FileLock lock = null;
try {
lockFile = new RandomAccessFile(new File(workTokDir, LOCK_FILE_NAME), "rw");
lock = lockFile.getChannel().lock();
if (lock != null) processTokenDirectory(workTokDir);
else if (log.isDebugEnabled())
log.debug(
"Token directory is being processed concurrently: " + workTokDir.getAbsolutePath());
} catch (OverlappingFileLockException ignored) {
if (log.isDebugEnabled())
log.debug(
"Token directory is being processed concurrently: " + workTokDir.getAbsolutePath());
} catch (FileLockInterruptionException ignored) {
Thread.currentThread().interrupt();
} catch (IOException e) {
U.error(log, "Failed to process directory: " + workTokDir.getAbsolutePath(), e);
} finally {
U.releaseQuiet(lock);
U.closeQuiet(lockFile);
}
}
/**
* @param cctx Context.
* @param id Partition ID.
*/
@SuppressWarnings("ExternalizableWithoutPublicNoArgConstructor")
GridDhtLocalPartition(GridCacheContext cctx, int id) {
assert cctx != null;
this.id = id;
this.cctx = cctx;
log = U.logger(cctx.kernalContext(), logRef, this);
rent =
new GridFutureAdapter<Object>() {
@Override
public String toString() {
return "PartitionRentFuture [part=" + GridDhtLocalPartition.this + ", map=" + map + ']';
}
};
map = new ConcurrentHashMap8<>(cctx.config().getStartSize() / cctx.affinity().partitions());
int delQueueSize =
CU.isSystemCache(cctx.name())
? 100
: Math.max(MAX_DELETE_QUEUE_SIZE / cctx.affinity().partitions(), 20);
rmvQueue = new GridCircularBuffer<>(U.ceilPow2(delQueueSize));
}
/** {@inheritDoc} */
@Override
void onCancelAtStop() {
super.onCancelAtStop();
for (GridCacheQueryFutureAdapter fut : futs.values())
try {
fut.cancel();
} catch (IgniteCheckedException e) {
U.error(log, "Failed to cancel running query future: " + fut, e);
}
U.interrupt(threads.values());
}
/**
* Waits until total number of events processed is equal or greater then argument passed.
*
* @param cnt Number of events to wait.
* @param timeout Timeout to wait.
* @return {@code True} if successfully waited, {@code false} if timeout happened.
* @throws InterruptedException If thread is interrupted.
*/
public synchronized boolean awaitEvents(int cnt, long timeout) throws InterruptedException {
long start = U.currentTimeMillis();
long now = start;
while (start + timeout > now) {
if (evtCnt >= cnt) return true;
wait(start + timeout - now);
now = U.currentTimeMillis();
}
return false;
}
/**
* Processes cache query response.
*
* @param sndId Sender node id.
* @param res Query response.
*/
@SuppressWarnings("unchecked")
private void processQueryResponse(UUID sndId, GridCacheQueryResponse res) {
if (log.isDebugEnabled()) log.debug("Received query response: " + res);
GridCacheQueryFutureAdapter fut = getQueryFuture(res.requestId());
if (fut != null)
if (res.fields())
((GridCacheDistributedFieldsQueryFuture) fut)
.onPage(
sndId,
res.metadata(),
(Collection<Map<String, Object>>) ((Collection) res.data()),
res.error(),
res.isFinished());
else fut.onPage(sndId, res.data(), res.error(), res.isFinished());
else if (!cancelled.contains(res.requestId()))
U.warn(
log,
"Received response for finished or unknown query [rmtNodeId="
+ sndId
+ ", res="
+ res
+ ']');
}
/** {@inheritDoc} */
@Override
public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr)
throws IgniteCheckedException {
super.finishUnmarshal(ctx, ldr);
if (writes != null) unmarshalTx(writes, false, ctx, ldr);
if (reads != null) unmarshalTx(reads, false, ctx, ldr);
if (grpLockKeyBytes != null && grpLockKey == null)
grpLockKey = ctx.marshaller().unmarshal(grpLockKeyBytes, ldr);
if (dhtVerKeys != null && dhtVers == null) {
assert dhtVerVals != null;
assert dhtVerKeys.size() == dhtVerVals.size();
Iterator<IgniteTxKey> keyIt = dhtVerKeys.iterator();
Iterator<GridCacheVersion> verIt = dhtVerVals.iterator();
dhtVers = U.newHashMap(dhtVerKeys.size());
while (keyIt.hasNext()) {
IgniteTxKey key = keyIt.next();
key.finishUnmarshal(ctx.cacheContext(key.cacheId()), ldr);
dhtVers.put(key, verIt.next());
}
}
if (txNodesBytes != null) txNodes = ctx.marshaller().unmarshal(txNodesBytes, ldr);
}
/** Clears swap entries for evicted partition. */
private void clearSwap() {
assert state() == EVICTED;
assert !GridQueryProcessor.isEnabled(cctx.config())
: "Indexing needs to have unswapped values.";
try {
GridCloseableIterator<Map.Entry<byte[], GridCacheSwapEntry>> it = cctx.swap().iterator(id);
boolean isLocStore = cctx.store().isLocal();
if (it != null) {
// We can safely remove these values because no entries will be created for evicted
// partition.
while (it.hasNext()) {
Map.Entry<byte[], GridCacheSwapEntry> entry = it.next();
byte[] keyBytes = entry.getKey();
KeyCacheObject key = cctx.toCacheKeyObject(keyBytes);
cctx.swap().remove(key);
if (isLocStore) cctx.store().remove(null, key.value(cctx.cacheObjectContext(), false));
}
}
} catch (IgniteCheckedException e) {
U.error(log, "Failed to clear swap for evicted partition: " + this, e);
}
}
/** @return Nodes to execute on. */
private Collection<ClusterNode> nodes() {
CacheMode cacheMode = cctx.config().getCacheMode();
switch (cacheMode) {
case LOCAL:
if (prj != null)
U.warn(
log,
"Ignoring query projection because it's executed over LOCAL cache "
+ "(only local node will be queried): "
+ this);
return Collections.singletonList(cctx.localNode());
case REPLICATED:
if (prj != null || partition() != null) return nodes(cctx, prj, partition());
return cctx.affinityNode()
? Collections.singletonList(cctx.localNode())
: Collections.singletonList(F.rand(nodes(cctx, null, partition())));
case PARTITIONED:
return nodes(cctx, prj, partition());
default:
throw new IllegalStateException("Unknown cache distribution mode: " + cacheMode);
}
}
private void recheck() {
// If this is the oldest node.
if (oldestNode.get().id().equals(cctx.localNodeId())) {
Collection<UUID> remaining = remaining();
if (!remaining.isEmpty()) {
try {
cctx.io()
.safeSend(
cctx.discovery().nodes(remaining),
new GridDhtPartitionsSingleRequest(exchId),
SYSTEM_POOL,
null);
} catch (IgniteCheckedException e) {
U.error(
log,
"Failed to request partitions from nodes [exchangeId="
+ exchId
+ ", nodes="
+ remaining
+ ']',
e);
}
}
// Resend full partition map because last attempt failed.
else {
if (spreadPartitions()) onDone(exchId.topologyVersion());
}
} else sendPartitions();
// Schedule another send.
scheduleRecheck();
}
/** {@inheritDoc} */
@Override
protected final boolean tryReleaseShared(int ignore) {
endTime = U.currentTimeMillis();
// Always signal after setting final done status.
return true;
}
/**
* @param out Output stream.
* @param err Error cause.
*/
private void sendErrorResponse(ObjectOutput out, Exception err) {
try {
out.writeObject(new IpcSharedMemoryInitResponse(err));
} catch (IOException e) {
U.error(log, "Failed to send error response to client.", e);
}
}
private void sendPartitions() {
ClusterNode oldestNode = this.oldestNode.get();
try {
sendLocalPartitions(oldestNode, exchId);
} catch (ClusterTopologyCheckedException ignore) {
if (log.isDebugEnabled())
log.debug(
"Oldest node left during partition exchange [nodeId="
+ oldestNode.id()
+ ", exchId="
+ exchId
+ ']');
} catch (IgniteCheckedException e) {
scheduleRecheck();
U.error(
log,
"Failed to send local partitions to oldest node (will retry after timeout) [oldestNodeId="
+ oldestNode.id()
+ ", exchId="
+ exchId
+ ']',
e);
}
}
/** {@inheritDoc} */
@Override
public void start() throws IgniteCheckedException {
IpcSharedMemoryNativeLoader.load(log);
pid = IpcSharedMemoryUtils.pid();
if (pid == -1) throw new IpcEndpointBindException("Failed to get PID of the current process.");
if (size <= 0) throw new IpcEndpointBindException("Space size should be positive: " + size);
String tokDirPath = this.tokDirPath;
if (F.isEmpty(tokDirPath)) throw new IpcEndpointBindException("Token directory path is empty.");
tokDirPath = tokDirPath + '/' + locNodeId.toString() + '-' + IpcSharedMemoryUtils.pid();
tokDir = U.resolveWorkDirectory(tokDirPath, false);
if (port <= 0 || port >= 0xffff)
throw new IpcEndpointBindException("Port value is illegal: " + port);
try {
srvSock = new ServerSocket();
// Always bind to loopback.
srvSock.bind(new InetSocketAddress("127.0.0.1", port));
} catch (IOException e) {
// Although empty socket constructor never throws exception, close it just in case.
U.closeQuiet(srvSock);
throw new IpcEndpointBindException(
"Failed to bind shared memory IPC endpoint (is port already " + "in use?): " + port, e);
}
gcWorker = new GcWorker(gridName, "ipc-shmem-gc", log);
new IgniteThread(gcWorker).start();
if (log.isInfoEnabled())
log.info(
"IPC shared memory server endpoint started [port="
+ port
+ ", tokDir="
+ tokDir.getAbsolutePath()
+ ']');
}
/** Perform cleanup of the trash directory. */
private void delete() {
IgfsFileInfo info = null;
try {
info = meta.info(TRASH_ID);
} catch (ClusterTopologyServerNotFoundException e) {
LT.warn(log, e, "Server nodes not found.");
} catch (IgniteCheckedException e) {
U.error(log, "Cannot obtain trash directory info.", e);
}
if (info != null) {
for (Map.Entry<String, IgfsListingEntry> entry : info.listing().entrySet()) {
IgniteUuid fileId = entry.getValue().fileId();
if (log.isDebugEnabled())
log.debug(
"Deleting IGFS trash entry [name=" + entry.getKey() + ", fileId=" + fileId + ']');
try {
if (!cancelled) {
if (delete(entry.getKey(), fileId)) {
if (log.isDebugEnabled())
log.debug(
"Sending delete confirmation message [name="
+ entry.getKey()
+ ", fileId="
+ fileId
+ ']');
sendDeleteMessage(new IgfsDeleteMessage(fileId));
}
} else break;
} catch (IgniteInterruptedCheckedException ignored) {
// Ignore this exception while stopping.
} catch (IgniteCheckedException e) {
U.error(log, "Failed to delete entry from the trash directory: " + entry.getKey(), e);
sendDeleteMessage(new IgfsDeleteMessage(fileId, e));
}
}
}
}
/**
* @param cacheCtx Cache context.
* @return {@code True} if local node can calculate affinity on it's own for this partition map
* exchange.
*/
private boolean canCalculateAffinity(GridCacheContext cacheCtx) {
AffinityFunction affFunc = cacheCtx.config().getAffinity();
// Do not request affinity from remote nodes if affinity function is not centralized.
if (!U.hasAnnotation(affFunc, AffinityCentralizedFunction.class)) return true;
// If local node did not initiate exchange or local node is the only cache node in grid.
Collection<ClusterNode> affNodes = CU.affinityNodes(cacheCtx, exchId.topologyVersion());
return !exchId.nodeId().equals(cctx.localNodeId())
|| (affNodes.size() == 1 && affNodes.contains(cctx.localNode()));
}
/** {@inheritDoc} */
@Override
public void onDiscoveryDataReceived(UUID nodeId, UUID rmtNodeId, Serializable data) {
Map<String, Serializable> discData = (Map<String, Serializable>) data;
if (discData != null) {
for (Map.Entry<String, Serializable> e : discData.entrySet()) {
PluginProvider provider = plugins.get(e.getKey());
if (provider != null) provider.receiveDiscoveryData(nodeId, e.getValue());
else U.warn(log, "Received discovery data for unknown plugin: " + e.getKey());
}
}
}
/**
* Notifies single listener.
*
* @param lsnr Listener.
*/
private void notifyListener(IgniteInClosure<? super IgniteInternalFuture<R>> lsnr) {
assert lsnr != null;
try {
lsnr.apply(this);
} catch (IllegalStateException e) {
U.error(
null,
"Failed to notify listener (is grid stopped?) [fut="
+ this
+ ", lsnr="
+ lsnr
+ ", err="
+ e.getMessage()
+ ']',
e);
} catch (RuntimeException | Error e) {
U.error(null, "Failed to notify listener: " + lsnr, e);
throw e;
}
}
/** {@inheritDoc} */
@Override
public void close() {
closed = true;
U.closeQuiet(srvSock);
if (gcWorker != null) {
U.cancel(gcWorker);
// This method may be called from already interrupted thread.
// Need to ensure cleaning on close.
boolean interrupted = Thread.interrupted();
try {
U.join(gcWorker);
} catch (IgniteInterruptedCheckedException e) {
U.warn(log, "Interrupted when stopping GC worker.", e);
} finally {
if (interrupted) Thread.currentThread().interrupt();
}
}
}
/**
* @param nanosTimeout Timeout (nanoseconds).
* @return Result.
* @throws InterruptedException If interrupted.
* @throws IgniteFutureTimeoutCheckedException If timeout reached before computation completed.
* @throws IgniteCheckedException If error occurred.
*/
@Nullable
protected R get0(long nanosTimeout) throws InterruptedException, IgniteCheckedException {
if (endTime == 0 && !tryAcquireSharedNanos(0, nanosTimeout))
throw new IgniteFutureTimeoutCheckedException(
"Timeout was reached before computation completed.");
if (getState() == CANCELLED)
throw new IgniteFutureCancelledCheckedException("Future was cancelled: " + this);
assert resFlag != 0;
if (resFlag == ERR) throw U.cast((Throwable) res);
return (R) res;
}
/**
* @param nodeId Node ID.
* @param retryCnt Number of retries.
*/
private void sendAllPartitions(final UUID nodeId, final int retryCnt) {
ClusterNode n = cctx.node(nodeId);
try {
if (n != null) sendAllPartitions(F.asList(n), exchId);
} catch (IgniteCheckedException e) {
if (e instanceof ClusterTopologyCheckedException || !cctx.discovery().alive(n)) {
log.debug(
"Failed to send full partition map to node, node left grid "
+ "[rmtNode="
+ nodeId
+ ", exchangeId="
+ exchId
+ ']');
return;
}
if (retryCnt > 0) {
long timeout = cctx.gridConfig().getNetworkSendRetryDelay();
LT.error(
log,
e,
"Failed to send full partition map to node (will retry after timeout) "
+ "[node="
+ nodeId
+ ", exchangeId="
+ exchId
+ ", timeout="
+ timeout
+ ']');
cctx.time()
.addTimeoutObject(
new GridTimeoutObjectAdapter(timeout) {
@Override
public void onTimeout() {
sendAllPartitions(nodeId, retryCnt - 1);
}
});
} else
U.error(
log,
"Failed to send full partition map [node=" + n + ", exchangeId=" + exchId + ']',
e);
}
}
/** {@inheritDoc} */
@Override
public String toString() {
return S.toString(
GridDhtLocalPartition.class,
this,
"state",
state(),
"reservations",
reservations(),
"empty",
map.isEmpty(),
"createTime",
U.format(createTime),
"mapPubSize",
mapPubSize);
}
/**
* Checks that cache is empty.
*
* @param cache Cache to check.
* @throws org.apache.ignite.internal.IgniteInterruptedCheckedException If interrupted while
* sleeping.
*/
@SuppressWarnings({"ErrorNotRethrown", "TypeMayBeWeakened"})
private void checkEmpty(IgniteCache<String, String> cache)
throws IgniteInterruptedCheckedException {
for (int i = 0; i < 3; i++) {
try {
assertTrue(!cache.iterator().hasNext());
break;
} catch (AssertionError e) {
if (i == 2) throw e;
info(">>> Cache is not empty, flushing evictions.");
U.sleep(1000);
}
}
}
/**
* Processes cache query request.
*
* @param sndId Sender node id.
* @param req Query request.
*/
@SuppressWarnings("unchecked")
@Override
void processQueryRequest(UUID sndId, GridCacheQueryRequest req) {
if (req.cancel()) {
cancelIds.add(new CancelMessageId(req.id(), sndId));
if (req.fields()) removeFieldsQueryResult(sndId, req.id());
else removeQueryResult(sndId, req.id());
} else {
if (!cancelIds.contains(new CancelMessageId(req.id(), sndId))) {
if (!F.eq(req.cacheName(), cctx.name())) {
GridCacheQueryResponse res =
new GridCacheQueryResponse(
cctx.cacheId(),
req.id(),
new IgniteCheckedException(
"Received request for incorrect cache [expected="
+ cctx.name()
+ ", actual="
+ req.cacheName()));
sendQueryResponse(sndId, res, 0);
} else {
threads.put(req.id(), Thread.currentThread());
try {
GridCacheQueryInfo info = distributedQueryInfo(sndId, req);
if (info == null) return;
if (req.fields()) runFieldsQuery(info);
else runQuery(info);
} catch (Throwable e) {
U.error(log(), "Failed to run query.", e);
sendQueryResponse(
sndId, new GridCacheQueryResponse(cctx.cacheId(), req.id(), e.getCause()), 0);
if (e instanceof Error) throw (Error) e;
} finally {
threads.remove(req.id());
}
}
}
}
}
/** {@inheritDoc} */
@Override
public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr)
throws IgniteCheckedException {
super.finishUnmarshal(ctx, ldr);
if (ownedValKeys != null && ownedVals == null) {
ownedVals = U.newHashMap(ownedValKeys.size());
assert ownedValKeys.size() == ownedValVals.size();
Iterator<IgniteTxKey> keyIter = ownedValKeys.iterator();
Iterator<CacheVersionedValue> valIter = ownedValVals.iterator();
while (keyIter.hasNext()) {
IgniteTxKey key = keyIter.next();
GridCacheContext cctx = ctx.cacheContext(key.cacheId());
CacheVersionedValue val = valIter.next();
key.finishUnmarshal(cctx, ldr);
val.finishUnmarshal(cctx, ldr);
ownedVals.put(key, val);
}
}
if (retVal != null && retVal.cacheId() != 0) {
GridCacheContext cctx = ctx.cacheContext(retVal.cacheId());
assert cctx != null : retVal.cacheId();
retVal.finishUnmarshal(cctx, ldr);
}
if (filterFailedKeys != null) {
for (IgniteTxKey key : filterFailedKeys) {
GridCacheContext cctx = ctx.cacheContext(key.cacheId());
key.finishUnmarshal(cctx, ldr);
}
}
}
private void dumpPendingObjects() {
U.warn(
log,
"Failed to wait for partition release future. Dumping pending objects that might be the cause: "
+ cctx.localNodeId());
U.warn(log, "Pending transactions:");
for (IgniteInternalTx tx : cctx.tm().activeTransactions()) U.warn(log, ">>> " + tx);
U.warn(log, "Pending explicit locks:");
for (GridCacheExplicitLockSpan lockSpan : cctx.mvcc().activeExplicitLocks())
U.warn(log, ">>> " + lockSpan);
U.warn(log, "Pending cache futures:");
for (GridCacheFuture<?> fut : cctx.mvcc().activeFutures()) U.warn(log, ">>> " + fut);
U.warn(log, "Pending atomic cache futures:");
for (GridCacheFuture<?> fut : cctx.mvcc().atomicFutures()) U.warn(log, ">>> " + fut);
}
/**
* Executes command using {@code shell} channel.
*
* @param ses SSH session.
* @param cmd Command.
* @throws JSchException In case of SSH error.
* @throws IOException If IO error occurs.
* @throws IgniteInterruptedCheckedException If thread was interrupted while waiting.
*/
private void shell(Session ses, String cmd)
throws JSchException, IOException, IgniteInterruptedCheckedException {
ChannelShell ch = null;
try {
ch = (ChannelShell) ses.openChannel("shell");
ch.connect();
try (PrintStream out = new PrintStream(ch.getOutputStream(), true)) {
out.println(cmd);
U.sleep(1000);
}
} finally {
if (ch != null && ch.isConnected()) ch.disconnect();
}
}
/** Print plugins information. */
private void ackPluginsInfo() {
U.quietAndInfo(log, "Configured plugins:");
if (plugins.isEmpty()) {
U.quietAndInfo(log, " ^-- None");
U.quietAndInfo(log, "");
} else {
for (PluginProvider plugin : plugins.values()) {
U.quietAndInfo(log, " ^-- " + plugin.name() + " " + plugin.version());
U.quietAndInfo(log, " ^-- " + plugin.copyright());
U.quietAndInfo(log, "");
}
}
}
/** @return {@code True} if succeeded. */
private boolean spreadPartitions() {
try {
sendAllPartitions(rmtNodes, exchId);
return true;
} catch (IgniteCheckedException e) {
scheduleRecheck();
if (!X.hasCause(e, InterruptedException.class))
U.error(
log,
"Failed to send full partition map to nodes (will retry after timeout) [nodes="
+ F.nodeId8s(rmtNodes)
+ ", exchangeId="
+ exchId
+ ']',
e);
return false;
}
}
/**
* Send delete message to all meta cache nodes in the grid.
*
* @param msg Message to send.
*/
private void sendDeleteMessage(IgfsDeleteMessage msg) {
assert msg != null;
Collection<ClusterNode> nodes = meta.metaCacheNodes();
for (ClusterNode node : nodes) {
try {
igfsCtx.send(node, topic, msg, GridIoPolicy.SYSTEM_POOL);
} catch (IgniteCheckedException e) {
U.warn(
log,
"Failed to send IGFS delete message to node [nodeId="
+ node.id()
+ ", msg="
+ msg
+ ", err="
+ e.getMessage()
+ ']');
}
}
}
/** {@inheritDoc} */
@Override
public R get() throws IgniteCheckedException {
try {
if (endTime == 0) {
if (ignoreInterrupts) acquireShared(0);
else acquireSharedInterruptibly(0);
}
if (getState() == CANCELLED)
throw new IgniteFutureCancelledCheckedException("Future was cancelled: " + this);
assert resFlag != 0;
if (resFlag == ERR) throw U.cast((Throwable) res);
return (R) res;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new IgniteInterruptedCheckedException(e);
}
}
