/**
* Log task mapped.
*
* @param log Logger.
* @param clazz Task class.
* @param nodes Mapped nodes.
*/
public static void logMapped(
@Nullable IgniteLogger log, Class<?> clazz, Collection<ClusterNode> nodes) {
log0(
log,
U.currentTimeMillis(),
String.format("[%s]: MAPPED: %s", clazz.getSimpleName(), U.toShortString(nodes)));
}
/**
* Message received callback.
*
* @param src Sender node ID.
* @param msg Received message.
* @return {@code True}.
*/
public boolean onMessageReceived(UUID src, GridHadoopMessage msg) {
if (msg instanceof GridHadoopShuffleMessage) {
GridHadoopShuffleMessage m = (GridHadoopShuffleMessage) msg;
try {
job(m.jobId()).onShuffleMessage(m);
} catch (GridException e) {
U.error(log, "Message handling failed.", e);
}
try {
// Reply with ack.
send0(src, new GridHadoopShuffleAck(m.id(), m.jobId()));
} catch (GridException e) {
U.error(
log,
"Failed to reply back to shuffle message sender [snd=" + src + ", msg=" + msg + ']',
e);
}
} else if (msg instanceof GridHadoopShuffleAck) {
GridHadoopShuffleAck m = (GridHadoopShuffleAck) msg;
try {
job(m.jobId()).onShuffleAck(m);
} catch (GridException e) {
U.error(log, "Message handling failed.", e);
}
} else
throw new IllegalStateException(
"Unknown message type received to Hadoop shuffle [src=" + src + ", msg=" + msg + ']');
return true;
}
/**
* @param cctx Context.
* @param id Partition ID.
*/
@SuppressWarnings("ExternalizableWithoutPublicNoArgConstructor")
GridDhtLocalPartition(GridCacheContext cctx, int id) {
assert cctx != null;
this.id = id;
this.cctx = cctx;
log = U.logger(cctx.kernalContext(), logRef, this);
rent =
new GridFutureAdapter<Object>() {
@Override
public String toString() {
return "PartitionRentFuture [part=" + GridDhtLocalPartition.this + ", map=" + map + ']';
}
};
map = new ConcurrentHashMap8<>(cctx.config().getStartSize() / cctx.affinity().partitions());
int delQueueSize =
CU.isSystemCache(cctx.name())
? 100
: Math.max(MAX_DELETE_QUEUE_SIZE / cctx.affinity().partitions(), 20);
rmvQueue = new GridCircularBuffer<>(U.ceilPow2(delQueueSize));
}
/** @param e Error. */
void onError(Throwable e) {
tx.commitError(e);
if (err.compareAndSet(null, e)) {
boolean marked = tx.setRollbackOnly();
if (e instanceof GridCacheTxRollbackException) {
if (marked) {
try {
tx.rollback();
} catch (GridException ex) {
U.error(log, "Failed to automatically rollback transaction: " + tx, ex);
}
}
} else if (tx.implicit()
&& tx.isSystemInvalidate()) { // Finish implicit transaction on heuristic error.
try {
tx.close();
} catch (GridException ex) {
U.error(log, "Failed to invalidate transaction: " + tx, ex);
}
}
onComplete();
}
}
/**
* Waits for all workers to finish.
*
* @param cancel Flag to indicate whether workers should be cancelled before waiting for them to
* finish.
*/
public void join(boolean cancel) {
if (cancel) U.cancel(workers);
// Record current interrupted status of calling thread.
boolean interrupted = Thread.interrupted();
try {
U.join(workers, log);
} finally {
// Reset interrupted flag on calling thread.
if (interrupted) Thread.currentThread().interrupt();
}
}
/** {@inheritDoc} */
@Override
void onCancelAtStop() {
super.onCancelAtStop();
for (GridCacheQueryFutureAdapter fut : futs.values())
try {
fut.cancel();
} catch (IgniteCheckedException e) {
U.error(log, "Failed to cancel running query future: " + fut, e);
}
U.interrupt(threads.values());
}
/**
* Waits until total number of events processed is equal or greater then argument passed.
*
* @param cnt Number of events to wait.
* @param timeout Timeout to wait.
* @return {@code True} if successfully waited, {@code false} if timeout happened.
* @throws InterruptedException If thread is interrupted.
*/
public synchronized boolean awaitEvents(int cnt, long timeout) throws InterruptedException {
long start = U.currentTimeMillis();
long now = start;
while (start + timeout > now) {
if (evtCnt >= cnt) return true;
wait(start + timeout - now);
now = U.currentTimeMillis();
}
return false;
}
/** {@inheritDoc} */
@Override
public void loadCache(GridBiInClosure<K, V> c, @Nullable Object... args) throws GridException {
ExecutorService exec =
new ThreadPoolExecutor(
threadsCnt,
threadsCnt,
0L,
MILLISECONDS,
new ArrayBlockingQueue<Runnable>(batchQueueSize),
new BlockingRejectedExecutionHandler());
Iterator<I> iter = inputIterator(args);
Collection<I> buf = new ArrayList<>(batchSize);
try {
while (iter.hasNext()) {
if (Thread.currentThread().isInterrupted()) {
U.warn(log, "Working thread was interrupted while loading data.");
break;
}
buf.add(iter.next());
if (buf.size() == batchSize) {
exec.submit(new Worker(c, buf, args));
buf = new ArrayList<>(batchSize);
}
}
if (!buf.isEmpty()) exec.submit(new Worker(c, buf, args));
} catch (RejectedExecutionException ignored) {
// Because of custom RejectedExecutionHandler.
assert false : "RejectedExecutionException was thrown while it shouldn't.";
} finally {
exec.shutdown();
try {
exec.awaitTermination(Long.MAX_VALUE, MILLISECONDS);
} catch (InterruptedException ignored) {
U.warn(log, "Working thread was interrupted while waiting for put operations to complete.");
Thread.currentThread().interrupt();
}
}
}
/**
* Processes cache query response.
*
* @param sndId Sender node id.
* @param res Query response.
*/
@SuppressWarnings("unchecked")
private void processQueryResponse(UUID sndId, GridCacheQueryResponse res) {
if (log.isDebugEnabled()) log.debug("Received query response: " + res);
GridCacheQueryFutureAdapter fut = getQueryFuture(res.requestId());
if (fut != null)
if (res.fields())
((GridCacheDistributedFieldsQueryFuture) fut)
.onPage(
sndId,
res.metadata(),
(Collection<Map<String, Object>>) ((Collection) res.data()),
res.error(),
res.isFinished());
else fut.onPage(sndId, res.data(), res.error(), res.isFinished());
else if (!cancelled.contains(res.requestId()))
U.warn(
log,
"Received response for finished or unknown query [rmtNodeId="
+ sndId
+ ", res="
+ res
+ ']');
}
/** {@inheritDoc} */
@Override
public boolean onDone(GridCacheTx tx, Throwable err) {
if ((initialized() || err != null) && super.onDone(tx, err)) {
if (error() instanceof GridCacheTxHeuristicException) {
long topVer = this.tx.topologyVersion();
for (GridCacheTxEntry<K, V> e : this.tx.writeMap().values()) {
try {
if (e.op() != NOOP && !cctx.affinity().localNode(e.key(), topVer)) {
GridCacheEntryEx<K, V> cacheEntry = cctx.cache().peekEx(e.key());
if (cacheEntry != null) cacheEntry.invalidate(null, this.tx.xidVersion());
}
} catch (Throwable t) {
U.error(log, "Failed to invalidate entry.", t);
if (t instanceof Error) throw (Error) t;
}
}
}
// Don't forget to clean up.
cctx.mvcc().removeFuture(this);
return true;
}
return false;
}
/** Initializes future. */
@SuppressWarnings({"unchecked"})
void finish() {
if (mappings != null) {
finish(mappings.values());
markInitialized();
if (!isSync()) {
boolean complete = true;
for (GridFuture<?> f : pending())
// Mini-future in non-sync mode gets done when message gets sent.
if (isMini(f) && !f.isDone()) complete = false;
if (complete) onComplete();
}
} else {
assert !commit;
try {
tx.rollback();
} catch (GridException e) {
U.error(log, "Failed to rollback empty transaction: " + tx, e);
}
markInitialized();
}
}
private void sendPartitions() {
ClusterNode oldestNode = this.oldestNode.get();
try {
sendLocalPartitions(oldestNode, exchId);
} catch (ClusterTopologyCheckedException ignore) {
if (log.isDebugEnabled())
log.debug(
"Oldest node left during partition exchange [nodeId="
+ oldestNode.id()
+ ", exchId="
+ exchId
+ ']');
} catch (IgniteCheckedException e) {
scheduleRecheck();
U.error(
log,
"Failed to send local partitions to oldest node (will retry after timeout) [oldestNodeId="
+ oldestNode.id()
+ ", exchId="
+ exchId
+ ']',
e);
}
}
/** @throws Exception If failed. */
@SuppressWarnings("unchecked")
public void testCancel() throws Exception {
Grid grid = G.grid(getTestGridName());
grid.compute()
.localDeployTask(GridCancelTestTask.class, U.detectClassLoader(GridCancelTestTask.class));
GridComputeTaskFuture<?> fut = grid.compute().execute(GridCancelTestTask.class.getName(), null);
// Wait until jobs begin execution.
boolean await = startSignal.await(WAIT_TIME, TimeUnit.MILLISECONDS);
assert await : "Jobs did not start.";
info("Test task result: " + fut);
assert fut != null;
// Only first job should successfully complete.
Object res = fut.get();
assert (Integer) res == 1;
// Wait for all jobs to finish.
await = stopSignal.await(WAIT_TIME, TimeUnit.MILLISECONDS);
assert await : "Jobs did not stop.";
// One is definitely processed. But there might be some more processed or cancelled or processed
// and cancelled.
// Thus total number should be at least SPLIT_COUNT and at most (SPLIT_COUNT - 1) *2 +1
assert (cancelCnt + processedCnt) >= SPLIT_COUNT
&& (cancelCnt + processedCnt) <= (SPLIT_COUNT - 1) * 2 + 1
: "Invalid cancel count value: " + cancelCnt;
}
/** Clears swap entries for evicted partition. */
private void clearSwap() {
assert state() == EVICTED;
assert !GridQueryProcessor.isEnabled(cctx.config())
: "Indexing needs to have unswapped values.";
try {
GridCloseableIterator<Map.Entry<byte[], GridCacheSwapEntry>> it = cctx.swap().iterator(id);
boolean isLocStore = cctx.store().isLocal();
if (it != null) {
// We can safely remove these values because no entries will be created for evicted
// partition.
while (it.hasNext()) {
Map.Entry<byte[], GridCacheSwapEntry> entry = it.next();
byte[] keyBytes = entry.getKey();
KeyCacheObject key = cctx.toCacheKeyObject(keyBytes);
cctx.swap().remove(key);
if (isLocStore) cctx.store().remove(null, key.value(cctx.cacheObjectContext(), false));
}
}
} catch (IgniteCheckedException e) {
U.error(log, "Failed to clear swap for evicted partition: " + this, e);
}
}
/** {@inheritDoc} */
@SuppressWarnings({"unchecked"})
@Override
public void listenAsync(@Nullable final GridInClosure<? super GridFuture<R>> lsnr) {
if (lsnr != null) {
checkValid();
boolean done;
synchronized (mux) {
done = this.done;
if (!done) lsnrs.add(lsnr);
}
if (done) {
try {
if (syncNotify) notifyListener(lsnr);
else
ctx.closure()
.runLocalSafe(
new GPR() {
@Override
public void run() {
notifyListener(lsnr);
}
},
true);
} catch (IllegalStateException ignore) {
U.warn(
null,
"Future notification will not proceed because grid is stopped: " + ctx.gridName());
}
}
}
}
private void recheck() {
// If this is the oldest node.
if (oldestNode.get().id().equals(cctx.localNodeId())) {
Collection<UUID> remaining = remaining();
if (!remaining.isEmpty()) {
try {
cctx.io()
.safeSend(
cctx.discovery().nodes(remaining),
new GridDhtPartitionsSingleRequest(exchId),
SYSTEM_POOL,
null);
} catch (IgniteCheckedException e) {
U.error(
log,
"Failed to request partitions from nodes [exchangeId="
+ exchId
+ ", nodes="
+ remaining
+ ']',
e);
}
}
// Resend full partition map because last attempt failed.
else {
if (spreadPartitions()) onDone(exchId.topologyVersion());
}
} else sendPartitions();
// Schedule another send.
scheduleRecheck();
}
/**
* Loads test configuration.
*
* @throws Exception if configuration is unawailable or broken.
*/
private void loadTestConfiguration() throws Exception {
assert TEST_CONFIGURATION_FILE.isFile();
InputStream in = null;
Properties p = new Properties();
try {
in = new FileInputStream(TEST_CONFIGURATION_FILE);
p.load(in);
} finally {
U.closeQuiet(in);
}
clientNodes = Integer.parseInt(p.getProperty("client.nodes.count"));
srvNodes = Integer.parseInt(p.getProperty("server.nodes.count"));
threadsPerClient = Integer.parseInt(p.getProperty("threads.per.client"));
cancelRate = Integer.parseInt(p.getProperty("cancel.rate"));
submitDelay = Long.parseLong(p.getProperty("submit.delay"));
taskParams =
new GridJobLoadTestParams(
Integer.parseInt(p.getProperty("jobs.count")),
Integer.parseInt(p.getProperty("jobs.test.duration")),
Integer.parseInt(p.getProperty("jobs.test.completion.delay")),
Double.parseDouble(p.getProperty("jobs.failure.probability")));
}
/** {@inheritDoc} */
@Override
protected final boolean tryReleaseShared(int ignore) {
endTime = U.currentTimeMillis();
// Always signal after setting final done status.
return true;
}
static {
try {
TEST_CONF_DIR =
new File(U.resolveGridGainUrl("/modules/core/src/test/config/job-loadtest").toURI());
} catch (URISyntaxException e) {
throw new RuntimeException("Failed to initialize directory.", e);
}
}
/**
* Delegates to {@link #call()} method.
*
* <p>{@inheritDoc}
*
* @return {@inheritDoc}
* @throws GridException {@inheritDoc}
*/
@Override
public final Object execute() throws GridException {
try {
return call();
} catch (Throwable e) {
throw U.cast(e);
}
}
/**
* Reconstructs object on demarshalling.
*
* @return Reconstructed object.
* @throws ObjectStreamException Thrown in case of demarshalling error.
*/
private Object readResolve() throws ObjectStreamException {
GridTuple2<GridCacheContext, String> t = stash.get();
try {
return t.get1().dataStructures().sequence(t.get2(), 0L, false, false);
} catch (GridException e) {
throw U.withCause(new InvalidObjectException(e.getMessage()), e);
}
}
/**
* Log finished.
*
* @param log Logger.
* @param clazz Class.
* @param start Start time.
*/
public static void logFinish(@Nullable IgniteLogger log, Class<?> clazz, long start) {
final long end = U.currentTimeMillis();
log0(
log,
end,
String.format(
"[%s]: FINISHED, duration: %s", clazz.getSimpleName(), formatDuration(end - start)));
}
/**
* Reconstructs object on demarshalling.
*
* @return Reconstructed object.
* @throws ObjectStreamException Thrown in case of demarshalling error.
*/
@SuppressWarnings({"ConstantConditions"})
private Object readResolve() throws ObjectStreamException {
GridTuple2<GridCacheContext, String> t = stash.get();
try {
return t.get1().dataStructures().countDownLatch(t.get2(), 0, false, false);
} catch (GridException e) {
throw U.withCause(new InvalidObjectException(e.getMessage()), e);
}
}
/** Perform cleanup of the trash directory. */
private void delete() {
IgfsFileInfo info = null;
try {
info = meta.info(TRASH_ID);
} catch (ClusterTopologyServerNotFoundException e) {
LT.warn(log, e, "Server nodes not found.");
} catch (IgniteCheckedException e) {
U.error(log, "Cannot obtain trash directory info.", e);
}
if (info != null) {
for (Map.Entry<String, IgfsListingEntry> entry : info.listing().entrySet()) {
IgniteUuid fileId = entry.getValue().fileId();
if (log.isDebugEnabled())
log.debug(
"Deleting IGFS trash entry [name=" + entry.getKey() + ", fileId=" + fileId + ']');
try {
if (!cancelled) {
if (delete(entry.getKey(), fileId)) {
if (log.isDebugEnabled())
log.debug(
"Sending delete confirmation message [name="
+ entry.getKey()
+ ", fileId="
+ fileId
+ ']');
sendDeleteMessage(new IgfsDeleteMessage(fileId));
}
} else break;
} catch (IgniteInterruptedCheckedException ignored) {
// Ignore this exception while stopping.
} catch (IgniteCheckedException e) {
U.error(log, "Failed to delete entry from the trash directory: " + entry.getKey(), e);
sendDeleteMessage(new IgfsDeleteMessage(fileId, e));
}
}
}
}
/** {@inheritDoc} */
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
try {
if (executor.isShutdown()) throw new RejectedExecutionException();
else executor.getQueue().put(r);
} catch (InterruptedException ignored) {
U.warn(log, "Working thread was interrupted while loading data.");
Thread.currentThread().interrupt();
}
}
/**
* Log message.
*
* @param log Logger.
* @param msg Message to log.
* @param clazz class.
* @param start start time.
* @return Time when message was logged.
*/
public static long log(@Nullable IgniteLogger log, String msg, Class<?> clazz, long start) {
final long end = U.currentTimeMillis();
log0(
log,
end,
String.format(
"[%s]: %s, duration: %s", clazz.getSimpleName(), msg, formatDuration(end - start)));
return end;
}
/** @param jobId Job id. */
public void jobFinished(GridHadoopJobId jobId) {
GridHadoopShuffleJob job = jobs.remove(jobId);
if (job != null) {
try {
job.close();
} catch (GridException e) {
U.error(log, "Failed to close job: " + jobId, e);
}
}
}
/**
* @param cacheCtx Cache context.
* @return {@code True} if local node can calculate affinity on it's own for this partition map
* exchange.
*/
private boolean canCalculateAffinity(GridCacheContext cacheCtx) {
AffinityFunction affFunc = cacheCtx.config().getAffinity();
// Do not request affinity from remote nodes if affinity function is not centralized.
if (!U.hasAnnotation(affFunc, AffinityCentralizedFunction.class)) return true;
// If local node did not initiate exchange or local node is the only cache node in grid.
Collection<ClusterNode> affNodes = CU.affinityNodes(cacheCtx, exchId.topologyVersion());
return !exchId.nodeId().equals(cctx.localNodeId())
|| (affNodes.size() == 1 && affNodes.contains(cctx.localNode()));
}
/**
* Reconstructs object on demarshalling.
*
* @return Reconstructed object.
* @throws ObjectStreamException Thrown in case of demarshalling error.
*/
@SuppressWarnings("unchecked")
private Object readResolve() throws ObjectStreamException {
try {
GridBiTuple<GridCacheContext, String> t = stash.get();
return t.get1().dataStructures().atomicReference(t.get2(), null, false);
} catch (GridException e) {
throw U.withCause(new InvalidObjectException(e.getMessage()), e);
} finally {
stash.remove();
}
}
/**
* Flushes every internal buffer if buffer was flushed before passed in threshold.
*
* <p>Does not wait for result and does not fail on errors assuming that this method should be
* called periodically.
*/
@Override
public void tryFlush() throws GridInterruptedException {
if (!busyLock.enterBusy()) return;
try {
for (Buffer buf : bufMappings.values()) buf.flush();
lastFlushTime = U.currentTimeMillis();
} finally {
leaveBusy();
}
}