public void cleanupCompletedTransactions() {
if (!completedTransactions.isEmpty()) {
try {
log.tracef(
"About to cleanup completed transaction. Initial size is %d",
completedTransactions.size());
// this iterator is weekly consistent and will never throw ConcurrentModificationException
Iterator<Map.Entry<GlobalTransaction, Long>> iterator =
completedTransactions.entrySet().iterator();
long timeout = configuration.transaction().completedTxTimeout();
int removedEntries = 0;
long beginning = timeService.time();
while (iterator.hasNext()) {
Map.Entry<GlobalTransaction, Long> e = iterator.next();
long ageMillis = timeService.timeDuration(e.getValue(), TimeUnit.MILLISECONDS);
if (ageMillis >= timeout) {
iterator.remove();
removedEntries++;
}
}
long duration = timeService.timeDuration(beginning, TimeUnit.MILLISECONDS);
log.tracef(
"Finished cleaning up completed transactions. %d transactions were removed, total duration was %d millis, "
+ "current number of completed transactions is %d",
removedEntries, duration, completedTransactions.size());
} catch (Exception e) {
log.errorf(e, "Failed to cleanup completed transactions: %s", e.getMessage());
}
}
}
public void cleanupTimedOutTransactions() {
if (trace)
log.tracef(
"About to cleanup remote transactions older than %d ms",
configuration.transaction().completedTxTimeout());
long beginning = timeService.time();
long cutoffCreationTime =
beginning - TimeUnit.MILLISECONDS.toNanos(configuration.transaction().completedTxTimeout());
List<GlobalTransaction> toKill = new ArrayList<>();
// Check remote transactions.
for (Map.Entry<GlobalTransaction, RemoteTransaction> e : remoteTransactions.entrySet()) {
GlobalTransaction gtx = e.getKey();
RemoteTransaction remoteTx = e.getValue();
if (remoteTx != null) {
if (trace) log.tracef("Checking transaction %s", gtx);
// Check the time.
if (remoteTx.getCreationTime() - cutoffCreationTime < 0) {
long duration =
timeService.timeDuration(
remoteTx.getCreationTime(), beginning, TimeUnit.MILLISECONDS);
log.remoteTransactionTimeout(gtx, duration);
toKill.add(gtx);
}
}
}
// Rollback the orphaned transactions and release any held locks.
for (GlobalTransaction gtx : toKill) {
killTransaction(gtx);
}
}
@Override
public CompletableFuture<Void> visitGetAllCommand(InvocationContext ctx, GetAllCommand command)
throws Throwable {
if (trace) {
log.tracef(
"Visit Get All Command %s. Is it in transaction scope? %s. Is it local? %s",
command, ctx.isInTxScope(), ctx.isOriginLocal());
}
if (!ctx.isInTxScope()) {
return ctx.continueInvocation();
}
long start = timeService.time();
return ctx.onReturn(
(rCtx, rCommand, rv, throwable) -> {
if (throwable != null) {
throw throwable;
}
long end = timeService.time();
initStatsIfNecessary(rCtx);
int numRemote = 0;
Collection<?> keys = ((GetAllCommand) rCommand).getKeys();
for (Object key : keys) {
if (isRemote(key)) numRemote++;
}
// TODO: tbh this seems like it doesn't work properly for statistics as each
// one will have the duration of all the time for all gets... Maybe do an average
// instead ? Either way this isn't very indicative
if (numRemote > 0) {
cacheStatisticManager.add(
NUM_REMOTE_GET, numRemote, getGlobalTransaction(rCtx), rCtx.isOriginLocal());
cacheStatisticManager.add(
REMOTE_GET_EXECUTION,
timeService.timeDuration(start, end, NANOSECONDS),
getGlobalTransaction(rCtx),
rCtx.isOriginLocal());
}
cacheStatisticManager.add(
ALL_GET_EXECUTION,
timeService.timeDuration(start, end, NANOSECONDS),
getGlobalTransaction(rCtx),
rCtx.isOriginLocal());
cacheStatisticManager.add(
NUM_GET, keys.size(), getGlobalTransaction(rCtx), rCtx.isOriginLocal());
return null;
});
}
@Override
public <KOut, VOut> Map<KOut, VOut> reduce(ReduceCommand<KOut, VOut> reduceCommand)
throws InterruptedException {
Cache<?, ?> cache = cacheManager.getCache(reduceCommand.getCacheName());
Set<KOut> keys = reduceCommand.getKeys();
String taskId = reduceCommand.getTaskId();
Reducer<KOut, VOut> reducer = reduceCommand.getReducer();
boolean useIntermediateKeys = reduceCommand.isEmitCompositeIntermediateKeys();
boolean noInputKeys = keys == null || keys.isEmpty();
Cache<Object, List<VOut>> tmpCache = cacheManager.getCache(reduceCommand.getCacheName());
Map<KOut, VOut> result = new HashMap<KOut, VOut>();
if (noInputKeys) {
// illegal state, raise exception
throw new IllegalStateException(
"Reduce phase of MapReduceTask "
+ taskId
+ " on node "
+ cdl.getAddress()
+ " executed with empty input keys");
} else {
// first hook into lifecycle
MapReduceTaskLifecycleService taskLifecycleService =
MapReduceTaskLifecycleService.getInstance();
log.tracef("For m/r task %s invoking %s at %s", taskId, reduceCommand, cdl.getAddress());
int interruptCount = 0;
long start = log.isTraceEnabled() ? timeService.time() : 0;
try {
taskLifecycleService.onPreExecute(reducer, cache);
for (KOut key : keys) {
interruptCount++;
if (checkInterrupt(interruptCount++) && Thread.currentThread().isInterrupted())
throw new InterruptedException();
// load result value from map phase
List<VOut> value;
if (useIntermediateKeys) {
value = tmpCache.get(new IntermediateCompositeKey<KOut>(taskId, key));
} else {
value = tmpCache.get(key);
}
// and reduce it
VOut reduced = reducer.reduce(key, value.iterator());
result.put(key, reduced);
log.tracef(
"For m/r task %s reduced %s to %s at %s ", taskId, key, reduced, cdl.getAddress());
}
} finally {
if (log.isTraceEnabled()) {
log.tracef(
"Reduce for task %s took %s milliseconds",
reduceCommand.getTaskId(), timeService.timeDuration(start, TimeUnit.MILLISECONDS));
}
taskLifecycleService.onPostExecute(reducer);
}
}
return result;
}
@Override
public CompletableFuture<Void> visitGetKeyValueCommand(
InvocationContext ctx, GetKeyValueCommand command) throws Throwable {
if (trace) {
log.tracef(
"Visit Get Key Value command %s. Is it in transaction scope? %s. Is it local? %s",
command, ctx.isInTxScope(), ctx.isOriginLocal());
}
if (!ctx.isInTxScope()) {
return ctx.continueInvocation();
}
long start = timeService.time();
return ctx.onReturn(
(rCtx, rCommand, rv, throwable) -> {
if (throwable != null) {
throw throwable;
}
long end = timeService.time();
initStatsIfNecessary(rCtx);
Object key = ((GetKeyValueCommand) rCommand).getKey();
if (isRemote(key)) {
cacheStatisticManager.increment(
NUM_REMOTE_GET, getGlobalTransaction(rCtx), rCtx.isOriginLocal());
cacheStatisticManager.add(
REMOTE_GET_EXECUTION,
timeService.timeDuration(start, end, NANOSECONDS),
getGlobalTransaction(rCtx),
rCtx.isOriginLocal());
}
cacheStatisticManager.add(
ALL_GET_EXECUTION,
timeService.timeDuration(start, end, NANOSECONDS),
getGlobalTransaction(rCtx),
rCtx.isOriginLocal());
cacheStatisticManager.increment(
NUM_GET, getGlobalTransaction(rCtx), rCtx.isOriginLocal());
return null;
});
}
@Override
@ManagedAttribute(
description = "Number of seconds since the cluster-wide cache statistics were last reset",
displayName = "Seconds since cluster-wide cache statistics were reset",
units = Units.SECONDS,
displayType = DisplayType.SUMMARY)
public long getTimeSinceReset() {
long result = -1;
if (isStatisticsEnabled()) {
result = ts.timeDuration(resetNanoseconds.get(), TimeUnit.SECONDS);
}
return result;
}
private void updateTime(
ExtendedStatistic duration,
ExtendedStatistic counter,
long initTime,
long endTime,
GlobalTransaction globalTransaction,
boolean local) {
cacheStatisticManager.add(
duration,
timeService.timeDuration(initTime, endTime, NANOSECONDS),
globalTransaction,
local);
cacheStatisticManager.increment(counter, globalTransaction, local);
}
private <KIn, VIn, KOut, VOut> Map<KOut, List<VOut>> combineForLocalReduction(
MapCombineCommand<KIn, VIn, KOut, VOut> mcc, CollectableCollector<KOut, VOut> collector) {
String taskId = mcc.getTaskId();
Reducer<KOut, VOut> combiner = mcc.getCombiner();
Map<KOut, List<VOut>> result = null;
if (combiner != null) {
result = new HashMap<KOut, List<VOut>>();
log.tracef("For m/r task %s invoking combiner %s at %s", taskId, mcc, cdl.getAddress());
MapReduceTaskLifecycleService taskLifecycleService =
MapReduceTaskLifecycleService.getInstance();
long start = log.isTraceEnabled() ? timeService.time() : 0;
try {
Cache<?, ?> cache = cacheManager.getCache(mcc.getCacheName());
taskLifecycleService.onPreExecute(combiner, cache);
Map<KOut, List<VOut>> collectedValues = collector.collectedValues();
for (Entry<KOut, List<VOut>> e : collectedValues.entrySet()) {
VOut combined;
List<VOut> list = e.getValue();
List<VOut> l = new LinkedList<VOut>();
if (list.size() > 1) {
combined = combiner.reduce(e.getKey(), list.iterator());
} else {
combined = list.get(0);
}
l.add(combined);
result.put(e.getKey(), l);
log.tracef(
"For m/r task %s combined %s to %s at %s",
taskId, e.getKey(), combined, cdl.getAddress());
}
} finally {
if (log.isTraceEnabled()) {
log.tracef(
"Combine for task %s took %s milliseconds",
mcc.getTaskId(), timeService.timeDuration(start, TimeUnit.MILLISECONDS));
}
taskLifecycleService.onPostExecute(combiner);
}
} else {
// Combiner not specified
result = collector.collectedValues();
}
return result;
}
private CompletableFuture<Void> visitWriteCommand(
InvocationContext ctx, WriteCommand command, Object key) throws Throwable {
if (trace) {
log.tracef(
"Visit write command %s. Is it in transaction scope? %s. Is it local? %s",
command, ctx.isInTxScope(), ctx.isOriginLocal());
}
if (!ctx.isInTxScope()) {
return ctx.continueInvocation();
}
long start = timeService.time();
return ctx.onReturn(
(rCtx, rCommand, rv, throwable) -> {
long end = timeService.time();
initStatsIfNecessary(rCtx);
if (throwable != null) {
processWriteException(rCtx, getGlobalTransaction(rCtx), throwable);
} else {
if (isRemote(key)) {
cacheStatisticManager.add(
REMOTE_PUT_EXECUTION,
timeService.timeDuration(start, end, NANOSECONDS),
getGlobalTransaction(rCtx),
rCtx.isOriginLocal());
cacheStatisticManager.increment(
NUM_REMOTE_PUT, getGlobalTransaction(rCtx), rCtx.isOriginLocal());
}
}
cacheStatisticManager.increment(
NUM_PUT, getGlobalTransaction(rCtx), rCtx.isOriginLocal());
cacheStatisticManager.markAsWriteTransaction(
getGlobalTransaction(rCtx), rCtx.isOriginLocal());
return null;
});
}
private boolean launchNewDistTask() {
long duration = ts.timeDuration(statsUpdateTimestamp, ts.time(), TimeUnit.MILLISECONDS);
return duration > staleStatsTreshold;
}
protected <KIn, VIn, KOut, VOut> Set<KOut> combine(
MapCombineCommand<KIn, VIn, KOut, VOut> mcc, CollectableCollector<KOut, VOut> collector)
throws Exception {
String taskId = mcc.getTaskId();
boolean emitCompositeIntermediateKeys = mcc.isEmitCompositeIntermediateKeys();
Reducer<KOut, VOut> combiner = mcc.getCombiner();
Set<KOut> mapPhaseKeys = new HashSet<KOut>();
Cache<Object, DeltaAwareList<VOut>> tmpCache = null;
if (emitCompositeIntermediateKeys) {
tmpCache = cacheManager.getCache(DEFAULT_TMP_CACHE_CONFIGURATION_NAME);
} else {
tmpCache = cacheManager.getCache(taskId);
}
if (tmpCache == null) {
throw new IllegalStateException(
"Temporary cache for MapReduceTask " + taskId + " not found on " + cdl.getAddress());
}
DistributionManager dm = tmpCache.getAdvancedCache().getDistributionManager();
if (combiner != null) {
Cache<?, ?> cache = cacheManager.getCache(mcc.getCacheName());
log.tracef("For m/r task %s invoking combiner %s at %s", taskId, mcc, cdl.getAddress());
MapReduceTaskLifecycleService taskLifecycleService =
MapReduceTaskLifecycleService.getInstance();
Map<KOut, VOut> combinedMap = new ConcurrentHashMap<KOut, VOut>();
long start = log.isTraceEnabled() ? timeService.time() : 0;
try {
taskLifecycleService.onPreExecute(combiner, cache);
Map<KOut, List<VOut>> collectedValues = collector.collectedValues();
for (Entry<KOut, List<VOut>> e : collectedValues.entrySet()) {
List<VOut> list = e.getValue();
VOut combined;
if (list.size() > 1) {
combined = combiner.reduce(e.getKey(), list.iterator());
combinedMap.put(e.getKey(), combined);
} else {
combined = list.get(0);
combinedMap.put(e.getKey(), combined);
}
log.tracef(
"For m/r task %s combined %s to %s at %s",
taskId, e.getKey(), combined, cdl.getAddress());
}
} finally {
if (log.isTraceEnabled()) {
log.tracef(
"Combine for task %s took %s milliseconds",
mcc.getTaskId(), timeService.timeDuration(start, TimeUnit.MILLISECONDS));
}
taskLifecycleService.onPostExecute(combiner);
}
Map<Address, List<KOut>> keysToNodes =
mapKeysToNodes(dm, taskId, combinedMap.keySet(), emitCompositeIntermediateKeys);
start = log.isTraceEnabled() ? timeService.time() : 0;
try {
for (Entry<Address, List<KOut>> entry : keysToNodes.entrySet()) {
List<KOut> keysHashedToAddress = entry.getValue();
try {
log.tracef(
"For m/r task %s migrating intermediate keys %s to %s",
taskId, keysHashedToAddress, entry.getKey());
for (KOut key : keysHashedToAddress) {
VOut value = combinedMap.get(key);
DeltaAwareList<VOut> delta = new DeltaAwareList<VOut>(value);
if (emitCompositeIntermediateKeys) {
tmpCache.put(new IntermediateCompositeKey<KOut>(taskId, key), delta);
} else {
tmpCache.put(key, delta);
}
mapPhaseKeys.add(key);
}
} catch (Exception e) {
throw new CacheException(
"Could not move intermediate keys/values for M/R task " + taskId, e);
}
}
} finally {
if (log.isTraceEnabled()) {
log.tracef(
"Migrating keys for task %s took %s milliseconds (Migrated %s keys)",
mcc.getTaskId(),
timeService.timeDuration(start, TimeUnit.MILLISECONDS),
mapPhaseKeys.size());
}
}
} else {
// Combiner not specified so lets insert each key/uncombined-List pair into tmp cache
Map<KOut, List<VOut>> collectedValues = collector.collectedValues();
Map<Address, List<KOut>> keysToNodes =
mapKeysToNodes(dm, taskId, collectedValues.keySet(), emitCompositeIntermediateKeys);
long start = log.isTraceEnabled() ? timeService.time() : 0;
try {
for (Entry<Address, List<KOut>> entry : keysToNodes.entrySet()) {
List<KOut> keysHashedToAddress = entry.getValue();
try {
log.tracef(
"For m/r task %s migrating intermediate keys %s to %s",
taskId, keysHashedToAddress, entry.getKey());
for (KOut key : keysHashedToAddress) {
List<VOut> value = collectedValues.get(key);
DeltaAwareList<VOut> delta = new DeltaAwareList<VOut>(value);
if (emitCompositeIntermediateKeys) {
tmpCache.put(new IntermediateCompositeKey<KOut>(taskId, key), delta);
} else {
tmpCache.put(key, delta);
}
mapPhaseKeys.add(key);
}
} catch (Exception e) {
throw new CacheException(
"Could not move intermediate keys/values for M/R task " + taskId, e);
}
}
} finally {
if (log.isTraceEnabled()) {
log.tracef(
"Migrating keys for task %s took %s milliseconds (Migrated %s keys)",
mcc.getTaskId(),
timeService.timeDuration(start, TimeUnit.MILLISECONDS),
mapPhaseKeys.size());
}
}
}
return mapPhaseKeys;
}
protected <KIn, VIn, KOut, VOut> CollectableCollector<KOut, VOut> map(
MapCombineCommand<KIn, VIn, KOut, VOut> mcc) throws InterruptedException {
Cache<KIn, VIn> cache = cacheManager.getCache(mcc.getCacheName());
Set<KIn> keys = mcc.getKeys();
Set<KIn> inputKeysCopy = null;
Mapper<KIn, VIn, KOut, VOut> mapper = mcc.getMapper();
DistributionManager dm = cache.getAdvancedCache().getDistributionManager();
boolean inputKeysSpecified = keys != null && !keys.isEmpty();
Set<KIn> inputKeys = keys;
if (!inputKeysSpecified) {
inputKeys = filterLocalPrimaryOwner(cache.keySet(), dm);
} else {
inputKeysCopy = new HashSet<KIn>(keys);
}
// hook map function into lifecycle and execute it
MapReduceTaskLifecycleService taskLifecycleService =
MapReduceTaskLifecycleService.getInstance();
DefaultCollector<KOut, VOut> collector = new DefaultCollector<KOut, VOut>();
log.tracef("For m/r task %s invoking %s with input keys %s", mcc.getTaskId(), mcc, inputKeys);
int interruptCount = 0;
long start = log.isTraceEnabled() ? timeService.time() : 0;
try {
taskLifecycleService.onPreExecute(mapper, cache);
for (KIn key : inputKeys) {
if (checkInterrupt(interruptCount++) && Thread.currentThread().isInterrupted())
throw new InterruptedException();
VIn value = cache.get(key);
mapper.map(key, value, collector);
if (inputKeysSpecified) {
inputKeysCopy.remove(key);
}
}
Set<KIn> keysFromCacheLoader = null;
if (inputKeysSpecified) {
// load only specified remaining input keys - iff in CL and pinned to this primary owner
keysFromCacheLoader = filterLocalPrimaryOwner(inputKeysCopy, dm);
} else {
// load everything from CL pinned to this primary owner
keysFromCacheLoader =
filterLocalPrimaryOwner(loadAllKeysFromCacheLoaderUsingFilter(inputKeys), dm);
}
log.tracef(
"For m/r task %s cache loader input keys %s", mcc.getTaskId(), keysFromCacheLoader);
interruptCount = 0;
for (KIn key : keysFromCacheLoader) {
if (checkInterrupt(interruptCount++) && Thread.currentThread().isInterrupted())
throw new InterruptedException();
VIn value = loadValueFromCacheLoader(key);
if (value != null) {
mapper.map(key, value, collector);
}
}
} finally {
if (log.isTraceEnabled()) {
log.tracef(
"Map phase for task %s took %s milliseconds",
mcc.getTaskId(), timeService.timeDuration(start, TimeUnit.MILLISECONDS));
}
taskLifecycleService.onPostExecute(mapper);
}
return collector;
}