private void recheck() {
// If this is the oldest node.
if (oldestNode.get().id().equals(cctx.localNodeId())) {
Collection<UUID> remaining = remaining();
if (!remaining.isEmpty()) {
try {
cctx.io()
.safeSend(
cctx.discovery().nodes(remaining),
new GridDhtPartitionsSingleRequest(exchId),
SYSTEM_POOL,
null);
} catch (IgniteCheckedException e) {
U.error(
log,
"Failed to request partitions from nodes [exchangeId="
+ exchId
+ ", nodes="
+ remaining
+ ']',
e);
}
}
// Resend full partition map because last attempt failed.
else {
if (spreadPartitions()) onDone(exchId.topologyVersion());
}
} else sendPartitions();
// Schedule another send.
scheduleRecheck();
}
/**
* Change topology.
*
* @param parent Grid to execute tasks on.
* @param add New nodes count.
* @param rmv Remove nodes count.
* @param type Type of nodes to manipulate.
*/
private static void changeTopology(Ignite parent, int add, int rmv, String type) {
Collection<ComputeTaskFuture<?>> tasks = new ArrayList<>();
IgniteCompute comp = parent.compute().withAsync();
// Start nodes in parallel.
while (add-- > 0) {
comp.execute(ClientStartNodeTask.class, type);
tasks.add(comp.future());
}
for (ComputeTaskFuture<?> task : tasks) task.get();
// Stop nodes in sequence.
while (rmv-- > 0) parent.compute().execute(ClientStopNodeTask.class, type);
// Wait for node stops.
// U.sleep(1000);
Collection<String> gridNames = new ArrayList<>();
for (Ignite g : G.allGrids()) gridNames.add(g.name());
parent.log().info(">>> Available grids: " + gridNames);
}
/**
* JUnit.
*
* @throws Exception If failed.
*/
public void testGetAndIncrement() throws Exception {
Collection<Long> res = new HashSet<>();
String seqName = UUID.randomUUID().toString();
for (int i = 0; i < GRID_CNT; i++) {
Set<Long> retVal =
compute(grid(i).cluster().forLocal()).call(new GetAndIncrementJob(seqName, RETRIES));
for (Long l : retVal) assert !res.contains(l) : "Value already was used " + l;
res.addAll(retVal);
}
assert res.size() == GRID_CNT * RETRIES;
int gapSize = 0;
for (long i = 0; i < GRID_CNT * RETRIES; i++) {
if (!res.contains(i)) gapSize++;
else gapSize = 0;
assert gapSize <= BATCH_SIZE + 1
: "Gap above id " + i + " is " + gapSize + " more than batch size: " + (BATCH_SIZE + 1);
}
}
/**
* Sends query request.
*
* @param fut Distributed future.
* @param req Request.
* @param nodes Nodes.
* @throws IgniteCheckedException In case of error.
*/
@SuppressWarnings("unchecked")
private void sendRequest(
final GridCacheDistributedQueryFuture<?, ?, ?> fut,
final GridCacheQueryRequest req,
Collection<ClusterNode> nodes)
throws IgniteCheckedException {
assert fut != null;
assert req != null;
assert nodes != null;
final UUID locNodeId = cctx.localNodeId();
ClusterNode locNode = null;
Collection<ClusterNode> rmtNodes = null;
for (ClusterNode n : nodes) {
if (n.id().equals(locNodeId)) locNode = n;
else {
if (rmtNodes == null) rmtNodes = new ArrayList<>(nodes.size());
rmtNodes.add(n);
}
}
// Request should be sent to remote nodes before the query is processed on the local node.
// For example, a remote reducer has a state, we should not serialize and then send
// the reducer changed by the local node.
if (!F.isEmpty(rmtNodes)) {
cctx.io()
.safeSend(
rmtNodes,
req,
cctx.ioPolicy(),
new P1<ClusterNode>() {
@Override
public boolean apply(ClusterNode node) {
fut.onNodeLeft(node.id());
return !fut.isDone();
}
});
}
if (locNode != null) {
cctx.closures()
.callLocalSafe(
new Callable<Object>() {
@Override
public Object call() throws Exception {
req.beforeLocalExecution(cctx);
processQueryRequest(locNodeId, req);
return null;
}
});
}
}
/** {@inheritDoc} */
@Override
public synchronized Iterable<String> getGroupNames() {
Collection<String> res = new HashSet<>();
for (HadoopCounter counter : cntrs.values()) res.add(counter.group());
return res;
}
/**
* Returns counters iterator for specified group.
*
* @param grpName Name of the group to iterate.
* @return Counters iterator.
*/
public Iterator<Counter> iterateGroup(String grpName) {
Collection<Counter> grpCounters = new ArrayList<>();
for (HadoopLongCounter counter : cntrs.values()) {
if (grpName.equals(counter.group())) grpCounters.add(new HadoopV2Counter(counter));
}
return grpCounters.iterator();
}
/** {@inheritDoc} */
@Override
public Collection<? extends ComputeJob> split(int gridSize, Serializable arg) {
if (log.isInfoEnabled())
log.info("Splitting task [task=" + this + ", gridSize=" + gridSize + ", arg=" + arg + ']');
Collection<GridCancelTestJob> jobs = new ArrayList<>(SPLIT_COUNT);
for (int i = 0; i < SPLIT_COUNT; i++) jobs.add(new GridCancelTestJob());
return jobs;
}
/** Cleans up resources to avoid excessive memory usage. */
public void cleanUp() {
topSnapshot.set(null);
singleMsgs.clear();
fullMsgs.clear();
rcvdIds.clear();
oldestNode.set(null);
partReleaseFut = null;
Collection<ClusterNode> rmtNodes = this.rmtNodes;
if (rmtNodes != null) rmtNodes.clear();
}
/**
* @param cacheCtx Cache context.
* @return {@code True} if local node can calculate affinity on it's own for this partition map
* exchange.
*/
private boolean canCalculateAffinity(GridCacheContext cacheCtx) {
AffinityFunction affFunc = cacheCtx.config().getAffinity();
// Do not request affinity from remote nodes if affinity function is not centralized.
if (!U.hasAnnotation(affFunc, AffinityCentralizedFunction.class)) return true;
// If local node did not initiate exchange or local node is the only cache node in grid.
Collection<ClusterNode> affNodes = CU.affinityNodes(cacheCtx, exchId.topologyVersion());
return !exchId.nodeId().equals(cctx.localNodeId())
|| (affNodes.size() == 1 && affNodes.contains(cctx.localNode()));
}
/**
* Returns compact class host.
*
* @param obj Object to compact.
* @return String.
*/
@Nullable
public static Object compactObject(Object obj) {
if (obj == null) return null;
if (obj instanceof Enum) return obj.toString();
if (obj instanceof String || obj instanceof Boolean || obj instanceof Number) return obj;
if (obj instanceof Collection) {
Collection col = (Collection) obj;
Object[] res = new Object[col.size()];
int i = 0;
for (Object elm : col) res[i++] = compactObject(elm);
return res;
}
if (obj.getClass().isArray()) {
Class<?> arrType = obj.getClass().getComponentType();
if (arrType.isPrimitive()) {
if (obj instanceof boolean[]) return Arrays.toString((boolean[]) obj);
if (obj instanceof byte[]) return Arrays.toString((byte[]) obj);
if (obj instanceof short[]) return Arrays.toString((short[]) obj);
if (obj instanceof int[]) return Arrays.toString((int[]) obj);
if (obj instanceof long[]) return Arrays.toString((long[]) obj);
if (obj instanceof float[]) return Arrays.toString((float[]) obj);
if (obj instanceof double[]) return Arrays.toString((double[]) obj);
}
Object[] arr = (Object[]) obj;
int iMax = arr.length - 1;
StringBuilder sb = new StringBuilder("[");
for (int i = 0; i <= iMax; i++) {
sb.append(compactObject(arr[i]));
if (i != iMax) sb.append(", ");
}
sb.append("]");
return sb.toString();
}
return U.compact(obj.getClass().getName());
}
/**
* @param rmtReducer Optional reducer.
* @param rmtTransform Optional transformer.
* @param args Arguments.
* @return Future.
*/
@SuppressWarnings("IfMayBeConditional")
private <R> CacheQueryFuture<R> execute(
@Nullable IgniteReducer<T, R> rmtReducer,
@Nullable IgniteClosure<T, R> rmtTransform,
@Nullable Object... args) {
Collection<ClusterNode> nodes = nodes();
cctx.checkSecurity(SecurityPermission.CACHE_READ);
if (nodes.isEmpty())
return new GridCacheQueryErrorFuture<>(
cctx.kernalContext(), new ClusterGroupEmptyCheckedException());
if (log.isDebugEnabled())
log.debug("Executing query [query=" + this + ", nodes=" + nodes + ']');
if (cctx.deploymentEnabled()) {
try {
cctx.deploy().registerClasses(filter, rmtReducer, rmtTransform);
cctx.deploy().registerClasses(args);
} catch (IgniteCheckedException e) {
return new GridCacheQueryErrorFuture<>(cctx.kernalContext(), e);
}
}
if (subjId == null) subjId = cctx.localNodeId();
taskHash = cctx.kernalContext().job().currentTaskNameHash();
final GridCacheQueryBean bean =
new GridCacheQueryBean(
this,
(IgniteReducer<Object, Object>) rmtReducer,
(IgniteClosure<Object, Object>) rmtTransform,
args);
final GridCacheQueryManager qryMgr = cctx.queries();
boolean loc = nodes.size() == 1 && F.first(nodes).id().equals(cctx.localNodeId());
if (type == SQL_FIELDS || type == SPI)
return (CacheQueryFuture<R>)
(loc ? qryMgr.queryFieldsLocal(bean) : qryMgr.queryFieldsDistributed(bean, nodes));
else if (type == SCAN && part != null && nodes.size() > 1)
return new CacheQueryFallbackFuture<>(nodes, bean, qryMgr);
else
return (CacheQueryFuture<R>)
(loc ? qryMgr.queryLocal(bean) : qryMgr.queryDistributed(bean, nodes));
}
/**
* Processes cache query response.
*
* @param sndId Sender node id.
* @param res Query response.
*/
@SuppressWarnings("unchecked")
private void processQueryResponse(UUID sndId, GridCacheQueryResponse res) {
if (log.isDebugEnabled()) log.debug("Received query response: " + res);
GridCacheQueryFutureAdapter fut = getQueryFuture(res.requestId());
if (fut != null)
if (res.fields())
((GridCacheDistributedFieldsQueryFuture) fut)
.onPage(
sndId,
res.metadata(),
(Collection<Map<String, Object>>) ((Collection) res.data()),
res.error(),
res.isFinished());
else fut.onPage(sndId, res.data(), res.error(), res.isFinished());
else if (!cancelled.contains(res.requestId()))
U.warn(
log,
"Received response for finished or unknown query [rmtNodeId="
+ sndId
+ ", res="
+ res
+ ']');
}
/**
* @param key Key to add to read set.
* @param val Value.
* @param drVer Data center replication version.
* @param skipStore Skip store flag.
* @throws IgniteCheckedException If failed.
* @return {@code True} if entry has been enlisted.
*/
public boolean addEntry(
GridCacheContext cacheCtx,
IgniteTxKey key,
GridCacheOperation op,
CacheObject val,
@Nullable GridCacheVersion drVer,
boolean skipStore)
throws IgniteCheckedException {
checkInternal(key);
GridNearCacheEntry cached = cacheCtx.near().peekExx(key.key());
try {
if (cached == null) {
evicted.add(key);
return false;
} else {
cached.unswap();
CacheObject peek = cached.peek(true, false, false, null);
if (peek == null && cached.evictInternal(false, xidVer, null)) {
cached.context().cache().removeIfObsolete(key.key());
evicted.add(key);
return false;
} else {
IgniteTxEntry txEntry =
new IgniteTxEntry(cacheCtx, this, op, val, -1L, -1L, cached, drVer, skipStore);
writeMap.put(key, txEntry);
return true;
}
}
} catch (GridCacheEntryRemovedException ignore) {
evicted.add(key);
if (log.isDebugEnabled())
log.debug(
"Got removed entry when adding reads to remote transaction (will ignore): " + cached);
return false;
}
}
/**
* @param p Partition.
* @param topVer Topology version ({@code -1} for all nodes).
* @param state Partition state.
* @param states Additional partition states.
* @return List of nodes for the partition.
*/
private List<ClusterNode> nodes(
int p,
AffinityTopologyVersion topVer,
GridDhtPartitionState state,
GridDhtPartitionState... states) {
Collection<UUID> allIds =
topVer.topologyVersion() > 0 ? F.nodeIds(CU.affinityNodes(cctx, topVer)) : null;
lock.readLock().lock();
try {
assert node2part != null && node2part.valid()
: "Invalid node-to-partitions map [topVer="
+ topVer
+ ", allIds="
+ allIds
+ ", node2part="
+ node2part
+ ", cache="
+ cctx.name()
+ ']';
Collection<UUID> nodeIds = part2node.get(p);
// Node IDs can be null if both, primary and backup, nodes disappear.
int size = nodeIds == null ? 0 : nodeIds.size();
if (size == 0) return Collections.emptyList();
List<ClusterNode> nodes = new ArrayList<>(size);
for (UUID id : nodeIds) {
if (topVer.topologyVersion() > 0 && !allIds.contains(id)) continue;
if (hasState(p, id, state, states)) {
ClusterNode n = cctx.discovery().node(id);
if (n != null && (topVer.topologyVersion() < 0 || n.order() <= topVer.topologyVersion()))
nodes.add(n);
}
}
return nodes;
} finally {
lock.readLock().unlock();
}
}
/**
* Processes cache query request.
*
* @param sndId Sender node id.
* @param req Query request.
*/
@SuppressWarnings("unchecked")
@Override
void processQueryRequest(UUID sndId, GridCacheQueryRequest req) {
if (req.cancel()) {
cancelIds.add(new CancelMessageId(req.id(), sndId));
if (req.fields()) removeFieldsQueryResult(sndId, req.id());
else removeQueryResult(sndId, req.id());
} else {
if (!cancelIds.contains(new CancelMessageId(req.id(), sndId))) {
if (!F.eq(req.cacheName(), cctx.name())) {
GridCacheQueryResponse res =
new GridCacheQueryResponse(
cctx.cacheId(),
req.id(),
new IgniteCheckedException(
"Received request for incorrect cache [expected="
+ cctx.name()
+ ", actual="
+ req.cacheName()));
sendQueryResponse(sndId, res, 0);
} else {
threads.put(req.id(), Thread.currentThread());
try {
GridCacheQueryInfo info = distributedQueryInfo(sndId, req);
if (info == null) return;
if (req.fields()) runFieldsQuery(info);
else runQuery(info);
} catch (Throwable e) {
U.error(log(), "Failed to run query.", e);
sendQueryResponse(
sndId, new GridCacheQueryResponse(cctx.cacheId(), req.id(), e.getCause()), 0);
if (e instanceof Error) throw (Error) e;
} finally {
threads.remove(req.id());
}
}
}
}
}
/** @return {@code True} if all replies are received. */
private boolean allReceived() {
Collection<UUID> rmtIds = this.rmtIds;
assert rmtIds != null : "Remote Ids can't be null: " + this;
synchronized (rcvdIds) {
return rcvdIds.containsAll(rmtIds);
}
}
/**
* @param entry Entry to enlist.
* @throws IgniteCheckedException If failed.
* @return {@code True} if entry was enlisted.
*/
private boolean addEntry(IgniteTxEntry entry) throws IgniteCheckedException {
checkInternal(entry.txKey());
GridCacheContext cacheCtx = entry.context();
if (!cacheCtx.isNear()) cacheCtx = cacheCtx.dht().near().context();
GridNearCacheEntry cached = cacheCtx.near().peekExx(entry.key());
if (cached == null) {
evicted.add(entry.txKey());
return false;
} else {
cached.unswap();
try {
CacheObject val = cached.peek(true, false, false, null);
if (val == null && cached.evictInternal(false, xidVer, null)) {
evicted.add(entry.txKey());
return false;
} else {
// Initialize cache entry.
entry.cached(cached);
writeMap.put(entry.txKey(), entry);
addExplicit(entry);
return true;
}
} catch (GridCacheEntryRemovedException ignore) {
evicted.add(entry.txKey());
if (log.isDebugEnabled())
log.debug("Got removed entry when adding to remote transaction (will ignore): " + cached);
return false;
}
}
}
/**
* This constructor is meant for optimistic transactions.
*
* @param ldr Class loader.
* @param nodeId Node ID.
* @param nearNodeId Near node ID.
* @param rmtThreadId Remote thread ID.
* @param xidVer XID version.
* @param commitVer Commit version.
* @param sys System flag.
* @param concurrency Concurrency level (should be pessimistic).
* @param isolation Transaction isolation.
* @param invalidate Invalidate flag.
* @param timeout Timeout.
* @param writeEntries Write entries.
* @param ctx Cache registry.
* @param txSize Expected transaction size.
* @throws IgniteCheckedException If unmarshalling failed.
*/
public GridNearTxRemote(
GridCacheSharedContext ctx,
ClassLoader ldr,
UUID nodeId,
UUID nearNodeId,
long rmtThreadId,
GridCacheVersion xidVer,
GridCacheVersion commitVer,
boolean sys,
byte plc,
TransactionConcurrency concurrency,
TransactionIsolation isolation,
boolean invalidate,
long timeout,
Collection<IgniteTxEntry> writeEntries,
int txSize,
@Nullable UUID subjId,
int taskNameHash)
throws IgniteCheckedException {
super(
ctx,
nodeId,
rmtThreadId,
xidVer,
commitVer,
sys,
plc,
concurrency,
isolation,
invalidate,
timeout,
txSize,
subjId,
taskNameHash);
assert nearNodeId != null;
this.nearNodeId = nearNodeId;
readMap = Collections.emptyMap();
writeMap =
new LinkedHashMap<>(
writeEntries != null ? Math.max(txSize, writeEntries.size()) : txSize, 1.0f);
if (writeEntries != null) {
for (IgniteTxEntry entry : writeEntries) {
entry.unmarshal(ctx, true, ldr);
addEntry(entry);
}
}
}
/** @throws Exception Thrown if test failed. */
public void testC() throws Exception {
final Collection<SampleBean> set = new GridConcurrentWeakHashSet<>();
int threadCnt = 2;
final int cnt = 5;
final CountDownLatch start = new CountDownLatch(1);
final CountDownLatch stop = new CountDownLatch(threadCnt);
Runnable r =
new Runnable() {
@Override
public void run() {
try {
start.await();
for (int i = 0; i < cnt; i++) {
for (int j = 0; j < cnt; j++) set.add(new SampleBean(i));
}
} catch (Exception e) {
error(e.getMessage());
}
stop.countDown();
}
};
for (int i = 0; i < threadCnt; i++) new Thread(r).start();
start.countDown();
stop.await();
assert set.size() == cnt;
gc();
assert set.isEmpty();
}
/**
* Decode file charset.
*
* @param f File to process.
* @return File charset.
* @throws IOException in case of error.
*/
public static Charset decode(File f) throws IOException {
SortedMap<String, Charset> charsets = Charset.availableCharsets();
String[] firstCharsets = {
Charset.defaultCharset().name(), "US-ASCII", "UTF-8", "UTF-16BE", "UTF-16LE"
};
Collection<Charset> orderedCharsets = U.newLinkedHashSet(charsets.size());
for (String c : firstCharsets)
if (charsets.containsKey(c)) orderedCharsets.add(charsets.get(c));
orderedCharsets.addAll(charsets.values());
try (RandomAccessFile raf = new RandomAccessFile(f, "r")) {
FileChannel ch = raf.getChannel();
ByteBuffer buf = ByteBuffer.allocate(4096);
ch.read(buf);
buf.flip();
for (Charset charset : orderedCharsets) {
CharsetDecoder decoder = charset.newDecoder();
decoder.reset();
try {
decoder.decode(buf);
return charset;
} catch (CharacterCodingException ignored) {
}
}
}
return Charset.defaultCharset();
}
/** {@inheritDoc} */
@Override
public Collection<ClusterNode> nodes(int p, AffinityTopologyVersion topVer) {
Collection<ClusterNode> affNodes = cctx.affinity().nodes(p, topVer);
lock.readLock().lock();
try {
assert node2part != null && node2part.valid()
: "Invalid node-to-partitions map [topVer1="
+ topVer
+ ", topVer2="
+ this.topVer
+ ", cache="
+ cctx.name()
+ ", node2part="
+ node2part
+ ']';
Collection<ClusterNode> nodes = null;
Collection<UUID> nodeIds = part2node.get(p);
if (!F.isEmpty(nodeIds)) {
Collection<UUID> affIds = new HashSet<>(F.viewReadOnly(affNodes, F.node2id()));
for (UUID nodeId : nodeIds) {
if (!affIds.contains(nodeId) && hasState(p, nodeId, OWNING, MOVING, RENTING)) {
ClusterNode n = cctx.discovery().node(nodeId);
if (n != null
&& (topVer.topologyVersion() < 0 || n.order() <= topVer.topologyVersion())) {
if (nodes == null) {
nodes = new ArrayList<>(affNodes.size() + 2);
nodes.addAll(affNodes);
}
nodes.add(n);
}
}
}
}
return nodes != null ? nodes : affNodes;
} finally {
lock.readLock().unlock();
}
}
/** @throws Exception If failed. */
public void testRangeSplit() throws Exception {
IgniteUuid affKey = IgniteUuid.randomUuid();
IgfsFileAffinityRange range = new IgfsFileAffinityRange(0, 9999, affKey);
Collection<IgfsFileAffinityRange> split = range.split(10000);
assertEquals(1, split.size());
assertTrue(range.regionEqual(F.first(split)));
split = range.split(5000);
assertEquals(2, split.size());
Iterator<IgfsFileAffinityRange> it = split.iterator();
IgfsFileAffinityRange part = it.next();
assertTrue(part.regionEqual(new IgfsFileAffinityRange(0, 4999, affKey)));
part = it.next();
assertTrue(part.regionEqual(new IgfsFileAffinityRange(5000, 9999, affKey)));
split = range.split(3000);
assertEquals(4, split.size());
it = split.iterator();
part = it.next();
assertTrue(part.regionEqual(new IgfsFileAffinityRange(0, 2999, affKey)));
part = it.next();
assertTrue(part.regionEqual(new IgfsFileAffinityRange(3000, 5999, affKey)));
part = it.next();
assertTrue(part.regionEqual(new IgfsFileAffinityRange(6000, 8999, affKey)));
part = it.next();
assertTrue(part.regionEqual(new IgfsFileAffinityRange(9000, 9999, affKey)));
}
/**
* @param updateSeq Update sequence.
* @return Checks if any of the local partitions need to be evicted.
*/
private boolean checkEvictions(long updateSeq) {
assert lock.isWriteLockedByCurrentThread();
boolean changed = false;
UUID locId = cctx.nodeId();
for (GridDhtLocalPartition part : locParts.values()) {
GridDhtPartitionState state = part.state();
if (state.active()) {
int p = part.id();
List<ClusterNode> affNodes = cctx.affinity().nodes(p, topVer);
if (!affNodes.contains(cctx.localNode())) {
Collection<UUID> nodeIds = F.nodeIds(nodes(p, topVer, OWNING));
// If all affinity nodes are owners, then evict partition from local node.
if (nodeIds.containsAll(F.nodeIds(affNodes))) {
part.rent(false);
updateLocal(part.id(), locId, part.state(), updateSeq);
changed = true;
if (log.isDebugEnabled())
log.debug("Evicted local partition (all affinity nodes are owners): " + part);
} else {
int ownerCnt = nodeIds.size();
int affCnt = affNodes.size();
if (ownerCnt > affCnt) {
List<ClusterNode> sorted = new ArrayList<>(cctx.discovery().nodes(nodeIds));
// Sort by node orders in ascending order.
Collections.sort(sorted, CU.nodeComparator(true));
int diff = sorted.size() - affCnt;
for (int i = 0; i < diff; i++) {
ClusterNode n = sorted.get(i);
if (locId.equals(n.id())) {
part.rent(false);
updateLocal(part.id(), locId, part.state(), updateSeq);
changed = true;
if (log.isDebugEnabled())
log.debug(
"Evicted local partition (this node is oldest non-affinity node): " + part);
break;
}
}
}
}
}
}
}
return changed;
}
/** {@inheritDoc} */
@Override
void onQueryFutureCanceled(long reqId) {
cancelled.add(reqId);
}
/**
* Grabs local events and detects if events was lost since last poll.
*
* @param ignite Target grid.
* @param evtOrderKey Unique key to take last order key from node local map.
* @param evtThrottleCntrKey Unique key to take throttle count from node local map.
* @param evtTypes Event types to collect.
* @param evtMapper Closure to map grid events to Visor data transfer objects.
* @return Collections of node events
*/
public static Collection<VisorGridEvent> collectEvents(
Ignite ignite,
String evtOrderKey,
String evtThrottleCntrKey,
final int[] evtTypes,
IgniteClosure<Event, VisorGridEvent> evtMapper) {
assert ignite != null;
assert evtTypes != null && evtTypes.length > 0;
ConcurrentMap<String, Long> nl = ignite.cluster().nodeLocalMap();
final long lastOrder = getOrElse(nl, evtOrderKey, -1L);
final long throttle = getOrElse(nl, evtThrottleCntrKey, 0L);
// When we first time arrive onto a node to get its local events,
// we'll grab only last those events that not older than given period to make sure we are
// not grabbing GBs of data accidentally.
final long notOlderThan = System.currentTimeMillis() - EVENTS_COLLECT_TIME_WINDOW;
// Flag for detecting gaps between events.
final AtomicBoolean lastFound = new AtomicBoolean(lastOrder < 0);
IgnitePredicate<Event> p =
new IgnitePredicate<Event>() {
/** */
private static final long serialVersionUID = 0L;
@Override
public boolean apply(Event e) {
// Detects that events were lost.
if (!lastFound.get() && (lastOrder == e.localOrder())) lastFound.set(true);
// Retains events by lastOrder, period and type.
return e.localOrder() > lastOrder
&& e.timestamp() > notOlderThan
&& F.contains(evtTypes, e.type());
}
};
Collection<Event> evts = ignite.events().localQuery(p);
// Update latest order in node local, if not empty.
if (!evts.isEmpty()) {
Event maxEvt = Collections.max(evts, EVTS_ORDER_COMPARATOR);
nl.put(evtOrderKey, maxEvt.localOrder());
}
// Update throttle counter.
if (!lastFound.get())
nl.put(evtThrottleCntrKey, throttle == 0 ? EVENTS_LOST_THROTTLE : throttle - 1);
boolean lost = !lastFound.get() && throttle == 0;
Collection<VisorGridEvent> res = new ArrayList<>(evts.size() + (lost ? 1 : 0));
if (lost) res.add(new VisorGridEventsLost(ignite.cluster().localNode().id()));
for (Event e : evts) {
VisorGridEvent visorEvt = evtMapper.apply(e);
if (visorEvt != null) res.add(visorEvt);
}
return res;
}
/**
* Remove particular entry from the trash directory or subdirectory.
*
* @param parentId Parent ID.
* @param id Entry id.
* @throws IgniteCheckedException If delete failed for some reason.
*/
private void deleteDirectory(IgniteUuid parentId, IgniteUuid id) throws IgniteCheckedException {
assert parentId != null;
assert id != null;
while (true) {
IgfsFileInfo info = meta.info(id);
if (info != null) {
assert info.isDirectory();
Map<String, IgfsListingEntry> listing = info.listing();
if (listing.isEmpty()) return; // Directory is empty.
Map<String, IgfsListingEntry> delListing;
if (listing.size() <= MAX_DELETE_BATCH) delListing = listing;
else {
delListing = new HashMap<>(MAX_DELETE_BATCH, 1.0f);
int i = 0;
for (Map.Entry<String, IgfsListingEntry> entry : listing.entrySet()) {
delListing.put(entry.getKey(), entry.getValue());
if (++i == MAX_DELETE_BATCH) break;
}
}
GridCompoundFuture<Object, ?> fut = new GridCompoundFuture<>();
// Delegate to child folders.
for (IgfsListingEntry entry : delListing.values()) {
if (!cancelled) {
if (entry.isDirectory()) deleteDirectory(id, entry.fileId());
else {
IgfsFileInfo fileInfo = meta.info(entry.fileId());
if (fileInfo != null) {
assert fileInfo.isFile();
fut.add(data.delete(fileInfo));
}
}
} else return;
}
fut.markInitialized();
// Wait for data cache to delete values before clearing meta cache.
try {
fut.get();
} catch (IgniteFutureCancelledCheckedException ignore) {
// This future can be cancelled only due to IGFS shutdown.
cancelled = true;
return;
}
// Actual delete of folder content.
Collection<IgniteUuid> delIds = meta.delete(id, delListing);
if (delListing == listing && delListing.size() == delIds.size())
break; // All entries were deleted.
} else break; // Entry was deleted concurrently.
}
}
/** @throws Exception Thrown if test failed. */
public void testA() throws Exception {
Collection<Integer> set = new GridConcurrentWeakHashSet<>();
Integer i = 1;
assert set.add(i);
assert !set.add(i);
assert set.contains(i);
assert set.size() == 1;
Collection<Integer> c = F.asList(2, 3, 4, 5);
assert set.addAll(c);
assert !set.addAll(c);
assert set.containsAll(c);
assert set.size() == 1 + c.size();
assert set.remove(i);
assert !set.remove(i);
assert !set.contains(i);
assert set.size() == c.size();
assert set.removeAll(c);
assert !set.removeAll(c);
assert !set.containsAll(c);
assert set.isEmpty();
Collection<Integer> c1 = Arrays.asList(1, 3, 5, 7, 9);
int cnt = 0;
for (Iterator<Integer> iter = set.iterator(); iter.hasNext(); cnt++) c1.contains(iter.next());
assert set.size() == cnt;
assert set.size() == set.toArray().length;
assert set.addAll(c1);
assert set.retainAll(c);
assert !set.retainAll(c);
Collection<Integer> c2 = F.retain(c1, true, c);
assert set.containsAll(c2);
assert !set.containsAll(c1);
assert !set.containsAll(c);
assert set.size() == c2.size();
set.clear();
assert set.isEmpty();
try {
set.iterator().next();
assert false;
} catch (NoSuchElementException ignored) {
assert true;
}
try {
set.add(null);
assert false;
} catch (NullPointerException ignored) {
assert true;
}
}
/** @throws Exception Thrown if test failed. */
public void testD() throws Exception {
final Collection<SampleBean> set = new GridConcurrentWeakHashSet<>();
final int cnt = 100;
final CountDownLatch start = new CountDownLatch(1);
final CountDownLatch stop = new CountDownLatch(3);
new Thread() {
@Override
public void run() {
try {
start.await();
for (int i = 0; i < cnt; i++) {
for (int j = 0; j < cnt; j++) set.add(new SampleBean(i));
}
} catch (Exception e) {
error(e.getMessage());
}
stop.countDown();
}
}.start();
new Thread() {
@Override
public void run() {
try {
start.await();
for (int i = 0; i < cnt; i++) {
for (int j = 0; j < cnt; j++) set.remove(new SampleBean(i));
}
} catch (Exception e) {
error(e.getMessage());
}
stop.countDown();
}
}.start();
new Thread() {
@SuppressWarnings({"UnusedDeclaration"})
@Override
public void run() {
try {
start.await();
while (stop.getCount() > 1) {
for (SampleBean b : set) {
// No-op.
}
}
} catch (Exception e) {
error(e.getMessage());
}
stop.countDown();
}
}.start();
start.countDown();
stop.await();
gc();
assert set.isEmpty();
}
/** @throws Exception Thrown if test failed. */
@SuppressWarnings({"UnusedAssignment"})
public void testB() throws Exception {
Collection<SampleBean> set = new GridConcurrentWeakHashSet<>();
SampleBean bean1 = new SampleBean(1);
assert set.add(bean1);
assert !set.add(bean1);
assert set.size() == 1;
assert set.contains(bean1);
bean1 = null;
gc();
assert set.isEmpty();
Collection<SampleBean> c =
F.asList(new SampleBean(1), new SampleBean(2), new SampleBean(3), new SampleBean(4));
assert set.addAll(c);
assert !set.addAll(c);
assert set.size() == c.size();
assert set.containsAll(c);
c = null;
gc();
assert set.isEmpty();
SampleBean b1 = new SampleBean(1);
SampleBean b2 = new SampleBean(2);
SampleBean b3 = new SampleBean(3);
SampleBean b4 = new SampleBean(4);
SampleBean b5 = new SampleBean(5);
set.add(b1);
set.add(b2);
set.add(b3);
set.add(b4);
set.add(b5);
Iterator iter = set.iterator();
assert iter.hasNext();
b2 = null;
b3 = null;
b4 = null;
gc();
int cnt = 0;
while (iter.hasNext()) {
info(iter.next().toString());
cnt++;
}
assert set.size() == cnt;
}
/**
* Adds evicted key bytes to evicted collection.
*
* @param key Evicted key.
*/
public void addEvicted(IgniteTxKey key) {
evicted.add(key);
}