/** {@inheritDoc} */
@Override
public Map<K, V> peekAll(
@Nullable Collection<? extends K> keys,
@Nullable GridPredicate<? super GridCacheEntry<K, V>>[] filter) {
if (keys == null || keys.isEmpty()) return emptyMap();
final Collection<K> skipped = new GridLeanSet<K>();
final Map<K, V> map = peekAll0(keys, filter, skipped);
if (map.size() + skipped.size() != keys.size()) {
map.putAll(
dht.peekAll(
F.view(
keys,
new P1<K>() {
@Override
public boolean apply(K k) {
return !map.containsKey(k) && !skipped.contains(k);
}
}),
filter));
}
return map;
}
/** {@inheritDoc} */
@Override
protected Collection<E> dequeue0(int cnt) {
WindowHolder tup = ref.get();
AtomicInteger size = tup.size();
Collection<T> evts = tup.collection();
Collection<E> resCol = new ArrayList<>(cnt);
while (true) {
int curSize = size.get();
if (curSize > 0) {
if (size.compareAndSet(curSize, curSize - 1)) {
E res = pollInternal(evts, tup.set());
if (res != null) {
resCol.add(res);
if (resCol.size() >= cnt) return resCol;
} else {
size.incrementAndGet();
return resCol;
}
}
} else return resCol;
}
}
/** @throws Exception If failed. */
public void testProjectionRun() throws Exception {
GridClientCompute dflt = client.compute();
Collection<? extends GridClientNode> nodes = dflt.nodes();
assertEquals(NODES_CNT, nodes.size());
for (int i = 0; i < NODES_CNT; i++) {
Grid g = grid(i);
assert g != null;
GridClientNode clientNode = dflt.node(g.localNode().id());
assertNotNull("Client node for " + g.localNode().id() + " was not found", clientNode);
GridClientCompute prj = dflt.projection(clientNode);
String res = prj.execute(TestTask.class.getName(), null);
assertNotNull(res);
assertEquals(g.localNode().id().toString(), res);
}
}
/** {@inheritDoc} */
@Nullable
@Override
public Map<String, Collection<?>> run(GridStreamerContext ctx, Collection<Quote> quotes)
throws GridException {
GridStreamerWindow win = ctx.window("stage1");
// Add numbers to window.
win.enqueueAll(quotes);
Collection<Quote> polled = win.pollEvictedBatch();
if (!polled.isEmpty()) {
Map<String, Bar> map = new HashMap<>();
for (Quote quote : polled) {
String symbol = quote.symbol();
Bar bar = map.get(symbol);
if (bar == null) map.put(symbol, bar = new Bar(symbol));
bar.update(quote.price());
}
return Collections.<String, Collection<?>>singletonMap(ctx.nextStageName(), map.values());
}
return null;
}
/** {@inheritDoc} */
@Override
public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr)
throws IgniteCheckedException {
super.finishUnmarshal(ctx, ldr);
if (writes != null) unmarshalTx(writes, false, ctx, ldr);
if (reads != null) unmarshalTx(reads, false, ctx, ldr);
if (grpLockKeyBytes != null && grpLockKey == null)
grpLockKey = ctx.marshaller().unmarshal(grpLockKeyBytes, ldr);
if (dhtVerKeys != null && dhtVers == null) {
assert dhtVerVals != null;
assert dhtVerKeys.size() == dhtVerVals.size();
Iterator<IgniteTxKey> keyIt = dhtVerKeys.iterator();
Iterator<GridCacheVersion> verIt = dhtVerVals.iterator();
dhtVers = U.newHashMap(dhtVerKeys.size());
while (keyIt.hasNext()) {
IgniteTxKey key = keyIt.next();
key.finishUnmarshal(ctx.cacheContext(key.cacheId()), ldr);
dhtVers.put(key, verIt.next());
}
}
if (txNodesBytes != null) txNodes = ctx.marshaller().unmarshal(txNodesBytes, ldr);
}
private void recheck() {
// If this is the oldest node.
if (oldestNode.get().id().equals(cctx.localNodeId())) {
Collection<UUID> remaining = remaining();
if (!remaining.isEmpty()) {
try {
cctx.io()
.safeSend(
cctx.discovery().nodes(remaining),
new GridDhtPartitionsSingleRequest(exchId),
SYSTEM_POOL,
null);
} catch (IgniteCheckedException e) {
U.error(
log,
"Failed to request partitions from nodes [exchangeId="
+ exchId
+ ", nodes="
+ remaining
+ ']',
e);
}
}
// Resend full partition map because last attempt failed.
else {
if (spreadPartitions()) onDone(exchId.topologyVersion());
}
} else sendPartitions();
// Schedule another send.
scheduleRecheck();
}
/** @param node Node to remove. */
public void removeMappedNode(GridNode node) {
if (mappedDhtNodes.contains(node))
mappedDhtNodes = new ArrayList<>(F.view(mappedDhtNodes, F.notEqualTo(node)));
if (mappedNearNodes != null && mappedNearNodes.contains(node))
mappedNearNodes = new ArrayList<>(F.view(mappedNearNodes, F.notEqualTo(node)));
}
/** {@inheritDoc} */
@SuppressWarnings("TypeMayBeWeakened")
@Nullable
private Collection<Object> unmarshalFieldsCollection(
@Nullable Collection<byte[]> byteCol, GridCacheContext<K, V> ctx, ClassLoader ldr)
throws GridException {
assert ctx != null;
assert ldr != null;
Collection<Object> col = unmarshalCollection(byteCol, ctx, ldr);
Collection<Object> col0 = null;
if (col != null) {
col0 = new ArrayList<>(col.size());
for (Object o : col) {
List<Object> list = (List<Object>) o;
List<Object> list0 = new ArrayList<>(list.size());
for (Object obj : list)
list0.add(obj != null ? ctx.marshaller().unmarshal((byte[]) obj, ldr) : null);
col0.add(list0);
}
}
return col0;
}
/** {@inheritDoc} */
@Override
public void explicitUndeploy(UUID nodeId, String rsrcName) {
Collection<SharedDeployment> undeployed = new LinkedList<SharedDeployment>();
synchronized (mux) {
for (Iterator<List<SharedDeployment>> i1 = cache.values().iterator(); i1.hasNext(); ) {
List<SharedDeployment> deps = i1.next();
for (Iterator<SharedDeployment> i2 = deps.iterator(); i2.hasNext(); ) {
SharedDeployment dep = i2.next();
if (dep.hasName(rsrcName)) {
if (!dep.isUndeployed()) {
dep.undeploy();
dep.onRemoved();
// Undeploy.
i2.remove();
undeployed.add(dep);
if (log.isInfoEnabled()) log.info("Undeployed per-version class loader: " + dep);
}
break;
}
}
if (deps.isEmpty()) i1.remove();
}
}
recordUndeployed(null, undeployed);
}
/** @throws Exception If failed. */
public void testCreateFileColocated() throws Exception {
GridGgfsPath path = new GridGgfsPath("/colocated");
UUID uuid = UUID.randomUUID();
GridUuid affKey;
long idx = 0;
while (true) {
affKey = new GridUuid(uuid, idx);
if (grid(0).mapKeyToNode(DATA_CACHE_NAME, affKey).id().equals(grid(0).localNode().id()))
break;
idx++;
}
try (GridGgfsOutputStream out = fs.create(path, 1024, true, affKey, 0, 1024, null)) {
// Write 5M, should be enough to test distribution.
for (int i = 0; i < 15; i++) out.write(new byte[1024 * 1024]);
}
GridGgfsFile info = fs.info(path);
Collection<GridGgfsBlockLocation> affNodes = fs.affinity(path, 0, info.length());
assertEquals(1, affNodes.size());
Collection<UUID> nodeIds = F.first(affNodes).nodeIds();
assertEquals(1, nodeIds.size());
assertEquals(grid(0).localNode().id(), F.first(nodeIds));
}
/** {@inheritDoc} */
@Override
protected Collection<? extends GridComputeJob> split(int gridSize, Object arg)
throws GridException {
Collection<GridComputeJobAdapter> jobs = new ArrayList<>(gridSize);
this.gridSize = gridSize;
final String locNodeId = grid.localNode().id().toString();
for (int i = 0; i < gridSize; i++) {
jobs.add(
new GridComputeJobAdapter() {
@SuppressWarnings("OverlyStrongTypeCast")
@Override
public Object execute() {
try {
Thread.sleep(1000);
} catch (InterruptedException ignored) {
Thread.currentThread().interrupt();
}
return new GridBiTuple<>(locNodeId, 1);
}
});
}
return jobs;
}
/** {@inheritDoc} */
@Nullable
@Override
public Map<String, Collection<?>> run(GridStreamerContext ctx, Collection<Bar> bars)
throws GridException {
ConcurrentMap<String, Bar> loc = ctx.localSpace();
GridStreamerWindow win = ctx.window("stage2");
// Add numbers to window.
win.enqueueAll(bars);
Collection<Bar> polled = win.pollEvictedBatch();
if (!polled.isEmpty()) {
Map<String, Bar> map = new HashMap<>();
for (Bar polledBar : polled) {
String symbol = polledBar.symbol();
Bar bar = map.get(symbol);
if (bar == null) map.put(symbol, bar = new Bar(symbol));
bar.update(polledBar);
}
loc.putAll(map);
}
return null;
}
/** @throws Exception If failed. */
public void testClientAffinity() throws Exception {
GridClientData partitioned = client.data(PARTITIONED_CACHE_NAME);
Collection<Object> keys = new ArrayList<>();
keys.addAll(Arrays.asList(Boolean.TRUE, Boolean.FALSE, 1, Integer.MAX_VALUE));
Random rnd = new Random();
StringBuilder sb = new StringBuilder();
// Generate some random strings.
for (int i = 0; i < 100; i++) {
sb.setLength(0);
for (int j = 0; j < 255; j++)
// Only printable ASCII symbols for test.
sb.append((char) (rnd.nextInt(0x7f - 0x20) + 0x20));
keys.add(sb.toString());
}
// Generate some more keys to achieve better coverage.
for (int i = 0; i < 100; i++) keys.add(UUID.randomUUID());
for (Object key : keys) {
UUID nodeId = grid(0).mapKeyToNode(PARTITIONED_CACHE_NAME, key).id();
UUID clientNodeId = partitioned.affinity(key);
assertEquals(
"Invalid affinity mapping for REST response for key: " + key, nodeId, clientNodeId);
}
}
/** {@inheritDoc} */
@Override
public Map<?, ?> waitForAttributes(Collection<?> keys, long timeout) throws InterruptedException {
A.notNull(keys, "keys");
if (keys.isEmpty()) return Collections.emptyMap();
if (timeout == 0) timeout = Long.MAX_VALUE;
long now = System.currentTimeMillis();
// Prevent overflow.
long end = now + timeout < 0 ? Long.MAX_VALUE : now + timeout;
// Don't wait longer than session timeout.
if (end > endTime) end = endTime;
synchronized (mux) {
while (!closed && !attrs.keySet().containsAll(keys) && now < end) {
mux.wait(end - now);
now = System.currentTimeMillis();
}
if (closed) throw new InterruptedException("Session was closed: " + this);
Map<Object, Object> retVal = new HashMap<Object, Object>(keys.size());
for (Object key : keys) retVal.put(key, attrs.get(key));
return retVal;
}
}
/** {@inheritDoc} */
@Override
public Map<K, V> peekAll(
@Nullable Collection<? extends K> keys, @Nullable Collection<GridCachePeekMode> modes)
throws GridException {
if (keys == null || keys.isEmpty()) return emptyMap();
final Collection<K> skipped = new GridLeanSet<K>();
final Map<K, V> map =
!modes.contains(PARTITIONED_ONLY)
? peekAll0(keys, modes, ctx.tm().localTxx(), skipped)
: new GridLeanMap<K, V>(0);
if (map.size() != keys.size() && !modes.contains(NEAR_ONLY)) {
map.putAll(
dht.peekAll(
F.view(
keys,
new P1<K>() {
@Override
public boolean apply(K k) {
return !map.containsKey(k) && !skipped.contains(k);
}
}),
modes));
}
return map;
}
/**
* Check whether provided path must be excluded from evictions.
*
* @param path Path.
* @return {@code True} in case non block of related file must be excluded.
* @throws GridException In case of faulty patterns.
*/
public boolean exclude(GridGgfsPath path) throws GridException {
assert path != null;
Collection<Pattern> excludePatterns0;
if (excludeRecompile.compareAndSet(true, false)) {
// Recompile.
Collection<String> excludePaths0 = excludePaths;
if (excludePaths0 != null) {
excludePatterns0 = new HashSet<>(excludePaths0.size(), 1.0f);
for (String excludePath : excludePaths0) {
try {
excludePatterns0.add(Pattern.compile(excludePath));
} catch (PatternSyntaxException ignore) {
throw new GridException("Invalid regex pattern: " + excludePath);
}
}
excludePatterns = excludePatterns0;
} else excludePatterns0 = excludePatterns = null;
} else excludePatterns0 = excludePatterns;
if (excludePatterns0 != null) {
String pathStr = path.toString();
for (Pattern pattern : excludePatterns0) {
if (pattern.matcher(pathStr).matches()) return true;
}
}
return false;
}
/**
* Sends query request.
*
* @param fut Distributed future.
* @param req Request.
* @param nodes Nodes.
* @throws IgniteCheckedException In case of error.
*/
@SuppressWarnings("unchecked")
private void sendRequest(
final GridCacheDistributedQueryFuture<?, ?, ?> fut,
final GridCacheQueryRequest req,
Collection<ClusterNode> nodes)
throws IgniteCheckedException {
assert fut != null;
assert req != null;
assert nodes != null;
final UUID locNodeId = cctx.localNodeId();
ClusterNode locNode = null;
Collection<ClusterNode> rmtNodes = null;
for (ClusterNode n : nodes) {
if (n.id().equals(locNodeId)) locNode = n;
else {
if (rmtNodes == null) rmtNodes = new ArrayList<>(nodes.size());
rmtNodes.add(n);
}
}
// Request should be sent to remote nodes before the query is processed on the local node.
// For example, a remote reducer has a state, we should not serialize and then send
// the reducer changed by the local node.
if (!F.isEmpty(rmtNodes)) {
cctx.io()
.safeSend(
rmtNodes,
req,
cctx.ioPolicy(),
new P1<ClusterNode>() {
@Override
public boolean apply(ClusterNode node) {
fut.onNodeLeft(node.id());
return !fut.isDone();
}
});
}
if (locNode != null) {
cctx.closures()
.callLocalSafe(
new Callable<Object>() {
@Override
public Object call() throws Exception {
req.beforeLocalExecution(cctx);
processQueryRequest(locNodeId, req);
return null;
}
});
}
}
/**
* @param part Partition.
* @param topVer Topology version.
* @return Primary node for given key.
*/
@Nullable
public GridNode primary(int part, long topVer) {
Collection<GridNode> nodes = nodes(part, topVer);
if (nodes.isEmpty()) return null;
return nodes.iterator().next();
}
/**
* @param part Partition.
* @param topVer Topology version.
* @return Backup nodes.
*/
public Collection<GridNode> backups(int part, long topVer) {
Collection<GridNode> nodes = nodes(part, topVer);
assert !F.isEmpty(nodes);
if (nodes.size() <= 1) return Collections.emptyList();
return F.view(nodes, F.notEqualTo(nodes.iterator().next()));
}
/** {@inheritDoc} */
@Override
protected Collection<? extends GridComputeJob> split(int gridSize, String arg)
throws GridException {
Collection<GridComputeJobAdapter> jobs = new ArrayList<>(jobCnt);
for (int i = 0; i < jobCnt; i++) jobs.add(new TestJob());
return jobs;
}
/** {@inheritDoc} */
@Override
public Collection<UUID> nodeIds() {
Collection<UUID> ids = new GridLeanSet<UUID>();
ids.add(cctx.nodeId());
ids.addAll(mappings.keySet());
return ids;
}
/**
* Removes locks regardless of whether they are owned or not for given version and keys.
*
* @param ver Lock version.
* @param keys Keys.
*/
@SuppressWarnings({"unchecked"})
public void removeLocks(GridCacheVersion ver, Collection<? extends K> keys) {
if (keys.isEmpty()) return;
Collection<GridRichNode> nodes = ctx.remoteNodes(keys);
try {
// Send request to remove from remote nodes.
GridDistributedUnlockRequest<K, V> req = new GridDistributedUnlockRequest<K, V>(keys.size());
req.version(ver);
for (K key : keys) {
while (true) {
GridDistributedCacheEntry<K, V> entry = peekexx(key);
try {
if (entry != null) {
GridCacheMvccCandidate<K> cand = entry.candidate(ver);
if (cand != null) {
// Remove candidate from local node first.
if (entry.removeLock(cand.version())) {
// If there is only local node in this lock's topology,
// then there is no reason to distribute the request.
if (nodes.isEmpty()) continue;
req.addKey(entry.key(), entry.getOrMarshalKeyBytes(), ctx);
}
}
}
break;
} catch (GridCacheEntryRemovedException ignored) {
if (log.isDebugEnabled())
log.debug(
"Attempted to remove lock from removed entry (will retry) [rmvVer="
+ ver
+ ", entry="
+ entry
+ ']');
}
}
}
if (nodes.isEmpty()) return;
req.completedVersions(ctx.tm().committedVersions(ver), ctx.tm().rolledbackVersions(ver));
if (!req.keyBytes().isEmpty())
// We don't wait for reply to this message.
ctx.io().safeSend(nodes, req, null);
} catch (GridException ex) {
U.error(log, "Failed to unlock the lock for keys: " + keys, ex);
}
}
/** {@inheritDoc} */
@Override
public Collection<GridDeployment> getDeployments() {
Collection<GridDeployment> deps = new LinkedList<GridDeployment>();
synchronized (mux) {
for (List<SharedDeployment> list : cache.values())
for (SharedDeployment d : list) deps.add(d);
}
return deps;
}
/** @param siblings Siblings. */
public void addJobSiblings(Collection<GridJobSibling> siblings) {
assert isTaskNode();
synchronized (mux) {
Collection<GridJobSibling> tmp = new ArrayList<GridJobSibling>(this.siblings);
tmp.addAll(siblings);
this.siblings = Collections.unmodifiableCollection(tmp);
}
}
/**
* @param cacheCtx Cache context.
* @return {@code True} if local node can calculate affinity on it's own for this partition map
* exchange.
*/
private boolean canCalculateAffinity(GridCacheContext cacheCtx) {
AffinityFunction affFunc = cacheCtx.config().getAffinity();
// Do not request affinity from remote nodes if affinity function is not centralized.
if (!U.hasAnnotation(affFunc, AffinityCentralizedFunction.class)) return true;
// If local node did not initiate exchange or local node is the only cache node in grid.
Collection<ClusterNode> affNodes = CU.affinityNodes(cacheCtx, exchId.topologyVersion());
return !exchId.nodeId().equals(cctx.localNodeId())
|| (affNodes.size() == 1 && affNodes.contains(cctx.localNode()));
}
/** Cleans up resources to avoid excessive memory usage. */
public void cleanUp() {
topSnapshot.set(null);
singleMsgs.clear();
fullMsgs.clear();
rcvdIds.clear();
oldestNode.set(null);
partReleaseFut = null;
Collection<ClusterNode> rmtNodes = this.rmtNodes;
if (rmtNodes != null) rmtNodes.clear();
}
/**
* Returns compact class host.
*
* @param obj Object to compact.
* @return String.
*/
@Nullable
public static Object compactObject(Object obj) {
if (obj == null) return null;
if (obj instanceof Enum) return obj.toString();
if (obj instanceof String || obj instanceof Boolean || obj instanceof Number) return obj;
if (obj instanceof Collection) {
Collection col = (Collection) obj;
Object[] res = new Object[col.size()];
int i = 0;
for (Object elm : col) res[i++] = compactObject(elm);
return res;
}
if (obj.getClass().isArray()) {
Class<?> arrType = obj.getClass().getComponentType();
if (arrType.isPrimitive()) {
if (obj instanceof boolean[]) return Arrays.toString((boolean[]) obj);
if (obj instanceof byte[]) return Arrays.toString((byte[]) obj);
if (obj instanceof short[]) return Arrays.toString((short[]) obj);
if (obj instanceof int[]) return Arrays.toString((int[]) obj);
if (obj instanceof long[]) return Arrays.toString((long[]) obj);
if (obj instanceof float[]) return Arrays.toString((float[]) obj);
if (obj instanceof double[]) return Arrays.toString((double[]) obj);
}
Object[] arr = (Object[]) obj;
int iMax = arr.length - 1;
StringBuilder sb = new StringBuilder("[");
for (int i = 0; i <= iMax; i++) {
sb.append(compactObject(arr[i]));
if (i != iMax) sb.append(", ");
}
sb.append("]");
return sb.toString();
}
return U.compact(obj.getClass().getName());
}
/** @throws Exception If failed. */
public void testTopologyListener() throws Exception {
final Collection<UUID> added = new ArrayList<>(1);
final Collection<UUID> rmvd = new ArrayList<>(1);
final CountDownLatch addedLatch = new CountDownLatch(1);
final CountDownLatch rmvLatch = new CountDownLatch(1);
assertEquals(NODES_CNT, client.compute().refreshTopology(false, false).size());
GridClientTopologyListener lsnr =
new GridClientTopologyListener() {
@Override
public void onNodeAdded(GridClientNode node) {
added.add(node.nodeId());
addedLatch.countDown();
}
@Override
public void onNodeRemoved(GridClientNode node) {
rmvd.add(node.nodeId());
rmvLatch.countDown();
}
};
client.addTopologyListener(lsnr);
try {
Grid g = startGrid(NODES_CNT + 1);
UUID id = g.localNode().id();
assertTrue(addedLatch.await(2 * TOP_REFRESH_FREQ, MILLISECONDS));
assertEquals(1, added.size());
assertEquals(id, F.first(added));
stopGrid(NODES_CNT + 1);
assertTrue(rmvLatch.await(2 * TOP_REFRESH_FREQ, MILLISECONDS));
assertEquals(1, rmvd.size());
assertEquals(id, F.first(rmvd));
} finally {
client.removeTopologyListener(lsnr);
stopGrid(NODES_CNT + 1);
}
}
/** {@inheritDoc} */
@Override
protected Collection<E> pollEvicted0(int cnt) {
Collection<E> res = new ArrayList<>(cnt);
for (int i = 0; i < cnt; i++) {
E evicted = pollEvictedInternal();
if (evicted == null) return res;
res.add(evicted);
}
return res;
}
/**
* @param rmtReducer Optional reducer.
* @param rmtTransform Optional transformer.
* @param args Arguments.
* @return Future.
*/
@SuppressWarnings("IfMayBeConditional")
private <R> CacheQueryFuture<R> execute(
@Nullable IgniteReducer<T, R> rmtReducer,
@Nullable IgniteClosure<T, R> rmtTransform,
@Nullable Object... args) {
Collection<ClusterNode> nodes = nodes();
cctx.checkSecurity(SecurityPermission.CACHE_READ);
if (nodes.isEmpty())
return new GridCacheQueryErrorFuture<>(
cctx.kernalContext(), new ClusterGroupEmptyCheckedException());
if (log.isDebugEnabled())
log.debug("Executing query [query=" + this + ", nodes=" + nodes + ']');
if (cctx.deploymentEnabled()) {
try {
cctx.deploy().registerClasses(filter, rmtReducer, rmtTransform);
cctx.deploy().registerClasses(args);
} catch (IgniteCheckedException e) {
return new GridCacheQueryErrorFuture<>(cctx.kernalContext(), e);
}
}
if (subjId == null) subjId = cctx.localNodeId();
taskHash = cctx.kernalContext().job().currentTaskNameHash();
final GridCacheQueryBean bean =
new GridCacheQueryBean(
this,
(IgniteReducer<Object, Object>) rmtReducer,
(IgniteClosure<Object, Object>) rmtTransform,
args);
final GridCacheQueryManager qryMgr = cctx.queries();
boolean loc = nodes.size() == 1 && F.first(nodes).id().equals(cctx.localNodeId());
if (type == SQL_FIELDS || type == SPI)
return (CacheQueryFuture<R>)
(loc ? qryMgr.queryFieldsLocal(bean) : qryMgr.queryFieldsDistributed(bean, nodes));
else if (type == SCAN && part != null && nodes.size() > 1)
return new CacheQueryFallbackFuture<>(nodes, bean, qryMgr);
else
return (CacheQueryFuture<R>)
(loc ? qryMgr.queryLocal(bean) : qryMgr.queryDistributed(bean, nodes));
}