/** {@inheritDoc} */
@Override
public Map<K, V> peekAll(
@Nullable Collection<? extends K> keys,
@Nullable GridPredicate<? super GridCacheEntry<K, V>>[] filter) {
if (keys == null || keys.isEmpty()) return emptyMap();
final Collection<K> skipped = new GridLeanSet<K>();
final Map<K, V> map = peekAll0(keys, filter, skipped);
if (map.size() + skipped.size() != keys.size()) {
map.putAll(
dht.peekAll(
F.view(
keys,
new P1<K>() {
@Override
public boolean apply(K k) {
return !map.containsKey(k) && !skipped.contains(k);
}
}),
filter));
}
return map;
}
/** @param node Node to remove. */
public void removeMappedNode(GridNode node) {
if (mappedDhtNodes.contains(node))
mappedDhtNodes = new ArrayList<>(F.view(mappedDhtNodes, F.notEqualTo(node)));
if (mappedNearNodes != null && mappedNearNodes.contains(node))
mappedNearNodes = new ArrayList<>(F.view(mappedNearNodes, F.notEqualTo(node)));
}
/** @throws Exception If failed. */
public void testCreateFileColocated() throws Exception {
GridGgfsPath path = new GridGgfsPath("/colocated");
UUID uuid = UUID.randomUUID();
GridUuid affKey;
long idx = 0;
while (true) {
affKey = new GridUuid(uuid, idx);
if (grid(0).mapKeyToNode(DATA_CACHE_NAME, affKey).id().equals(grid(0).localNode().id()))
break;
idx++;
}
try (GridGgfsOutputStream out = fs.create(path, 1024, true, affKey, 0, 1024, null)) {
// Write 5M, should be enough to test distribution.
for (int i = 0; i < 15; i++) out.write(new byte[1024 * 1024]);
}
GridGgfsFile info = fs.info(path);
Collection<GridGgfsBlockLocation> affNodes = fs.affinity(path, 0, info.length());
assertEquals(1, affNodes.size());
Collection<UUID> nodeIds = F.first(affNodes).nodeIds();
assertEquals(1, nodeIds.size());
assertEquals(grid(0).localNode().id(), F.first(nodeIds));
}
/** {@inheritDoc} */
@Override
public final Map<UUID, GridNodeMetrics> metrics(Collection<UUID> nodeIds)
throws GridSpiException {
assert !F.isEmpty(nodeIds);
long now = U.currentTimeMillis();
Collection<UUID> expired = new LinkedList<>();
for (UUID id : nodeIds) {
GridNodeMetrics nodeMetrics = metricsMap.get(id);
Long ts = tsMap.get(id);
if (nodeMetrics == null || ts == null || ts < now - metricsExpireTime) expired.add(id);
}
if (!expired.isEmpty()) {
Map<UUID, GridNodeMetrics> refreshed = metrics0(expired);
for (UUID id : refreshed.keySet()) tsMap.put(id, now);
metricsMap.putAll(refreshed);
}
return F.view(metricsMap, F.contains(nodeIds));
}
/**
* @param keys keys.
* @param topVer Topology version.
* @return Nodes for the keys.
*/
public Collection<GridNode> remoteNodes(Iterable<? extends K> keys, long topVer) {
Collection<Collection<GridNode>> colcol = new GridLeanSet<>();
for (K key : keys) colcol.add(nodes(key, topVer));
return F.view(F.flatCollections(colcol), F.remoteNodes(cctx.localNodeId()));
}
/**
* @param part Partition.
* @param topVer Topology version.
* @return Backup nodes.
*/
public Collection<GridNode> backups(int part, long topVer) {
Collection<GridNode> nodes = nodes(part, topVer);
assert !F.isEmpty(nodes);
if (nodes.size() <= 1) return Collections.emptyList();
return F.view(nodes, F.notEqualTo(nodes.iterator().next()));
}
/** @throws Exception If failed. */
public void testAffinityPut() throws Exception {
Thread.sleep(2 * TOP_REFRESH_FREQ);
assertEquals(NODES_CNT, client.compute().refreshTopology(false, false).size());
Map<UUID, Grid> gridsByLocNode = new HashMap<>(NODES_CNT);
GridClientData partitioned = client.data(PARTITIONED_CACHE_NAME);
GridClientCompute compute = client.compute();
for (int i = 0; i < NODES_CNT; i++) gridsByLocNode.put(grid(i).localNode().id(), grid(i));
for (int i = 0; i < 100; i++) {
String key = "key" + i;
UUID primaryNodeId = grid(0).mapKeyToNode(PARTITIONED_CACHE_NAME, key).id();
assertEquals("Affinity mismatch for key: " + key, primaryNodeId, partitioned.affinity(key));
assertEquals(primaryNodeId, partitioned.affinity(key));
// Must go to primary node only. Since backup count is 0, value must present on
// primary node only.
partitioned.put(key, "val" + key);
for (Map.Entry<UUID, Grid> entry : gridsByLocNode.entrySet()) {
Object val = entry.getValue().cache(PARTITIONED_CACHE_NAME).peek(key);
if (primaryNodeId.equals(entry.getKey())) assertEquals("val" + key, val);
else assertNull(val);
}
}
// Now check that we will see value in near cache in pinned mode.
for (int i = 100; i < 200; i++) {
String pinnedKey = "key" + i;
UUID primaryNodeId = grid(0).mapKeyToNode(PARTITIONED_CACHE_NAME, pinnedKey).id();
UUID pinnedNodeId = F.first(F.view(gridsByLocNode.keySet(), F.notEqualTo(primaryNodeId)));
GridClientNode node = compute.node(pinnedNodeId);
partitioned.pinNodes(node).put(pinnedKey, "val" + pinnedKey);
for (Map.Entry<UUID, Grid> entry : gridsByLocNode.entrySet()) {
Object val = entry.getValue().cache(PARTITIONED_CACHE_NAME).peek(pinnedKey);
if (primaryNodeId.equals(entry.getKey()) || pinnedNodeId.equals(entry.getKey()))
assertEquals("val" + pinnedKey, val);
else assertNull(val);
}
}
}
/** {@inheritDoc} */
@Override
public void stopListenAsync(@Nullable GridInClosure<? super GridFuture<R>>... lsnr) {
if (F.isEmpty(lsnr))
synchronized (mux) {
lsnrs.clear();
}
else
synchronized (mux) {
lsnrs.removeAll(F.asList(lsnr));
}
}
/** @throws Exception If failed. */
public void testTopologyListener() throws Exception {
final Collection<UUID> added = new ArrayList<>(1);
final Collection<UUID> rmvd = new ArrayList<>(1);
final CountDownLatch addedLatch = new CountDownLatch(1);
final CountDownLatch rmvLatch = new CountDownLatch(1);
assertEquals(NODES_CNT, client.compute().refreshTopology(false, false).size());
GridClientTopologyListener lsnr =
new GridClientTopologyListener() {
@Override
public void onNodeAdded(GridClientNode node) {
added.add(node.nodeId());
addedLatch.countDown();
}
@Override
public void onNodeRemoved(GridClientNode node) {
rmvd.add(node.nodeId());
rmvLatch.countDown();
}
};
client.addTopologyListener(lsnr);
try {
Grid g = startGrid(NODES_CNT + 1);
UUID id = g.localNode().id();
assertTrue(addedLatch.await(2 * TOP_REFRESH_FREQ, MILLISECONDS));
assertEquals(1, added.size());
assertEquals(id, F.first(added));
stopGrid(NODES_CNT + 1);
assertTrue(rmvLatch.await(2 * TOP_REFRESH_FREQ, MILLISECONDS));
assertEquals(1, rmvd.size());
assertEquals(id, F.first(rmvd));
} finally {
client.removeTopologyListener(lsnr);
stopGrid(NODES_CNT + 1);
}
}
/** {@inheritDoc} */
@Override
public GridDeployment getDeployment(final GridUuid ldrId) {
synchronized (mux) {
return F.find(
F.flat(cache.values()),
null,
new P1<SharedDeployment>() {
@Override
public boolean apply(SharedDeployment d) {
return d.classLoaderId().equals(ldrId);
}
});
}
}
/**
* Creates signature for data.
*
* @param alg Algorithm.
* @param prv Provider.
* @param key Private key.
* @param data Data.
* @return Signature.
* @throws GeneralSecurityException If any exception occurs while signature creation.
*/
public static byte[] createSignature(String alg, String prv, PrivateKey key, byte[] data)
throws GeneralSecurityException {
assert !F.isEmpty(alg);
assert !F.isEmpty(prv);
assert key != null;
assert data != null;
Signature sign = Signature.getInstance(alg, prv);
sign.initSign(key);
sign.update(data);
return sign.sign();
}
/**
* @param rmtReducer Optional reducer.
* @param rmtTransform Optional transformer.
* @param args Arguments.
* @return Future.
*/
@SuppressWarnings("IfMayBeConditional")
private <R> GridCacheQueryFuture<R> execute(
@Nullable GridReducer<T, R> rmtReducer,
@Nullable GridClosure<T, R> rmtTransform,
@Nullable Object... args) {
Collection<GridNode> nodes = nodes();
cctx.checkSecurity(GridSecurityPermission.CACHE_READ);
if (F.isEmpty(nodes))
return new GridCacheQueryErrorFuture<>(
cctx.kernalContext(),
new GridEmptyProjectionException("There are no data nodes for cache: " + cctx.namexx()));
if (log.isDebugEnabled())
log.debug("Executing query [query=" + this + ", nodes=" + nodes + ']');
if (cctx.deploymentEnabled()) {
try {
cctx.deploy().registerClasses(filter, rmtReducer, rmtTransform);
cctx.deploy().registerClasses(args);
} catch (GridException e) {
return new GridCacheQueryErrorFuture<>(cctx.kernalContext(), e);
}
}
if (subjId == null) subjId = cctx.localNodeId();
taskHash = cctx.kernalContext().job().currentTaskNameHash();
GridCacheQueryBean bean =
new GridCacheQueryBean(
this,
(GridReducer<Object, Object>) rmtReducer,
(GridClosure<Object, Object>) rmtTransform,
args);
GridCacheQueryManager qryMgr = cctx.queries();
boolean loc = nodes.size() == 1 && F.first(nodes).id().equals(cctx.localNodeId());
if (type == SQL_FIELDS)
return (GridCacheQueryFuture<R>)
(loc ? qryMgr.queryFieldsLocal(bean) : qryMgr.queryFieldsDistributed(bean, nodes));
else
return (GridCacheQueryFuture<R>)
(loc ? qryMgr.queryLocal(bean) : qryMgr.queryDistributed(bean, nodes));
}
/** {@inheritDoc} */
@Override
public boolean apply(@Nullable T1 t1, @Nullable T2 t2) {
lazyCompile();
if (expr != null) {
JexlContext ctx = new MapContext();
ctx.set(var1, t1);
ctx.set(var2, t2);
for (Map.Entry<String, Object> e : map.entrySet()) {
ctx.set(e.getKey(), e.getValue());
}
try {
Object obj = expr.evaluate(ctx);
if (obj instanceof Boolean) {
return (Boolean) obj;
}
} catch (Exception ex) {
throw F.wrap(ex);
}
}
return false;
}
/** @return Nodes to execute on. */
private Collection<GridNode> nodes() {
GridCacheMode cacheMode = cctx.config().getCacheMode();
switch (cacheMode) {
case LOCAL:
if (prj != null)
U.warn(
log,
"Ignoring query projection because it's executed over LOCAL cache "
+ "(only local node will be queried): "
+ this);
return Collections.singletonList(cctx.localNode());
case REPLICATED:
if (prj != null) return nodes(cctx, prj);
GridCacheDistributionMode mode = cctx.config().getDistributionMode();
return mode == PARTITIONED_ONLY || mode == NEAR_PARTITIONED
? Collections.singletonList(cctx.localNode())
: Collections.singletonList(F.rand(nodes(cctx, null)));
case PARTITIONED:
return nodes(cctx, prj);
default:
throw new IllegalStateException("Unknown cache distribution mode: " + cacheMode);
}
}
/**
* @return Query info.
* @throws GridException In case of error.
*/
@SuppressWarnings({"unchecked"})
private GridCacheQueryInfo localQueryInfo() throws GridException {
GridCacheQueryBean qry = query();
GridPredicate<GridCacheEntry<Object, Object>> prjPred =
qry.query().projectionFilter() == null
? F.<GridCacheEntry<Object, Object>>alwaysTrue()
: qry.query().projectionFilter();
GridMarshaller marsh = cctx.marshaller();
GridReducer<Object, Object> rdc =
qry.reducer() != null
? marsh.<GridReducer<Object, Object>>unmarshal(marsh.marshal(qry.reducer()), null)
: null;
GridClosure<Object, Object> trans =
qry.transform() != null
? marsh.<GridClosure<Object, Object>>unmarshal(marsh.marshal(qry.transform()), null)
: null;
return new GridCacheQueryInfo(
true,
prjPred,
trans,
rdc,
qry.query(),
GridCacheLocalQueryFuture.this,
ctx.localNodeId(),
cctx.io().nextIoId(),
qry.query().includeMetadata(),
true,
qry.arguments());
}
/**
* Verifies signature for data.
*
* @param alg Algorithm.
* @param prv Provider.
* @param key Public key.
* @param data Data.
* @param sign Signature.
* @return {@code true} if signature was verified, {@code false} - otherwise.
* @throws GeneralSecurityException If any exception occurs while verifying.
*/
public static boolean verifySignature(
String alg, String prv, PublicKey key, byte[] data, byte[] sign)
throws GeneralSecurityException {
assert !F.isEmpty(alg);
assert !F.isEmpty(prv);
assert key != null;
assert data != null;
assert sign != null;
Signature sign0 = Signature.getInstance(alg, prv);
sign0.initVerify(key);
sign0.update(data);
return sign0.verify(sign);
}
/** {@inheritDoc} */
@Override
public V unswap(K key) throws GridException {
ctx.denyOnFlags(F.asList(READ, SKIP_SWAP));
// Unswap only from DHT. Near cache does not have swap storage.
return dht.unswap(key);
}
/**
* @param mapping Mapping to order.
* @param committedVers Committed versions.
* @param rolledbackVers Rolled back versions.
*/
void orderCompleted(
GridDistributedTxMapping<K, V> mapping,
Collection<GridCacheVersion> committedVers,
Collection<GridCacheVersion> rolledbackVers) {
for (GridCacheTxEntry<K, V> txEntry : F.concat(false, mapping.reads(), mapping.writes())) {
while (true) {
GridDistributedCacheEntry<K, V> entry = (GridDistributedCacheEntry<K, V>) txEntry.cached();
try {
// Handle explicit locks.
GridCacheVersion base =
txEntry.explicitVersion() != null ? txEntry.explicitVersion() : xidVer;
entry.doneRemote(xidVer, base, committedVers, rolledbackVers);
if (ec()) entry.recheck();
break;
} catch (GridCacheEntryRemovedException ignored) {
assert entry.obsoleteVersion() != null;
if (log.isDebugEnabled())
log.debug(
"Replacing obsolete entry in remote transaction [entry="
+ entry
+ ", tx="
+ this
+ ']');
// Replace the entry.
txEntry.cached(cctx.cache().entryEx(txEntry.key()), entry.keyBytes());
}
}
}
}
/**
* Prints a phrase on the grid nodes running anonymous closure objects and calculating total
* number of letters.
*
* @param phrase Phrase to print on of the grid nodes.
* @throws GridException If failed.
*/
private static void countLettersClosure(String phrase) throws GridException {
X.println(">>> Starting countLettersClosure() example...");
// Explicitly execute the collection of callable objects and receive a result.
Collection<Integer> results =
G.grid()
.call(
SPREAD,
new GridClosure<String, Integer>() { // Create executable logic.
@Override
public Integer apply(String word) {
// Print out a given word, just so we can
// see which node is doing what.
X.println(">>> Executing word: " + word);
// Return the length of a given word, i.e. number of letters.
return word.length();
}
},
Arrays.asList(phrase.split(" "))); // Collection of arguments for closures.
// Add up all results using convenience 'sum()' method.
int letterCnt = F.sum(results);
X.println(">>>");
X.println(">>> Finished execution of counting letters with closure based on GridGain 3.0 API.");
X.println(">>> You should see the phrase '" + phrase + "' printed out on the nodes.");
X.println(">>> Total number of letters in the phrase is '" + letterCnt + "'.");
X.println(">>> Check all nodes for output (this node is also part of the grid).");
X.println(">>>");
}
/** {@inheritDoc} */
@Override
public Map<K, V> peekAll(
@Nullable Collection<? extends K> keys, @Nullable Collection<GridCachePeekMode> modes)
throws GridException {
if (keys == null || keys.isEmpty()) return emptyMap();
final Collection<K> skipped = new GridLeanSet<K>();
final Map<K, V> map =
!modes.contains(PARTITIONED_ONLY)
? peekAll0(keys, modes, ctx.tm().localTxx(), skipped)
: new GridLeanMap<K, V>(0);
if (map.size() != keys.size() && !modes.contains(NEAR_ONLY)) {
map.putAll(
dht.peekAll(
F.view(
keys,
new P1<K>() {
@Override
public boolean apply(K k) {
return !map.containsKey(k) && !skipped.contains(k);
}
}),
modes));
}
return map;
}
/**
* Prints a phrase on the grid nodes running anonymous callable objects and calculating total
* number of letters.
*
* @param phrase Phrase to print on of the grid nodes.
* @throws GridException If failed.
*/
private static void countLettersCallable(String phrase) throws GridException {
X.println(">>> Starting countLettersCallable() example...");
Collection<Callable<Integer>> calls = new HashSet<Callable<Integer>>();
for (final String word : phrase.split(" "))
calls.add(
new GridCallable<Integer>() { // Create executable logic.
@Override
public Integer call() throws Exception {
// Print out a given word, just so we can
// see which node is doing what.
X.println(">>> Executing word: " + word);
// Return the length of a given word, i.e. number of letters.
return word.length();
}
});
// Explicitly execute the collection of callable objects and receive a result.
Collection<Integer> results = G.grid().call(SPREAD, calls);
// Add up all results using convenience 'sum()' method on GridFunc class.
int letterCnt = F.sum(results);
X.println(">>>");
X.println(
">>> Finished execution of counting letters with callables based on GridGain 3.0 API.");
X.println(">>> You should see the phrase '" + phrase + "' printed out on the nodes.");
X.println(">>> Total number of letters in the phrase is '" + letterCnt + "'.");
X.println(">>> Check all nodes for output (this node is also part of the grid).");
X.println(">>>");
}
/**
* Prints every word a phrase on different nodes.
*
* @param phrase Phrase from which to print words on different nodes.
* @throws GridException If failed.
*/
private static void spreadWordsClosure(String phrase) throws GridException {
X.println(">>> Starting spreadWordsClosure() example...");
// Splits the passed in phrase into words and prints every word
// on a individual grid node. If there are more words than nodes -
// some nodes will print more than one word.
G.grid()
.run(
SPREAD,
F.yield(
phrase.split(" "),
new GridInClosure<String>() {
@Override
public void apply(String word) {
X.println(word);
}
}));
// NOTE:
//
// Alternatively, you can use existing closure 'F.println()' to
// print any yield result in 'F.yield()' like so:
//
// G.grid().run(SPREAD, F.yield(phrase.split(" "), F.println()));
//
X.println(">>>");
X.println(
">>> Finished printing individual words on different nodes based on GridGain 3.0 API.");
X.println(">>> Check all nodes for output (this node is also part of the grid).");
X.println(">>>");
}
/** {@inheritDoc} */
@Override
public boolean apply() {
try {
return applyx();
} catch (GridException ex) {
throw F.wrap(ex);
}
}
/** {@inheritDoc} */
@Override
public R apply() {
try {
return applyx();
} catch (GridException e) {
throw F.wrap(e);
}
}
/** {@inheritDoc} */
@Override
public <T> T affinityKey() {
try {
return (T) job.getDeployment().annotatedValue(job.getJob(), GridCacheAffinityMapped.class);
} catch (GridException e) {
throw F.wrap(e);
}
}
/** {@inheritDoc} */
@Override
public String cacheName() {
try {
return (String) job.getDeployment().annotatedValue(job.getJob(), GridCacheName.class);
} catch (GridException e) {
throw F.wrap(e);
}
}
/**
* Creates node predicate that evaluates to {@code true} for all provided node IDs. Implementation
* will make a defensive copy.
*
* @param ids Optional node IDs. If none provided - predicate will always return {@code false}.
*/
public GridNodePredicate(@Nullable UUID... ids) {
if (F.isEmpty(ids)) this.ids = Collections.emptySet();
else if (ids.length == 1) this.ids = Collections.singleton(ids[0]);
else {
this.ids = new HashSet<>(ids.length);
Collections.addAll(this.ids, ids);
}
}
/**
* Creates node predicate that evaluates to {@code true} for all provided nodes. Implementation
* will make a defensive copy.
*
* @param nodes Optional grid nodes. If none provided - predicate will always return {@code
* false}.
*/
public GridNodePredicate(@Nullable GridNode... nodes) {
if (F.isEmpty(nodes)) ids = Collections.emptySet();
else if (nodes.length == 1) ids = Collections.singleton(nodes[0].id());
else {
ids = new HashSet<>(nodes.length);
for (GridNode n : nodes) ids.add(n.id());
}
}
/**
* Add information about key and version to request.
*
* <p>Other version has to be provided if suspect lock is DHT local.
*
* @param key Key.
* @param cand Candidate.
*/
@SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
void addCandidate(K key, GridCacheDgcLockCandidate cand) {
Collection<GridCacheDgcLockCandidate> col =
F.addIfAbsent(map, key, new ArrayList<GridCacheDgcLockCandidate>());
assert col != null;
col.add(cand);
}
/**
* @param g Grid.
* @return Non-system caches.
*/
private Collection<GridCacheConfiguration> caches(Grid g) {
return F.view(
Arrays.asList(g.configuration().getCacheConfiguration()),
new GridPredicate<GridCacheConfiguration>() {
@Override
public boolean apply(GridCacheConfiguration c) {
return c.getName() == null || !c.getName().equals(CU.UTILITY_CACHE_NAME);
}
});
}