/** {@inheritDoc} */
@Override
public final Map<UUID, GridNodeMetrics> metrics(Collection<UUID> nodeIds)
throws GridSpiException {
assert !F.isEmpty(nodeIds);
long now = U.currentTimeMillis();
Collection<UUID> expired = new LinkedList<>();
for (UUID id : nodeIds) {
GridNodeMetrics nodeMetrics = metricsMap.get(id);
Long ts = tsMap.get(id);
if (nodeMetrics == null || ts == null || ts < now - metricsExpireTime) expired.add(id);
}
if (!expired.isEmpty()) {
Map<UUID, GridNodeMetrics> refreshed = metrics0(expired);
for (UUID id : refreshed.keySet()) tsMap.put(id, now);
metricsMap.putAll(refreshed);
}
return F.view(metricsMap, F.contains(nodeIds));
}
/** @throws Exception If failed. */
public void testCreateFileColocated() throws Exception {
GridGgfsPath path = new GridGgfsPath("/colocated");
UUID uuid = UUID.randomUUID();
GridUuid affKey;
long idx = 0;
while (true) {
affKey = new GridUuid(uuid, idx);
if (grid(0).mapKeyToNode(DATA_CACHE_NAME, affKey).id().equals(grid(0).localNode().id()))
break;
idx++;
}
try (GridGgfsOutputStream out = fs.create(path, 1024, true, affKey, 0, 1024, null)) {
// Write 5M, should be enough to test distribution.
for (int i = 0; i < 15; i++) out.write(new byte[1024 * 1024]);
}
GridGgfsFile info = fs.info(path);
Collection<GridGgfsBlockLocation> affNodes = fs.affinity(path, 0, info.length());
assertEquals(1, affNodes.size());
Collection<UUID> nodeIds = F.first(affNodes).nodeIds();
assertEquals(1, nodeIds.size());
assertEquals(grid(0).localNode().id(), F.first(nodeIds));
}
/** @param node Node to remove. */
public void removeMappedNode(GridNode node) {
if (mappedDhtNodes.contains(node))
mappedDhtNodes = new ArrayList<>(F.view(mappedDhtNodes, F.notEqualTo(node)));
if (mappedNearNodes != null && mappedNearNodes.contains(node))
mappedNearNodes = new ArrayList<>(F.view(mappedNearNodes, F.notEqualTo(node)));
}
/**
* @param rmtReducer Optional reducer.
* @param rmtTransform Optional transformer.
* @param args Arguments.
* @return Future.
*/
@SuppressWarnings("IfMayBeConditional")
private <R> GridCacheQueryFuture<R> execute(
@Nullable GridReducer<T, R> rmtReducer,
@Nullable GridClosure<T, R> rmtTransform,
@Nullable Object... args) {
Collection<GridNode> nodes = nodes();
cctx.checkSecurity(GridSecurityPermission.CACHE_READ);
if (F.isEmpty(nodes))
return new GridCacheQueryErrorFuture<>(
cctx.kernalContext(),
new GridEmptyProjectionException("There are no data nodes for cache: " + cctx.namexx()));
if (log.isDebugEnabled())
log.debug("Executing query [query=" + this + ", nodes=" + nodes + ']');
if (cctx.deploymentEnabled()) {
try {
cctx.deploy().registerClasses(filter, rmtReducer, rmtTransform);
cctx.deploy().registerClasses(args);
} catch (GridException e) {
return new GridCacheQueryErrorFuture<>(cctx.kernalContext(), e);
}
}
if (subjId == null) subjId = cctx.localNodeId();
taskHash = cctx.kernalContext().job().currentTaskNameHash();
GridCacheQueryBean bean =
new GridCacheQueryBean(
this,
(GridReducer<Object, Object>) rmtReducer,
(GridClosure<Object, Object>) rmtTransform,
args);
GridCacheQueryManager qryMgr = cctx.queries();
boolean loc = nodes.size() == 1 && F.first(nodes).id().equals(cctx.localNodeId());
if (type == SQL_FIELDS)
return (GridCacheQueryFuture<R>)
(loc ? qryMgr.queryFieldsLocal(bean) : qryMgr.queryFieldsDistributed(bean, nodes));
else
return (GridCacheQueryFuture<R>)
(loc ? qryMgr.queryLocal(bean) : qryMgr.queryDistributed(bean, nodes));
}
/** @return Nodes to execute on. */
private Collection<GridNode> nodes() {
GridCacheMode cacheMode = cctx.config().getCacheMode();
switch (cacheMode) {
case LOCAL:
if (prj != null)
U.warn(
log,
"Ignoring query projection because it's executed over LOCAL cache "
+ "(only local node will be queried): "
+ this);
return Collections.singletonList(cctx.localNode());
case REPLICATED:
if (prj != null) return nodes(cctx, prj);
GridCacheDistributionMode mode = cctx.config().getDistributionMode();
return mode == PARTITIONED_ONLY || mode == NEAR_PARTITIONED
? Collections.singletonList(cctx.localNode())
: Collections.singletonList(F.rand(nodes(cctx, null)));
case PARTITIONED:
return nodes(cctx, prj);
default:
throw new IllegalStateException("Unknown cache distribution mode: " + cacheMode);
}
}
/** {@inheritDoc} */
@Override
public <T extends GridEvent> Collection<T> localEvents(GridPredicate<T> p) {
A.notNull(p, "p");
cleanupQueue();
return F.retain((Collection<T>) evts, true, p);
}
/**
* Add information about key and version to request.
*
* <p>Other version has to be provided if suspect lock is DHT local.
*
* @param key Key.
* @param cand Candidate.
*/
@SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
void addCandidate(K key, GridCacheDgcLockCandidate cand) {
Collection<GridCacheDgcLockCandidate> col =
F.addIfAbsent(map, key, new ArrayList<GridCacheDgcLockCandidate>());
assert col != null;
col.add(cand);
}
/**
* Creates node predicate that evaluates to {@code true} for all provided nodes. Implementation
* will make a defensive copy.
*
* @param nodes Optional grid nodes. If none provided - predicate will always return {@code
* false}.
*/
public GridNodePredicate(@Nullable GridNode... nodes) {
if (F.isEmpty(nodes)) ids = Collections.emptySet();
else if (nodes.length == 1) ids = Collections.singleton(nodes[0].id());
else {
ids = new HashSet<>(nodes.length);
for (GridNode n : nodes) ids.add(n.id());
}
}
/**
* Creates node predicate that evaluates to {@code true} for all provided node IDs. Implementation
* will make a defensive copy.
*
* @param ids Optional node IDs. If none provided - predicate will always return {@code false}.
*/
public GridNodePredicate(@Nullable UUID... ids) {
if (F.isEmpty(ids)) this.ids = Collections.emptySet();
else if (ids.length == 1) this.ids = Collections.singleton(ids[0]);
else {
this.ids = new HashSet<>(ids.length);
Collections.addAll(this.ids, ids);
}
}
/**
* @param g Grid.
* @return Non-system caches.
*/
private Collection<GridCacheConfiguration> caches(Grid g) {
return F.view(
Arrays.asList(g.configuration().getCacheConfiguration()),
new GridPredicate<GridCacheConfiguration>() {
@Override
public boolean apply(GridCacheConfiguration c) {
return c.getName() == null || !c.getName().equals(CU.UTILITY_CACHE_NAME);
}
});
}
/**
* Checks entry for empty value.
*
* @param entry Entry to check.
* @return {@code True} if entry is empty.
*/
private boolean empty(GridCacheEntry<K, V> entry) {
try {
return entry.peek(F.asList(GLOBAL)) == null;
} catch (GridException e) {
U.error(null, e.getMessage(), e);
assert false : "Should never happen: " + e;
return false;
}
}
/** {@inheritDoc} */
@Override
public final void removeMetrics(Collection<UUID> nodeIds) throws GridSpiException {
assert !F.isEmpty(nodeIds);
for (UUID id : nodeIds) {
metricsMap.remove(id);
tsMap.remove(id);
}
removeMetrics0(nodeIds);
}
/**
* Parses HTTP parameters in an appropriate format and return back map of values to predefined
* list of names.
*
* @param req Request.
* @return Map of parsed parameters.
*/
@SuppressWarnings({"unchecked"})
private Map<String, Object> parameters(ServletRequest req) {
Map<String, String[]> params = req.getParameterMap();
if (F.isEmpty(params)) return Collections.emptyMap();
Map<String, Object> map = U.newHashMap(params.size());
for (Map.Entry<String, String[]> entry : params.entrySet())
map.put(entry.getKey(), parameter(entry.getValue()));
return map;
}
/** @return Participating nodes. */
@Override
public Collection<? extends GridNode> nodes() {
return F.viewReadOnly(
futures(),
new GridClosure<GridFuture<?>, GridNode>() {
@Nullable
@Override
public GridNode apply(GridFuture<?> f) {
if (isMini(f)) return ((MiniFuture) f).node();
return cctx.discovery().localNode();
}
});
}
/** @throws GridException If failed. */
public void testAffinity() throws GridException {
Grid g1 = grid(1);
Grid g2 = grid(2);
assert caches(g1).size() == 0;
assert F.first(caches(g2)).getCacheMode() == PARTITIONED;
Map<GridNode, Collection<String>> map = g1.mapKeysToNodes(null, F.asList("1"));
assertNotNull(map);
assertEquals("Invalid map size: " + map.size(), 1, map.size());
assertEquals(F.first(map.keySet()), g2.localNode());
UUID id1 = g1.mapKeyToNode(null, "2").id();
assertNotNull(id1);
assertEquals(g2.localNode().id(), id1);
UUID id2 = g1.mapKeyToNode(null, "3").id();
assertNotNull(id2);
assertEquals(g2.localNode().id(), id2);
}
/**
* Checks {@link GridSystemProperties#GG_JETTY_PORT} system property and overrides default
* connector port if it present. Then initializes {@code port} with the found value.
*
* @param con Jetty connector.
*/
private void override(AbstractNetworkConnector con) {
String host = System.getProperty(GG_JETTY_HOST);
if (!F.isEmpty(host)) con.setHost(host);
int currPort = con.getPort();
Integer overridePort = Integer.getInteger(GG_JETTY_PORT);
if (overridePort != null && overridePort != 0) currPort = overridePort;
con.setPort(currPort);
port = currPort;
}
/**
* @param cctx Cache context.
* @param prj Projection (optional).
* @return Collection of data nodes in provided projection (if any).
*/
private static Collection<GridNode> nodes(
final GridCacheContext<?, ?> cctx, @Nullable final GridProjection prj) {
assert cctx != null;
return F.view(
CU.allNodes(cctx),
new P1<GridNode>() {
@Override
public boolean apply(GridNode n) {
GridCacheDistributionMode mode = U.distributionMode(n, cctx.name());
return (mode == PARTITIONED_ONLY || mode == NEAR_PARTITIONED)
&& (prj == null || prj.node(n.id()) != null);
}
});
}
/**
* @param cctx Context.
* @param tx Transaction.
* @param commit Commit flag.
*/
public GridNearTxFinishFuture(
GridCacheContext<K, V> cctx, GridNearTxLocal<K, V> tx, boolean commit) {
super(cctx.kernalContext(), F.<GridCacheTx>identityReducer(tx));
this.cctx = cctx;
this.tx = tx;
this.commit = commit;
ignoreInterrupts(true);
mappings = tx.mappings();
futId = GridUuid.randomUuid();
log = U.logger(ctx, logRef, GridNearTxFinishFuture.class);
}
/** {@inheritDoc} */
@Override
public void isolated(boolean isolated) throws GridException {
if (isolated()) return;
GridNode node = F.first(ctx.grid().forCache(cacheName).nodes());
if (node == null) throw new GridException("Failed to get node for cache: " + cacheName);
GridCacheAttributes a = U.cacheAttributes(node, cacheName);
assert a != null;
updater =
a.atomicityMode() == GridCacheAtomicityMode.ATOMIC
? GridDataLoadCacheUpdaters.<K, V>batched()
: GridDataLoadCacheUpdaters.<K, V>groupLocked();
}
/** {@inheritDoc} */
@Override
public void prepareMarshal(GridCacheContext<K, V> ctx) throws GridException {
super.prepareMarshal(ctx);
if (err != null) errBytes = ctx.marshaller().marshal(err);
metaDataBytes = marshalCollection(metadata, ctx);
dataBytes = fields ? marshalFieldsCollection(data, ctx) : marshalCollection(data, ctx);
if (ctx.deploymentEnabled() && !F.isEmpty(data)) {
for (Object o : data) {
if (o instanceof Map.Entry) {
Map.Entry e = (Map.Entry) o;
prepareObject(e.getKey(), ctx);
prepareObject(e.getValue(), ctx);
}
}
}
}
@Override
protected GridBiTuple<GridCacheContext, String> initialValue() {
return F.t2();
}
/**
* Creates node predicate that evaluates to {@code true} for all provided node IDs. Implementation
* will make a defensive copy.
*
* @param ids Optional node IDs. If none provided - predicate will always return {@code false}.
*/
public GridNodePredicate(@Nullable Collection<UUID> ids) {
this.ids =
F.isEmpty(ids)
? Collections.<UUID>emptySet()
: ids.size() == 1 ? Collections.singleton(F.first(ids)) : new HashSet<>(ids);
}
/**
* Initializes store.
*
* @throws GridException If failed to initialize.
*/
private void init() throws GridException {
if (initGuard.compareAndSet(false, true)) {
if (log.isDebugEnabled()) log.debug("Initializing cache store.");
try {
if (sesFactory != null)
// Session factory has been provided - nothing to do.
return;
if (!F.isEmpty(hibernateCfgPath)) {
try {
URL url = new URL(hibernateCfgPath);
sesFactory = new Configuration().configure(url).buildSessionFactory();
if (log.isDebugEnabled()) log.debug("Configured session factory using URL: " + url);
// Session factory has been successfully initialized.
return;
} catch (MalformedURLException e) {
if (log.isDebugEnabled())
log.debug("Caught malformed URL exception: " + e.getMessage());
}
// Provided path is not a valid URL. File?
File cfgFile = new File(hibernateCfgPath);
if (cfgFile.exists()) {
sesFactory = new Configuration().configure(cfgFile).buildSessionFactory();
if (log.isDebugEnabled())
log.debug("Configured session factory using file: " + hibernateCfgPath);
// Session factory has been successfully initialized.
return;
}
// Provided path is not a file. Classpath resource?
sesFactory = new Configuration().configure(hibernateCfgPath).buildSessionFactory();
if (log.isDebugEnabled())
log.debug("Configured session factory using classpath resource: " + hibernateCfgPath);
} else {
if (hibernateProps == null) {
U.warn(
log, "No Hibernate configuration has been provided for store (will use default).");
hibernateProps = new Properties();
hibernateProps.setProperty("hibernate.connection.url", DFLT_CONN_URL);
hibernateProps.setProperty("hibernate.show_sql", DFLT_SHOW_SQL);
hibernateProps.setProperty("hibernate.hbm2ddl.auto", DFLT_HBM2DDL_AUTO);
}
Configuration cfg = new Configuration();
cfg.setProperties(hibernateProps);
assert resourceAvailable(MAPPING_RESOURCE);
cfg.addResource(MAPPING_RESOURCE);
sesFactory = cfg.buildSessionFactory();
if (log.isDebugEnabled())
log.debug("Configured session factory using properties: " + hibernateProps);
}
} catch (HibernateException e) {
throw new GridException("Failed to initialize store.", e);
} finally {
initLatch.countDown();
}
} else if (initLatch.getCount() > 0) U.await(initLatch);
if (sesFactory == null) throw new GridException("Cache store was not properly initialized.");
}
/** {@inheritDoc} */
@Override
public GridFuture<?> addData(Map.Entry<K, V> entry) throws GridException, IllegalStateException {
A.notNull(entry, "entry");
return addData(F.asList(entry));
}
/** @throws Exception If failed. */
public void testCreateFileFragmented() throws Exception {
GridGgfsEx impl = (GridGgfsEx) grid(0).ggfs("ggfs");
GridGgfsFragmentizerManager fragmentizer = impl.context().fragmentizer();
GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", false);
GridGgfsPath path = new GridGgfsPath("/file");
try {
GridGgfs fs0 = grid(0).ggfs("ggfs");
GridGgfs fs1 = grid(1).ggfs("ggfs");
GridGgfs fs2 = grid(2).ggfs("ggfs");
try (GridGgfsOutputStream out =
fs0.create(
path,
128,
false,
1,
CFG_GRP_SIZE,
F.asMap(GridGgfs.PROP_PREFER_LOCAL_WRITES, "true"))) {
// 1.5 blocks
byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];
Arrays.fill(data, (byte) 1);
out.write(data);
}
try (GridGgfsOutputStream out = fs1.append(path, false)) {
// 1.5 blocks.
byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];
Arrays.fill(data, (byte) 2);
out.write(data);
}
// After this we should have first two block colocated with grid 0 and last block colocated
// with grid 1.
GridGgfsFileImpl fileImpl = (GridGgfsFileImpl) fs.info(path);
GridCache<Object, Object> metaCache = grid(0).cachex(META_CACHE_NAME);
GridGgfsFileInfo fileInfo = (GridGgfsFileInfo) metaCache.get(fileImpl.fileId());
GridGgfsFileMap map = fileInfo.fileMap();
List<GridGgfsFileAffinityRange> ranges = map.ranges();
assertEquals(2, ranges.size());
assertTrue(ranges.get(0).startOffset() == 0);
assertTrue(ranges.get(0).endOffset() == 2 * CFG_BLOCK_SIZE - 1);
assertTrue(ranges.get(1).startOffset() == 2 * CFG_BLOCK_SIZE);
assertTrue(ranges.get(1).endOffset() == 3 * CFG_BLOCK_SIZE - 1);
// Validate data read after colocated writes.
try (GridGgfsInputStream in = fs2.open(path)) {
// Validate first part of file.
for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 1, in.read());
// Validate second part of file.
for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 2, in.read());
assertEquals(-1, in.read());
}
} finally {
GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", true);
boolean hasData = false;
for (int i = 0; i < NODES_CNT; i++) hasData |= !grid(i).cachex(DATA_CACHE_NAME).isEmpty();
assertTrue(hasData);
fs.delete(path, true);
}
GridTestUtils.retryAssert(
log,
ASSERT_RETRIES,
ASSERT_RETRY_INTERVAL,
new CAX() {
@Override
public void applyx() {
for (int i = 0; i < NODES_CNT; i++)
assertTrue(grid(i).cachex(DATA_CACHE_NAME).isEmpty());
}
});
}
/**
* @param nodeId Node ID to send message to.
* @param msg Message to send.
* @throws GridException If send failed.
*/
private void send0(UUID nodeId, Object msg) throws GridException {
GridNode node = ctx.kernalContext().discovery().node(nodeId);
ctx.kernalContext().io().sendUserMessage(F.asList(node), msg, GridTopic.TOPIC_HADOOP, false, 0);
}
/**
* Add key to request.
*
* @param key Key to evict.
* @param ver Entry version.
* @param near {@code true} if key should be evicted from near cache.
*/
void addKey(K key, GridCacheVersion ver, boolean near) {
assert key != null;
assert ver != null;
entries.add(F.t(key, ver, near));
}
/** {@inheritDoc} */
@Override
public void execute(@Nullable GridProjection prj) throws GridException {
if (cb == null)
throw new IllegalStateException("Mandatory local callback is not set for the query: " + this);
if (prj == null) prj = ctx.grid();
prj = prj.forCache(ctx.name());
if (prj.nodes().isEmpty())
throw new GridTopologyException("Failed to execute query (projection is empty): " + this);
GridCacheMode mode = ctx.config().getCacheMode();
if (mode == LOCAL || mode == REPLICATED) {
Collection<GridNode> nodes = prj.nodes();
GridNode node = nodes.contains(ctx.localNode()) ? ctx.localNode() : F.rand(nodes);
assert node != null;
if (nodes.size() > 1 && !ctx.cache().isDrSystemCache()) {
if (node.id().equals(ctx.localNodeId()))
U.warn(
log,
"Continuous query for "
+ mode
+ " cache can be run only on local node. "
+ "Will execute query locally: "
+ this);
else
U.warn(
log,
"Continuous query for "
+ mode
+ " cache can be run only on single node. "
+ "Will execute query on remote node [qry="
+ this
+ ", node="
+ node
+ ']');
}
prj = prj.forNode(node);
}
closeLock.lock();
try {
if (routineId != null)
throw new IllegalStateException("Continuous query can't be executed twice.");
guard.block();
GridContinuousHandler hnd =
new GridCacheContinuousQueryHandler<>(ctx.name(), topic, cb, filter, prjPred);
routineId =
ctx.kernalContext()
.continuous()
.startRoutine(hnd, bufSize, timeInterval, autoUnsubscribe, prj.predicate())
.get();
} finally {
closeLock.unlock();
}
}
/**
* Creates a new set with the same elements as the given collection. The collection is created
* with a capacity of twice the number of mappings in the given map or 11 (whichever is greater),
* and a default load factor and concurrencyLevel.
*
* @param c Collection to add.
*/
public GridLeanSet(Collection<E> c) {
super(new GridLeanMap<>(F.zip(c, VAL)));
}