/** * @param rmtReducer Optional reducer. * @param rmtTransform Optional transformer. * @param args Arguments. * @return Future. */ @SuppressWarnings("IfMayBeConditional") private <R> CacheQueryFuture<R> execute( @Nullable IgniteReducer<T, R> rmtReducer, @Nullable IgniteClosure<T, R> rmtTransform, @Nullable Object... args) { Collection<ClusterNode> nodes = nodes(); cctx.checkSecurity(SecurityPermission.CACHE_READ); if (nodes.isEmpty()) return new GridCacheQueryErrorFuture<>( cctx.kernalContext(), new ClusterGroupEmptyCheckedException()); if (log.isDebugEnabled()) log.debug("Executing query [query=" + this + ", nodes=" + nodes + ']'); if (cctx.deploymentEnabled()) { try { cctx.deploy().registerClasses(filter, rmtReducer, rmtTransform); cctx.deploy().registerClasses(args); } catch (IgniteCheckedException e) { return new GridCacheQueryErrorFuture<>(cctx.kernalContext(), e); } } if (subjId == null) subjId = cctx.localNodeId(); taskHash = cctx.kernalContext().job().currentTaskNameHash(); final GridCacheQueryBean bean = new GridCacheQueryBean( this, (IgniteReducer<Object, Object>) rmtReducer, (IgniteClosure<Object, Object>) rmtTransform, args); final GridCacheQueryManager qryMgr = cctx.queries(); boolean loc = nodes.size() == 1 && F.first(nodes).id().equals(cctx.localNodeId()); if (type == SQL_FIELDS || type == SPI) return (CacheQueryFuture<R>) (loc ? qryMgr.queryFieldsLocal(bean) : qryMgr.queryFieldsDistributed(bean, nodes)); else if (type == SCAN && part != null && nodes.size() > 1) return new CacheQueryFallbackFuture<>(nodes, bean, qryMgr); else return (CacheQueryFuture<R>) (loc ? qryMgr.queryLocal(bean) : qryMgr.queryDistributed(bean, nodes)); }
/** * JUnit. * * @throws Exception If failed. */ public void testGetAndIncrement() throws Exception { Collection<Long> res = new HashSet<>(); String seqName = UUID.randomUUID().toString(); for (int i = 0; i < GRID_CNT; i++) { Set<Long> retVal = compute(grid(i).cluster().forLocal()).call(new GetAndIncrementJob(seqName, RETRIES)); for (Long l : retVal) assert !res.contains(l) : "Value already was used " + l; res.addAll(retVal); } assert res.size() == GRID_CNT * RETRIES; int gapSize = 0; for (long i = 0; i < GRID_CNT * RETRIES; i++) { if (!res.contains(i)) gapSize++; else gapSize = 0; assert gapSize <= BATCH_SIZE + 1 : "Gap above id " + i + " is " + gapSize + " more than batch size: " + (BATCH_SIZE + 1); } }
/** * Sends query request. * * @param fut Distributed future. * @param req Request. * @param nodes Nodes. * @throws IgniteCheckedException In case of error. */ @SuppressWarnings("unchecked") private void sendRequest( final GridCacheDistributedQueryFuture<?, ?, ?> fut, final GridCacheQueryRequest req, Collection<ClusterNode> nodes) throws IgniteCheckedException { assert fut != null; assert req != null; assert nodes != null; final UUID locNodeId = cctx.localNodeId(); ClusterNode locNode = null; Collection<ClusterNode> rmtNodes = null; for (ClusterNode n : nodes) { if (n.id().equals(locNodeId)) locNode = n; else { if (rmtNodes == null) rmtNodes = new ArrayList<>(nodes.size()); rmtNodes.add(n); } } // Request should be sent to remote nodes before the query is processed on the local node. // For example, a remote reducer has a state, we should not serialize and then send // the reducer changed by the local node. if (!F.isEmpty(rmtNodes)) { cctx.io() .safeSend( rmtNodes, req, cctx.ioPolicy(), new P1<ClusterNode>() { @Override public boolean apply(ClusterNode node) { fut.onNodeLeft(node.id()); return !fut.isDone(); } }); } if (locNode != null) { cctx.closures() .callLocalSafe( new Callable<Object>() { @Override public Object call() throws Exception { req.beforeLocalExecution(cctx); processQueryRequest(locNodeId, req); return null; } }); } }
/** @throws Exception If failed. */ public void testRangeSplit() throws Exception { IgniteUuid affKey = IgniteUuid.randomUuid(); IgfsFileAffinityRange range = new IgfsFileAffinityRange(0, 9999, affKey); Collection<IgfsFileAffinityRange> split = range.split(10000); assertEquals(1, split.size()); assertTrue(range.regionEqual(F.first(split))); split = range.split(5000); assertEquals(2, split.size()); Iterator<IgfsFileAffinityRange> it = split.iterator(); IgfsFileAffinityRange part = it.next(); assertTrue(part.regionEqual(new IgfsFileAffinityRange(0, 4999, affKey))); part = it.next(); assertTrue(part.regionEqual(new IgfsFileAffinityRange(5000, 9999, affKey))); split = range.split(3000); assertEquals(4, split.size()); it = split.iterator(); part = it.next(); assertTrue(part.regionEqual(new IgfsFileAffinityRange(0, 2999, affKey))); part = it.next(); assertTrue(part.regionEqual(new IgfsFileAffinityRange(3000, 5999, affKey))); part = it.next(); assertTrue(part.regionEqual(new IgfsFileAffinityRange(6000, 8999, affKey))); part = it.next(); assertTrue(part.regionEqual(new IgfsFileAffinityRange(9000, 9999, affKey))); }
/** * @param cacheCtx Cache context. * @return {@code True} if local node can calculate affinity on it's own for this partition map * exchange. */ private boolean canCalculateAffinity(GridCacheContext cacheCtx) { AffinityFunction affFunc = cacheCtx.config().getAffinity(); // Do not request affinity from remote nodes if affinity function is not centralized. if (!U.hasAnnotation(affFunc, AffinityCentralizedFunction.class)) return true; // If local node did not initiate exchange or local node is the only cache node in grid. Collection<ClusterNode> affNodes = CU.affinityNodes(cacheCtx, exchId.topologyVersion()); return !exchId.nodeId().equals(cctx.localNodeId()) || (affNodes.size() == 1 && affNodes.contains(cctx.localNode())); }
/** * This constructor is meant for optimistic transactions. * * @param ldr Class loader. * @param nodeId Node ID. * @param nearNodeId Near node ID. * @param rmtThreadId Remote thread ID. * @param xidVer XID version. * @param commitVer Commit version. * @param sys System flag. * @param concurrency Concurrency level (should be pessimistic). * @param isolation Transaction isolation. * @param invalidate Invalidate flag. * @param timeout Timeout. * @param writeEntries Write entries. * @param ctx Cache registry. * @param txSize Expected transaction size. * @throws IgniteCheckedException If unmarshalling failed. */ public GridNearTxRemote( GridCacheSharedContext ctx, ClassLoader ldr, UUID nodeId, UUID nearNodeId, long rmtThreadId, GridCacheVersion xidVer, GridCacheVersion commitVer, boolean sys, byte plc, TransactionConcurrency concurrency, TransactionIsolation isolation, boolean invalidate, long timeout, Collection<IgniteTxEntry> writeEntries, int txSize, @Nullable UUID subjId, int taskNameHash) throws IgniteCheckedException { super( ctx, nodeId, rmtThreadId, xidVer, commitVer, sys, plc, concurrency, isolation, invalidate, timeout, txSize, subjId, taskNameHash); assert nearNodeId != null; this.nearNodeId = nearNodeId; readMap = Collections.emptyMap(); writeMap = new LinkedHashMap<>( writeEntries != null ? Math.max(txSize, writeEntries.size()) : txSize, 1.0f); if (writeEntries != null) { for (IgniteTxEntry entry : writeEntries) { entry.unmarshal(ctx, true, ldr); addEntry(entry); } } }
/** * Returns compact class host. * * @param obj Object to compact. * @return String. */ @Nullable public static Object compactObject(Object obj) { if (obj == null) return null; if (obj instanceof Enum) return obj.toString(); if (obj instanceof String || obj instanceof Boolean || obj instanceof Number) return obj; if (obj instanceof Collection) { Collection col = (Collection) obj; Object[] res = new Object[col.size()]; int i = 0; for (Object elm : col) res[i++] = compactObject(elm); return res; } if (obj.getClass().isArray()) { Class<?> arrType = obj.getClass().getComponentType(); if (arrType.isPrimitive()) { if (obj instanceof boolean[]) return Arrays.toString((boolean[]) obj); if (obj instanceof byte[]) return Arrays.toString((byte[]) obj); if (obj instanceof short[]) return Arrays.toString((short[]) obj); if (obj instanceof int[]) return Arrays.toString((int[]) obj); if (obj instanceof long[]) return Arrays.toString((long[]) obj); if (obj instanceof float[]) return Arrays.toString((float[]) obj); if (obj instanceof double[]) return Arrays.toString((double[]) obj); } Object[] arr = (Object[]) obj; int iMax = arr.length - 1; StringBuilder sb = new StringBuilder("["); for (int i = 0; i <= iMax; i++) { sb.append(compactObject(arr[i])); if (i != iMax) sb.append(", "); } sb.append("]"); return sb.toString(); } return U.compact(obj.getClass().getName()); }
/** {@inheritDoc} */ @Override public Collection<ClusterNode> nodes(int p, AffinityTopologyVersion topVer) { Collection<ClusterNode> affNodes = cctx.affinity().nodes(p, topVer); lock.readLock().lock(); try { assert node2part != null && node2part.valid() : "Invalid node-to-partitions map [topVer1=" + topVer + ", topVer2=" + this.topVer + ", cache=" + cctx.name() + ", node2part=" + node2part + ']'; Collection<ClusterNode> nodes = null; Collection<UUID> nodeIds = part2node.get(p); if (!F.isEmpty(nodeIds)) { Collection<UUID> affIds = new HashSet<>(F.viewReadOnly(affNodes, F.node2id())); for (UUID nodeId : nodeIds) { if (!affIds.contains(nodeId) && hasState(p, nodeId, OWNING, MOVING, RENTING)) { ClusterNode n = cctx.discovery().node(nodeId); if (n != null && (topVer.topologyVersion() < 0 || n.order() <= topVer.topologyVersion())) { if (nodes == null) { nodes = new ArrayList<>(affNodes.size() + 2); nodes.addAll(affNodes); } nodes.add(n); } } } } return nodes != null ? nodes : affNodes; } finally { lock.readLock().unlock(); } }
/** * @param p Partition. * @param topVer Topology version ({@code -1} for all nodes). * @param state Partition state. * @param states Additional partition states. * @return List of nodes for the partition. */ private List<ClusterNode> nodes( int p, AffinityTopologyVersion topVer, GridDhtPartitionState state, GridDhtPartitionState... states) { Collection<UUID> allIds = topVer.topologyVersion() > 0 ? F.nodeIds(CU.affinityNodes(cctx, topVer)) : null; lock.readLock().lock(); try { assert node2part != null && node2part.valid() : "Invalid node-to-partitions map [topVer=" + topVer + ", allIds=" + allIds + ", node2part=" + node2part + ", cache=" + cctx.name() + ']'; Collection<UUID> nodeIds = part2node.get(p); // Node IDs can be null if both, primary and backup, nodes disappear. int size = nodeIds == null ? 0 : nodeIds.size(); if (size == 0) return Collections.emptyList(); List<ClusterNode> nodes = new ArrayList<>(size); for (UUID id : nodeIds) { if (topVer.topologyVersion() > 0 && !allIds.contains(id)) continue; if (hasState(p, id, state, states)) { ClusterNode n = cctx.discovery().node(id); if (n != null && (topVer.topologyVersion() < 0 || n.order() <= topVer.topologyVersion())) nodes.add(n); } } return nodes; } finally { lock.readLock().unlock(); } }
/** @throws Exception Thrown if test failed. */ public void testC() throws Exception { final Collection<SampleBean> set = new GridConcurrentWeakHashSet<>(); int threadCnt = 2; final int cnt = 5; final CountDownLatch start = new CountDownLatch(1); final CountDownLatch stop = new CountDownLatch(threadCnt); Runnable r = new Runnable() { @Override public void run() { try { start.await(); for (int i = 0; i < cnt; i++) { for (int j = 0; j < cnt; j++) set.add(new SampleBean(i)); } } catch (Exception e) { error(e.getMessage()); } stop.countDown(); } }; for (int i = 0; i < threadCnt; i++) new Thread(r).start(); start.countDown(); stop.await(); assert set.size() == cnt; gc(); assert set.isEmpty(); }
/** * Remove particular entry from the trash directory or subdirectory. * * @param parentId Parent ID. * @param id Entry id. * @throws IgniteCheckedException If delete failed for some reason. */ private void deleteDirectory(IgniteUuid parentId, IgniteUuid id) throws IgniteCheckedException { assert parentId != null; assert id != null; while (true) { IgfsFileInfo info = meta.info(id); if (info != null) { assert info.isDirectory(); Map<String, IgfsListingEntry> listing = info.listing(); if (listing.isEmpty()) return; // Directory is empty. Map<String, IgfsListingEntry> delListing; if (listing.size() <= MAX_DELETE_BATCH) delListing = listing; else { delListing = new HashMap<>(MAX_DELETE_BATCH, 1.0f); int i = 0; for (Map.Entry<String, IgfsListingEntry> entry : listing.entrySet()) { delListing.put(entry.getKey(), entry.getValue()); if (++i == MAX_DELETE_BATCH) break; } } GridCompoundFuture<Object, ?> fut = new GridCompoundFuture<>(); // Delegate to child folders. for (IgfsListingEntry entry : delListing.values()) { if (!cancelled) { if (entry.isDirectory()) deleteDirectory(id, entry.fileId()); else { IgfsFileInfo fileInfo = meta.info(entry.fileId()); if (fileInfo != null) { assert fileInfo.isFile(); fut.add(data.delete(fileInfo)); } } } else return; } fut.markInitialized(); // Wait for data cache to delete values before clearing meta cache. try { fut.get(); } catch (IgniteFutureCancelledCheckedException ignore) { // This future can be cancelled only due to IGFS shutdown. cancelled = true; return; } // Actual delete of folder content. Collection<IgniteUuid> delIds = meta.delete(id, delListing); if (delListing == listing && delListing.size() == delIds.size()) break; // All entries were deleted. } else break; // Entry was deleted concurrently. } }
/** @throws Exception Thrown if test failed. */ public void testA() throws Exception { Collection<Integer> set = new GridConcurrentWeakHashSet<>(); Integer i = 1; assert set.add(i); assert !set.add(i); assert set.contains(i); assert set.size() == 1; Collection<Integer> c = F.asList(2, 3, 4, 5); assert set.addAll(c); assert !set.addAll(c); assert set.containsAll(c); assert set.size() == 1 + c.size(); assert set.remove(i); assert !set.remove(i); assert !set.contains(i); assert set.size() == c.size(); assert set.removeAll(c); assert !set.removeAll(c); assert !set.containsAll(c); assert set.isEmpty(); Collection<Integer> c1 = Arrays.asList(1, 3, 5, 7, 9); int cnt = 0; for (Iterator<Integer> iter = set.iterator(); iter.hasNext(); cnt++) c1.contains(iter.next()); assert set.size() == cnt; assert set.size() == set.toArray().length; assert set.addAll(c1); assert set.retainAll(c); assert !set.retainAll(c); Collection<Integer> c2 = F.retain(c1, true, c); assert set.containsAll(c2); assert !set.containsAll(c1); assert !set.containsAll(c); assert set.size() == c2.size(); set.clear(); assert set.isEmpty(); try { set.iterator().next(); assert false; } catch (NoSuchElementException ignored) { assert true; } try { set.add(null); assert false; } catch (NullPointerException ignored) { assert true; } }
/** @throws Exception Thrown if test failed. */ @SuppressWarnings({"UnusedAssignment"}) public void testB() throws Exception { Collection<SampleBean> set = new GridConcurrentWeakHashSet<>(); SampleBean bean1 = new SampleBean(1); assert set.add(bean1); assert !set.add(bean1); assert set.size() == 1; assert set.contains(bean1); bean1 = null; gc(); assert set.isEmpty(); Collection<SampleBean> c = F.asList(new SampleBean(1), new SampleBean(2), new SampleBean(3), new SampleBean(4)); assert set.addAll(c); assert !set.addAll(c); assert set.size() == c.size(); assert set.containsAll(c); c = null; gc(); assert set.isEmpty(); SampleBean b1 = new SampleBean(1); SampleBean b2 = new SampleBean(2); SampleBean b3 = new SampleBean(3); SampleBean b4 = new SampleBean(4); SampleBean b5 = new SampleBean(5); set.add(b1); set.add(b2); set.add(b3); set.add(b4); set.add(b5); Iterator iter = set.iterator(); assert iter.hasNext(); b2 = null; b3 = null; b4 = null; gc(); int cnt = 0; while (iter.hasNext()) { info(iter.next().toString()); cnt++; } assert set.size() == cnt; }
/** * @param updateSeq Update sequence. * @return Checks if any of the local partitions need to be evicted. */ private boolean checkEvictions(long updateSeq) { assert lock.isWriteLockedByCurrentThread(); boolean changed = false; UUID locId = cctx.nodeId(); for (GridDhtLocalPartition part : locParts.values()) { GridDhtPartitionState state = part.state(); if (state.active()) { int p = part.id(); List<ClusterNode> affNodes = cctx.affinity().nodes(p, topVer); if (!affNodes.contains(cctx.localNode())) { Collection<UUID> nodeIds = F.nodeIds(nodes(p, topVer, OWNING)); // If all affinity nodes are owners, then evict partition from local node. if (nodeIds.containsAll(F.nodeIds(affNodes))) { part.rent(false); updateLocal(part.id(), locId, part.state(), updateSeq); changed = true; if (log.isDebugEnabled()) log.debug("Evicted local partition (all affinity nodes are owners): " + part); } else { int ownerCnt = nodeIds.size(); int affCnt = affNodes.size(); if (ownerCnt > affCnt) { List<ClusterNode> sorted = new ArrayList<>(cctx.discovery().nodes(nodeIds)); // Sort by node orders in ascending order. Collections.sort(sorted, CU.nodeComparator(true)); int diff = sorted.size() - affCnt; for (int i = 0; i < diff; i++) { ClusterNode n = sorted.get(i); if (locId.equals(n.id())) { part.rent(false); updateLocal(part.id(), locId, part.state(), updateSeq); changed = true; if (log.isDebugEnabled()) log.debug( "Evicted local partition (this node is oldest non-affinity node): " + part); break; } } } } } } } return changed; }
/** * Grabs local events and detects if events was lost since last poll. * * @param ignite Target grid. * @param evtOrderKey Unique key to take last order key from node local map. * @param evtThrottleCntrKey Unique key to take throttle count from node local map. * @param evtTypes Event types to collect. * @param evtMapper Closure to map grid events to Visor data transfer objects. * @return Collections of node events */ public static Collection<VisorGridEvent> collectEvents( Ignite ignite, String evtOrderKey, String evtThrottleCntrKey, final int[] evtTypes, IgniteClosure<Event, VisorGridEvent> evtMapper) { assert ignite != null; assert evtTypes != null && evtTypes.length > 0; ConcurrentMap<String, Long> nl = ignite.cluster().nodeLocalMap(); final long lastOrder = getOrElse(nl, evtOrderKey, -1L); final long throttle = getOrElse(nl, evtThrottleCntrKey, 0L); // When we first time arrive onto a node to get its local events, // we'll grab only last those events that not older than given period to make sure we are // not grabbing GBs of data accidentally. final long notOlderThan = System.currentTimeMillis() - EVENTS_COLLECT_TIME_WINDOW; // Flag for detecting gaps between events. final AtomicBoolean lastFound = new AtomicBoolean(lastOrder < 0); IgnitePredicate<Event> p = new IgnitePredicate<Event>() { /** */ private static final long serialVersionUID = 0L; @Override public boolean apply(Event e) { // Detects that events were lost. if (!lastFound.get() && (lastOrder == e.localOrder())) lastFound.set(true); // Retains events by lastOrder, period and type. return e.localOrder() > lastOrder && e.timestamp() > notOlderThan && F.contains(evtTypes, e.type()); } }; Collection<Event> evts = ignite.events().localQuery(p); // Update latest order in node local, if not empty. if (!evts.isEmpty()) { Event maxEvt = Collections.max(evts, EVTS_ORDER_COMPARATOR); nl.put(evtOrderKey, maxEvt.localOrder()); } // Update throttle counter. if (!lastFound.get()) nl.put(evtThrottleCntrKey, throttle == 0 ? EVENTS_LOST_THROTTLE : throttle - 1); boolean lost = !lastFound.get() && throttle == 0; Collection<VisorGridEvent> res = new ArrayList<>(evts.size() + (lost ? 1 : 0)); if (lost) res.add(new VisorGridEventsLost(ignite.cluster().localNode().id())); for (Event e : evts) { VisorGridEvent visorEvt = evtMapper.apply(e); if (visorEvt != null) res.add(visorEvt); } return res; }
/** * Sets data configurations. * * @param dataCfgs Data configurations. */ public void setDataConfigurations(Collection<? extends GridClientDataConfiguration> dataCfgs) { this.dataCfgs = U.newHashMap(dataCfgs.size()); for (GridClientDataConfiguration dataCfg : dataCfgs) this.dataCfgs.put(dataCfg.getName(), new GridClientDataConfiguration(dataCfg)); }