private static void testPotato( Class<? extends Collection> implClazz, Class<? extends List> argClazz) throws Throwable { try { System.out.printf("implClazz=%s, argClazz=%s\n", implClazz.getName(), argClazz.getName()); final int iterations = 100000; final List<Integer> list = (List<Integer>) argClazz.newInstance(); final Integer one = Integer.valueOf(1); final List<Integer> oneElementList = Collections.singletonList(one); final Constructor<? extends Collection> constr = implClazz.getConstructor(Collection.class); final Thread t = new CheckedThread() { public void realRun() { for (int i = 0; i < iterations; i++) { list.add(one); list.remove(one); } } }; t.setDaemon(true); t.start(); for (int i = 0; i < iterations; i++) { Collection<?> coll = constr.newInstance(list); Object[] elts = coll.toArray(); check(elts.length == 0 || (elts.length == 1 && elts[0] == one)); } } catch (Throwable t) { unexpected(t); } }
/** {@inheritDoc} */ @Override protected Collection<? extends GridComputeJob> split(int gridSize, Object arg) throws GridException { Collection<GridComputeJobAdapter> jobs = new ArrayList<>(gridSize); this.gridSize = gridSize; final String locNodeId = grid.localNode().id().toString(); for (int i = 0; i < gridSize; i++) { jobs.add( new GridComputeJobAdapter() { @SuppressWarnings("OverlyStrongTypeCast") @Override public Object execute() { try { Thread.sleep(1000); } catch (InterruptedException ignored) { Thread.currentThread().interrupt(); } return new GridBiTuple<>(locNodeId, 1); } }); } return jobs; }
/** @throws Exception If failed. */ public void testProjectionRun() throws Exception { GridClientCompute dflt = client.compute(); Collection<? extends GridClientNode> nodes = dflt.nodes(); assertEquals(NODES_CNT, nodes.size()); for (int i = 0; i < NODES_CNT; i++) { Grid g = grid(i); assert g != null; GridClientNode clientNode = dflt.node(g.localNode().id()); assertNotNull("Client node for " + g.localNode().id() + " was not found", clientNode); GridClientCompute prj = dflt.projection(clientNode); String res = prj.execute(TestTask.class.getName(), null); assertNotNull(res); assertEquals(g.localNode().id().toString(), res); } }
/** @throws Exception If failed. */ public void testClientAffinity() throws Exception { GridClientData partitioned = client.data(PARTITIONED_CACHE_NAME); Collection<Object> keys = new ArrayList<>(); keys.addAll(Arrays.asList(Boolean.TRUE, Boolean.FALSE, 1, Integer.MAX_VALUE)); Random rnd = new Random(); StringBuilder sb = new StringBuilder(); // Generate some random strings. for (int i = 0; i < 100; i++) { sb.setLength(0); for (int j = 0; j < 255; j++) // Only printable ASCII symbols for test. sb.append((char) (rnd.nextInt(0x7f - 0x20) + 0x20)); keys.add(sb.toString()); } // Generate some more keys to achieve better coverage. for (int i = 0; i < 100; i++) keys.add(UUID.randomUUID()); for (Object key : keys) { UUID nodeId = grid(0).mapKeyToNode(PARTITIONED_CACHE_NAME, key).id(); UUID clientNodeId = partitioned.affinity(key); assertEquals( "Invalid affinity mapping for REST response for key: " + key, nodeId, clientNodeId); } }
/** * Makes <tt>RTCPSRPacket</tt>s for all the RTP streams that we're sending. * * @return a <tt>List</tt> of <tt>RTCPSRPacket</tt> for all the RTP streams that we're sending. */ private Collection<RTCPSRPacket> makeRTCPSRPackets(long time) { Collection<RTCPSRPacket> srPackets = new ArrayList<RTCPSRPacket>(); for (RTPStatsEntry rtpStatsEntry : rtpStatsMap.values()) { int ssrc = rtpStatsEntry.getSsrc(); RemoteClock estimate = remoteClockEstimator.estimate(ssrc, time); if (estimate == null) { // We're not going to go far without an estimate.. continue; } RTCPSRPacket srPacket = new RTCPSRPacket(ssrc, MIN_RTCP_REPORTS_BLOCKS_ARRAY); // Set the NTP timestamp for this SR. long estimatedRemoteTime = estimate.getRemoteTime(); long secs = estimatedRemoteTime / 1000L; double fraction = (estimatedRemoteTime - secs * 1000L) / 1000D; srPacket.ntptimestamplsw = (int) (fraction * 4294967296D); srPacket.ntptimestampmsw = secs; // Set the RTP timestamp. srPacket.rtptimestamp = estimate.getRtpTimestamp(); // Fill-in packet and octet send count. srPacket.packetcount = rtpStatsEntry.getPacketsSent(); srPacket.octetcount = rtpStatsEntry.getBytesSent(); srPackets.add(srPacket); } return srPackets; }
/** @throws Exception If failed. */ public void testCreateFileColocated() throws Exception { GridGgfsPath path = new GridGgfsPath("/colocated"); UUID uuid = UUID.randomUUID(); GridUuid affKey; long idx = 0; while (true) { affKey = new GridUuid(uuid, idx); if (grid(0).mapKeyToNode(DATA_CACHE_NAME, affKey).id().equals(grid(0).localNode().id())) break; idx++; } try (GridGgfsOutputStream out = fs.create(path, 1024, true, affKey, 0, 1024, null)) { // Write 5M, should be enough to test distribution. for (int i = 0; i < 15; i++) out.write(new byte[1024 * 1024]); } GridGgfsFile info = fs.info(path); Collection<GridGgfsBlockLocation> affNodes = fs.affinity(path, 0, info.length()); assertEquals(1, affNodes.size()); Collection<UUID> nodeIds = F.first(affNodes).nodeIds(); assertEquals(1, nodeIds.size()); assertEquals(grid(0).localNode().id(), F.first(nodeIds)); }
private void recheck() { // If this is the oldest node. if (oldestNode.get().id().equals(cctx.localNodeId())) { Collection<UUID> remaining = remaining(); if (!remaining.isEmpty()) { try { cctx.io() .safeSend( cctx.discovery().nodes(remaining), new GridDhtPartitionsSingleRequest(exchId), SYSTEM_POOL, null); } catch (IgniteCheckedException e) { U.error( log, "Failed to request partitions from nodes [exchangeId=" + exchId + ", nodes=" + remaining + ']', e); } } // Resend full partition map because last attempt failed. else { if (spreadPartitions()) onDone(exchId.topologyVersion()); } } else sendPartitions(); // Schedule another send. scheduleRecheck(); }
/** * Makes an <tt>RTCPREMBPacket</tt> that provides receiver feedback to the endpoint from which we * receive. * * @return an <tt>RTCPREMBPacket</tt> that provides receiver feedback to the endpoint from which * we receive. */ private RTCPREMBPacket makeRTCPREMBPacket() { // TODO we should only make REMBs if REMB support has been advertised. // Destination RemoteBitrateEstimator remoteBitrateEstimator = ((VideoMediaStream) getStream()).getRemoteBitrateEstimator(); Collection<Integer> ssrcs = remoteBitrateEstimator.getSsrcs(); // TODO(gp) intersect with SSRCs from signaled simulcast layers // NOTE(gp) The Google Congestion Control algorithm (sender side) // doesn't seem to care about the SSRCs in the dest field. long[] dest = new long[ssrcs.size()]; int i = 0; for (Integer ssrc : ssrcs) dest[i++] = ssrc & 0xFFFFFFFFL; // Exp & mantissa long bitrate = remoteBitrateEstimator.getLatestEstimate(); if (bitrate == -1) return null; if (logger.isDebugEnabled()) logger.debug("Estimated bitrate: " + bitrate); // Create and return the packet. // We use the stream's local source ID (SSRC) as the SSRC of packet // sender. long streamSSRC = getLocalSSRC(); return new RTCPREMBPacket(streamSSRC, /* mediaSSRC */ 0L, bitrate, dest); }
/** {@inheritDoc} */ @Nullable @Override public Map<String, Collection<?>> run(GridStreamerContext ctx, Collection<Bar> bars) throws GridException { ConcurrentMap<String, Bar> loc = ctx.localSpace(); GridStreamerWindow win = ctx.window("stage2"); // Add numbers to window. win.enqueueAll(bars); Collection<Bar> polled = win.pollEvictedBatch(); if (!polled.isEmpty()) { Map<String, Bar> map = new HashMap<>(); for (Bar polledBar : polled) { String symbol = polledBar.symbol(); Bar bar = map.get(symbol); if (bar == null) map.put(symbol, bar = new Bar(symbol)); bar.update(polledBar); } loc.putAll(map); } return null; }
/** {@inheritDoc} */ @Nullable @Override public Map<String, Collection<?>> run(GridStreamerContext ctx, Collection<Quote> quotes) throws GridException { GridStreamerWindow win = ctx.window("stage1"); // Add numbers to window. win.enqueueAll(quotes); Collection<Quote> polled = win.pollEvictedBatch(); if (!polled.isEmpty()) { Map<String, Bar> map = new HashMap<>(); for (Quote quote : polled) { String symbol = quote.symbol(); Bar bar = map.get(symbol); if (bar == null) map.put(symbol, bar = new Bar(symbol)); bar.update(quote.price()); } return Collections.<String, Collection<?>>singletonMap(ctx.nextStageName(), map.values()); } return null; }
/** * Schedules runnable task for execution. * * @param w Runnable task. * @throws GridException Thrown if any exception occurred. */ @SuppressWarnings({"CatchGenericClass", "ProhibitedExceptionThrown"}) public void execute(final GridWorker w) throws GridException { workers.add(w); try { exec.execute( new Runnable() { @Override public void run() { try { w.run(); } finally { workers.remove(w); } } }); } catch (RejectedExecutionException e) { workers.remove(w); throw new GridComputeExecutionRejectedException( "Failed to execute worker due to execution rejection.", e); } catch (RuntimeException e) { workers.remove(w); throw new GridException("Failed to execute worker due to runtime exception.", e); } catch (Error e) { workers.remove(w); throw e; } }
/** * Sends query request. * * @param fut Distributed future. * @param req Request. * @param nodes Nodes. * @throws IgniteCheckedException In case of error. */ @SuppressWarnings("unchecked") private void sendRequest( final GridCacheDistributedQueryFuture<?, ?, ?> fut, final GridCacheQueryRequest req, Collection<ClusterNode> nodes) throws IgniteCheckedException { assert fut != null; assert req != null; assert nodes != null; final UUID locNodeId = cctx.localNodeId(); ClusterNode locNode = null; Collection<ClusterNode> rmtNodes = null; for (ClusterNode n : nodes) { if (n.id().equals(locNodeId)) locNode = n; else { if (rmtNodes == null) rmtNodes = new ArrayList<>(nodes.size()); rmtNodes.add(n); } } // Request should be sent to remote nodes before the query is processed on the local node. // For example, a remote reducer has a state, we should not serialize and then send // the reducer changed by the local node. if (!F.isEmpty(rmtNodes)) { cctx.io() .safeSend( rmtNodes, req, cctx.ioPolicy(), new P1<ClusterNode>() { @Override public boolean apply(ClusterNode node) { fut.onNodeLeft(node.id()); return !fut.isDone(); } }); } if (locNode != null) { cctx.closures() .callLocalSafe( new Callable<Object>() { @Override public Object call() throws Exception { req.beforeLocalExecution(cctx); processQueryRequest(locNodeId, req); return null; } }); } }
private static void testImplementation(Class<? extends Collection> implClazz) throws Throwable { testPotato(implClazz, Vector.class); testPotato(implClazz, CopyOnWriteArrayList.class); final Constructor<? extends Collection> constr = implClazz.getConstructor(Collection.class); final Collection<Object> coll = constr.newInstance(Arrays.asList(new String[] {})); coll.add(1); equal(coll.toString(), "[1]"); }
/** {@inheritDoc} */ @Override public Collection<UUID> nodeIds() { Collection<UUID> ids = new GridLeanSet<UUID>(); ids.add(cctx.nodeId()); ids.addAll(mappings.keySet()); return ids; }
/** * Removes receiver and sender feedback from RTCP packets. * * @param inPacket the <tt>RTCPCompoundPacket</tt> to filter. * @return the filtered <tt>RawPacket</tt>. */ public RawPacket gateway(RTCPCompoundPacket inPacket) { if (inPacket == null || inPacket.packets == null || inPacket.packets.length == 0) { logger.info("Ignoring empty RTCP packet."); return null; } ArrayList<RTCPPacket> outPackets = new ArrayList<RTCPPacket>(inPacket.packets.length); for (RTCPPacket p : inPacket.packets) { switch (p.type) { case RTCPPacket.RR: case RTCPPacket.SR: case RTCPPacket.SDES: // We generate our own RR/SR/SDES packets. We only want // to forward NACKs/PLIs/etc. break; case RTCPFBPacket.PSFB: RTCPFBPacket psfb = (RTCPFBPacket) p; switch (psfb.fmt) { case RTCPREMBPacket.FMT: // We generate its own REMB packets. break; default: // We let through everything else, like NACK // packets. outPackets.add(psfb); break; } break; default: // We let through everything else, like BYE and APP // packets. outPackets.add(p); break; } } if (outPackets.size() == 0) { return null; } // We have feedback messages to send. Pack them in a compound // RR and send them. TODO Use RFC5506 Reduced-Size RTCP, if the // receiver supports it. Collection<RTCPRRPacket> rrPackets = makeRTCPRRPackets(System.currentTimeMillis()); if (rrPackets != null && rrPackets.size() != 0) { outPackets.addAll(0, rrPackets); } else { logger.warn("We might be sending invalid RTCPs."); } RTCPPacket[] pkts = outPackets.toArray(new RTCPPacket[outPackets.size()]); RTCPCompoundPacket outPacket = new RTCPCompoundPacket(pkts); return generator.apply(outPacket); }
// Check all iterators for correct size counts private void checkSizes(int expectedSize) { assertEquals("size()", _nbhm.size(), expectedSize); Collection<String> vals = _nbhm.values(); checkSizes("values()", vals.size(), vals.iterator(), expectedSize); Set<String> keys = _nbhm.keySet(); checkSizes("keySet()", keys.size(), keys.iterator(), expectedSize); Set<Entry<String, String>> ents = _nbhm.entrySet(); checkSizes("entrySet()", ents.size(), ents.iterator(), expectedSize); }
/** {@inheritDoc} */ @Override protected Collection<? extends GridComputeJob> split(int gridSize, String arg) throws GridException { Collection<GridComputeJobAdapter> jobs = new ArrayList<>(jobCnt); for (int i = 0; i < jobCnt; i++) jobs.add(new TestJob()); return jobs; }
/** values collection contains all values */ public void testDescendingValues() { ConcurrentNavigableMap map = dmap5(); Collection s = map.values(); assertEquals(5, s.size()); assertTrue(s.contains("A")); assertTrue(s.contains("B")); assertTrue(s.contains("C")); assertTrue(s.contains("D")); assertTrue(s.contains("E")); }
protected void handleMapValues() { Collection set = getMap().values(); Iterator it = set.iterator(); int count = 0; while (it.hasNext()) { count++; println(it.next()); } println("Total " + count); }
private static Collection<Node> getNodeMatching(Node body, String regexp) { final Collection<Node> nodes = new ArrayList<>(); if (body.getNodeName().matches(regexp)) nodes.add(body); if (body.getChildNodes().getLength() == 0) return nodes; NodeList returnList = body.getChildNodes(); for (int k = 0; k < returnList.getLength(); k++) { final Node node = returnList.item(k); nodes.addAll(getNodeMatching(node, regexp)); } return nodes; }
/** {@inheritDoc} */ @Override protected Collection<? extends GridComputeJob> split(int gridSize, Object arg) throws GridException { if (log.isInfoEnabled()) log.info("Splitting job [job=" + this + ", gridSize=" + gridSize + ", arg=" + arg + ']'); Collection<GridComputeJob> jobs = new ArrayList<>(SPLIT_COUNT); for (int i = 1; i <= SPLIT_COUNT; i++) jobs.add(new GridCancelTestJob(i)); return jobs; }
/** * @param cacheCtx Cache context. * @return {@code True} if local node can calculate affinity on it's own for this partition map * exchange. */ private boolean canCalculateAffinity(GridCacheContext cacheCtx) { AffinityFunction affFunc = cacheCtx.config().getAffinity(); // Do not request affinity from remote nodes if affinity function is not centralized. if (!U.hasAnnotation(affFunc, AffinityCentralizedFunction.class)) return true; // If local node did not initiate exchange or local node is the only cache node in grid. Collection<ClusterNode> affNodes = CU.affinityNodes(cacheCtx, exchId.topologyVersion()); return !exchId.nodeId().equals(cctx.localNodeId()) || (affNodes.size() == 1 && affNodes.contains(cctx.localNode())); }
/** Cleans up resources to avoid excessive memory usage. */ public void cleanUp() { topSnapshot.set(null); singleMsgs.clear(); fullMsgs.clear(); rcvdIds.clear(); oldestNode.set(null); partReleaseFut = null; Collection<ClusterNode> rmtNodes = this.rmtNodes; if (rmtNodes != null) rmtNodes.clear(); }
/** Values.toArray contains all values */ public void testDescendingValuesToArray() { ConcurrentNavigableMap map = dmap5(); Collection v = map.values(); Object[] ar = v.toArray(); ArrayList s = new ArrayList(Arrays.asList(ar)); assertEquals(5, ar.length); assertTrue(s.contains("A")); assertTrue(s.contains("B")); assertTrue(s.contains("C")); assertTrue(s.contains("D")); assertTrue(s.contains("E")); }
/** * Returns compact class host. * * @param obj Object to compact. * @return String. */ @Nullable public static Object compactObject(Object obj) { if (obj == null) return null; if (obj instanceof Enum) return obj.toString(); if (obj instanceof String || obj instanceof Boolean || obj instanceof Number) return obj; if (obj instanceof Collection) { Collection col = (Collection) obj; Object[] res = new Object[col.size()]; int i = 0; for (Object elm : col) res[i++] = compactObject(elm); return res; } if (obj.getClass().isArray()) { Class<?> arrType = obj.getClass().getComponentType(); if (arrType.isPrimitive()) { if (obj instanceof boolean[]) return Arrays.toString((boolean[]) obj); if (obj instanceof byte[]) return Arrays.toString((byte[]) obj); if (obj instanceof short[]) return Arrays.toString((short[]) obj); if (obj instanceof int[]) return Arrays.toString((int[]) obj); if (obj instanceof long[]) return Arrays.toString((long[]) obj); if (obj instanceof float[]) return Arrays.toString((float[]) obj); if (obj instanceof double[]) return Arrays.toString((double[]) obj); } Object[] arr = (Object[]) obj; int iMax = arr.length - 1; StringBuilder sb = new StringBuilder("["); for (int i = 0; i <= iMax; i++) { sb.append(compactObject(arr[i])); if (i != iMax) sb.append(", "); } sb.append("]"); return sb.toString(); } return U.compact(obj.getClass().getName()); }
/** @throws Exception If failed. */ public void testTopologyListener() throws Exception { final Collection<UUID> added = new ArrayList<>(1); final Collection<UUID> rmvd = new ArrayList<>(1); final CountDownLatch addedLatch = new CountDownLatch(1); final CountDownLatch rmvLatch = new CountDownLatch(1); assertEquals(NODES_CNT, client.compute().refreshTopology(false, false).size()); GridClientTopologyListener lsnr = new GridClientTopologyListener() { @Override public void onNodeAdded(GridClientNode node) { added.add(node.nodeId()); addedLatch.countDown(); } @Override public void onNodeRemoved(GridClientNode node) { rmvd.add(node.nodeId()); rmvLatch.countDown(); } }; client.addTopologyListener(lsnr); try { Grid g = startGrid(NODES_CNT + 1); UUID id = g.localNode().id(); assertTrue(addedLatch.await(2 * TOP_REFRESH_FREQ, MILLISECONDS)); assertEquals(1, added.size()); assertEquals(id, F.first(added)); stopGrid(NODES_CNT + 1); assertTrue(rmvLatch.await(2 * TOP_REFRESH_FREQ, MILLISECONDS)); assertEquals(1, rmvd.size()); assertEquals(id, F.first(rmvd)); } finally { client.removeTopologyListener(lsnr); stopGrid(NODES_CNT + 1); } }
/** {@inheritDoc} */ @Override public void loadCache(GridBiInClosure<K, V> c, @Nullable Object... args) throws GridException { ExecutorService exec = new ThreadPoolExecutor( threadsCnt, threadsCnt, 0L, MILLISECONDS, new ArrayBlockingQueue<Runnable>(batchQueueSize), new BlockingRejectedExecutionHandler()); Iterator<I> iter = inputIterator(args); Collection<I> buf = new ArrayList<>(batchSize); try { while (iter.hasNext()) { if (Thread.currentThread().isInterrupted()) { U.warn(log, "Working thread was interrupted while loading data."); break; } buf.add(iter.next()); if (buf.size() == batchSize) { exec.submit(new Worker(c, buf, args)); buf = new ArrayList<>(batchSize); } } if (!buf.isEmpty()) exec.submit(new Worker(c, buf, args)); } catch (RejectedExecutionException ignored) { // Because of custom RejectedExecutionHandler. assert false : "RejectedExecutionException was thrown while it shouldn't."; } finally { exec.shutdown(); try { exec.awaitTermination(Long.MAX_VALUE, MILLISECONDS); } catch (InterruptedException ignored) { U.warn(log, "Working thread was interrupted while waiting for put operations to complete."); Thread.currentThread().interrupt(); } } }
/** * Processes cache query response. * * @param sndId Sender node id. * @param res Query response. */ @SuppressWarnings("unchecked") private void processQueryResponse(UUID sndId, GridCacheQueryResponse res) { if (log.isDebugEnabled()) log.debug("Received query response: " + res); GridCacheQueryFutureAdapter fut = getQueryFuture(res.requestId()); if (fut != null) if (res.fields()) ((GridCacheDistributedFieldsQueryFuture) fut) .onPage( sndId, res.metadata(), (Collection<Map<String, Object>>) ((Collection) res.data()), res.error(), res.isFinished()); else fut.onPage(sndId, res.data(), res.error(), res.isFinished()); else if (!cancelled.contains(res.requestId())) U.warn( log, "Received response for finished or unknown query [rmtNodeId=" + sndId + ", res=" + res + ']'); }
private static Collection<String> parseSoapResponseForUrls(byte[] data) throws SOAPException, IOException { // System.out.println(new String(data)); final Collection<String> urls = new ArrayList<>(); MessageFactory factory = MessageFactory.newInstance(WS_DISCOVERY_SOAP_VERSION); final MimeHeaders headers = new MimeHeaders(); headers.addHeader("Content-type", WS_DISCOVERY_CONTENT_TYPE); SOAPMessage message = factory.createMessage(headers, new ByteArrayInputStream(data)); SOAPBody body = message.getSOAPBody(); for (Node node : getNodeMatching(body, ".*:XAddrs")) { if (node.getTextContent().length() > 0) { urls.addAll(Arrays.asList(node.getTextContent().split(" "))); } } return urls; }
public void makeSnapshots(Collection<InetAddress> endpoints) { try { snapshotLatch = new CountDownLatch(endpoints.size()); IAsyncCallback callback = new IAsyncCallback() { @Override public boolean isLatencyForSnitch() { return false; } @Override public void response(Message msg) { RepairJob.this.snapshotLatch.countDown(); } }; for (InetAddress endpoint : endpoints) MessagingService.instance() .sendRR( new SnapshotCommand(tablename, cfname, sessionName, false), endpoint, callback); snapshotLatch.await(); snapshotLatch = null; } catch (InterruptedException e) { throw new RuntimeException(e); } }