/** {@inheritDoc} */ @Nullable @Override public Map<String, Collection<?>> run(GridStreamerContext ctx, Collection<Bar> bars) throws GridException { ConcurrentMap<String, Bar> loc = ctx.localSpace(); GridStreamerWindow win = ctx.window("stage2"); // Add numbers to window. win.enqueueAll(bars); Collection<Bar> polled = win.pollEvictedBatch(); if (!polled.isEmpty()) { Map<String, Bar> map = new HashMap<>(); for (Bar polledBar : polled) { String symbol = polledBar.symbol(); Bar bar = map.get(symbol); if (bar == null) map.put(symbol, bar = new Bar(symbol)); bar.update(polledBar); } loc.putAll(map); } return null; }
private void recheck() { // If this is the oldest node. if (oldestNode.get().id().equals(cctx.localNodeId())) { Collection<UUID> remaining = remaining(); if (!remaining.isEmpty()) { try { cctx.io() .safeSend( cctx.discovery().nodes(remaining), new GridDhtPartitionsSingleRequest(exchId), SYSTEM_POOL, null); } catch (IgniteCheckedException e) { U.error( log, "Failed to request partitions from nodes [exchangeId=" + exchId + ", nodes=" + remaining + ']', e); } } // Resend full partition map because last attempt failed. else { if (spreadPartitions()) onDone(exchId.topologyVersion()); } } else sendPartitions(); // Schedule another send. scheduleRecheck(); }
/** {@inheritDoc} */ @Nullable @Override public Map<String, Collection<?>> run(GridStreamerContext ctx, Collection<Quote> quotes) throws GridException { GridStreamerWindow win = ctx.window("stage1"); // Add numbers to window. win.enqueueAll(quotes); Collection<Quote> polled = win.pollEvictedBatch(); if (!polled.isEmpty()) { Map<String, Bar> map = new HashMap<>(); for (Quote quote : polled) { String symbol = quote.symbol(); Bar bar = map.get(symbol); if (bar == null) map.put(symbol, bar = new Bar(symbol)); bar.update(quote.price()); } return Collections.<String, Collection<?>>singletonMap(ctx.nextStageName(), map.values()); } return null; }
public void checkUnknown() throws MojoExecutionException { if (!unknownFiles.isEmpty()) { String msg = "Unable to find a comment style definition for some " + "files. You may want to add a custom mapping for the relevant file extensions."; if (failIfUnknown) { throw new MojoExecutionException(msg); } getLog().warn(msg); } }
/** {@inheritDoc} */ @Override public void loadCache(GridBiInClosure<K, V> c, @Nullable Object... args) throws GridException { ExecutorService exec = new ThreadPoolExecutor( threadsCnt, threadsCnt, 0L, MILLISECONDS, new ArrayBlockingQueue<Runnable>(batchQueueSize), new BlockingRejectedExecutionHandler()); Iterator<I> iter = inputIterator(args); Collection<I> buf = new ArrayList<>(batchSize); try { while (iter.hasNext()) { if (Thread.currentThread().isInterrupted()) { U.warn(log, "Working thread was interrupted while loading data."); break; } buf.add(iter.next()); if (buf.size() == batchSize) { exec.submit(new Worker(c, buf, args)); buf = new ArrayList<>(batchSize); } } if (!buf.isEmpty()) exec.submit(new Worker(c, buf, args)); } catch (RejectedExecutionException ignored) { // Because of custom RejectedExecutionHandler. assert false : "RejectedExecutionException was thrown while it shouldn't."; } finally { exec.shutdown(); try { exec.awaitTermination(Long.MAX_VALUE, MILLISECONDS); } catch (InterruptedException ignored) { U.warn(log, "Working thread was interrupted while waiting for put operations to complete."); Thread.currentThread().interrupt(); } } }
/** @throws Exception Thrown if test failed. */ public void testC() throws Exception { final Collection<SampleBean> set = new GridConcurrentWeakHashSet<>(); int threadCnt = 2; final int cnt = 5; final CountDownLatch start = new CountDownLatch(1); final CountDownLatch stop = new CountDownLatch(threadCnt); Runnable r = new Runnable() { @Override public void run() { try { start.await(); for (int i = 0; i < cnt; i++) { for (int j = 0; j < cnt; j++) set.add(new SampleBean(i)); } } catch (Exception e) { error(e.getMessage()); } stop.countDown(); } }; for (int i = 0; i < threadCnt; i++) new Thread(r).start(); start.countDown(); stop.await(); assert set.size() == cnt; gc(); assert set.isEmpty(); }
/** * Test file creation. * * @param path Path to file to store. * @param size Size of file to store. * @param salt Salt for file content generation. * @throws Exception In case of any exception. */ private void testCreateFile(final GridGgfsPath path, final long size, final int salt) throws Exception { info("Create file [path=" + path + ", size=" + size + ", salt=" + salt + ']'); final AtomicInteger cnt = new AtomicInteger(0); final Collection<GridGgfsPath> cleanUp = new ConcurrentLinkedQueue<>(); long time = runMultiThreaded( new Callable<Object>() { @Override public Object call() throws Exception { int id = cnt.incrementAndGet(); GridGgfsPath f = new GridGgfsPath(path.parent(), "asdf" + (id > 1 ? "-" + id : "")); try (GridGgfsOutputStream out = fs.create(f, 0, true, null, 0, 1024, null)) { assertNotNull(out); cleanUp.add(f); // Add all created into cleanup list. U.copy(new GridGgfsTestInputStream(size, salt), out); } return null; } }, WRITING_THREADS_CNT, "perform-multi-thread-writing"); if (time > 0) { double rate = size * 1000. / time / 1024 / 1024; info( String.format( "Write file [path=%s, size=%d kB, rate=%2.1f MB/s]", path, WRITING_THREADS_CNT * size / 1024, WRITING_THREADS_CNT * rate)); } info("Read and validate saved file: " + path); final InputStream expIn = new GridGgfsTestInputStream(size, salt); final GridGgfsInputStream actIn = fs.open(path, CFG_BLOCK_SIZE * READING_THREADS_CNT * 11 / 10); // Validate continuous reading of whole file. assertEqualStreams(expIn, actIn, size, null); // Validate random seek and reading. final Random rnd = new Random(); runMultiThreaded( new Callable<Object>() { @Override public Object call() throws Exception { long skip = Math.abs(rnd.nextLong() % (size + 1)); long range = Math.min(size - skip, rnd.nextInt(CFG_BLOCK_SIZE * 400)); assertEqualStreams(new GridGgfsTestInputStream(size, salt), actIn, range, skip); return null; } }, READING_THREADS_CNT, "validate-multi-thread-reading"); expIn.close(); actIn.close(); info("Get stored file info: " + path); GridGgfsFile desc = fs.info(path); info("Validate stored file info: " + desc); assertNotNull(desc); if (log.isDebugEnabled()) log.debug("File descriptor: " + desc); Collection<GridGgfsBlockLocation> aff = fs.affinity(path, 0, desc.length()); assertFalse("Affinity: " + aff, desc.length() != 0 && aff.isEmpty()); int blockSize = desc.blockSize(); assertEquals("File size", size, desc.length()); assertEquals("Binary block size", CFG_BLOCK_SIZE, blockSize); // assertEquals("Permission", "rwxr-xr-x", desc.getPermission().toString()); // assertEquals("Permission sticky bit marks this is file", false, // desc.getPermission().getStickyBit()); assertEquals("Type", true, desc.isFile()); assertEquals("Type", false, desc.isDirectory()); info("Cleanup files: " + cleanUp); for (GridGgfsPath f : cleanUp) { fs.delete(f, true); assertNull(fs.info(f)); } }
/** @throws Exception Thrown if test failed. */ public void testA() throws Exception { Collection<Integer> set = new GridConcurrentWeakHashSet<>(); Integer i = 1; assert set.add(i); assert !set.add(i); assert set.contains(i); assert set.size() == 1; Collection<Integer> c = F.asList(2, 3, 4, 5); assert set.addAll(c); assert !set.addAll(c); assert set.containsAll(c); assert set.size() == 1 + c.size(); assert set.remove(i); assert !set.remove(i); assert !set.contains(i); assert set.size() == c.size(); assert set.removeAll(c); assert !set.removeAll(c); assert !set.containsAll(c); assert set.isEmpty(); Collection<Integer> c1 = Arrays.asList(1, 3, 5, 7, 9); int cnt = 0; for (Iterator<Integer> iter = set.iterator(); iter.hasNext(); cnt++) c1.contains(iter.next()); assert set.size() == cnt; assert set.size() == set.toArray().length; assert set.addAll(c1); assert set.retainAll(c); assert !set.retainAll(c); Collection<Integer> c2 = F.retain(c1, true, c); assert set.containsAll(c2); assert !set.containsAll(c1); assert !set.containsAll(c); assert set.size() == c2.size(); set.clear(); assert set.isEmpty(); try { set.iterator().next(); assert false; } catch (NoSuchElementException ignored) { assert true; } try { set.add(null); assert false; } catch (NullPointerException ignored) { assert true; } }
/** @throws Exception Thrown if test failed. */ public void testD() throws Exception { final Collection<SampleBean> set = new GridConcurrentWeakHashSet<>(); final int cnt = 100; final CountDownLatch start = new CountDownLatch(1); final CountDownLatch stop = new CountDownLatch(3); new Thread() { @Override public void run() { try { start.await(); for (int i = 0; i < cnt; i++) { for (int j = 0; j < cnt; j++) set.add(new SampleBean(i)); } } catch (Exception e) { error(e.getMessage()); } stop.countDown(); } }.start(); new Thread() { @Override public void run() { try { start.await(); for (int i = 0; i < cnt; i++) { for (int j = 0; j < cnt; j++) set.remove(new SampleBean(i)); } } catch (Exception e) { error(e.getMessage()); } stop.countDown(); } }.start(); new Thread() { @SuppressWarnings({"UnusedDeclaration"}) @Override public void run() { try { start.await(); while (stop.getCount() > 1) { for (SampleBean b : set) { // No-op. } } } catch (Exception e) { error(e.getMessage()); } stop.countDown(); } }.start(); start.countDown(); stop.await(); gc(); assert set.isEmpty(); }
/** @throws Exception Thrown if test failed. */ @SuppressWarnings({"UnusedAssignment"}) public void testB() throws Exception { Collection<SampleBean> set = new GridConcurrentWeakHashSet<>(); SampleBean bean1 = new SampleBean(1); assert set.add(bean1); assert !set.add(bean1); assert set.size() == 1; assert set.contains(bean1); bean1 = null; gc(); assert set.isEmpty(); Collection<SampleBean> c = F.asList(new SampleBean(1), new SampleBean(2), new SampleBean(3), new SampleBean(4)); assert set.addAll(c); assert !set.addAll(c); assert set.size() == c.size(); assert set.containsAll(c); c = null; gc(); assert set.isEmpty(); SampleBean b1 = new SampleBean(1); SampleBean b2 = new SampleBean(2); SampleBean b3 = new SampleBean(3); SampleBean b4 = new SampleBean(4); SampleBean b5 = new SampleBean(5); set.add(b1); set.add(b2); set.add(b3); set.add(b4); set.add(b5); Iterator iter = set.iterator(); assert iter.hasNext(); b2 = null; b3 = null; b4 = null; gc(); int cnt = 0; while (iter.hasNext()) { info(iter.next().toString()); cnt++; } assert set.size() == cnt; }
/** * Grabs local events and detects if events was lost since last poll. * * @param ignite Target grid. * @param evtOrderKey Unique key to take last order key from node local map. * @param evtThrottleCntrKey Unique key to take throttle count from node local map. * @param evtTypes Event types to collect. * @param evtMapper Closure to map grid events to Visor data transfer objects. * @return Collections of node events */ public static Collection<VisorGridEvent> collectEvents( Ignite ignite, String evtOrderKey, String evtThrottleCntrKey, final int[] evtTypes, IgniteClosure<Event, VisorGridEvent> evtMapper) { assert ignite != null; assert evtTypes != null && evtTypes.length > 0; ConcurrentMap<String, Long> nl = ignite.cluster().nodeLocalMap(); final long lastOrder = getOrElse(nl, evtOrderKey, -1L); final long throttle = getOrElse(nl, evtThrottleCntrKey, 0L); // When we first time arrive onto a node to get its local events, // we'll grab only last those events that not older than given period to make sure we are // not grabbing GBs of data accidentally. final long notOlderThan = System.currentTimeMillis() - EVENTS_COLLECT_TIME_WINDOW; // Flag for detecting gaps between events. final AtomicBoolean lastFound = new AtomicBoolean(lastOrder < 0); IgnitePredicate<Event> p = new IgnitePredicate<Event>() { /** */ private static final long serialVersionUID = 0L; @Override public boolean apply(Event e) { // Detects that events were lost. if (!lastFound.get() && (lastOrder == e.localOrder())) lastFound.set(true); // Retains events by lastOrder, period and type. return e.localOrder() > lastOrder && e.timestamp() > notOlderThan && F.contains(evtTypes, e.type()); } }; Collection<Event> evts = ignite.events().localQuery(p); // Update latest order in node local, if not empty. if (!evts.isEmpty()) { Event maxEvt = Collections.max(evts, EVTS_ORDER_COMPARATOR); nl.put(evtOrderKey, maxEvt.localOrder()); } // Update throttle counter. if (!lastFound.get()) nl.put(evtThrottleCntrKey, throttle == 0 ? EVENTS_LOST_THROTTLE : throttle - 1); boolean lost = !lastFound.get() && throttle == 0; Collection<VisorGridEvent> res = new ArrayList<>(evts.size() + (lost ? 1 : 0)); if (lost) res.add(new VisorGridEventsLost(ignite.cluster().localNode().id())); for (Event e : evts) { VisorGridEvent visorEvt = evtMapper.apply(e); if (visorEvt != null) res.add(visorEvt); } return res; }