/** {@inheritDoc} */
@Nullable
@Override
public Map<String, Collection<?>> run(GridStreamerContext ctx, Collection<Bar> bars)
throws GridException {
ConcurrentMap<String, Bar> loc = ctx.localSpace();
GridStreamerWindow win = ctx.window("stage2");
// Add numbers to window.
win.enqueueAll(bars);
Collection<Bar> polled = win.pollEvictedBatch();
if (!polled.isEmpty()) {
Map<String, Bar> map = new HashMap<>();
for (Bar polledBar : polled) {
String symbol = polledBar.symbol();
Bar bar = map.get(symbol);
if (bar == null) map.put(symbol, bar = new Bar(symbol));
bar.update(polledBar);
}
loc.putAll(map);
}
return null;
}
private void recheck() {
// If this is the oldest node.
if (oldestNode.get().id().equals(cctx.localNodeId())) {
Collection<UUID> remaining = remaining();
if (!remaining.isEmpty()) {
try {
cctx.io()
.safeSend(
cctx.discovery().nodes(remaining),
new GridDhtPartitionsSingleRequest(exchId),
SYSTEM_POOL,
null);
} catch (IgniteCheckedException e) {
U.error(
log,
"Failed to request partitions from nodes [exchangeId="
+ exchId
+ ", nodes="
+ remaining
+ ']',
e);
}
}
// Resend full partition map because last attempt failed.
else {
if (spreadPartitions()) onDone(exchId.topologyVersion());
}
} else sendPartitions();
// Schedule another send.
scheduleRecheck();
}
/** {@inheritDoc} */
@Nullable
@Override
public Map<String, Collection<?>> run(GridStreamerContext ctx, Collection<Quote> quotes)
throws GridException {
GridStreamerWindow win = ctx.window("stage1");
// Add numbers to window.
win.enqueueAll(quotes);
Collection<Quote> polled = win.pollEvictedBatch();
if (!polled.isEmpty()) {
Map<String, Bar> map = new HashMap<>();
for (Quote quote : polled) {
String symbol = quote.symbol();
Bar bar = map.get(symbol);
if (bar == null) map.put(symbol, bar = new Bar(symbol));
bar.update(quote.price());
}
return Collections.<String, Collection<?>>singletonMap(ctx.nextStageName(), map.values());
}
return null;
}
public void checkUnknown() throws MojoExecutionException {
if (!unknownFiles.isEmpty()) {
String msg =
"Unable to find a comment style definition for some "
+ "files. You may want to add a custom mapping for the relevant file extensions.";
if (failIfUnknown) {
throw new MojoExecutionException(msg);
}
getLog().warn(msg);
}
}
/** {@inheritDoc} */
@Override
public void loadCache(GridBiInClosure<K, V> c, @Nullable Object... args) throws GridException {
ExecutorService exec =
new ThreadPoolExecutor(
threadsCnt,
threadsCnt,
0L,
MILLISECONDS,
new ArrayBlockingQueue<Runnable>(batchQueueSize),
new BlockingRejectedExecutionHandler());
Iterator<I> iter = inputIterator(args);
Collection<I> buf = new ArrayList<>(batchSize);
try {
while (iter.hasNext()) {
if (Thread.currentThread().isInterrupted()) {
U.warn(log, "Working thread was interrupted while loading data.");
break;
}
buf.add(iter.next());
if (buf.size() == batchSize) {
exec.submit(new Worker(c, buf, args));
buf = new ArrayList<>(batchSize);
}
}
if (!buf.isEmpty()) exec.submit(new Worker(c, buf, args));
} catch (RejectedExecutionException ignored) {
// Because of custom RejectedExecutionHandler.
assert false : "RejectedExecutionException was thrown while it shouldn't.";
} finally {
exec.shutdown();
try {
exec.awaitTermination(Long.MAX_VALUE, MILLISECONDS);
} catch (InterruptedException ignored) {
U.warn(log, "Working thread was interrupted while waiting for put operations to complete.");
Thread.currentThread().interrupt();
}
}
}
/** @throws Exception Thrown if test failed. */
public void testC() throws Exception {
final Collection<SampleBean> set = new GridConcurrentWeakHashSet<>();
int threadCnt = 2;
final int cnt = 5;
final CountDownLatch start = new CountDownLatch(1);
final CountDownLatch stop = new CountDownLatch(threadCnt);
Runnable r =
new Runnable() {
@Override
public void run() {
try {
start.await();
for (int i = 0; i < cnt; i++) {
for (int j = 0; j < cnt; j++) set.add(new SampleBean(i));
}
} catch (Exception e) {
error(e.getMessage());
}
stop.countDown();
}
};
for (int i = 0; i < threadCnt; i++) new Thread(r).start();
start.countDown();
stop.await();
assert set.size() == cnt;
gc();
assert set.isEmpty();
}
/**
* Test file creation.
*
* @param path Path to file to store.
* @param size Size of file to store.
* @param salt Salt for file content generation.
* @throws Exception In case of any exception.
*/
private void testCreateFile(final GridGgfsPath path, final long size, final int salt)
throws Exception {
info("Create file [path=" + path + ", size=" + size + ", salt=" + salt + ']');
final AtomicInteger cnt = new AtomicInteger(0);
final Collection<GridGgfsPath> cleanUp = new ConcurrentLinkedQueue<>();
long time =
runMultiThreaded(
new Callable<Object>() {
@Override
public Object call() throws Exception {
int id = cnt.incrementAndGet();
GridGgfsPath f = new GridGgfsPath(path.parent(), "asdf" + (id > 1 ? "-" + id : ""));
try (GridGgfsOutputStream out = fs.create(f, 0, true, null, 0, 1024, null)) {
assertNotNull(out);
cleanUp.add(f); // Add all created into cleanup list.
U.copy(new GridGgfsTestInputStream(size, salt), out);
}
return null;
}
},
WRITING_THREADS_CNT,
"perform-multi-thread-writing");
if (time > 0) {
double rate = size * 1000. / time / 1024 / 1024;
info(
String.format(
"Write file [path=%s, size=%d kB, rate=%2.1f MB/s]",
path, WRITING_THREADS_CNT * size / 1024, WRITING_THREADS_CNT * rate));
}
info("Read and validate saved file: " + path);
final InputStream expIn = new GridGgfsTestInputStream(size, salt);
final GridGgfsInputStream actIn = fs.open(path, CFG_BLOCK_SIZE * READING_THREADS_CNT * 11 / 10);
// Validate continuous reading of whole file.
assertEqualStreams(expIn, actIn, size, null);
// Validate random seek and reading.
final Random rnd = new Random();
runMultiThreaded(
new Callable<Object>() {
@Override
public Object call() throws Exception {
long skip = Math.abs(rnd.nextLong() % (size + 1));
long range = Math.min(size - skip, rnd.nextInt(CFG_BLOCK_SIZE * 400));
assertEqualStreams(new GridGgfsTestInputStream(size, salt), actIn, range, skip);
return null;
}
},
READING_THREADS_CNT,
"validate-multi-thread-reading");
expIn.close();
actIn.close();
info("Get stored file info: " + path);
GridGgfsFile desc = fs.info(path);
info("Validate stored file info: " + desc);
assertNotNull(desc);
if (log.isDebugEnabled()) log.debug("File descriptor: " + desc);
Collection<GridGgfsBlockLocation> aff = fs.affinity(path, 0, desc.length());
assertFalse("Affinity: " + aff, desc.length() != 0 && aff.isEmpty());
int blockSize = desc.blockSize();
assertEquals("File size", size, desc.length());
assertEquals("Binary block size", CFG_BLOCK_SIZE, blockSize);
// assertEquals("Permission", "rwxr-xr-x", desc.getPermission().toString());
// assertEquals("Permission sticky bit marks this is file", false,
// desc.getPermission().getStickyBit());
assertEquals("Type", true, desc.isFile());
assertEquals("Type", false, desc.isDirectory());
info("Cleanup files: " + cleanUp);
for (GridGgfsPath f : cleanUp) {
fs.delete(f, true);
assertNull(fs.info(f));
}
}
/** @throws Exception Thrown if test failed. */
public void testA() throws Exception {
Collection<Integer> set = new GridConcurrentWeakHashSet<>();
Integer i = 1;
assert set.add(i);
assert !set.add(i);
assert set.contains(i);
assert set.size() == 1;
Collection<Integer> c = F.asList(2, 3, 4, 5);
assert set.addAll(c);
assert !set.addAll(c);
assert set.containsAll(c);
assert set.size() == 1 + c.size();
assert set.remove(i);
assert !set.remove(i);
assert !set.contains(i);
assert set.size() == c.size();
assert set.removeAll(c);
assert !set.removeAll(c);
assert !set.containsAll(c);
assert set.isEmpty();
Collection<Integer> c1 = Arrays.asList(1, 3, 5, 7, 9);
int cnt = 0;
for (Iterator<Integer> iter = set.iterator(); iter.hasNext(); cnt++) c1.contains(iter.next());
assert set.size() == cnt;
assert set.size() == set.toArray().length;
assert set.addAll(c1);
assert set.retainAll(c);
assert !set.retainAll(c);
Collection<Integer> c2 = F.retain(c1, true, c);
assert set.containsAll(c2);
assert !set.containsAll(c1);
assert !set.containsAll(c);
assert set.size() == c2.size();
set.clear();
assert set.isEmpty();
try {
set.iterator().next();
assert false;
} catch (NoSuchElementException ignored) {
assert true;
}
try {
set.add(null);
assert false;
} catch (NullPointerException ignored) {
assert true;
}
}
/** @throws Exception Thrown if test failed. */
public void testD() throws Exception {
final Collection<SampleBean> set = new GridConcurrentWeakHashSet<>();
final int cnt = 100;
final CountDownLatch start = new CountDownLatch(1);
final CountDownLatch stop = new CountDownLatch(3);
new Thread() {
@Override
public void run() {
try {
start.await();
for (int i = 0; i < cnt; i++) {
for (int j = 0; j < cnt; j++) set.add(new SampleBean(i));
}
} catch (Exception e) {
error(e.getMessage());
}
stop.countDown();
}
}.start();
new Thread() {
@Override
public void run() {
try {
start.await();
for (int i = 0; i < cnt; i++) {
for (int j = 0; j < cnt; j++) set.remove(new SampleBean(i));
}
} catch (Exception e) {
error(e.getMessage());
}
stop.countDown();
}
}.start();
new Thread() {
@SuppressWarnings({"UnusedDeclaration"})
@Override
public void run() {
try {
start.await();
while (stop.getCount() > 1) {
for (SampleBean b : set) {
// No-op.
}
}
} catch (Exception e) {
error(e.getMessage());
}
stop.countDown();
}
}.start();
start.countDown();
stop.await();
gc();
assert set.isEmpty();
}
/** @throws Exception Thrown if test failed. */
@SuppressWarnings({"UnusedAssignment"})
public void testB() throws Exception {
Collection<SampleBean> set = new GridConcurrentWeakHashSet<>();
SampleBean bean1 = new SampleBean(1);
assert set.add(bean1);
assert !set.add(bean1);
assert set.size() == 1;
assert set.contains(bean1);
bean1 = null;
gc();
assert set.isEmpty();
Collection<SampleBean> c =
F.asList(new SampleBean(1), new SampleBean(2), new SampleBean(3), new SampleBean(4));
assert set.addAll(c);
assert !set.addAll(c);
assert set.size() == c.size();
assert set.containsAll(c);
c = null;
gc();
assert set.isEmpty();
SampleBean b1 = new SampleBean(1);
SampleBean b2 = new SampleBean(2);
SampleBean b3 = new SampleBean(3);
SampleBean b4 = new SampleBean(4);
SampleBean b5 = new SampleBean(5);
set.add(b1);
set.add(b2);
set.add(b3);
set.add(b4);
set.add(b5);
Iterator iter = set.iterator();
assert iter.hasNext();
b2 = null;
b3 = null;
b4 = null;
gc();
int cnt = 0;
while (iter.hasNext()) {
info(iter.next().toString());
cnt++;
}
assert set.size() == cnt;
}
/**
* Grabs local events and detects if events was lost since last poll.
*
* @param ignite Target grid.
* @param evtOrderKey Unique key to take last order key from node local map.
* @param evtThrottleCntrKey Unique key to take throttle count from node local map.
* @param evtTypes Event types to collect.
* @param evtMapper Closure to map grid events to Visor data transfer objects.
* @return Collections of node events
*/
public static Collection<VisorGridEvent> collectEvents(
Ignite ignite,
String evtOrderKey,
String evtThrottleCntrKey,
final int[] evtTypes,
IgniteClosure<Event, VisorGridEvent> evtMapper) {
assert ignite != null;
assert evtTypes != null && evtTypes.length > 0;
ConcurrentMap<String, Long> nl = ignite.cluster().nodeLocalMap();
final long lastOrder = getOrElse(nl, evtOrderKey, -1L);
final long throttle = getOrElse(nl, evtThrottleCntrKey, 0L);
// When we first time arrive onto a node to get its local events,
// we'll grab only last those events that not older than given period to make sure we are
// not grabbing GBs of data accidentally.
final long notOlderThan = System.currentTimeMillis() - EVENTS_COLLECT_TIME_WINDOW;
// Flag for detecting gaps between events.
final AtomicBoolean lastFound = new AtomicBoolean(lastOrder < 0);
IgnitePredicate<Event> p =
new IgnitePredicate<Event>() {
/** */
private static final long serialVersionUID = 0L;
@Override
public boolean apply(Event e) {
// Detects that events were lost.
if (!lastFound.get() && (lastOrder == e.localOrder())) lastFound.set(true);
// Retains events by lastOrder, period and type.
return e.localOrder() > lastOrder
&& e.timestamp() > notOlderThan
&& F.contains(evtTypes, e.type());
}
};
Collection<Event> evts = ignite.events().localQuery(p);
// Update latest order in node local, if not empty.
if (!evts.isEmpty()) {
Event maxEvt = Collections.max(evts, EVTS_ORDER_COMPARATOR);
nl.put(evtOrderKey, maxEvt.localOrder());
}
// Update throttle counter.
if (!lastFound.get())
nl.put(evtThrottleCntrKey, throttle == 0 ? EVENTS_LOST_THROTTLE : throttle - 1);
boolean lost = !lastFound.get() && throttle == 0;
Collection<VisorGridEvent> res = new ArrayList<>(evts.size() + (lost ? 1 : 0));
if (lost) res.add(new VisorGridEventsLost(ignite.cluster().localNode().id()));
for (Event e : evts) {
VisorGridEvent visorEvt = evtMapper.apply(e);
if (visorEvt != null) res.add(visorEvt);
}
return res;
}
