@Test public void testLong() throws Throwable { final String LINE = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAASDSSSSSSSSSSSSSSSSSSSSSSSDDDDDD"; final int NUM = 10000; final int FACTOR = 5; StringBuilder longText = new StringBuilder(); for (int i = 0; i < NUM / FACTOR; i++) { longText.append(LINE).append("\n"); } List<String> expect = new LinkedList<String>(); expect.add("start"); for (int i = 0; i < NUM; i++) { expect.add(LINE); } List<String> send = new LinkedList<String>(); for (int i = 0; i < NUM; i++) { if (i % (NUM / FACTOR) == 0) { send.add(longText.toString()); } else { send.add(null); } } Sink sink = new Sink(expect.toArray(new String[0]), send.toArray(new String[0])); _sshdialog.connect(); _sshdialog.authenticate(); _sshdialog.executeCommand(sink, "echo start && sleep 4 && cat", null); sink.exception(); }
public void set(Map<String, Sink> newSinkMap) { try { for (Map.Entry<String, Sink> sink : sinkMap.entrySet()) { if (!newSinkMap.containsKey(sink.getKey())) { // removed Sink removedSink = sinkMap.remove(sink.getKey()); if (removedSink != null) { log.info(String.format("Removing sink '%s'", sink.getKey())); removedSink.close(); } } } for (Map.Entry<String, Sink> sink : newSinkMap.entrySet()) { if (!sinkMap.containsKey(sink.getKey())) { // added log.info(String.format("Adding sink '%s'", sink.getKey())); sink.getValue().open(); sinkMap.put(sink.getKey(), sink.getValue()); } } } catch (Exception e) { log.error("Exception on building SinkManager: " + e.getMessage(), e); if (sinkMap.isEmpty()) { throw new RuntimeException("At least one sink is needed"); } } }
@Test public void demonstrateBroadcast() { final Sink<Author, CompletionStage<Done>> writeAuthors = Sink.ignore(); final Sink<Hashtag, CompletionStage<Done>> writeHashtags = Sink.ignore(); // #flow-graph-broadcast RunnableGraph.fromGraph( GraphDSL.create( b -> { final UniformFanOutShape<Tweet, Tweet> bcast = b.add(Broadcast.create(2)); final FlowShape<Tweet, Author> toAuthor = b.add(Flow.of(Tweet.class).map(t -> t.author)); final FlowShape<Tweet, Hashtag> toTags = b.add( Flow.of(Tweet.class) .mapConcat(t -> new ArrayList<Hashtag>(t.hashtags()))); final SinkShape<Author> authors = b.add(writeAuthors); final SinkShape<Hashtag> hashtags = b.add(writeHashtags); b.from(b.add(tweets)).viaFanOut(bcast).via(toAuthor).to(authors); b.from(bcast).via(toTags).to(hashtags); return ClosedShape.getInstance(); })) .run(mat); // #flow-graph-broadcast }
/** Write a message that has been serialized to a sequence of buffers. */ private void writeBufferChain(BufferChainOutputStream bufferChain, boolean compressed) { ByteBuffer header = ByteBuffer.wrap(headerScratch); header.put(compressed ? COMPRESSED : UNCOMPRESSED); int messageLength = bufferChain.readableBytes(); header.putInt(messageLength); WritableBuffer writeableHeader = bufferAllocator.allocate(HEADER_LENGTH); writeableHeader.write(headerScratch, 0, header.position()); if (messageLength == 0) { // the payload had 0 length so make the header the current buffer. buffer = writeableHeader; return; } // Note that we are always delivering a small message to the transport here which // may incur transport framing overhead as it may be sent separately to the contents // of the GRPC frame. sink.deliverFrame(writeableHeader, false, false); // Commit all except the last buffer to the sink List<WritableBuffer> bufferList = bufferChain.bufferList; for (int i = 0; i < bufferList.size() - 1; i++) { sink.deliverFrame(bufferList.get(i), false, false); } // Assign the current buffer to the last in the chain so it can be used // for future writes or written with end-of-stream=true on close. buffer = bufferList.get(bufferList.size() - 1); }
@Test public void reusingComponents() throws Exception { final Source<Integer, NotUsed> nestedSource = Source.single(0) // An atomic source .map(i -> i + 1) // an atomic processing stage .named("nestedSource"); // wraps up the current Source and gives it a name final Flow<Integer, Integer, NotUsed> nestedFlow = Flow.of(Integer.class) .filter(i -> i != 0) // an atomic processing stage .map(i -> i - 2) // another atomic processing stage .named("nestedFlow"); // wraps up the Flow, and gives it a name final Sink<Integer, NotUsed> nestedSink = nestedFlow .to(Sink.fold(0, (acc, i) -> acc + i)) // wire an atomic sink to the nestedFlow .named("nestedSink"); // wrap it up // #reuse // Create a RunnableGraph from our components final RunnableGraph<NotUsed> runnableGraph = nestedSource.to(nestedSink); // Usage is uniform, no matter if modules are composite or atomic final RunnableGraph<NotUsed> runnableGraph2 = Source.single(0).to(Sink.fold(0, (acc, i) -> acc + i)); // #reuse }
@Test(expected = TimeLimitExceededException.class) public void testTimeout() throws Throwable { Sink sink = new Sink(new String[] {"start"}, new String[] {}); _sshdialog.setSoftTimeout(1 * 1000); _sshdialog.connect(); _sshdialog.authenticate(); _sshdialog.executeCommand(sink, "cat", null); sink.exception(); }
@Test public void mustWorkFromFuture() throws Exception { final Iterable<String> input = Arrays.asList("A", "B", "C"); CompletionStage<String> future1 = Source.from(input).runWith(Sink.<String>head(), materializer); CompletionStage<String> future2 = Source.fromCompletionStage(future1).runWith(Sink.<String>head(), materializer); String result = future2.toCompletableFuture().get(3, TimeUnit.SECONDS); assertEquals("A", result); }
@Test(expected = RuntimeException.class) public void testStderr() throws Throwable { try (final InputStream start = new ByteArrayInputStream("start\n".getBytes("UTF-8"))) { Sink sink = new Sink(new String[] {"start", "text1", "text2"}, new String[] {"text1", "text2"}); _sshdialog.connect(); _sshdialog.authenticate(); _sshdialog.executeCommand(sink, "echo message >&2 && cat", new InputStream[] {start}); sink.exception(); } }
@Test public void testSimple() throws Throwable { try (final InputStream start = new ByteArrayInputStream("start\n".getBytes("UTF-8"))) { Sink sink = new Sink(new String[] {"start", "text1", "text2"}, new String[] {"text1", "text2"}); _sshdialog.connect(); _sshdialog.authenticate(); _sshdialog.executeCommand(sink, "cat", new InputStream[] {start}); sink.exception(); } }
@Test public void partialGraph() throws Exception { // #partial-graph final Graph<FlowShape<Integer, Integer>, NotUsed> partial = GraphDSL.create( builder -> { final UniformFanOutShape<Integer, Integer> B = builder.add(Broadcast.create(2)); final UniformFanInShape<Integer, Integer> C = builder.add(Merge.create(2)); final UniformFanOutShape<Integer, Integer> E = builder.add(Balance.create(2)); final UniformFanInShape<Integer, Integer> F = builder.add(Merge.create(2)); builder.from(F.out()).toInlet(C.in(0)); builder.from(B).viaFanIn(C).toFanIn(F); builder .from(B) .via(builder.add(Flow.of(Integer.class).map(i -> i + 1))) .viaFanOut(E) .toFanIn(F); return new FlowShape<Integer, Integer>(B.in(), E.out(1)); }); // #partial-graph // #partial-use Source.single(0).via(partial).to(Sink.ignore()); // #partial-use // #partial-flow-dsl // Convert the partial graph of FlowShape to a Flow to get // access to the fluid DSL (for example to be able to call .filter()) final Flow<Integer, Integer, NotUsed> flow = Flow.fromGraph(partial); // Simple way to create a graph backed Source final Source<Integer, NotUsed> source = Source.fromGraph( GraphDSL.create( builder -> { final UniformFanInShape<Integer, Integer> merge = builder.add(Merge.create(2)); builder.from(builder.add(Source.single(0))).toFanIn(merge); builder.from(builder.add(Source.from(Arrays.asList(2, 3, 4)))).toFanIn(merge); // Exposing exactly one output port return new SourceShape<Integer>(merge.out()); })); // Building a Sink with a nested Flow, using the fluid DSL final Sink<Integer, NotUsed> sink = Flow.of(Integer.class).map(i -> i * 2).drop(10).named("nestedFlow").to(Sink.head()); // Putting all together final RunnableGraph<NotUsed> closed = source.via(flow.filter(i -> i > 1)).to(sink); // #partial-flow-dsl }
@Test public void sinkFromOutputStream() throws Exception { Buffer data = new Buffer(); data.writeUtf8("a"); data.writeUtf8(repeat('b', 9998)); data.writeUtf8("c"); ByteArrayOutputStream out = new ByteArrayOutputStream(); Sink sink = Okio.sink(out); sink.write(data, 3); assertEquals("abb", out.toString("UTF-8")); sink.write(data, data.size()); assertEquals("a" + repeat('b', 9998) + "c", out.toString("UTF-8")); }
@SuppressWarnings("serial") private void testLinker(int addressCount, int messageCount) { server.setMaxConnections(addressCount); Linker linker = new Linker(); linker.start(reactor); for (int i : range(addressCount)) { for (int j : range(messageCount)) { Sender snd = linker.sender(server.address(i), null); assertEquals("sender per address", i + 1, linker.sendersSize()); final int fi = i, fj = j; snd.send(dict("i", fi, "j", fj)); } } linker.close(); reactor.run(); HashMap<Integer, ArrayList<Map<?, ?>>> by_addr = new HashMap<>(); for (int i : range(addressCount)) { by_addr.put(i, new ArrayList<Map<?, ?>>()); } ArrayList<Map<?, ?>> messagesDicts = sink.getMessagesDicts(); for (Map<?, ?> m : messagesDicts) { assertNotNull("did not receive a dict", m); by_addr.get(m.get("i")).add(m); } for (int i : range(addressCount)) { ArrayList<Map<?, ?>> actual = by_addr.get(i); ArrayList<Map<?, ?>> expected = new ArrayList<>(messageCount); for (int j : range(messageCount)) { expected.add(dict("i", i, "j", j)); } assertEquals(String.format("Messages for address %d", i), expected, actual); } assertEquals("total messages", addressCount * messageCount, messagesDicts.size()); }
@Test public void mustProduceTicks() throws Exception { final JavaTestKit probe = new JavaTestKit(system); Source<String, Cancellable> tickSource = Source.tick( FiniteDuration.create(1, TimeUnit.SECONDS), FiniteDuration.create(500, TimeUnit.MILLISECONDS), "tick"); @SuppressWarnings("unused") Cancellable cancellable = tickSource .to( Sink.foreach( new Procedure<String>() { public void apply(String elem) { probe.getRef().tell(elem, ActorRef.noSender()); } })) .run(materializer); probe.expectNoMsg(FiniteDuration.create(600, TimeUnit.MILLISECONDS)); probe.expectMsgEquals("tick"); probe.expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS)); probe.expectMsgEquals("tick"); probe.expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS)); }
/** * Initialize the lists used to record input data. * * @exception IllegalActionException If the parent class throws it. */ public void initialize() throws IllegalActionException { super.initialize(); _records = new LinkedList(); _timeRecord = new LinkedList(); _latest = null; _count = 0; }
@Test public void mustBeAbleToRecover() throws Exception { final ManualProbe<Integer> publisherProbe = TestPublisher.manualProbe(true, system); final JavaTestKit probe = new JavaTestKit(system); final Source<Integer, NotUsed> source = Source.fromPublisher(publisherProbe) .map( elem -> { if (elem == 1) throw new RuntimeException("ex"); else return elem; }) .recover(new PFBuilder<Throwable, Integer>().matchAny(ex -> 0).build()); final CompletionStage<Done> future = source.runWith( Sink.foreach(elem -> probe.getRef().tell(elem, ActorRef.noSender())), materializer); final PublisherProbeSubscription<Integer> s = publisherProbe.expectSubscription(); s.sendNext(0); probe.expectMsgEquals(0); s.sendNext(1); probe.expectMsgEquals(0); future.toCompletableFuture().get(200, TimeUnit.MILLISECONDS); }
@Test public void mustBeAbleToUseFlatMapMerge() throws Exception { final JavaTestKit probe = new JavaTestKit(system); final Iterable<Integer> input1 = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); final Iterable<Integer> input2 = Arrays.asList(10, 11, 12, 13, 14, 15, 16, 17, 18, 19); final Iterable<Integer> input3 = Arrays.asList(20, 21, 22, 23, 24, 25, 26, 27, 28, 29); final Iterable<Integer> input4 = Arrays.asList(30, 31, 32, 33, 34, 35, 36, 37, 38, 39); final List<Source<Integer, NotUsed>> mainInputs = new ArrayList<Source<Integer, NotUsed>>(); mainInputs.add(Source.from(input1)); mainInputs.add(Source.from(input2)); mainInputs.add(Source.from(input3)); mainInputs.add(Source.from(input4)); CompletionStage<List<Integer>> future = Source.from(mainInputs) .flatMapMerge(3, ConstantFun.<Source<Integer, NotUsed>>javaIdentityFunction()) .grouped(60) .runWith(Sink.<List<Integer>>head(), materializer); List<Integer> result = future.toCompletableFuture().get(3, TimeUnit.SECONDS); final Set<Integer> set = new HashSet<Integer>(); for (Integer i : result) { set.add(i); } final Set<Integer> expected = new HashSet<Integer>(); for (int i = 0; i < 40; ++i) { expected.add(i); } assertEquals(expected, set); }
@Test public void complexGraph() throws Exception { // #complex-graph RunnableGraph.fromGraph( GraphDSL.create( builder -> { final Outlet<Integer> A = builder.add(Source.single(0)).out(); final UniformFanOutShape<Integer, Integer> B = builder.add(Broadcast.create(2)); final UniformFanInShape<Integer, Integer> C = builder.add(Merge.create(2)); final FlowShape<Integer, Integer> D = builder.add(Flow.of(Integer.class).map(i -> i + 1)); final UniformFanOutShape<Integer, Integer> E = builder.add(Balance.create(2)); final UniformFanInShape<Integer, Integer> F = builder.add(Merge.create(2)); final Inlet<Integer> G = builder.add(Sink.<Integer>foreach(System.out::println)).in(); builder.from(F).toFanIn(C); builder.from(A).viaFanOut(B).viaFanIn(C).toFanIn(F); builder.from(B).via(D).viaFanOut(E).toFanIn(F); builder.from(E).toInlet(G); return ClosedShape.getInstance(); })); // #complex-graph // #complex-graph-alt RunnableGraph.fromGraph( GraphDSL.create( builder -> { final SourceShape<Integer> A = builder.add(Source.single(0)); final UniformFanOutShape<Integer, Integer> B = builder.add(Broadcast.create(2)); final UniformFanInShape<Integer, Integer> C = builder.add(Merge.create(2)); final FlowShape<Integer, Integer> D = builder.add(Flow.of(Integer.class).map(i -> i + 1)); final UniformFanOutShape<Integer, Integer> E = builder.add(Balance.create(2)); final UniformFanInShape<Integer, Integer> F = builder.add(Merge.create(2)); final SinkShape<Integer> G = builder.add(Sink.foreach(System.out::println)); builder.from(F.out()).toInlet(C.in(0)); builder.from(A).toInlet(B.in()); builder.from(B.out(0)).toInlet(C.in(1)); builder.from(C.out()).toInlet(F.in(0)); builder.from(B.out(1)).via(D).toInlet(E.in()); builder.from(E.out(0)).toInlet(F.in(1)); builder.from(E.out(1)).to(G); return ClosedShape.getInstance(); })); // #complex-graph-alt }
@Test public void mustRepeat() throws Exception { final CompletionStage<List<Integer>> f = Source.repeat(42).grouped(10000).runWith(Sink.<List<Integer>>head(), materializer); final List<Integer> result = f.toCompletableFuture().get(3, TimeUnit.SECONDS); assertEquals(result.size(), 10000); for (Integer i : result) assertEquals(i, (Integer) 42); }
private ChannelTree getChannelTree(String pattern) { ChannelTree tr = null; try { sMap = new ChannelMap(); if ((pattern != null) && (pattern.length() > 0)) sMap.Add(pattern); sink.RequestRegistration(sMap); sMap = sink.Fetch(-1, sMap); tr = ChannelTree.createFromChannelMap(sMap); } catch (SAPIException se) { se.printStackTrace(); } return tr; }
@Test public void mustBeAbleToUseToFuture() throws Exception { final JavaTestKit probe = new JavaTestKit(system); final Iterable<String> input = Arrays.asList("A", "B", "C"); CompletionStage<String> future = Source.from(input).runWith(Sink.<String>head(), materializer); String result = future.toCompletableFuture().get(3, TimeUnit.SECONDS); assertEquals("A", result); }
@Test public void mustBeAbleToUsePrefixAndTail() throws Exception { final JavaTestKit probe = new JavaTestKit(system); final Iterable<Integer> input = Arrays.asList(1, 2, 3, 4, 5, 6); CompletionStage<Pair<List<Integer>, Source<Integer, NotUsed>>> future = Source.from(input) .prefixAndTail(3) .runWith(Sink.<Pair<List<Integer>, Source<Integer, NotUsed>>>head(), materializer); Pair<List<Integer>, Source<Integer, NotUsed>> result = future.toCompletableFuture().get(3, TimeUnit.SECONDS); assertEquals(Arrays.asList(1, 2, 3), result.first()); CompletionStage<List<Integer>> tailFuture = result.second().limit(4).runWith(Sink.<Integer>seq(), materializer); List<Integer> tailResult = tailFuture.toCompletableFuture().get(3, TimeUnit.SECONDS); assertEquals(Arrays.asList(4, 5, 6), tailResult); }
@Override final boolean forEachWithCancel(Spliterator<Long> spliterator, Sink<Long> sink) { Spliterator.OfLong spl = adapt(spliterator); LongConsumer adaptedSink = adapt(sink); boolean cancelled; do {} while (!(cancelled = sink.cancellationRequested()) && spl.tryAdvance(adaptedSink)); return cancelled; }
public void go() { source.start(); source.waitFor(); if (sink != null) { sink.close(); } stats.print(); }
private void testConnect() { try { // test the connection to the server Sink sink = new Sink(); sink.OpenRBNBConnection(getServer(), sinkName); connected = true; System.out.println( "DataVideoGather: Test connection made to server = " + getServer() + " as " + sinkName + "."); sink.CloseRBNBConnection(); } catch (SAPIException se) { se.printStackTrace(); } }
@Test public void mustWorkFromRange() throws Exception { CompletionStage<List<Integer>> f = Source.range(0, 10).grouped(20).runWith(Sink.<List<Integer>>head(), materializer); final List<Integer> result = f.toCompletableFuture().get(3, TimeUnit.SECONDS); assertEquals(11, result.size()); Integer counter = 0; for (Integer i : result) assertEquals(i, counter++); }
@Test public void demonstrateSlowProcessing() { // #tweets-slow-consumption-dropHead tweets .buffer(10, OverflowStrategy.dropHead()) .map(t -> slowComputation(t)) .runWith(Sink.ignore(), mat); // #tweets-slow-consumption-dropHead }
public void appendChannelListFromPattern() { try { // Create a sink and connect: Sink sink = new Sink(); sink.OpenRBNBConnection(getServer(), sinkName); // get all the channel paths that match the pattern ChannelMap sMap = new ChannelMap(); sink.RequestRegistration(); sMap = sink.Fetch(-1, sMap); ChannelTree tree = ChannelTree.createFromChannelMap(sMap); Pattern p = Pattern.compile(channelPathPattern); // for each channel path, check match, collect matches... Iterator nodes = tree.iterator(); while (nodes.hasNext()) { ChannelTree.Node n = (ChannelTree.Node) nodes.next(); // System.out.println("Checking " + n.getFullName() + ";" + n.getName()); if (!includeHidden && n.getFullName().startsWith("_")) continue; if (n.getType() != ChannelTree.CHANNEL) continue; String name = n.getFullName(); Matcher m = p.matcher(name); if (m.matches()) { // System.out.println("Matches"); boolean isSource = false; ChannelTree.Node upNode = n.getParent(); while ((!isSource) || (upNode != null)) { if (upNode.getType() == ChannelTree.SOURCE) isSource = true; upNode = upNode.getParent(); } if (isSource) { // System.out.println("... and is a source."); channelPathList.add(n); } else { // System.out.println("... and is NOT a source."); } } } } catch (SAPIException se) { se.printStackTrace(); } } // appendChannelListFromPattern
public void appendChannelListFromString() { try { StringTokenizer st = new StringTokenizer(channelPathListString, ","); // Create a sink and connect: Sink sink = new Sink(); sink.OpenRBNBConnection(getServer(), sinkName); // get all the channel paths that match the pattern ChannelMap sMap = new ChannelMap(); sink.RequestRegistration(); sMap = sink.Fetch(-1, sMap); ChannelTree tree = ChannelTree.createFromChannelMap(sMap); Pattern p = Pattern.compile(channelPathPattern); // for each channel path, check match, collect matches... while (st.hasMoreTokens()) { String path = st.nextToken(); // System.out.println("Checking " + path); ChannelTree.Node n = tree.findNode(path); if (n == null) continue; if (n.getType() != ChannelTree.CHANNEL) continue; String name = n.getFullName(); // System.out.println("Found it..."); boolean isSource = false; ChannelTree.Node upNode = n.getParent(); while ((!isSource) || (upNode != null)) { if (upNode.getType() == ChannelTree.SOURCE) isSource = true; upNode = upNode.getParent(); } if (isSource) { // System.out.println("... and is a source."); channelPathList.add(n); } else { // System.out.println("... and is NOT a source."); } } // while next token } catch (SAPIException se) { se.printStackTrace(); } } // appendChannelListFromString
public void demonstrateACustomMaterializedValue() throws Exception { // tests: RunnableGraph<CompletionStage<Integer>> flow = Source.from(Arrays.asList(1, 2, 3)) .viaMat(new FirstValue(), Keep.right()) .to(Sink.ignore()); CompletionStage<Integer> result = flow.run(mat); assertEquals(new Integer(1), result.toCompletableFuture().get(3, TimeUnit.SECONDS)); }
private void testSampler(final int count, float frequency) { server.setMaxConnections(1); final ArrayList<String> expected = new ArrayList<String>(count); Timeout gen = new Timeout() { Tag tag = new SimpleTag(0); int sent = 0; @Override public void onSample(DatawireEvent e) { String body = template.render(sent); expected.add(body); Message message = Message.Factory.create(); message.setBody(new AmqpValue(body)); DatawireUtils.send(e.getLink(), tag, message); sent += 1; if (sent >= count) { e.getLink().close(); cancel(); } } }; Sender snd = Sender.Builder() .withTarget(server.address()) .withHandlers(new Sampler(gen, frequency)) .create(); reactor.getHandler().add(snd); gen.setTimeout(reactor, 2000); snd.start(reactor); reactor.run(); assertTrue("Sampling timed out", gen.isCancelled()); assertEquals("Expected messages", expected, sink.getMessages()); }