@Test
public void testLong() throws Throwable {
final String LINE = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAASDSSSSSSSSSSSSSSSSSSSSSSSDDDDDD";
final int NUM = 10000;
final int FACTOR = 5;
StringBuilder longText = new StringBuilder();
for (int i = 0; i < NUM / FACTOR; i++) {
longText.append(LINE).append("\n");
}
List<String> expect = new LinkedList<String>();
expect.add("start");
for (int i = 0; i < NUM; i++) {
expect.add(LINE);
}
List<String> send = new LinkedList<String>();
for (int i = 0; i < NUM; i++) {
if (i % (NUM / FACTOR) == 0) {
send.add(longText.toString());
} else {
send.add(null);
}
}
Sink sink = new Sink(expect.toArray(new String[0]), send.toArray(new String[0]));
_sshdialog.connect();
_sshdialog.authenticate();
_sshdialog.executeCommand(sink, "echo start && sleep 4 && cat", null);
sink.exception();
}
public void set(Map<String, Sink> newSinkMap) {
try {
for (Map.Entry<String, Sink> sink : sinkMap.entrySet()) {
if (!newSinkMap.containsKey(sink.getKey())) { // removed
Sink removedSink = sinkMap.remove(sink.getKey());
if (removedSink != null) {
log.info(String.format("Removing sink '%s'", sink.getKey()));
removedSink.close();
}
}
}
for (Map.Entry<String, Sink> sink : newSinkMap.entrySet()) {
if (!sinkMap.containsKey(sink.getKey())) { // added
log.info(String.format("Adding sink '%s'", sink.getKey()));
sink.getValue().open();
sinkMap.put(sink.getKey(), sink.getValue());
}
}
} catch (Exception e) {
log.error("Exception on building SinkManager: " + e.getMessage(), e);
if (sinkMap.isEmpty()) {
throw new RuntimeException("At least one sink is needed");
}
}
}
@Test
public void demonstrateBroadcast() {
final Sink<Author, CompletionStage<Done>> writeAuthors = Sink.ignore();
final Sink<Hashtag, CompletionStage<Done>> writeHashtags = Sink.ignore();
// #flow-graph-broadcast
RunnableGraph.fromGraph(
GraphDSL.create(
b -> {
final UniformFanOutShape<Tweet, Tweet> bcast = b.add(Broadcast.create(2));
final FlowShape<Tweet, Author> toAuthor =
b.add(Flow.of(Tweet.class).map(t -> t.author));
final FlowShape<Tweet, Hashtag> toTags =
b.add(
Flow.of(Tweet.class)
.mapConcat(t -> new ArrayList<Hashtag>(t.hashtags())));
final SinkShape<Author> authors = b.add(writeAuthors);
final SinkShape<Hashtag> hashtags = b.add(writeHashtags);
b.from(b.add(tweets)).viaFanOut(bcast).via(toAuthor).to(authors);
b.from(bcast).via(toTags).to(hashtags);
return ClosedShape.getInstance();
}))
.run(mat);
// #flow-graph-broadcast
}
/** Write a message that has been serialized to a sequence of buffers. */
private void writeBufferChain(BufferChainOutputStream bufferChain, boolean compressed) {
ByteBuffer header = ByteBuffer.wrap(headerScratch);
header.put(compressed ? COMPRESSED : UNCOMPRESSED);
int messageLength = bufferChain.readableBytes();
header.putInt(messageLength);
WritableBuffer writeableHeader = bufferAllocator.allocate(HEADER_LENGTH);
writeableHeader.write(headerScratch, 0, header.position());
if (messageLength == 0) {
// the payload had 0 length so make the header the current buffer.
buffer = writeableHeader;
return;
}
// Note that we are always delivering a small message to the transport here which
// may incur transport framing overhead as it may be sent separately to the contents
// of the GRPC frame.
sink.deliverFrame(writeableHeader, false, false);
// Commit all except the last buffer to the sink
List<WritableBuffer> bufferList = bufferChain.bufferList;
for (int i = 0; i < bufferList.size() - 1; i++) {
sink.deliverFrame(bufferList.get(i), false, false);
}
// Assign the current buffer to the last in the chain so it can be used
// for future writes or written with end-of-stream=true on close.
buffer = bufferList.get(bufferList.size() - 1);
}
@Test
public void reusingComponents() throws Exception {
final Source<Integer, NotUsed> nestedSource =
Source.single(0) // An atomic source
.map(i -> i + 1) // an atomic processing stage
.named("nestedSource"); // wraps up the current Source and gives it a name
final Flow<Integer, Integer, NotUsed> nestedFlow =
Flow.of(Integer.class)
.filter(i -> i != 0) // an atomic processing stage
.map(i -> i - 2) // another atomic processing stage
.named("nestedFlow"); // wraps up the Flow, and gives it a name
final Sink<Integer, NotUsed> nestedSink =
nestedFlow
.to(Sink.fold(0, (acc, i) -> acc + i)) // wire an atomic sink to the nestedFlow
.named("nestedSink"); // wrap it up
// #reuse
// Create a RunnableGraph from our components
final RunnableGraph<NotUsed> runnableGraph = nestedSource.to(nestedSink);
// Usage is uniform, no matter if modules are composite or atomic
final RunnableGraph<NotUsed> runnableGraph2 =
Source.single(0).to(Sink.fold(0, (acc, i) -> acc + i));
// #reuse
}
@Test(expected = TimeLimitExceededException.class)
public void testTimeout() throws Throwable {
Sink sink = new Sink(new String[] {"start"}, new String[] {});
_sshdialog.setSoftTimeout(1 * 1000);
_sshdialog.connect();
_sshdialog.authenticate();
_sshdialog.executeCommand(sink, "cat", null);
sink.exception();
}
@Test
public void mustWorkFromFuture() throws Exception {
final Iterable<String> input = Arrays.asList("A", "B", "C");
CompletionStage<String> future1 = Source.from(input).runWith(Sink.<String>head(), materializer);
CompletionStage<String> future2 =
Source.fromCompletionStage(future1).runWith(Sink.<String>head(), materializer);
String result = future2.toCompletableFuture().get(3, TimeUnit.SECONDS);
assertEquals("A", result);
}
@Test(expected = RuntimeException.class)
public void testStderr() throws Throwable {
try (final InputStream start = new ByteArrayInputStream("start\n".getBytes("UTF-8"))) {
Sink sink =
new Sink(new String[] {"start", "text1", "text2"}, new String[] {"text1", "text2"});
_sshdialog.connect();
_sshdialog.authenticate();
_sshdialog.executeCommand(sink, "echo message >&2 && cat", new InputStream[] {start});
sink.exception();
}
}
@Test
public void testSimple() throws Throwable {
try (final InputStream start = new ByteArrayInputStream("start\n".getBytes("UTF-8"))) {
Sink sink =
new Sink(new String[] {"start", "text1", "text2"}, new String[] {"text1", "text2"});
_sshdialog.connect();
_sshdialog.authenticate();
_sshdialog.executeCommand(sink, "cat", new InputStream[] {start});
sink.exception();
}
}
@Test
public void partialGraph() throws Exception {
// #partial-graph
final Graph<FlowShape<Integer, Integer>, NotUsed> partial =
GraphDSL.create(
builder -> {
final UniformFanOutShape<Integer, Integer> B = builder.add(Broadcast.create(2));
final UniformFanInShape<Integer, Integer> C = builder.add(Merge.create(2));
final UniformFanOutShape<Integer, Integer> E = builder.add(Balance.create(2));
final UniformFanInShape<Integer, Integer> F = builder.add(Merge.create(2));
builder.from(F.out()).toInlet(C.in(0));
builder.from(B).viaFanIn(C).toFanIn(F);
builder
.from(B)
.via(builder.add(Flow.of(Integer.class).map(i -> i + 1)))
.viaFanOut(E)
.toFanIn(F);
return new FlowShape<Integer, Integer>(B.in(), E.out(1));
});
// #partial-graph
// #partial-use
Source.single(0).via(partial).to(Sink.ignore());
// #partial-use
// #partial-flow-dsl
// Convert the partial graph of FlowShape to a Flow to get
// access to the fluid DSL (for example to be able to call .filter())
final Flow<Integer, Integer, NotUsed> flow = Flow.fromGraph(partial);
// Simple way to create a graph backed Source
final Source<Integer, NotUsed> source =
Source.fromGraph(
GraphDSL.create(
builder -> {
final UniformFanInShape<Integer, Integer> merge = builder.add(Merge.create(2));
builder.from(builder.add(Source.single(0))).toFanIn(merge);
builder.from(builder.add(Source.from(Arrays.asList(2, 3, 4)))).toFanIn(merge);
// Exposing exactly one output port
return new SourceShape<Integer>(merge.out());
}));
// Building a Sink with a nested Flow, using the fluid DSL
final Sink<Integer, NotUsed> sink =
Flow.of(Integer.class).map(i -> i * 2).drop(10).named("nestedFlow").to(Sink.head());
// Putting all together
final RunnableGraph<NotUsed> closed = source.via(flow.filter(i -> i > 1)).to(sink);
// #partial-flow-dsl
}
@Test
public void sinkFromOutputStream() throws Exception {
Buffer data = new Buffer();
data.writeUtf8("a");
data.writeUtf8(repeat('b', 9998));
data.writeUtf8("c");
ByteArrayOutputStream out = new ByteArrayOutputStream();
Sink sink = Okio.sink(out);
sink.write(data, 3);
assertEquals("abb", out.toString("UTF-8"));
sink.write(data, data.size());
assertEquals("a" + repeat('b', 9998) + "c", out.toString("UTF-8"));
}
@SuppressWarnings("serial")
private void testLinker(int addressCount, int messageCount) {
server.setMaxConnections(addressCount);
Linker linker = new Linker();
linker.start(reactor);
for (int i : range(addressCount)) {
for (int j : range(messageCount)) {
Sender snd = linker.sender(server.address(i), null);
assertEquals("sender per address", i + 1, linker.sendersSize());
final int fi = i, fj = j;
snd.send(dict("i", fi, "j", fj));
}
}
linker.close();
reactor.run();
HashMap<Integer, ArrayList<Map<?, ?>>> by_addr = new HashMap<>();
for (int i : range(addressCount)) {
by_addr.put(i, new ArrayList<Map<?, ?>>());
}
ArrayList<Map<?, ?>> messagesDicts = sink.getMessagesDicts();
for (Map<?, ?> m : messagesDicts) {
assertNotNull("did not receive a dict", m);
by_addr.get(m.get("i")).add(m);
}
for (int i : range(addressCount)) {
ArrayList<Map<?, ?>> actual = by_addr.get(i);
ArrayList<Map<?, ?>> expected = new ArrayList<>(messageCount);
for (int j : range(messageCount)) {
expected.add(dict("i", i, "j", j));
}
assertEquals(String.format("Messages for address %d", i), expected, actual);
}
assertEquals("total messages", addressCount * messageCount, messagesDicts.size());
}
@Test
public void mustProduceTicks() throws Exception {
final JavaTestKit probe = new JavaTestKit(system);
Source<String, Cancellable> tickSource =
Source.tick(
FiniteDuration.create(1, TimeUnit.SECONDS),
FiniteDuration.create(500, TimeUnit.MILLISECONDS),
"tick");
@SuppressWarnings("unused")
Cancellable cancellable =
tickSource
.to(
Sink.foreach(
new Procedure<String>() {
public void apply(String elem) {
probe.getRef().tell(elem, ActorRef.noSender());
}
}))
.run(materializer);
probe.expectNoMsg(FiniteDuration.create(600, TimeUnit.MILLISECONDS));
probe.expectMsgEquals("tick");
probe.expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
probe.expectMsgEquals("tick");
probe.expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
}
/**
* Initialize the lists used to record input data.
*
* @exception IllegalActionException If the parent class throws it.
*/
public void initialize() throws IllegalActionException {
super.initialize();
_records = new LinkedList();
_timeRecord = new LinkedList();
_latest = null;
_count = 0;
}
@Test
public void mustBeAbleToRecover() throws Exception {
final ManualProbe<Integer> publisherProbe = TestPublisher.manualProbe(true, system);
final JavaTestKit probe = new JavaTestKit(system);
final Source<Integer, NotUsed> source =
Source.fromPublisher(publisherProbe)
.map(
elem -> {
if (elem == 1) throw new RuntimeException("ex");
else return elem;
})
.recover(new PFBuilder<Throwable, Integer>().matchAny(ex -> 0).build());
final CompletionStage<Done> future =
source.runWith(
Sink.foreach(elem -> probe.getRef().tell(elem, ActorRef.noSender())), materializer);
final PublisherProbeSubscription<Integer> s = publisherProbe.expectSubscription();
s.sendNext(0);
probe.expectMsgEquals(0);
s.sendNext(1);
probe.expectMsgEquals(0);
future.toCompletableFuture().get(200, TimeUnit.MILLISECONDS);
}
@Test
public void mustBeAbleToUseFlatMapMerge() throws Exception {
final JavaTestKit probe = new JavaTestKit(system);
final Iterable<Integer> input1 = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
final Iterable<Integer> input2 = Arrays.asList(10, 11, 12, 13, 14, 15, 16, 17, 18, 19);
final Iterable<Integer> input3 = Arrays.asList(20, 21, 22, 23, 24, 25, 26, 27, 28, 29);
final Iterable<Integer> input4 = Arrays.asList(30, 31, 32, 33, 34, 35, 36, 37, 38, 39);
final List<Source<Integer, NotUsed>> mainInputs = new ArrayList<Source<Integer, NotUsed>>();
mainInputs.add(Source.from(input1));
mainInputs.add(Source.from(input2));
mainInputs.add(Source.from(input3));
mainInputs.add(Source.from(input4));
CompletionStage<List<Integer>> future =
Source.from(mainInputs)
.flatMapMerge(3, ConstantFun.<Source<Integer, NotUsed>>javaIdentityFunction())
.grouped(60)
.runWith(Sink.<List<Integer>>head(), materializer);
List<Integer> result = future.toCompletableFuture().get(3, TimeUnit.SECONDS);
final Set<Integer> set = new HashSet<Integer>();
for (Integer i : result) {
set.add(i);
}
final Set<Integer> expected = new HashSet<Integer>();
for (int i = 0; i < 40; ++i) {
expected.add(i);
}
assertEquals(expected, set);
}
@Test
public void complexGraph() throws Exception {
// #complex-graph
RunnableGraph.fromGraph(
GraphDSL.create(
builder -> {
final Outlet<Integer> A = builder.add(Source.single(0)).out();
final UniformFanOutShape<Integer, Integer> B = builder.add(Broadcast.create(2));
final UniformFanInShape<Integer, Integer> C = builder.add(Merge.create(2));
final FlowShape<Integer, Integer> D =
builder.add(Flow.of(Integer.class).map(i -> i + 1));
final UniformFanOutShape<Integer, Integer> E = builder.add(Balance.create(2));
final UniformFanInShape<Integer, Integer> F = builder.add(Merge.create(2));
final Inlet<Integer> G = builder.add(Sink.<Integer>foreach(System.out::println)).in();
builder.from(F).toFanIn(C);
builder.from(A).viaFanOut(B).viaFanIn(C).toFanIn(F);
builder.from(B).via(D).viaFanOut(E).toFanIn(F);
builder.from(E).toInlet(G);
return ClosedShape.getInstance();
}));
// #complex-graph
// #complex-graph-alt
RunnableGraph.fromGraph(
GraphDSL.create(
builder -> {
final SourceShape<Integer> A = builder.add(Source.single(0));
final UniformFanOutShape<Integer, Integer> B = builder.add(Broadcast.create(2));
final UniformFanInShape<Integer, Integer> C = builder.add(Merge.create(2));
final FlowShape<Integer, Integer> D =
builder.add(Flow.of(Integer.class).map(i -> i + 1));
final UniformFanOutShape<Integer, Integer> E = builder.add(Balance.create(2));
final UniformFanInShape<Integer, Integer> F = builder.add(Merge.create(2));
final SinkShape<Integer> G = builder.add(Sink.foreach(System.out::println));
builder.from(F.out()).toInlet(C.in(0));
builder.from(A).toInlet(B.in());
builder.from(B.out(0)).toInlet(C.in(1));
builder.from(C.out()).toInlet(F.in(0));
builder.from(B.out(1)).via(D).toInlet(E.in());
builder.from(E.out(0)).toInlet(F.in(1));
builder.from(E.out(1)).to(G);
return ClosedShape.getInstance();
}));
// #complex-graph-alt
}
@Test
public void mustRepeat() throws Exception {
final CompletionStage<List<Integer>> f =
Source.repeat(42).grouped(10000).runWith(Sink.<List<Integer>>head(), materializer);
final List<Integer> result = f.toCompletableFuture().get(3, TimeUnit.SECONDS);
assertEquals(result.size(), 10000);
for (Integer i : result) assertEquals(i, (Integer) 42);
}
private ChannelTree getChannelTree(String pattern) {
ChannelTree tr = null;
try {
sMap = new ChannelMap();
if ((pattern != null) && (pattern.length() > 0)) sMap.Add(pattern);
sink.RequestRegistration(sMap);
sMap = sink.Fetch(-1, sMap);
tr = ChannelTree.createFromChannelMap(sMap);
} catch (SAPIException se) {
se.printStackTrace();
}
return tr;
}
@Test
public void mustBeAbleToUseToFuture() throws Exception {
final JavaTestKit probe = new JavaTestKit(system);
final Iterable<String> input = Arrays.asList("A", "B", "C");
CompletionStage<String> future = Source.from(input).runWith(Sink.<String>head(), materializer);
String result = future.toCompletableFuture().get(3, TimeUnit.SECONDS);
assertEquals("A", result);
}
@Test
public void mustBeAbleToUsePrefixAndTail() throws Exception {
final JavaTestKit probe = new JavaTestKit(system);
final Iterable<Integer> input = Arrays.asList(1, 2, 3, 4, 5, 6);
CompletionStage<Pair<List<Integer>, Source<Integer, NotUsed>>> future =
Source.from(input)
.prefixAndTail(3)
.runWith(Sink.<Pair<List<Integer>, Source<Integer, NotUsed>>>head(), materializer);
Pair<List<Integer>, Source<Integer, NotUsed>> result =
future.toCompletableFuture().get(3, TimeUnit.SECONDS);
assertEquals(Arrays.asList(1, 2, 3), result.first());
CompletionStage<List<Integer>> tailFuture =
result.second().limit(4).runWith(Sink.<Integer>seq(), materializer);
List<Integer> tailResult = tailFuture.toCompletableFuture().get(3, TimeUnit.SECONDS);
assertEquals(Arrays.asList(4, 5, 6), tailResult);
}
@Override
final boolean forEachWithCancel(Spliterator<Long> spliterator, Sink<Long> sink) {
Spliterator.OfLong spl = adapt(spliterator);
LongConsumer adaptedSink = adapt(sink);
boolean cancelled;
do {} while (!(cancelled = sink.cancellationRequested()) && spl.tryAdvance(adaptedSink));
return cancelled;
}
public void go() {
source.start();
source.waitFor();
if (sink != null) {
sink.close();
}
stats.print();
}
private void testConnect() {
try {
// test the connection to the server
Sink sink = new Sink();
sink.OpenRBNBConnection(getServer(), sinkName);
connected = true;
System.out.println(
"DataVideoGather: Test connection made to server = "
+ getServer()
+ " as "
+ sinkName
+ ".");
sink.CloseRBNBConnection();
} catch (SAPIException se) {
se.printStackTrace();
}
}
@Test
public void mustWorkFromRange() throws Exception {
CompletionStage<List<Integer>> f =
Source.range(0, 10).grouped(20).runWith(Sink.<List<Integer>>head(), materializer);
final List<Integer> result = f.toCompletableFuture().get(3, TimeUnit.SECONDS);
assertEquals(11, result.size());
Integer counter = 0;
for (Integer i : result) assertEquals(i, counter++);
}
@Test
public void demonstrateSlowProcessing() {
// #tweets-slow-consumption-dropHead
tweets
.buffer(10, OverflowStrategy.dropHead())
.map(t -> slowComputation(t))
.runWith(Sink.ignore(), mat);
// #tweets-slow-consumption-dropHead
}
public void appendChannelListFromPattern() {
try {
// Create a sink and connect:
Sink sink = new Sink();
sink.OpenRBNBConnection(getServer(), sinkName);
// get all the channel paths that match the pattern
ChannelMap sMap = new ChannelMap();
sink.RequestRegistration();
sMap = sink.Fetch(-1, sMap);
ChannelTree tree = ChannelTree.createFromChannelMap(sMap);
Pattern p = Pattern.compile(channelPathPattern);
// for each channel path, check match, collect matches...
Iterator nodes = tree.iterator();
while (nodes.hasNext()) {
ChannelTree.Node n = (ChannelTree.Node) nodes.next();
// System.out.println("Checking " + n.getFullName() + ";" + n.getName());
if (!includeHidden && n.getFullName().startsWith("_")) continue;
if (n.getType() != ChannelTree.CHANNEL) continue;
String name = n.getFullName();
Matcher m = p.matcher(name);
if (m.matches()) {
// System.out.println("Matches");
boolean isSource = false;
ChannelTree.Node upNode = n.getParent();
while ((!isSource) || (upNode != null)) {
if (upNode.getType() == ChannelTree.SOURCE) isSource = true;
upNode = upNode.getParent();
}
if (isSource) {
// System.out.println("... and is a source.");
channelPathList.add(n);
} else {
// System.out.println("... and is NOT a source.");
}
}
}
} catch (SAPIException se) {
se.printStackTrace();
}
} // appendChannelListFromPattern
public void appendChannelListFromString() {
try {
StringTokenizer st = new StringTokenizer(channelPathListString, ",");
// Create a sink and connect:
Sink sink = new Sink();
sink.OpenRBNBConnection(getServer(), sinkName);
// get all the channel paths that match the pattern
ChannelMap sMap = new ChannelMap();
sink.RequestRegistration();
sMap = sink.Fetch(-1, sMap);
ChannelTree tree = ChannelTree.createFromChannelMap(sMap);
Pattern p = Pattern.compile(channelPathPattern);
// for each channel path, check match, collect matches...
while (st.hasMoreTokens()) {
String path = st.nextToken();
// System.out.println("Checking " + path);
ChannelTree.Node n = tree.findNode(path);
if (n == null) continue;
if (n.getType() != ChannelTree.CHANNEL) continue;
String name = n.getFullName();
// System.out.println("Found it...");
boolean isSource = false;
ChannelTree.Node upNode = n.getParent();
while ((!isSource) || (upNode != null)) {
if (upNode.getType() == ChannelTree.SOURCE) isSource = true;
upNode = upNode.getParent();
}
if (isSource) {
// System.out.println("... and is a source.");
channelPathList.add(n);
} else {
// System.out.println("... and is NOT a source.");
}
} // while next token
} catch (SAPIException se) {
se.printStackTrace();
}
} // appendChannelListFromString
public void demonstrateACustomMaterializedValue() throws Exception {
// tests:
RunnableGraph<CompletionStage<Integer>> flow =
Source.from(Arrays.asList(1, 2, 3))
.viaMat(new FirstValue(), Keep.right())
.to(Sink.ignore());
CompletionStage<Integer> result = flow.run(mat);
assertEquals(new Integer(1), result.toCompletableFuture().get(3, TimeUnit.SECONDS));
}
private void testSampler(final int count, float frequency) {
server.setMaxConnections(1);
final ArrayList<String> expected = new ArrayList<String>(count);
Timeout gen =
new Timeout() {
Tag tag = new SimpleTag(0);
int sent = 0;
@Override
public void onSample(DatawireEvent e) {
String body = template.render(sent);
expected.add(body);
Message message = Message.Factory.create();
message.setBody(new AmqpValue(body));
DatawireUtils.send(e.getLink(), tag, message);
sent += 1;
if (sent >= count) {
e.getLink().close();
cancel();
}
}
};
Sender snd =
Sender.Builder()
.withTarget(server.address())
.withHandlers(new Sampler(gen, frequency))
.create();
reactor.getHandler().add(snd);
gen.setTimeout(reactor, 2000);
snd.start(reactor);
reactor.run();
assertTrue("Sampling timed out", gen.isCancelled());
assertEquals("Expected messages", expected, sink.getMessages());
}
