/** Entry point for a streaming Fizz Buzz! */ public static void main(String[] args) throws Exception { Topology topology = new Topology(); // Declare an infinite stream of Long values TStream<Long> counting = BeaconStreams.longBeacon(topology); // Throttle the rate to allow the output to be seen easier counting = counting.throttle(100, TimeUnit.MILLISECONDS); // Print the tuples to standard output playFizzBuzz(counting).print(); // At this point the streaming topology (streaming) is // declared, but no data is flowing. The topology // must be submitted to a StreamsContext to be executed. // Since this is an streaming graph with an endless // data source it will run for ever Future<?> runningTopology = StreamsContextFactory.getEmbedded().submit(topology); // Run for one minute before canceling. Thread.sleep(TimeUnit.MINUTES.toMillis(1)); runningTopology.cancel(true); }
@Test public void testSimpleTagsSink() { assumeTrue(isMainRun()); Topology t = newTopology(); TStream<String> s = t.strings("3"); testSimpleTags(s.print()); }
/** Json Subscribe feeding a filter */ @Test public void testPublishJsonFilter() throws Exception { Random r = new Random(); final Topology t = new Topology(); includePythonApp(t, "json_filter_json.py", "json_filter_json::json_filter_json"); JSONObject j1 = new JSONObject(); j1.put("a", 23523L); j1.put("b", "Hello:" + r.nextInt(200)); JSONObject j2 = new JSONObject(); j2.put("a", 7L); j2.put("b", "Goodbye:" + r.nextInt(200)); JSONObject j3 = new JSONObject(); j3.put("a", 101L); j3.put("b", "So long:" + r.nextInt(200)); String s2 = "R" + j2.get("a") + "X" + j2.get("b"); TStream<JSONObject> source = t.constants(Arrays.asList(j1, j2, j3)); source = addStartupDelay(source).asType(JSONObject.class); source.publish("pytest/json/filter"); TStream<JSONObject> subscribe = t.subscribe("pytest/json/filter/result", JSONObject.class); TStream<String> asString = subscribe.transform(j -> "R" + j.get("a") + "X" + j.get("b")); completeAndValidate(asString, 30, s2); }
@Test public void testFuseThenTagStreamSink() { assumeTrue(isMainRun()); Topology t = newTopology(); TStream<String> s1 = t.strings("3"); TStream<String> s2 = t.strings("3"); testFuseThenTag(s1, s2.print()); }
@Test public void testTagBothThenFuseSinkStream() { assumeTrue(isMainRun()); Topology t = newTopology(); TStream<String> s1 = t.strings("3"); TStream<String> s2 = t.strings("3"); testTagBothThenFuse(s1.print(), s2); }
/** * Test with a distributed execution with explicit colocation of two functions end up on the same * container. */ @Test public void testSimpleDistributedColocate() throws Exception { assumeTrue(SC_OK); assumeTrue(getTesterType() == StreamsContext.Type.DISTRIBUTED_TESTER); Topology t = newTopology(); TStream<String> sa = t.strings("a"); TStream<String> sb = t.strings("b"); sa = sa.transform(IsolateTest.getContainerId()); sb = sb.transform(IsolateTest.getContainerId()); sa.colocate(sb); sa = sa.isolate().filter(new AllowAll<String>()); sb = sb.isolate().filter(new AllowAll<String>()); sa = sa.union(sb); Condition<List<String>> pes = t.getTester().stringContents(sa, ""); Condition<Long> tc = t.getTester().tupleCount(sa, 2); complete(t.getTester(), tc, 10, TimeUnit.SECONDS); Set<String> singlePe = new HashSet<>(pes.getResult()); assertTrue(pes.getResult().toString(), singlePe.size() == 1); }
@Test(expected = IllegalStateException.class) public void testColocateIsolateViolation() throws Exception { assumeTrue(isMainRun()); // verify s1.isolate().modify().colocate(s1) is disallowed Topology t = newTopology("testColocateIsolateViolation"); TStream<String> s1 = t.strings("a"); s1.isolate() .modify(getContainerIdAppend()) .colocate(s1) // throws ISE: can't colocate isolated stream with parent ; }
/** * Publish some messages to a topic; subscribe to the topic and report received messages. * * @param contextType string value of a {@code StreamsContext.Type} * @throws Exception */ public void publishSubscribe(String contextType) throws Exception { Map<String, Object> contextConfig = new HashMap<>(); initContextConfig(contextConfig); Topology top = new Topology("mqttSample"); // A compile time MQTT topic value. Supplier<String> topic = new Value<String>(TOPIC); // Create the MQTT connector MqttStreams mqtt = new MqttStreams(top, createMqttConfig()); // Create a stream of messages and for the sample, give the // consumer a change to become ready TStream<Message> msgs = makeStreamToPublish(top).modify(initialDelayFunc(PUB_DELAY_MSEC)); // Publish the message stream to the topic mqtt.publish(msgs, topic); // Subscribe to the topic and report received messages TStream<Message> rcvdMsgs = mqtt.subscribe(topic); rcvdMsgs.print(); // Submit the topology, to send and receive the messages. Future<?> future = StreamsContextFactory.getStreamsContext(contextType).submit(top, contextConfig); if (contextType.contains("DISTRIBUTED")) { System.out.println( "\nSee the job's PE console logs for the topology output.\n" + "Use Streams Studio or streamtool. e.g.,\n" + " # identify the job's \"Print\" PE\n" + " streamtool lspes --jobs " + future.get() + "\n" + " # print the PE's console log\n" + " streamtool viewlog --print --console --pe <the-peid>" + "\n"); System.out.println( "Cancel the job using Streams Studio or streamtool. e.g.,\n" + " streamtool canceljob " + future.get() + "\n"); } else if (contextType.contains("STANDALONE")) { Thread.sleep(15000); future.cancel(true); } }
/** Return a stream that plays Fizz Buzz based upon the values in the input stream. */ public static TStream<String> playFizzBuzz(TStream<Long> counting) { /* * Transform an input stream of longs TStream<Long> to a * stream of strings TStream<String> that follow * the Fizz Buzz rules based upon each value in the * input stream. */ TStream<String> shouts = counting.transform( new Function<Long, String>() { private static final long serialVersionUID = 1L; @Override public String apply(Long v) { // Skip 0, humans count from 1! if (v == 0) return null; StringBuilder sb = new StringBuilder(); if (v % 3 == 0) sb.append("Fizz"); if (v % 5 == 0) sb.append("Buzz"); if (sb.length() == 0) sb.append(Long.toString(v)); else sb.append("!"); return sb.toString(); } }); return shouts; }
@SuppressWarnings("unused") @Test(expected = IllegalStateException.class) public void testColocateLowLatencyRegions() throws Exception { assumeTrue(isMainRun()); // ensure colocating two low latency regions doesn't break lowLatancy // and colocating is achieved. Topology t = newTopology("testColocateLowLatencyRegions"); Tester tester = t.getTester(); // getConfig().put(ContextProperties.KEEP_ARTIFACTS, true); TStream<String> s1 = t.strings("a").lowLatency().modify(getContainerIdAppend()).modify(getContainerIdAppend()); s1.endLowLatency(); TStream<String> s2 = t.strings("A").lowLatency().modify(getContainerIdAppend()).modify(getContainerIdAppend()); s2.endLowLatency(); s1.colocate(s2); // expect throw ISE: colocate in a low latency region // once it's supported... (today it breaks the low latency guarantee) // and adjust isMainRun() too // // Given the default fuse-island behavior, expect islands to continue // // to be fused, now both in a single container. // // // Today FAILING in an interesting way. // // There are 2 PEs: // // - one has just the single colocated s1 and s2 modify ops // // - the other has everything else // // TStream<String> all = s1.union(s2); // all.print(); // Condition<Long> nTuples = tester.tupleCount(all.filter(new AllowAll<String>()), 2); // Condition<List<String>> contents = tester.stringContents( // all.filter(new AllowAll<String>()), ""); // // complete(tester, nTuples, 10, TimeUnit.SECONDS); // // Set<String> ids = getContainerIds(contents.getResult()); // assertEquals("ids: "+ids, 1, ids.size()); }
/** Json Subscribe feeding a flat map */ @Test public void testPublishJsonFlatMap() throws Exception { Random r = new Random(); final Topology t = new Topology(); JSONObject j1 = new JSONObject(); j1.put("a", r.nextLong()); j1.put("b", "Hello:" + r.nextInt(200)); JSONObject j2 = new JSONObject(); j2.put("a", r.nextLong()); j2.put("b", "Goodbye:" + r.nextInt(200)); JSONObject j3 = new JSONObject(); j3.put("a", r.nextLong()); j3.put("b", "So long:" + r.nextInt(200)); String s1a = j1.get("a").toString(); String s1b = j1.get("b").toString(); String s2a = j2.get("a").toString(); String s2b = j2.get("b").toString(); String s3a = j3.get("a").toString(); String s3b = j3.get("b").toString(); includePythonApp(t, "json_flatmap_string.py", "json_flatmap_str::json_flatmap_str"); TStream<JSONObject> source = t.constants(Arrays.asList(j1, j2, j3)); source = addStartupDelay(source).asType(JSONObject.class); source.publish("pytest/json/flatmap"); TStream<String> subscribe = t.subscribe("pytest/json/flatmap/result", String.class); completeAndValidate(subscribe, 60, s1a, s1b, s2a, s2b, s3a, s3b); }
/** * Test with a distributed execution with explicit colocation of two functions end up on the same * container. */ @Test @Ignore("Need to figure out how to get the tags set by the test") // TODO public void testSimpleDistributedHostTags() throws Exception { assumeTrue(SC_OK); assumeTrue(getTesterType() == StreamsContext.Type.DISTRIBUTED_TESTER); Topology t = newTopology(); TStream<String> sa = t.strings("a"); sa.addResourceTags("application"); sa = sa.filter(new AllowAll<String>()); sa.addResourceTags("application"); getConfig().put(ContextProperties.KEEP_ARTIFACTS, Boolean.TRUE); Condition<List<String>> aout = t.getTester().stringContents(sa, "a"); complete(t.getTester(), aout, 10, TimeUnit.SECONDS); assertTrue(aout.getResult().toString(), aout.valid()); }
@Test public void testTags() { assumeTrue(isMainRun()); Topology t = newTopology(); TStream<String> s1 = t.strings("3"); TStream<String> s2 = t.strings("3"); TStream<String> s3 = t.strings("3"); s1.addResourceTags(); assertNull(getResourceTags(s1)); s2.addResourceTags("A", "B"); Set<String> s2s = getResourceTags(s2); assertEquals(2, s2s.size()); assertTrue(s2s.contains("A")); assertTrue(s2s.contains("B")); s3.addResourceTags("C", "D", "E"); Set<String> s3s = getResourceTags(s3); assertEquals(3, s3s.size()); assertTrue(s3s.contains("C")); assertTrue(s3s.contains("D")); assertTrue(s3s.contains("E")); s2s = getResourceTags(s2); assertEquals(2, s2s.size()); assertTrue(s2s.contains("A")); assertTrue(s2s.contains("B")); s2.addResourceTags("X", "Y"); s2s = getResourceTags(s2); assertEquals(4, s2s.size()); assertTrue(s2s.contains("A")); assertTrue(s2s.contains("B")); assertTrue(s2s.contains("X")); assertTrue(s2s.contains("Y")); // Colocating means the s1 will inherit // s3 resource tags s1.colocate(s3); Set<String> s1s = getResourceTags(s1); assertEquals(3, s1s.size()); assertTrue(s1s.contains("C")); assertTrue(s1s.contains("D")); assertTrue(s1s.contains("E")); }
@Test public void testNonplaceable() { assumeTrue(isMainRun()); Topology t = newTopology(); TStream<String> s1 = t.strings("3"); TStream<String> s2 = t.strings("3"); assertFalse(s1.union(s2).isPlaceable()); assertFalse(s1.isolate().isPlaceable()); TStream<String> sp = s1.parallel(3); assertFalse(sp.isPlaceable()); assertFalse(sp.endParallel().isPlaceable()); }
@Test public void testFusing() { assumeTrue(isMainRun()); Topology t = newTopology(); TStream<String> s1 = t.strings("3"); TStream<String> s2 = t.strings("3"); TStream<String> snf = t.strings("3"); assertTrue(s1.isPlaceable()); assertSame(s1.colocate(s2), s1); String id1 = getFusingId(s1); String id2 = getFusingId(s2); assertNotNull(id1); assertFalse(id1.isEmpty()); assertEquals(id1, id2); TStream<String> s3 = t.strings("3"); TStream<String> s4 = t.strings("3"); TSink s5 = s4.print(); assertTrue(s5.isPlaceable()); assertSame(s3.colocate(s4, s5), s3); assertEquals(getFusingId(s3), getFusingId(s4)); assertEquals(getFusingId(s3), getColocate(s5.operator())); assertFalse(getFusingId(s1).equals(getFusingId(s3))); assertNull(getFusingId(snf)); TStream<String> s6 = StringStreams.toString(s4); s1.colocate(s6); assertEquals(getFusingId(s1), getFusingId(s6)); }
private static Set<String> getResourceTags(TStream<?> s) { BOperator bop = ((BOutputPort) s.output()).operator(); return getResourceTags(bop); }
private static String getFusingId(TStream<?> s) { BOperator bop = ((BOutputPort) s.output()).operator(); return getColocate(bop); }