/** Entry point for a streaming Fizz Buzz! */
public static void main(String[] args) throws Exception {
Topology topology = new Topology();
// Declare an infinite stream of Long values
TStream<Long> counting = BeaconStreams.longBeacon(topology);
// Throttle the rate to allow the output to be seen easier
counting = counting.throttle(100, TimeUnit.MILLISECONDS);
// Print the tuples to standard output
playFizzBuzz(counting).print();
// At this point the streaming topology (streaming) is
// declared, but no data is flowing. The topology
// must be submitted to a StreamsContext to be executed.
// Since this is an streaming graph with an endless
// data source it will run for ever
Future<?> runningTopology = StreamsContextFactory.getEmbedded().submit(topology);
// Run for one minute before canceling.
Thread.sleep(TimeUnit.MINUTES.toMillis(1));
runningTopology.cancel(true);
}
@Test
public void testSimpleTagsSink() {
assumeTrue(isMainRun());
Topology t = newTopology();
TStream<String> s = t.strings("3");
testSimpleTags(s.print());
}
/** Json Subscribe feeding a filter */
@Test
public void testPublishJsonFilter() throws Exception {
Random r = new Random();
final Topology t = new Topology();
includePythonApp(t, "json_filter_json.py", "json_filter_json::json_filter_json");
JSONObject j1 = new JSONObject();
j1.put("a", 23523L);
j1.put("b", "Hello:" + r.nextInt(200));
JSONObject j2 = new JSONObject();
j2.put("a", 7L);
j2.put("b", "Goodbye:" + r.nextInt(200));
JSONObject j3 = new JSONObject();
j3.put("a", 101L);
j3.put("b", "So long:" + r.nextInt(200));
String s2 = "R" + j2.get("a") + "X" + j2.get("b");
TStream<JSONObject> source = t.constants(Arrays.asList(j1, j2, j3));
source = addStartupDelay(source).asType(JSONObject.class);
source.publish("pytest/json/filter");
TStream<JSONObject> subscribe = t.subscribe("pytest/json/filter/result", JSONObject.class);
TStream<String> asString = subscribe.transform(j -> "R" + j.get("a") + "X" + j.get("b"));
completeAndValidate(asString, 30, s2);
}
@Test
public void testFuseThenTagStreamSink() {
assumeTrue(isMainRun());
Topology t = newTopology();
TStream<String> s1 = t.strings("3");
TStream<String> s2 = t.strings("3");
testFuseThenTag(s1, s2.print());
}
@Test
public void testTagBothThenFuseSinkStream() {
assumeTrue(isMainRun());
Topology t = newTopology();
TStream<String> s1 = t.strings("3");
TStream<String> s2 = t.strings("3");
testTagBothThenFuse(s1.print(), s2);
}
/**
* Test with a distributed execution with explicit colocation of two functions end up on the same
* container.
*/
@Test
public void testSimpleDistributedColocate() throws Exception {
assumeTrue(SC_OK);
assumeTrue(getTesterType() == StreamsContext.Type.DISTRIBUTED_TESTER);
Topology t = newTopology();
TStream<String> sa = t.strings("a");
TStream<String> sb = t.strings("b");
sa = sa.transform(IsolateTest.getContainerId());
sb = sb.transform(IsolateTest.getContainerId());
sa.colocate(sb);
sa = sa.isolate().filter(new AllowAll<String>());
sb = sb.isolate().filter(new AllowAll<String>());
sa = sa.union(sb);
Condition<List<String>> pes = t.getTester().stringContents(sa, "");
Condition<Long> tc = t.getTester().tupleCount(sa, 2);
complete(t.getTester(), tc, 10, TimeUnit.SECONDS);
Set<String> singlePe = new HashSet<>(pes.getResult());
assertTrue(pes.getResult().toString(), singlePe.size() == 1);
}
@Test(expected = IllegalStateException.class)
public void testColocateIsolateViolation() throws Exception {
assumeTrue(isMainRun());
// verify s1.isolate().modify().colocate(s1) is disallowed
Topology t = newTopology("testColocateIsolateViolation");
TStream<String> s1 = t.strings("a");
s1.isolate()
.modify(getContainerIdAppend())
.colocate(s1) // throws ISE: can't colocate isolated stream with parent
;
}
/**
* Publish some messages to a topic; subscribe to the topic and report received messages.
*
* @param contextType string value of a {@code StreamsContext.Type}
* @throws Exception
*/
public void publishSubscribe(String contextType) throws Exception {
Map<String, Object> contextConfig = new HashMap<>();
initContextConfig(contextConfig);
Topology top = new Topology("mqttSample");
// A compile time MQTT topic value.
Supplier<String> topic = new Value<String>(TOPIC);
// Create the MQTT connector
MqttStreams mqtt = new MqttStreams(top, createMqttConfig());
// Create a stream of messages and for the sample, give the
// consumer a change to become ready
TStream<Message> msgs = makeStreamToPublish(top).modify(initialDelayFunc(PUB_DELAY_MSEC));
// Publish the message stream to the topic
mqtt.publish(msgs, topic);
// Subscribe to the topic and report received messages
TStream<Message> rcvdMsgs = mqtt.subscribe(topic);
rcvdMsgs.print();
// Submit the topology, to send and receive the messages.
Future<?> future =
StreamsContextFactory.getStreamsContext(contextType).submit(top, contextConfig);
if (contextType.contains("DISTRIBUTED")) {
System.out.println(
"\nSee the job's PE console logs for the topology output.\n"
+ "Use Streams Studio or streamtool. e.g.,\n"
+ " # identify the job's \"Print\" PE\n"
+ " streamtool lspes --jobs "
+ future.get()
+ "\n"
+ " # print the PE's console log\n"
+ " streamtool viewlog --print --console --pe <the-peid>"
+ "\n");
System.out.println(
"Cancel the job using Streams Studio or streamtool. e.g.,\n"
+ " streamtool canceljob "
+ future.get()
+ "\n");
} else if (contextType.contains("STANDALONE")) {
Thread.sleep(15000);
future.cancel(true);
}
}
/** Return a stream that plays Fizz Buzz based upon the values in the input stream. */
public static TStream<String> playFizzBuzz(TStream<Long> counting) {
/*
* Transform an input stream of longs TStream<Long> to a
* stream of strings TStream<String> that follow
* the Fizz Buzz rules based upon each value in the
* input stream.
*/
TStream<String> shouts =
counting.transform(
new Function<Long, String>() {
private static final long serialVersionUID = 1L;
@Override
public String apply(Long v) {
// Skip 0, humans count from 1!
if (v == 0) return null;
StringBuilder sb = new StringBuilder();
if (v % 3 == 0) sb.append("Fizz");
if (v % 5 == 0) sb.append("Buzz");
if (sb.length() == 0) sb.append(Long.toString(v));
else sb.append("!");
return sb.toString();
}
});
return shouts;
}
@SuppressWarnings("unused")
@Test(expected = IllegalStateException.class)
public void testColocateLowLatencyRegions() throws Exception {
assumeTrue(isMainRun());
// ensure colocating two low latency regions doesn't break lowLatancy
// and colocating is achieved.
Topology t = newTopology("testColocateLowLatencyRegions");
Tester tester = t.getTester();
// getConfig().put(ContextProperties.KEEP_ARTIFACTS, true);
TStream<String> s1 =
t.strings("a").lowLatency().modify(getContainerIdAppend()).modify(getContainerIdAppend());
s1.endLowLatency();
TStream<String> s2 =
t.strings("A").lowLatency().modify(getContainerIdAppend()).modify(getContainerIdAppend());
s2.endLowLatency();
s1.colocate(s2); // expect throw ISE: colocate in a low latency region
// once it's supported... (today it breaks the low latency guarantee)
// and adjust isMainRun() too
// // Given the default fuse-island behavior, expect islands to continue
// // to be fused, now both in a single container.
//
// // Today FAILING in an interesting way.
// // There are 2 PEs:
// // - one has just the single colocated s1 and s2 modify ops
// // - the other has everything else
//
// TStream<String> all = s1.union(s2);
// all.print();
// Condition<Long> nTuples = tester.tupleCount(all.filter(new AllowAll<String>()), 2);
// Condition<List<String>> contents = tester.stringContents(
// all.filter(new AllowAll<String>()), "");
//
// complete(tester, nTuples, 10, TimeUnit.SECONDS);
//
// Set<String> ids = getContainerIds(contents.getResult());
// assertEquals("ids: "+ids, 1, ids.size());
}
/** Json Subscribe feeding a flat map */
@Test
public void testPublishJsonFlatMap() throws Exception {
Random r = new Random();
final Topology t = new Topology();
JSONObject j1 = new JSONObject();
j1.put("a", r.nextLong());
j1.put("b", "Hello:" + r.nextInt(200));
JSONObject j2 = new JSONObject();
j2.put("a", r.nextLong());
j2.put("b", "Goodbye:" + r.nextInt(200));
JSONObject j3 = new JSONObject();
j3.put("a", r.nextLong());
j3.put("b", "So long:" + r.nextInt(200));
String s1a = j1.get("a").toString();
String s1b = j1.get("b").toString();
String s2a = j2.get("a").toString();
String s2b = j2.get("b").toString();
String s3a = j3.get("a").toString();
String s3b = j3.get("b").toString();
includePythonApp(t, "json_flatmap_string.py", "json_flatmap_str::json_flatmap_str");
TStream<JSONObject> source = t.constants(Arrays.asList(j1, j2, j3));
source = addStartupDelay(source).asType(JSONObject.class);
source.publish("pytest/json/flatmap");
TStream<String> subscribe = t.subscribe("pytest/json/flatmap/result", String.class);
completeAndValidate(subscribe, 60, s1a, s1b, s2a, s2b, s3a, s3b);
}
/**
* Test with a distributed execution with explicit colocation of two functions end up on the same
* container.
*/
@Test
@Ignore("Need to figure out how to get the tags set by the test") // TODO
public void testSimpleDistributedHostTags() throws Exception {
assumeTrue(SC_OK);
assumeTrue(getTesterType() == StreamsContext.Type.DISTRIBUTED_TESTER);
Topology t = newTopology();
TStream<String> sa = t.strings("a");
sa.addResourceTags("application");
sa = sa.filter(new AllowAll<String>());
sa.addResourceTags("application");
getConfig().put(ContextProperties.KEEP_ARTIFACTS, Boolean.TRUE);
Condition<List<String>> aout = t.getTester().stringContents(sa, "a");
complete(t.getTester(), aout, 10, TimeUnit.SECONDS);
assertTrue(aout.getResult().toString(), aout.valid());
}
@Test
public void testTags() {
assumeTrue(isMainRun());
Topology t = newTopology();
TStream<String> s1 = t.strings("3");
TStream<String> s2 = t.strings("3");
TStream<String> s3 = t.strings("3");
s1.addResourceTags();
assertNull(getResourceTags(s1));
s2.addResourceTags("A", "B");
Set<String> s2s = getResourceTags(s2);
assertEquals(2, s2s.size());
assertTrue(s2s.contains("A"));
assertTrue(s2s.contains("B"));
s3.addResourceTags("C", "D", "E");
Set<String> s3s = getResourceTags(s3);
assertEquals(3, s3s.size());
assertTrue(s3s.contains("C"));
assertTrue(s3s.contains("D"));
assertTrue(s3s.contains("E"));
s2s = getResourceTags(s2);
assertEquals(2, s2s.size());
assertTrue(s2s.contains("A"));
assertTrue(s2s.contains("B"));
s2.addResourceTags("X", "Y");
s2s = getResourceTags(s2);
assertEquals(4, s2s.size());
assertTrue(s2s.contains("A"));
assertTrue(s2s.contains("B"));
assertTrue(s2s.contains("X"));
assertTrue(s2s.contains("Y"));
// Colocating means the s1 will inherit
// s3 resource tags
s1.colocate(s3);
Set<String> s1s = getResourceTags(s1);
assertEquals(3, s1s.size());
assertTrue(s1s.contains("C"));
assertTrue(s1s.contains("D"));
assertTrue(s1s.contains("E"));
}
@Test
public void testNonplaceable() {
assumeTrue(isMainRun());
Topology t = newTopology();
TStream<String> s1 = t.strings("3");
TStream<String> s2 = t.strings("3");
assertFalse(s1.union(s2).isPlaceable());
assertFalse(s1.isolate().isPlaceable());
TStream<String> sp = s1.parallel(3);
assertFalse(sp.isPlaceable());
assertFalse(sp.endParallel().isPlaceable());
}
@Test
public void testFusing() {
assumeTrue(isMainRun());
Topology t = newTopology();
TStream<String> s1 = t.strings("3");
TStream<String> s2 = t.strings("3");
TStream<String> snf = t.strings("3");
assertTrue(s1.isPlaceable());
assertSame(s1.colocate(s2), s1);
String id1 = getFusingId(s1);
String id2 = getFusingId(s2);
assertNotNull(id1);
assertFalse(id1.isEmpty());
assertEquals(id1, id2);
TStream<String> s3 = t.strings("3");
TStream<String> s4 = t.strings("3");
TSink s5 = s4.print();
assertTrue(s5.isPlaceable());
assertSame(s3.colocate(s4, s5), s3);
assertEquals(getFusingId(s3), getFusingId(s4));
assertEquals(getFusingId(s3), getColocate(s5.operator()));
assertFalse(getFusingId(s1).equals(getFusingId(s3)));
assertNull(getFusingId(snf));
TStream<String> s6 = StringStreams.toString(s4);
s1.colocate(s6);
assertEquals(getFusingId(s1), getFusingId(s6));
}
private static Set<String> getResourceTags(TStream<?> s) {
BOperator bop = ((BOutputPort) s.output()).operator();
return getResourceTags(bop);
}
private static String getFusingId(TStream<?> s) {
BOperator bop = ((BOutputPort) s.output()).operator();
return getColocate(bop);
}