/*
* Test that Metrics are automatically unregistered after the job is closed
*/
@Test
public void automaticMetricCleanup1() throws Exception {
// Declare topology with custom metric oplet
Topology t = newTopology();
AtomicInteger n = new AtomicInteger(0);
TStream<Integer> ints = t.poll(() -> n.incrementAndGet(), 10, TimeUnit.MILLISECONDS);
ints.pipe(new TestOplet<Integer>());
// Submit job
Future<? extends Job> fj = getSubmitter().submit(t);
Job job = fj.get();
Thread.sleep(TimeUnit.MILLISECONDS.toMillis(50));
// At least one tuple was processed
int tupleCount = n.get();
assertTrue("Expected more tuples than " + tupleCount, tupleCount > 0);
// Each test oplet registers two metrics
Map<String, Metric> all = metricRegistry.getMetrics();
assertEquals(2, all.size());
// After close all metrics have been unregistered
job.stateChange(Job.Action.CLOSE);
assertEquals(0, all.size());
}
private void _testFanoutWithPeek(boolean after) throws Exception {
Topology t = newTopology();
Graph g = t.graph();
/* -- Filter -- Sink(.)
* /
* Source -- Peek -- FanOut ---- Modify -- Sink(@)
*
*/
TStream<Integer> d = integers(t, 1, 2, 3);
List<Integer> peekedValues = new ArrayList<>();
if (!after) d.peek(tuple -> peekedValues.add(tuple));
TStream<Integer> df = d.filter(tuple -> tuple.intValue() > 0);
TStream<Integer> dm = d.modify(tuple -> new Integer(tuple.intValue() + 1));
if (after) d.peek(tuple -> peekedValues.add(tuple));
df.sink(tuple -> System.out.print("."));
dm.sink(tuple -> System.out.print("@"));
assertEquals(7, g.getVertices().size());
assertEquals(6, g.getEdges().size());
// Insert counter metrics into all the streams
Metrics.counter(t);
printGraph(g);
assertEquals(10, g.getVertices().size());
assertEquals(9, g.getEdges().size());
}
@Test
public void metricsEverywhereMultiplePeek() throws Exception {
Topology t = newTopology();
Graph g = t.graph();
TStream<String> s = t.strings("a", "b", "c");
List<String> peekedValues = new ArrayList<>();
TStream<String> speek = s.peek(tuple -> peekedValues.add(tuple + "1st"));
TStream<String> speek2 = speek.peek(tuple -> peekedValues.add(tuple + "2nd"));
TStream<String> speek3 = speek2.peek(tuple -> peekedValues.add(tuple + "3rd"));
speek3.sink(tuple -> System.out.print("."));
Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> vertices = g.getVertices();
assertEquals(5, vertices.size());
Collection<Edge> edges = g.getEdges();
assertEquals(4, edges.size());
Metrics.counter(t);
printGraph(g);
// One single counter inserted after the 3rd peek
vertices = g.getVertices();
assertEquals(6, vertices.size());
edges = g.getEdges();
assertEquals(5, edges.size());
}
@Test
public void metricsEverywhereFanOut() {
Topology t = newTopology();
Graph g = t.graph();
/* -- OP_3 (Sink)
* /
* OP_0 -- FanOut ----- OP_4 (Sink)
*/
TStream<Integer> d = integers(t, 1, 2, 3);
d.sink(tuple -> System.out.print("."));
d.sink(tuple -> System.out.print("@"));
// Insert counter metrics into all the topology streams
Metrics.counter(t);
printGraph(g);
Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> vertices = g.getVertices();
assertEquals(5, vertices.size());
Collection<Edge> edges = g.getEdges();
assertEquals(4, edges.size());
}
public static void main(String[] args) throws Exception {
DevelopmentProvider dtp = new DevelopmentProvider();
Topology t = dtp.newTopology("DevelopmentSampleJobMXBean");
Random r = new Random();
TStream<Double> d = t.poll(() -> r.nextGaussian(), 100, TimeUnit.MILLISECONDS);
d.sink(tuple -> System.out.print("."));
dtp.submit(t);
System.out.println(dtp.getServices().getService(HttpServer.class).getConsoleUrl());
MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
StringBuffer sbuf = new StringBuffer();
sbuf.append(DevelopmentProvider.JMX_DOMAIN);
sbuf.append(":interface=");
sbuf.append(ObjectName.quote("quarks.execution.mbeans.JobMXBean"));
sbuf.append(",type=");
sbuf.append(ObjectName.quote("job"));
sbuf.append(",*");
System.out.println("Looking for MBeans of type job: " + sbuf.toString());
ObjectName jobObjName = new ObjectName(sbuf.toString());
Set<ObjectInstance> jobInstances = mBeanServer.queryMBeans(jobObjName, null);
Iterator<ObjectInstance> jobIterator = jobInstances.iterator();
while (jobIterator.hasNext()) {
ObjectInstance jobInstance = jobIterator.next();
ObjectName objectName = jobInstance.getObjectName();
String jobId = (String) mBeanServer.getAttribute(objectName, "Id");
String jobName = (String) mBeanServer.getAttribute(objectName, "Name");
String jobCurState = (String) mBeanServer.getAttribute(objectName, "CurrentState");
String jobNextState = (String) mBeanServer.getAttribute(objectName, "NextState");
String jobHealth = (String) mBeanServer.getAttribute(objectName, "Health");
String jobLastError = (String) mBeanServer.getAttribute(objectName, "LastError");
System.out.println(
"Found a job with JobId: "
+ jobId
+ " Name: "
+ jobName
+ " CurrentState: "
+ jobCurState
+ " NextState: "
+ jobNextState
+ " Health: "
+ jobHealth
+ " LastError: \""
+ jobLastError
+ "\"");
}
}
// Apply Metrics on all streams, graph with split oplet and Metric oplet
@Test
public void metricsEverywhereSplit() throws Exception {
/* -- OP_2 (Counter) --- OP_3 (Sink)
* /
* OP_0 -- OP_1(Split) ----- OP_4 (Sink)
* \
* -- OP_5 (Sink)
*/
Topology t = newTopology();
Graph g = t.graph();
// Source
TStream<Integer> d = integers(t, 1, 2, 3);
// Split
List<TStream<Integer>> splits =
d.split(
3,
tuple -> {
switch (tuple.intValue()) {
case 0:
return 0;
case 1:
return 1;
default:
return 2;
}
});
// Insert counter metric for the zeroes stream
Metrics.counter(splits.get(0)).sink(tuple -> System.out.print("."));
splits.get(1).sink(tuple -> System.out.print("#"));
splits.get(2).sink(tuple -> System.out.print("@"));
// Insert counter metrics into all the topology streams
Metrics.counter(t);
printGraph(g);
Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> vertices = g.getVertices();
assertEquals(11, vertices.size());
Collection<Edge> edges = g.getEdges();
assertEquals(10, edges.size());
}
// Apply Metrics on all streams, simple graph
@Test
public void metricsEverywhereSimple() throws Exception {
Topology t = newTopology();
Graph g = t.graph();
// Source
TStream<Integer> d = integers(t, 1, 2, 3);
d.sink(tuple -> System.out.print("."));
// Insert counter metrics into all the topology streams
Metrics.counter(t);
printGraph(g);
Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> vertices = g.getVertices();
assertEquals(3, vertices.size());
Collection<Edge> edges = g.getEdges();
assertEquals(2, edges.size());
}
private <T> TStream<T> integers(Topology t, @SuppressWarnings("unchecked") T... values) {
return t.source(() -> Arrays.asList(values));
}