/* * Test that Metrics are automatically unregistered after the job is closed */ @Test public void automaticMetricCleanup1() throws Exception { // Declare topology with custom metric oplet Topology t = newTopology(); AtomicInteger n = new AtomicInteger(0); TStream<Integer> ints = t.poll(() -> n.incrementAndGet(), 10, TimeUnit.MILLISECONDS); ints.pipe(new TestOplet<Integer>()); // Submit job Future<? extends Job> fj = getSubmitter().submit(t); Job job = fj.get(); Thread.sleep(TimeUnit.MILLISECONDS.toMillis(50)); // At least one tuple was processed int tupleCount = n.get(); assertTrue("Expected more tuples than " + tupleCount, tupleCount > 0); // Each test oplet registers two metrics Map<String, Metric> all = metricRegistry.getMetrics(); assertEquals(2, all.size()); // After close all metrics have been unregistered job.stateChange(Job.Action.CLOSE); assertEquals(0, all.size()); }
private void _testFanoutWithPeek(boolean after) throws Exception { Topology t = newTopology(); Graph g = t.graph(); /* -- Filter -- Sink(.) * / * Source -- Peek -- FanOut ---- Modify -- Sink(@) * */ TStream<Integer> d = integers(t, 1, 2, 3); List<Integer> peekedValues = new ArrayList<>(); if (!after) d.peek(tuple -> peekedValues.add(tuple)); TStream<Integer> df = d.filter(tuple -> tuple.intValue() > 0); TStream<Integer> dm = d.modify(tuple -> new Integer(tuple.intValue() + 1)); if (after) d.peek(tuple -> peekedValues.add(tuple)); df.sink(tuple -> System.out.print(".")); dm.sink(tuple -> System.out.print("@")); assertEquals(7, g.getVertices().size()); assertEquals(6, g.getEdges().size()); // Insert counter metrics into all the streams Metrics.counter(t); printGraph(g); assertEquals(10, g.getVertices().size()); assertEquals(9, g.getEdges().size()); }
@Test public void metricsEverywhereMultiplePeek() throws Exception { Topology t = newTopology(); Graph g = t.graph(); TStream<String> s = t.strings("a", "b", "c"); List<String> peekedValues = new ArrayList<>(); TStream<String> speek = s.peek(tuple -> peekedValues.add(tuple + "1st")); TStream<String> speek2 = speek.peek(tuple -> peekedValues.add(tuple + "2nd")); TStream<String> speek3 = speek2.peek(tuple -> peekedValues.add(tuple + "3rd")); speek3.sink(tuple -> System.out.print(".")); Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> vertices = g.getVertices(); assertEquals(5, vertices.size()); Collection<Edge> edges = g.getEdges(); assertEquals(4, edges.size()); Metrics.counter(t); printGraph(g); // One single counter inserted after the 3rd peek vertices = g.getVertices(); assertEquals(6, vertices.size()); edges = g.getEdges(); assertEquals(5, edges.size()); }
@Test public void metricsEverywhereFanOut() { Topology t = newTopology(); Graph g = t.graph(); /* -- OP_3 (Sink) * / * OP_0 -- FanOut ----- OP_4 (Sink) */ TStream<Integer> d = integers(t, 1, 2, 3); d.sink(tuple -> System.out.print(".")); d.sink(tuple -> System.out.print("@")); // Insert counter metrics into all the topology streams Metrics.counter(t); printGraph(g); Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> vertices = g.getVertices(); assertEquals(5, vertices.size()); Collection<Edge> edges = g.getEdges(); assertEquals(4, edges.size()); }
public static void main(String[] args) throws Exception { DevelopmentProvider dtp = new DevelopmentProvider(); Topology t = dtp.newTopology("DevelopmentSampleJobMXBean"); Random r = new Random(); TStream<Double> d = t.poll(() -> r.nextGaussian(), 100, TimeUnit.MILLISECONDS); d.sink(tuple -> System.out.print(".")); dtp.submit(t); System.out.println(dtp.getServices().getService(HttpServer.class).getConsoleUrl()); MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); StringBuffer sbuf = new StringBuffer(); sbuf.append(DevelopmentProvider.JMX_DOMAIN); sbuf.append(":interface="); sbuf.append(ObjectName.quote("quarks.execution.mbeans.JobMXBean")); sbuf.append(",type="); sbuf.append(ObjectName.quote("job")); sbuf.append(",*"); System.out.println("Looking for MBeans of type job: " + sbuf.toString()); ObjectName jobObjName = new ObjectName(sbuf.toString()); Set<ObjectInstance> jobInstances = mBeanServer.queryMBeans(jobObjName, null); Iterator<ObjectInstance> jobIterator = jobInstances.iterator(); while (jobIterator.hasNext()) { ObjectInstance jobInstance = jobIterator.next(); ObjectName objectName = jobInstance.getObjectName(); String jobId = (String) mBeanServer.getAttribute(objectName, "Id"); String jobName = (String) mBeanServer.getAttribute(objectName, "Name"); String jobCurState = (String) mBeanServer.getAttribute(objectName, "CurrentState"); String jobNextState = (String) mBeanServer.getAttribute(objectName, "NextState"); String jobHealth = (String) mBeanServer.getAttribute(objectName, "Health"); String jobLastError = (String) mBeanServer.getAttribute(objectName, "LastError"); System.out.println( "Found a job with JobId: " + jobId + " Name: " + jobName + " CurrentState: " + jobCurState + " NextState: " + jobNextState + " Health: " + jobHealth + " LastError: \"" + jobLastError + "\""); } }
// Apply Metrics on all streams, graph with split oplet and Metric oplet @Test public void metricsEverywhereSplit() throws Exception { /* -- OP_2 (Counter) --- OP_3 (Sink) * / * OP_0 -- OP_1(Split) ----- OP_4 (Sink) * \ * -- OP_5 (Sink) */ Topology t = newTopology(); Graph g = t.graph(); // Source TStream<Integer> d = integers(t, 1, 2, 3); // Split List<TStream<Integer>> splits = d.split( 3, tuple -> { switch (tuple.intValue()) { case 0: return 0; case 1: return 1; default: return 2; } }); // Insert counter metric for the zeroes stream Metrics.counter(splits.get(0)).sink(tuple -> System.out.print(".")); splits.get(1).sink(tuple -> System.out.print("#")); splits.get(2).sink(tuple -> System.out.print("@")); // Insert counter metrics into all the topology streams Metrics.counter(t); printGraph(g); Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> vertices = g.getVertices(); assertEquals(11, vertices.size()); Collection<Edge> edges = g.getEdges(); assertEquals(10, edges.size()); }
// Apply Metrics on all streams, simple graph @Test public void metricsEverywhereSimple() throws Exception { Topology t = newTopology(); Graph g = t.graph(); // Source TStream<Integer> d = integers(t, 1, 2, 3); d.sink(tuple -> System.out.print(".")); // Insert counter metrics into all the topology streams Metrics.counter(t); printGraph(g); Collection<Vertex<? extends Oplet<?, ?>, ?, ?>> vertices = g.getVertices(); assertEquals(3, vertices.size()); Collection<Edge> edges = g.getEdges(); assertEquals(2, edges.size()); }
private <T> TStream<T> integers(Topology t, @SuppressWarnings("unchecked") T... values) { return t.source(() -> Arrays.asList(values)); }