public void failingListenerTest(FailingFlowListener.OnFail onFail) throws Exception { if (!new File(inputFileLower).exists()) fail("data file not found"); copyFromLocal(inputFileLower); copyFromLocal(inputFileUpper); Tap sourceLower = new Hfs(new TextLine(new Fields("offset", "line")), inputFileLower); Tap sourceUpper = new Hfs(new TextLine(new Fields("offset", "line")), inputFileUpper); Map sources = new HashMap(); sources.put("lower", sourceLower); sources.put("upper", sourceUpper); Function splitter = new RegexSplitter(new Fields("num", "char"), " "); // using null pos so all fields are written Tap sink = new Hfs(new TextLine(), outputPath + "/stopped/", true); Pipe pipeLower = new Each(new Pipe("lower"), new Fields("line"), splitter); if (onFail == FailingFlowListener.OnFail.THROWABLE) { pipeLower = new Each( pipeLower, new Debug() { @Override public boolean isRemove(FlowProcess flowProcess, FilterCall filterCall) { throw new RuntimeException("failing inside pipe assembly intentionally"); } }); } pipeLower = new GroupBy(pipeLower, new Fields("num")); Pipe pipeUpper = new Each(new Pipe("upper"), new Fields("line"), splitter); pipeUpper = new GroupBy(pipeUpper, new Fields("num")); Pipe splice = new CoGroup(pipeLower, new Fields("num"), pipeUpper, new Fields("num"), Fields.size(4)); Flow flow = new FlowConnector(getProperties()).connect(sources, sink, splice); // countFlow.writeDOT( "stopped.dot" ); FailingFlowListener listener = new FailingFlowListener(onFail); flow.addListener(listener); System.out.println("calling start"); flow.start(); assertTrue("did not start", listener.started.tryAcquire(120, TimeUnit.SECONDS)); if (onFail == FailingFlowListener.OnFail.STOPPING) { while (true) { System.out.println("testing if running"); Thread.sleep(1000); Map<String, Callable<Throwable>> map = flow.getJobsMap(); if (map == null || map.values().size() == 0) continue; if (((FlowStepJob) map.values().iterator().next()).wasStarted()) break; } System.out.println("calling stop"); flow.stop(); } assertTrue("did not complete", listener.completed.tryAcquire(120, TimeUnit.SECONDS)); assertTrue("did not stop", listener.stopped.tryAcquire(120, TimeUnit.SECONDS)); try { flow.complete(); fail("did not rethrow exception from listener"); } catch (Exception exception) { // ignore } }
public void testStop() throws Exception { if (!new File(inputFileLower).exists()) fail("data file not found"); copyFromLocal(inputFileLower); copyFromLocal(inputFileUpper); Tap sourceLower = new Hfs(new TextLine(new Fields("offset", "line")), inputFileLower); Tap sourceUpper = new Hfs(new TextLine(new Fields("offset", "line")), inputFileUpper); Map sources = new HashMap(); sources.put("lower", sourceLower); sources.put("upper", sourceUpper); Function splitter = new RegexSplitter(new Fields("num", "char"), " "); // using null pos so all fields are written Tap sink = new Hfs(new TextLine(), outputPath + "/stopped/", true); Pipe pipeLower = new Each(new Pipe("lower"), new Fields("line"), splitter); pipeLower = new GroupBy(pipeLower, new Fields("num")); Pipe pipeUpper = new Each(new Pipe("upper"), new Fields("line"), splitter); pipeUpper = new GroupBy(pipeUpper, new Fields("num")); Pipe splice = new CoGroup(pipeLower, new Fields("num"), pipeUpper, new Fields("num"), Fields.size(4)); Flow flow = new FlowConnector(getProperties()).connect(sources, sink, splice); // countFlow.writeDOT( "stopped.dot" ); LockingFlowListener listener = new LockingFlowListener(); flow.addListener(listener); System.out.println("calling start"); flow.start(); assertTrue("did not start", listener.started.tryAcquire(60, TimeUnit.SECONDS)); while (true) { System.out.println("testing if running"); Thread.sleep(1000); Map<String, Callable<Throwable>> map = flow.getJobsMap(); if (map == null || map.values().size() == 0) continue; if (((FlowStepJob) map.values().iterator().next()).wasStarted()) break; } System.out.println("calling stop"); flow.stop(); assertTrue("did not stop", listener.stopped.tryAcquire(60, TimeUnit.SECONDS)); assertTrue("did not complete", listener.completed.tryAcquire(60, TimeUnit.SECONDS)); }