public void testStartStopRace() throws Exception {
if (!new File(inputFileLower).exists()) fail("data file not found");
copyFromLocal(inputFileLower);
Tap sourceLower = new Hfs(new TextLine(new Fields("offset", "line")), inputFileLower);
Map sources = new HashMap();
sources.put("lower", sourceLower);
Function splitter = new RegexSplitter(new Fields("num", "char"), " ");
// using null pos so all fields are written
Tap sink = new Hfs(new TextLine(), outputPath + "/startstop/", true);
Pipe pipeLower = new Each(new Pipe("lower"), new Fields("line"), splitter);
pipeLower = new GroupBy(pipeLower, new Fields("num"));
Flow flow = new FlowConnector(getProperties()).connect(sources, sink, pipeLower);
flow.start();
flow.stop(); // should not fail
}
public void failingListenerTest(FailingFlowListener.OnFail onFail) throws Exception {
if (!new File(inputFileLower).exists()) fail("data file not found");
copyFromLocal(inputFileLower);
copyFromLocal(inputFileUpper);
Tap sourceLower = new Hfs(new TextLine(new Fields("offset", "line")), inputFileLower);
Tap sourceUpper = new Hfs(new TextLine(new Fields("offset", "line")), inputFileUpper);
Map sources = new HashMap();
sources.put("lower", sourceLower);
sources.put("upper", sourceUpper);
Function splitter = new RegexSplitter(new Fields("num", "char"), " ");
// using null pos so all fields are written
Tap sink = new Hfs(new TextLine(), outputPath + "/stopped/", true);
Pipe pipeLower = new Each(new Pipe("lower"), new Fields("line"), splitter);
if (onFail == FailingFlowListener.OnFail.THROWABLE) {
pipeLower =
new Each(
pipeLower,
new Debug() {
@Override
public boolean isRemove(FlowProcess flowProcess, FilterCall filterCall) {
throw new RuntimeException("failing inside pipe assembly intentionally");
}
});
}
pipeLower = new GroupBy(pipeLower, new Fields("num"));
Pipe pipeUpper = new Each(new Pipe("upper"), new Fields("line"), splitter);
pipeUpper = new GroupBy(pipeUpper, new Fields("num"));
Pipe splice =
new CoGroup(pipeLower, new Fields("num"), pipeUpper, new Fields("num"), Fields.size(4));
Flow flow = new FlowConnector(getProperties()).connect(sources, sink, splice);
// countFlow.writeDOT( "stopped.dot" );
FailingFlowListener listener = new FailingFlowListener(onFail);
flow.addListener(listener);
System.out.println("calling start");
flow.start();
assertTrue("did not start", listener.started.tryAcquire(120, TimeUnit.SECONDS));
if (onFail == FailingFlowListener.OnFail.STOPPING) {
while (true) {
System.out.println("testing if running");
Thread.sleep(1000);
Map<String, Callable<Throwable>> map = flow.getJobsMap();
if (map == null || map.values().size() == 0) continue;
if (((FlowStepJob) map.values().iterator().next()).wasStarted()) break;
}
System.out.println("calling stop");
flow.stop();
}
assertTrue("did not complete", listener.completed.tryAcquire(120, TimeUnit.SECONDS));
assertTrue("did not stop", listener.stopped.tryAcquire(120, TimeUnit.SECONDS));
try {
flow.complete();
fail("did not rethrow exception from listener");
} catch (Exception exception) {
// ignore
}
}
public void testStop() throws Exception {
if (!new File(inputFileLower).exists()) fail("data file not found");
copyFromLocal(inputFileLower);
copyFromLocal(inputFileUpper);
Tap sourceLower = new Hfs(new TextLine(new Fields("offset", "line")), inputFileLower);
Tap sourceUpper = new Hfs(new TextLine(new Fields("offset", "line")), inputFileUpper);
Map sources = new HashMap();
sources.put("lower", sourceLower);
sources.put("upper", sourceUpper);
Function splitter = new RegexSplitter(new Fields("num", "char"), " ");
// using null pos so all fields are written
Tap sink = new Hfs(new TextLine(), outputPath + "/stopped/", true);
Pipe pipeLower = new Each(new Pipe("lower"), new Fields("line"), splitter);
pipeLower = new GroupBy(pipeLower, new Fields("num"));
Pipe pipeUpper = new Each(new Pipe("upper"), new Fields("line"), splitter);
pipeUpper = new GroupBy(pipeUpper, new Fields("num"));
Pipe splice =
new CoGroup(pipeLower, new Fields("num"), pipeUpper, new Fields("num"), Fields.size(4));
Flow flow = new FlowConnector(getProperties()).connect(sources, sink, splice);
// countFlow.writeDOT( "stopped.dot" );
LockingFlowListener listener = new LockingFlowListener();
flow.addListener(listener);
System.out.println("calling start");
flow.start();
assertTrue("did not start", listener.started.tryAcquire(60, TimeUnit.SECONDS));
while (true) {
System.out.println("testing if running");
Thread.sleep(1000);
Map<String, Callable<Throwable>> map = flow.getJobsMap();
if (map == null || map.values().size() == 0) continue;
if (((FlowStepJob) map.values().iterator().next()).wasStarted()) break;
}
System.out.println("calling stop");
flow.stop();
assertTrue("did not stop", listener.stopped.tryAcquire(60, TimeUnit.SECONDS));
assertTrue("did not complete", listener.completed.tryAcquire(60, TimeUnit.SECONDS));
}
