public void testFailedSerialization() throws Exception {
if (!new File(inputFileLower).exists()) fail("data file not found");
copyFromLocal(inputFileLower);
Tap sourceLower = new Hfs(new TextLine(new Fields("offset", "line")), inputFileLower);
Map sources = new HashMap();
sources.put("lower", sourceLower);
Function splitter = new RegexSplitter(new Fields("num", "char"), " ");
// using null pos so all fields are written
Tap sink = new Hfs(new TextLine(), outputPath + "/badserialization/", true);
Pipe pipeLower = new Each(new Pipe("lower"), new Fields("line"), splitter);
pipeLower = new Each(pipeLower, new Fields("num"), new BadFilter());
pipeLower = new GroupBy(pipeLower, new Fields("num"));
Flow flow = new FlowConnector(getProperties()).connect(sources, sink, pipeLower);
// countFlow.writeDOT( "stopped.dot" );
LockingFlowListener listener = new LockingFlowListener();
flow.addListener(listener);
try {
flow.complete();
fail("did not throw serialization exception");
} catch (Exception exception) {
// ignore
}
assertTrue("not marked failed", flow.getFlowStats().isFailed());
}
@Test
public void testSimpleClasspath() throws Exception {
getPlatform().copyFromLocal(inputFileApache);
Tap source = getPlatform().getTextFile(new Fields("offset", "line"), inputFileApache);
Pipe pipe = new Pipe("test");
pipe = new Each(pipe, new Fields("line"), new TestFunction(), Fields.RESULTS);
Tap sink = getPlatform().getTextFile(getOutputPath("classpath"), SinkMode.REPLACE);
FlowDef flowDef =
FlowDef.flowDef()
.addSource("test", source)
.addTailSink(pipe, sink)
.addToClassPath(testClasspathJar);
Flow flow = getPlatform().getFlowConnector().connect(flowDef);
flow.complete();
validateLength(flow, 10);
}
public void failingListenerTest(FailingFlowListener.OnFail onFail) throws Exception {
if (!new File(inputFileLower).exists()) fail("data file not found");
copyFromLocal(inputFileLower);
copyFromLocal(inputFileUpper);
Tap sourceLower = new Hfs(new TextLine(new Fields("offset", "line")), inputFileLower);
Tap sourceUpper = new Hfs(new TextLine(new Fields("offset", "line")), inputFileUpper);
Map sources = new HashMap();
sources.put("lower", sourceLower);
sources.put("upper", sourceUpper);
Function splitter = new RegexSplitter(new Fields("num", "char"), " ");
// using null pos so all fields are written
Tap sink = new Hfs(new TextLine(), outputPath + "/stopped/", true);
Pipe pipeLower = new Each(new Pipe("lower"), new Fields("line"), splitter);
if (onFail == FailingFlowListener.OnFail.THROWABLE) {
pipeLower =
new Each(
pipeLower,
new Debug() {
@Override
public boolean isRemove(FlowProcess flowProcess, FilterCall filterCall) {
throw new RuntimeException("failing inside pipe assembly intentionally");
}
});
}
pipeLower = new GroupBy(pipeLower, new Fields("num"));
Pipe pipeUpper = new Each(new Pipe("upper"), new Fields("line"), splitter);
pipeUpper = new GroupBy(pipeUpper, new Fields("num"));
Pipe splice =
new CoGroup(pipeLower, new Fields("num"), pipeUpper, new Fields("num"), Fields.size(4));
Flow flow = new FlowConnector(getProperties()).connect(sources, sink, splice);
// countFlow.writeDOT( "stopped.dot" );
FailingFlowListener listener = new FailingFlowListener(onFail);
flow.addListener(listener);
System.out.println("calling start");
flow.start();
assertTrue("did not start", listener.started.tryAcquire(120, TimeUnit.SECONDS));
if (onFail == FailingFlowListener.OnFail.STOPPING) {
while (true) {
System.out.println("testing if running");
Thread.sleep(1000);
Map<String, Callable<Throwable>> map = flow.getJobsMap();
if (map == null || map.values().size() == 0) continue;
if (((FlowStepJob) map.values().iterator().next()).wasStarted()) break;
}
System.out.println("calling stop");
flow.stop();
}
assertTrue("did not complete", listener.completed.tryAcquire(120, TimeUnit.SECONDS));
assertTrue("did not stop", listener.stopped.tryAcquire(120, TimeUnit.SECONDS));
try {
flow.complete();
fail("did not rethrow exception from listener");
} catch (Exception exception) {
// ignore
}
}
