@Test
public void testIndexPattern() throws Exception {
Properties props = HdpBootstrap.asProperties(CascadingHadoopSuite.configuration);
Tap in = sourceTap();
Tap out =
new EsTap("cascading-hadoop/pattern-{id}", new Fields("id", "name", "url", "picture"));
Pipe pipe = new Pipe("copy");
StatsUtils.proxy(new HadoopFlowConnector(props).connect(in, out, pipe)).complete();
}
@Test
public void testWriteToES() throws Exception {
Tap in = sourceTap();
Tap out = new EsTap("cascading-hadoop/artists", new Fields("name", "url", "picture"));
Pipe pipe = new Pipe("copy");
FlowDef flowDef = FlowDef.flowDef().addSource(pipe, in).addTailSink(pipe, out);
StatsUtils.proxy(
new HadoopFlowConnector(HdpBootstrap.asProperties(CascadingHadoopSuite.configuration))
.connect(flowDef))
.complete();
}
@Test
public void testWriteToESWithAlias() throws Exception {
Tap in = sourceTap();
Tap out = new EsTap("cascading-hadoop/alias", "", new Fields("name", "url", "picture"));
Pipe pipe = new Pipe("copy");
// rename "id" -> "garbage"
pipe = new Each(pipe, new Identity(new Fields("garbage", "name", "url", "picture", "ts")));
Properties props = HdpBootstrap.asProperties(CascadingHadoopSuite.configuration);
props.setProperty("es.mapping.names", "url:address");
StatsUtils.proxy(new HadoopFlowConnector(props).connect(in, out, pipe)).complete();
}
@Test
public void testCascadeConnector() {
Pipe copy = new Pipe("copy");
Properties cfg = HdpBootstrap.asProperties(CascadingHadoopSuite.configuration);
FlowDef flow =
new FlowDef()
.addSource(copy, sourceTap())
.addTailSink(copy, new EsTap("cascading-hadoop/cascade-connector"));
FlowConnector connector = new HadoopFlowConnector(cfg);
Flow[] flows = new Flow[] {connector.connect(flow)};
CascadeConnector cascadeConnector = new CascadeConnector(cfg);
cascadeConnector.connect(flows).complete();
}