@Test public void testIndexPattern() throws Exception { Properties props = HdpBootstrap.asProperties(CascadingHadoopSuite.configuration); Tap in = sourceTap(); Tap out = new EsTap("cascading-hadoop/pattern-{id}", new Fields("id", "name", "url", "picture")); Pipe pipe = new Pipe("copy"); StatsUtils.proxy(new HadoopFlowConnector(props).connect(in, out, pipe)).complete(); }
@Test public void testWriteToES() throws Exception { Tap in = sourceTap(); Tap out = new EsTap("cascading-hadoop/artists", new Fields("name", "url", "picture")); Pipe pipe = new Pipe("copy"); FlowDef flowDef = FlowDef.flowDef().addSource(pipe, in).addTailSink(pipe, out); StatsUtils.proxy( new HadoopFlowConnector(HdpBootstrap.asProperties(CascadingHadoopSuite.configuration)) .connect(flowDef)) .complete(); }
@Test public void testWriteToESWithAlias() throws Exception { Tap in = sourceTap(); Tap out = new EsTap("cascading-hadoop/alias", "", new Fields("name", "url", "picture")); Pipe pipe = new Pipe("copy"); // rename "id" -> "garbage" pipe = new Each(pipe, new Identity(new Fields("garbage", "name", "url", "picture", "ts"))); Properties props = HdpBootstrap.asProperties(CascadingHadoopSuite.configuration); props.setProperty("es.mapping.names", "url:address"); StatsUtils.proxy(new HadoopFlowConnector(props).connect(in, out, pipe)).complete(); }
@Test public void testCascadeConnector() { Pipe copy = new Pipe("copy"); Properties cfg = HdpBootstrap.asProperties(CascadingHadoopSuite.configuration); FlowDef flow = new FlowDef() .addSource(copy, sourceTap()) .addTailSink(copy, new EsTap("cascading-hadoop/cascade-connector")); FlowConnector connector = new HadoopFlowConnector(cfg); Flow[] flows = new Flow[] {connector.connect(flow)}; CascadeConnector cascadeConnector = new CascadeConnector(cfg); cascadeConnector.connect(flows).complete(); }