@Test
public void testNestedTuple() throws Exception {
RestUtils.postData(
"pig/nestedtuple", "{\"my_array\" : [\"1.a\",\"1.b\"]}".getBytes(StringUtils.UTF_8));
RestUtils.postData(
"pig/nestedtuple", "{\"my_array\" : [\"2.a\",\"2.b\"]}".getBytes(StringUtils.UTF_8));
RestUtils.waitForYellow("pig");
}
@Test
public void testEsSchemaRDD1WriteWithMappingExclude() throws Exception {
DataFrame dataFrame = artistsAsDataFrame();
String target = "sparksql-test/scala-basic-write-exclude-mapping";
JavaEsSparkSQL.saveToEs(dataFrame, target, ImmutableMap.of(ES_MAPPING_EXCLUDE, "url"));
assertTrue(RestUtils.exists(target));
assertThat(RestUtils.get(target + "/_search?"), not(containsString("url")));
}
@Test
public void testEsdataFrame1Write() throws Exception {
DataFrame dataFrame = artistsAsDataFrame();
String target = "sparksql-test/scala-basic-write";
JavaEsSparkSQL.saveToEs(dataFrame, target);
assertTrue(RestUtils.exists(target));
assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
}
@Test
public void testEsdataFrame1WriteWithId() throws Exception {
DataFrame dataFrame = artistsAsDataFrame();
String target = "sparksql-test/scala-basic-write-id-mapping";
JavaEsSparkSQL.saveToEs(dataFrame, target, ImmutableMap.of(ES_MAPPING_ID, "id"));
assertTrue(RestUtils.exists(target));
assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
assertThat(RestUtils.exists(target + "/1"), is(true));
}
@Test
public void testIndexPatternFormatMapping() throws Exception {
assertThat(
RestUtils.getMapping("pig/pattern-format-2001-10-06").skipHeaders().toString(),
is(
"pattern-format-2001-10-06=[id=LONG, name=STRING, picture=STRING, timestamp=DATE, url=STRING]"));
}
@Test
public void testWriteToESWithAliasMapping() throws Exception {
assertThat(
RestUtils.getMapping("cascading-hadoop/alias").toString(),
VERSION.onOrAfter(V_5_X)
? is("alias=[address=TEXT, name=TEXT, picture=TEXT]")
: is("alias=[address=STRING, name=STRING, picture=STRING]"));
}
@Test
public void testWriteToESMapping() throws Exception {
assertThat(
RestUtils.getMapping("cascading-hadoop/artists").toString(),
VERSION.onOrAfter(V_5_X)
? is("artists=[name=TEXT, picture=TEXT, url=TEXT]")
: is("artists=[name=STRING, picture=STRING, url=STRING]"));
}
@Test
public void testIndexPatternMapping() throws Exception {
assertThat(
RestUtils.getMapping("cascading-hadoop/pattern-12").toString(),
VERSION.onOrAfter(V_5_X)
? is("pattern-12=[id=TEXT, name=TEXT, picture=TEXT, url=TEXT]")
: is("pattern-12=[id=STRING, name=STRING, picture=STRING, url=STRING]"));
}
@Test
public void testIndexPatternWithFormatMapping() throws Exception {
assertThat(
RestUtils.getMapping("cascading-hadoop/pattern-format-2012-10-06").toString(),
VERSION.onOrAfter(V_5_X)
? is("pattern-format-2012-10-06=[id=TEXT, name=TEXT, picture=TEXT, ts=DATE, url=TEXT]")
: is(
"pattern-format-2012-10-06=[id=STRING, name=STRING, picture=STRING, ts=DATE, url=STRING]"));
}
@Test
public void testNestedField() throws Exception {
String data = "{ \"data\" : { \"map\" : { \"key\" : [ 10, 20 ] } } }";
RestUtils.putData(indexPrefix + "cascading-local/nestedmap", StringUtils.toUTF(data));
RestUtils.refresh(indexPrefix + "cascading-local");
Properties cfg = cfg();
cfg.setProperty("es.mapping.names", "nested:data.map.key");
Tap in = new EsTap(indexPrefix + "cascading-local/nestedmap", new Fields("nested"));
Pipe pipe = new Pipe("copy");
pipe = new Each(pipe, new FilterNotNull());
pipe = new Each(pipe, AssertionLevel.STRICT, new AssertSizeLessThan(2));
// print out
Tap out = new OutputStreamTap(new TextLine(), OUT);
build(cfg, in, out, pipe);
}
@Test
public void testParentChild() throws Exception {
RestUtils.putMapping("pig/child", "org/elasticsearch/hadoop/integration/mr-child.json");
String script =
"REGISTER "
+ Provisioner.ESHADOOP_TESTING_JAR
+ ";"
+ loadSource()
+ "B = FOREACH A GENERATE id, name, TOBAG(url, picture) AS links;"
+ "STORE B INTO 'pig/child' USING org.elasticsearch.hadoop.pig.EsStorage('"
+ ConfigurationOptions.ES_MAPPING_PARENT
+ "=id','"
+ ConfigurationOptions.ES_INDEX_AUTO_CREATE
+ "=no');";
pig.executeScript(script);
}
@Test
public void testTupleMapping() throws Exception {
assertThat(
RestUtils.getMapping("pig/tupleartists").skipHeaders().toString(),
is("tupleartists=[links=STRING, name=STRING]"));
}
@Test
public void testDynamicPattern() throws Exception {
Assert.assertTrue(RestUtils.exists(indexPrefix + "cascading-local/pattern-1"));
Assert.assertTrue(RestUtils.exists(indexPrefix + "cascading-local/pattern-500"));
Assert.assertTrue(RestUtils.exists(indexPrefix + "cascading-local/pattern-990"));
}
@Test
public void testDynamicPatternWithFormat() throws Exception {
Assert.assertTrue(RestUtils.exists(indexPrefix + "cascading-local/pattern-format-2001-10-06"));
Assert.assertTrue(RestUtils.exists(indexPrefix + "cascading-local/pattern-format-2198-10-06"));
Assert.assertTrue(RestUtils.exists(indexPrefix + "cascading-local/pattern-format-2890-10-06"));
}
@Test
public void testParentChildMapping() throws Exception {
assertThat(
RestUtils.getMapping("pig/child").skipHeaders().toString(),
is("child=[id=LONG, links=STRING, name=STRING]"));
}
@Test
public void testUpdateWithIdMapping() throws Exception {
assertThat(
RestUtils.getMapping("pig/update").skipHeaders().toString(),
is("update=[id=LONG, links=STRING, name=STRING]"));
}
@Test
public void testEmptyComplexStructuresMapping() throws Exception {
assertThat(
RestUtils.getMapping("pig/emptyconst").skipHeaders().toString(), is("emptyconst=[]"));
}
@Test
public void testCaseSensitivityMapping() throws Exception {
assertThat(
RestUtils.getMapping("pig/casesensitivity").skipHeaders().toString(),
is("casesensitivity=[Name=STRING, pIctUre=STRING, uRL=STRING]"));
}
@Test
public void testFieldAliasMapping() throws Exception {
assertThat(
RestUtils.getMapping("pig/fieldalias").skipHeaders().toString(),
is("fieldalias=[@timestamp=DATE, name=STRING, picture=STRING, url=STRING]"));
}
@Test
public void testTimestampMapping() throws Exception {
String mapping = RestUtils.getMapping("pig/timestamp").skipHeaders().toString();
assertThat(mapping, startsWith("timestamp=[name=STRING, org.apache.pig.builtin.todate_"));
assertThat(mapping, endsWith("=DATE, url=STRING]"));
}
@Before
public void before() throws Exception {
RestUtils.refresh(indexPrefix + "cascading-local");
}
