コード例 #1
0
 @Test
 public void testNestedTuple() throws Exception {
   RestUtils.postData(
       "pig/nestedtuple", "{\"my_array\" : [\"1.a\",\"1.b\"]}".getBytes(StringUtils.UTF_8));
   RestUtils.postData(
       "pig/nestedtuple", "{\"my_array\" : [\"2.a\",\"2.b\"]}".getBytes(StringUtils.UTF_8));
   RestUtils.waitForYellow("pig");
 }
コード例 #2
0
  @Test
  public void testEsSchemaRDD1WriteWithMappingExclude() throws Exception {
    DataFrame dataFrame = artistsAsDataFrame();

    String target = "sparksql-test/scala-basic-write-exclude-mapping";
    JavaEsSparkSQL.saveToEs(dataFrame, target, ImmutableMap.of(ES_MAPPING_EXCLUDE, "url"));
    assertTrue(RestUtils.exists(target));
    assertThat(RestUtils.get(target + "/_search?"), not(containsString("url")));
  }
コード例 #3
0
  @Test
  public void testEsdataFrame1Write() throws Exception {
    DataFrame dataFrame = artistsAsDataFrame();

    String target = "sparksql-test/scala-basic-write";
    JavaEsSparkSQL.saveToEs(dataFrame, target);
    assertTrue(RestUtils.exists(target));
    assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
  }
コード例 #4
0
  @Test
  public void testEsdataFrame1WriteWithId() throws Exception {
    DataFrame dataFrame = artistsAsDataFrame();

    String target = "sparksql-test/scala-basic-write-id-mapping";
    JavaEsSparkSQL.saveToEs(dataFrame, target, ImmutableMap.of(ES_MAPPING_ID, "id"));
    assertTrue(RestUtils.exists(target));
    assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
    assertThat(RestUtils.exists(target + "/1"), is(true));
  }
コード例 #5
0
 @Test
 public void testIndexPatternFormatMapping() throws Exception {
   assertThat(
       RestUtils.getMapping("pig/pattern-format-2001-10-06").skipHeaders().toString(),
       is(
           "pattern-format-2001-10-06=[id=LONG, name=STRING, picture=STRING, timestamp=DATE, url=STRING]"));
 }
 @Test
 public void testWriteToESWithAliasMapping() throws Exception {
   assertThat(
       RestUtils.getMapping("cascading-hadoop/alias").toString(),
       VERSION.onOrAfter(V_5_X)
           ? is("alias=[address=TEXT, name=TEXT, picture=TEXT]")
           : is("alias=[address=STRING, name=STRING, picture=STRING]"));
 }
 @Test
 public void testWriteToESMapping() throws Exception {
   assertThat(
       RestUtils.getMapping("cascading-hadoop/artists").toString(),
       VERSION.onOrAfter(V_5_X)
           ? is("artists=[name=TEXT, picture=TEXT, url=TEXT]")
           : is("artists=[name=STRING, picture=STRING, url=STRING]"));
 }
 @Test
 public void testIndexPatternMapping() throws Exception {
   assertThat(
       RestUtils.getMapping("cascading-hadoop/pattern-12").toString(),
       VERSION.onOrAfter(V_5_X)
           ? is("pattern-12=[id=TEXT, name=TEXT, picture=TEXT, url=TEXT]")
           : is("pattern-12=[id=STRING, name=STRING, picture=STRING, url=STRING]"));
 }
 @Test
 public void testIndexPatternWithFormatMapping() throws Exception {
   assertThat(
       RestUtils.getMapping("cascading-hadoop/pattern-format-2012-10-06").toString(),
       VERSION.onOrAfter(V_5_X)
           ? is("pattern-format-2012-10-06=[id=TEXT, name=TEXT, picture=TEXT, ts=DATE, url=TEXT]")
           : is(
               "pattern-format-2012-10-06=[id=STRING, name=STRING, picture=STRING, ts=DATE, url=STRING]"));
 }
  @Test
  public void testNestedField() throws Exception {
    String data = "{ \"data\" : { \"map\" : { \"key\" : [ 10, 20 ] } } }";
    RestUtils.putData(indexPrefix + "cascading-local/nestedmap", StringUtils.toUTF(data));

    RestUtils.refresh(indexPrefix + "cascading-local");

    Properties cfg = cfg();
    cfg.setProperty("es.mapping.names", "nested:data.map.key");

    Tap in = new EsTap(indexPrefix + "cascading-local/nestedmap", new Fields("nested"));
    Pipe pipe = new Pipe("copy");
    pipe = new Each(pipe, new FilterNotNull());
    pipe = new Each(pipe, AssertionLevel.STRICT, new AssertSizeLessThan(2));

    // print out
    Tap out = new OutputStreamTap(new TextLine(), OUT);
    build(cfg, in, out, pipe);
  }
コード例 #11
0
  @Test
  public void testParentChild() throws Exception {
    RestUtils.putMapping("pig/child", "org/elasticsearch/hadoop/integration/mr-child.json");

    String script =
        "REGISTER "
            + Provisioner.ESHADOOP_TESTING_JAR
            + ";"
            + loadSource()
            + "B = FOREACH A GENERATE id, name, TOBAG(url, picture) AS links;"
            + "STORE B INTO 'pig/child' USING org.elasticsearch.hadoop.pig.EsStorage('"
            + ConfigurationOptions.ES_MAPPING_PARENT
            + "=id','"
            + ConfigurationOptions.ES_INDEX_AUTO_CREATE
            + "=no');";
    pig.executeScript(script);
  }
コード例 #12
0
 @Test
 public void testTupleMapping() throws Exception {
   assertThat(
       RestUtils.getMapping("pig/tupleartists").skipHeaders().toString(),
       is("tupleartists=[links=STRING, name=STRING]"));
 }
 @Test
 public void testDynamicPattern() throws Exception {
   Assert.assertTrue(RestUtils.exists(indexPrefix + "cascading-local/pattern-1"));
   Assert.assertTrue(RestUtils.exists(indexPrefix + "cascading-local/pattern-500"));
   Assert.assertTrue(RestUtils.exists(indexPrefix + "cascading-local/pattern-990"));
 }
 @Test
 public void testDynamicPatternWithFormat() throws Exception {
   Assert.assertTrue(RestUtils.exists(indexPrefix + "cascading-local/pattern-format-2001-10-06"));
   Assert.assertTrue(RestUtils.exists(indexPrefix + "cascading-local/pattern-format-2198-10-06"));
   Assert.assertTrue(RestUtils.exists(indexPrefix + "cascading-local/pattern-format-2890-10-06"));
 }
コード例 #15
0
 @Test
 public void testParentChildMapping() throws Exception {
   assertThat(
       RestUtils.getMapping("pig/child").skipHeaders().toString(),
       is("child=[id=LONG, links=STRING, name=STRING]"));
 }
コード例 #16
0
 @Test
 public void testUpdateWithIdMapping() throws Exception {
   assertThat(
       RestUtils.getMapping("pig/update").skipHeaders().toString(),
       is("update=[id=LONG, links=STRING, name=STRING]"));
 }
コード例 #17
0
 @Test
 public void testEmptyComplexStructuresMapping() throws Exception {
   assertThat(
       RestUtils.getMapping("pig/emptyconst").skipHeaders().toString(), is("emptyconst=[]"));
 }
コード例 #18
0
 @Test
 public void testCaseSensitivityMapping() throws Exception {
   assertThat(
       RestUtils.getMapping("pig/casesensitivity").skipHeaders().toString(),
       is("casesensitivity=[Name=STRING, pIctUre=STRING, uRL=STRING]"));
 }
コード例 #19
0
 @Test
 public void testFieldAliasMapping() throws Exception {
   assertThat(
       RestUtils.getMapping("pig/fieldalias").skipHeaders().toString(),
       is("fieldalias=[@timestamp=DATE, name=STRING, picture=STRING, url=STRING]"));
 }
コード例 #20
0
 @Test
 public void testTimestampMapping() throws Exception {
   String mapping = RestUtils.getMapping("pig/timestamp").skipHeaders().toString();
   assertThat(mapping, startsWith("timestamp=[name=STRING, org.apache.pig.builtin.todate_"));
   assertThat(mapping, endsWith("=DATE, url=STRING]"));
 }
 @Before
 public void before() throws Exception {
   RestUtils.refresh(indexPrefix + "cascading-local");
 }