@Test
  public void testDefaultApplied() throws IOException {
    createIndex("test1", ImmutableSettings.settingsBuilder().build());
    createIndex("test2", ImmutableSettings.settingsBuilder().build());
    XContentBuilder defaultMapping =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject(MapperService.DEFAULT_MAPPING)
            .startObject("_size")
            .field("enabled", true)
            .endObject()
            .endObject()
            .endObject();
    client()
        .admin()
        .indices()
        .preparePutMapping()
        .setType(MapperService.DEFAULT_MAPPING)
        .setSource(defaultMapping)
        .get();
    XContentBuilder typeMapping =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type")
            .startObject("_all")
            .field("enabled", false)
            .endObject()
            .endObject()
            .endObject();
    client()
        .admin()
        .indices()
        .preparePutMapping("test1")
        .setType("type")
        .setSource(typeMapping)
        .get();
    client()
        .admin()
        .indices()
        .preparePutMapping("test1", "test2")
        .setType("type")
        .setSource(typeMapping)
        .get();

    GetMappingsResponse response = client().admin().indices().prepareGetMappings("test2").get();
    assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_all"));
    assertFalse(
        (Boolean)
            ((LinkedHashMap)
                    response.getMappings().get("test2").get("type").getSourceAsMap().get("_all"))
                .get("enabled"));
    assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_size"));
    assertTrue(
        (Boolean)
            ((LinkedHashMap)
                    response.getMappings().get("test2").get("type").getSourceAsMap().get("_size"))
                .get("enabled"));
  }
  public void testTypeNotCreatedOnIndexFailure() throws IOException, InterruptedException {
    XContentBuilder mapping =
        jsonBuilder()
            .startObject()
            .startObject("_default_")
            .field("dynamic", "strict")
            .endObject()
            .endObject();

    IndexService indexService = createIndex("test", Settings.EMPTY, "_default_", mapping);

    try {
      client()
          .prepareIndex()
          .setIndex("test")
          .setType("type")
          .setSource(jsonBuilder().startObject().field("test", "test").endObject())
          .get();
      fail();
    } catch (StrictDynamicMappingException e) {

    }

    GetMappingsResponse getMappingsResponse =
        client().admin().indices().prepareGetMappings("test").get();
    assertNull(getMappingsResponse.getMappings().get("test").get("type"));
  }
 private static void assertMappingsHaveField(
     GetMappingsResponse mappings, String index, String type, String field) throws IOException {
   ImmutableOpenMap<String, MappingMetaData> indexMappings = mappings.getMappings().get("index");
   assertNotNull(indexMappings);
   MappingMetaData typeMappings = indexMappings.get(type);
   assertNotNull(typeMappings);
   Map<String, Object> typeMappingsMap = typeMappings.getSourceAsMap();
   Map<String, Object> properties = (Map<String, Object>) typeMappingsMap.get("properties");
   assertTrue(
       "Could not find [" + field + "] in " + typeMappingsMap.toString(),
       properties.containsKey(field));
 }
Example #4
0
  private void assertTTLMappingEnabled(String index, String type) throws IOException {
    String errMsg =
        String.format(
            Locale.ROOT, "Expected ttl field mapping to be enabled for %s/%s", index, type);

    GetMappingsResponse getMappingsResponse =
        client().admin().indices().prepareGetMappings(index).addTypes(type).get();
    Map<String, Object> mappingSource =
        getMappingsResponse.getMappings().get(index).get(type).getSourceAsMap();
    assertThat(errMsg, mappingSource, hasKey("_ttl"));
    String ttlAsString = mappingSource.get("_ttl").toString();
    assertThat(ttlAsString, is(notNullValue()));
    assertThat(errMsg, ttlAsString, is("{enabled=true}"));
  }
  @Test
  public void testMappingNodeClient() throws Exception {
    final NodeClient es =
        new NodeClient().flushInterval(TimeValue.timeValueSeconds(5)).newClient(client("1"));
    es.addMapping("test", "{\"test\":{\"properties\":{\"location\":{\"type\":\"geo_point\"}}}}");
    es.newIndex("test");

    GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test");
    GetMappingsResponse getMappingsResponse =
        es.client().admin().indices().getMappings(getMappingsRequest).actionGet();

    logger.info("mappings={}", getMappingsResponse.getMappings());

    es.shutdown();
    if (es.hasThrowable()) {
      logger.error("error", es.getThrowable());
    }
    assertFalse(es.hasThrowable());
  }
  public void testMappingsPropagatedToMasterNodeImmediately() throws IOException {
    createIndex("index");

    // works when the type has been dynamically created
    client().prepareIndex("index", "type", "1").setSource("foo", 3).get();
    GetMappingsResponse mappings =
        client().admin().indices().prepareGetMappings("index").setTypes("type").get();
    assertMappingsHaveField(mappings, "index", "type", "foo");

    // works if the type already existed
    client().prepareIndex("index", "type", "1").setSource("bar", "baz").get();
    mappings = client().admin().indices().prepareGetMappings("index").setTypes("type").get();
    assertMappingsHaveField(mappings, "index", "type", "bar");

    // works if we indexed an empty document
    client().prepareIndex("index", "type2", "1").setSource().get();
    mappings = client().admin().indices().prepareGetMappings("index").setTypes("type2").get();
    assertTrue(
        mappings.getMappings().get("index").toString(),
        mappings.getMappings().get("index").containsKey("type2"));
  }
  /**
   * Until we can verify that all instances have moved over to our new mapping, we need to handcheck
   * all index mappings to make sure they have doc.associations.assoc_index as a doc value
   *
   * @param communityIdStrs
   * @return
   */
  @SuppressWarnings("unchecked")
  private static boolean validateAssociationMapping(String[] communityIdStrs) {
    // get all index mappings associated with these commids
    String[] mappings = new String[communityIdStrs.length];
    StringBuilder sb = new StringBuilder(", ");
    for (int i = 0; i < communityIdStrs.length; i++) {
      String s = communityIdStrs[i];
      mappings[i] = "doc_" + s + "*";
      sb.append("doc_").append(s).append("*, ");
    }
    ElasticSearchManager esm = ElasticSearchManager.getIndex(sb.substring(2, sb.length()));
    GetMappingsResponse response =
        esm.getRawClient().admin().indices().prepareGetMappings(mappings).get();
    for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> mapping :
        response.getMappings()) {
      ImmutableOpenMap<String, MappingMetaData> mappingVal = mapping.value;
      MappingMetaData mapping_meta = mappingVal.get("document_index");
      try {
        Map<String, Object> map = mapping_meta.getSourceAsMap();
        Map<String, Object> props = (Map<String, Object>) map.get("properties");
        Map<String, Object> assocs = (Map<String, Object>) props.get(DocumentPojo.associations_);
        Map<String, Object> assocs_props = (Map<String, Object>) assocs.get("properties");
        Map<String, Object> assoc_index =
            (Map<String, Object>) assocs_props.get(AssociationPojo.assoc_index_);
        if (!assoc_index.containsKey("doc_values") || !((Boolean) assoc_index.get("doc_values"))) {
          // doc values doesn't exist in mapping or was false
          return false;
        }

      } catch (Exception ex) {
        // failed somehow
        return false;
      }
    }
    // if we fell through, all the checked indexes had the doc_value field set
    return true;
  }
  @Test
  public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception {
    String masterNode = startMasterNode();
    String redNodeDataPath = createTempDir().toString();
    String redNode = startDataNode("red", redNodeDataPath);
    // create red_index on red_node and same for red
    client()
        .admin()
        .cluster()
        .health(clusterHealthRequest().waitForYellowStatus().waitForNodes("2"))
        .get();
    assertAcked(
        prepareCreate("red_index")
            .setSettings(
                Settings.builder()
                    .put("index.number_of_replicas", 0)
                    .put(FilterAllocationDecider.INDEX_ROUTING_INCLUDE_GROUP + "color", "red")));
    index(
        "red_index",
        "doc",
        "1",
        jsonBuilder().startObject().field("text", "some text").endObject());

    logger.info("--> wait for green red_index");
    ensureGreen();
    logger.info("--> wait for meta state written for red_index");
    assertIndexInMetaState(redNode, "red_index");
    assertIndexInMetaState(masterNode, "red_index");

    waitForConcreteMappingsOnAll("red_index", "doc", "text");

    logger.info("--> close red_index");
    client().admin().indices().prepareClose("red_index").get();
    // close the index
    ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
    assertThat(
        clusterStateResponse.getState().getMetaData().index("red_index").getState().name(),
        equalTo(IndexMetaData.State.CLOSE.name()));

    logger.info("--> restart red node");
    stopNode(redNode);
    redNode = startDataNode("red", redNodeDataPath);
    client()
        .admin()
        .indices()
        .preparePutMapping("red_index")
        .setType("doc")
        .setSource(
            jsonBuilder()
                .startObject()
                .startObject("properties")
                .startObject("integer_field")
                .field("type", "integer")
                .endObject()
                .endObject()
                .endObject())
        .get();

    GetMappingsResponse getMappingsResponse =
        client().admin().indices().prepareGetMappings("red_index").addTypes("doc").get();
    assertNotNull(
        ((LinkedHashMap)
                (getMappingsResponse
                    .getMappings()
                    .get("red_index")
                    .get("doc")
                    .getSourceAsMap()
                    .get("properties")))
            .get("integer_field"));
    // restart master with empty data folder and maybe red node
    ((InternalTestCluster) cluster()).stopCurrentMasterNode();
    masterNode = startMasterNode();

    ensureGreen("red_index");
    assertIndexInMetaState(redNode, "red_index");
    assertIndexInMetaState(masterNode, "red_index");
    clusterStateResponse = client().admin().cluster().prepareState().get();
    assertThat(
        clusterStateResponse.getState().getMetaData().index("red_index").getState().name(),
        equalTo(IndexMetaData.State.CLOSE.name()));
    getMappingsResponse =
        client().admin().indices().prepareGetMappings("red_index").addTypes("doc").get();
    assertNotNull(
        ((LinkedHashMap)
                (getMappingsResponse
                    .getMappings()
                    .get("red_index")
                    .get("doc")
                    .getSourceAsMap()
                    .get("properties")))
            .get("integer_field"));
  }
  @SuppressWarnings("unchecked")
  @Test
  public void updateDefaultMappingSettings() throws Exception {

    logger.info("Creating index with _default_ mappings");
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .addMapping(
            MapperService.DEFAULT_MAPPING,
            JsonXContent.contentBuilder()
                .startObject()
                .startObject(MapperService.DEFAULT_MAPPING)
                .field("date_detection", false)
                .endObject()
                .endObject())
        .get();

    GetMappingsResponse getResponse =
        client()
            .admin()
            .indices()
            .prepareGetMappings("test")
            .addTypes(MapperService.DEFAULT_MAPPING)
            .get();
    Map<String, Object> defaultMapping =
        getResponse.getMappings().get("test").get(MapperService.DEFAULT_MAPPING).sourceAsMap();
    assertThat(defaultMapping, hasKey("date_detection"));

    logger.info("Emptying _default_ mappings");
    // now remove it
    PutMappingResponse putResponse =
        client()
            .admin()
            .indices()
            .preparePutMapping("test")
            .setType(MapperService.DEFAULT_MAPPING)
            .setSource(
                JsonXContent.contentBuilder()
                    .startObject()
                    .startObject(MapperService.DEFAULT_MAPPING)
                    .endObject()
                    .endObject())
            .get();
    assertThat(putResponse.isAcknowledged(), equalTo(true));
    logger.info("Done Emptying _default_ mappings");

    getResponse =
        client()
            .admin()
            .indices()
            .prepareGetMappings("test")
            .addTypes(MapperService.DEFAULT_MAPPING)
            .get();
    defaultMapping =
        getResponse.getMappings().get("test").get(MapperService.DEFAULT_MAPPING).sourceAsMap();
    assertThat(defaultMapping, not(hasKey("date_detection")));

    // now test you can change stuff that are normally unchangeable
    logger.info("Creating _default_ mappings with an analyzed field");
    putResponse =
        client()
            .admin()
            .indices()
            .preparePutMapping("test")
            .setType(MapperService.DEFAULT_MAPPING)
            .setSource(
                JsonXContent.contentBuilder()
                    .startObject()
                    .startObject(MapperService.DEFAULT_MAPPING)
                    .startObject("properties")
                    .startObject("f")
                    .field("type", "string")
                    .field("index", "analyzed")
                    .endObject()
                    .endObject()
                    .endObject()
                    .endObject())
            .get();
    assertThat(putResponse.isAcknowledged(), equalTo(true));

    logger.info("Changing _default_ mappings field from analyzed to non-analyzed");
    putResponse =
        client()
            .admin()
            .indices()
            .preparePutMapping("test")
            .setType(MapperService.DEFAULT_MAPPING)
            .setSource(
                JsonXContent.contentBuilder()
                    .startObject()
                    .startObject(MapperService.DEFAULT_MAPPING)
                    .startObject("properties")
                    .startObject("f")
                    .field("type", "string")
                    .field("index", "not_analyzed")
                    .endObject()
                    .endObject()
                    .endObject()
                    .endObject())
            .get();
    assertThat(putResponse.isAcknowledged(), equalTo(true));
    logger.info("Done changing _default_ mappings field from analyzed to non-analyzed");

    getResponse =
        client()
            .admin()
            .indices()
            .prepareGetMappings("test")
            .addTypes(MapperService.DEFAULT_MAPPING)
            .get();
    defaultMapping =
        getResponse.getMappings().get("test").get(MapperService.DEFAULT_MAPPING).sourceAsMap();
    Map<String, Object> fieldSettings =
        (Map<String, Object>) ((Map) defaultMapping.get("properties")).get("f");
    assertThat(fieldSettings, hasEntry("index", (Object) "not_analyzed"));

    // but we still validate the _default_ type
    logger.info("Confirming _default_ mappings validation");
    assertThrows(
        client()
            .admin()
            .indices()
            .preparePutMapping("test")
            .setType(MapperService.DEFAULT_MAPPING)
            .setSource(
                JsonXContent.contentBuilder()
                    .startObject()
                    .startObject(MapperService.DEFAULT_MAPPING)
                    .startObject("properties")
                    .startObject("f")
                    .field("type", "DOESNT_EXIST")
                    .endObject()
                    .endObject()
                    .endObject()
                    .endObject()),
        MapperParsingException.class);
  }
  @Test
  public void test_ageOut() throws IOException, InterruptedException, ExecutionException {

    // Call test_endToEnd_autoTime to create 5 time based indexes
    // 2015-01-01 -> 2015-05-01
    // How far is now from 2015-05-03
    final Date d = TimeUtils.getDateFromSuffix("2015-03-02").success();
    final long total_time_ms = new Date().getTime() - d.getTime();
    final long total_days = total_time_ms / (1000L * 3600L * 24L);
    final String age_out = ErrorUtils.get("{0} days", total_days);

    final DataBucketBean bucket =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("full_name", "/test/end-end/auto-time")
            .with(
                DataBucketBean::data_schema,
                BeanTemplateUtils.build(DataSchemaBean.class)
                    .with(
                        DataSchemaBean::temporal_schema,
                        BeanTemplateUtils.build(TemporalSchemaBean.class)
                            .with(TemporalSchemaBean::exist_age_max, age_out)
                            .done()
                            .get())
                    .done()
                    .get())
            .done()
            .get();

    final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket);

    test_endToEnd_autoTime(false);

    _index_service
        ._crud_factory
        .getClient()
        .admin()
        .indices()
        .prepareCreate(template_name + "_2015-03-01_1")
        .execute()
        .actionGet();

    final GetMappingsResponse gmr =
        _index_service
            ._crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareGetMappings(template_name + "*")
            .execute()
            .actionGet();
    assertEquals(6, gmr.getMappings().keys().size());

    CompletableFuture<BasicMessageBean> cf =
        _index_service.getDataService().get().handleAgeOutRequest(bucket);

    BasicMessageBean res = cf.get();

    assertEquals(true, res.success());
    assertTrue("sensible message: " + res.message(), res.message().contains(" 2 "));

    assertTrue(
        "Message marked as loggable: " + res.details(),
        Optional.ofNullable(res.details()).filter(m -> m.containsKey("loggable")).isPresent());

    System.out.println("Return from to delete: " + res.message());

    Thread.sleep(5000L); // give the indexes time to delete

    final GetMappingsResponse gmr2 =
        _index_service
            ._crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareGetMappings(template_name + "*")
            .execute()
            .actionGet();
    assertEquals(3, gmr2.getMappings().keys().size());

    // Check some edge cases:

    // 1) Run it again, returns success but not loggable:

    CompletableFuture<BasicMessageBean> cf2 =
        _index_service.getDataService().get().handleAgeOutRequest(bucket);

    BasicMessageBean res2 = cf2.get();

    assertEquals(true, res2.success());
    assertTrue("sensible message: " + res2.message(), res2.message().contains(" 0 "));
    assertTrue(
        "Message _not_ marked as loggable: " + res2.details(),
        !Optional.ofNullable(res2.details()).map(m -> m.get("loggable")).isPresent());

    // 2) No temporal settings

    final DataBucketBean bucket3 =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("full_name", "/test/handle/age/out/delete/not/temporal")
            .with(
                DataBucketBean::data_schema,
                BeanTemplateUtils.build(DataSchemaBean.class).done().get())
            .done()
            .get();

    CompletableFuture<BasicMessageBean> cf3 =
        _index_service.getDataService().get().handleAgeOutRequest(bucket3);
    BasicMessageBean res3 = cf3.get();
    // no temporal settings => returns success
    assertEquals(true, res3.success());

    // 3) Unparseable temporal settings (in theory won't validate but we can test here)

    final DataBucketBean bucket4 =
        BeanTemplateUtils.build(DataBucketBean.class)
            .with("full_name", "/test/handle/age/out/delete/temporal/malformed")
            .with(
                DataBucketBean::data_schema,
                BeanTemplateUtils.build(DataSchemaBean.class)
                    .with(
                        DataSchemaBean::temporal_schema,
                        BeanTemplateUtils.build(TemporalSchemaBean.class)
                            .with(TemporalSchemaBean::exist_age_max, "bananas")
                            .done()
                            .get())
                    .done()
                    .get())
            .done()
            .get();

    CompletableFuture<BasicMessageBean> cf4 =
        _index_service.getDataService().get().handleAgeOutRequest(bucket4);
    BasicMessageBean res4 = cf4.get();
    // no temporal settings => returns success
    assertEquals(false, res4.success());
  }
  @Test
  public void test_endToEnd_fixedFixed()
      throws IOException, InterruptedException, ExecutionException {
    final Calendar time_setter = GregorianCalendar.getInstance();
    time_setter.set(2015, 1, 1, 13, 0, 0);
    final String bucket_str =
        Resources.toString(
            Resources.getResource(
                "com/ikanow/aleph2/search_service/elasticsearch/services/test_end_2_end_bucket2.json"),
            Charsets.UTF_8);
    final DataBucketBean bucket =
        BeanTemplateUtils.build(bucket_str, DataBucketBean.class)
            .with("_id", "2b_test_end_2_end")
            .with("full_name", "/test/end-end/fixed/fixed")
            .with("modified", time_setter.getTime())
            .done()
            .get();

    final String template_name = ElasticsearchIndexUtils.getBaseIndexName(bucket);

    // Check starting from clean

    {
      try {
        _crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareDeleteTemplate(template_name)
            .execute()
            .actionGet();
      } catch (Exception e) {
      } // (This is fine, just means it doesn't exist)
      try {
        _crud_factory
            .getClient()
            .admin()
            .indices()
            .prepareDelete(template_name + "*")
            .execute()
            .actionGet();
      } catch (Exception e) {
      } // (This is fine, just means it doesn't exist)

      final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr =
          _crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
      assertTrue("No templates to start with", gtr.getIndexTemplates().isEmpty());
    }

    final ICrudService<JsonNode> index_service_crud =
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, bucket, Optional.empty(), Optional.empty()))
            .flatMap(IDataWriteService::getCrudService)
            .get();

    // Check template added:

    {
      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr2.getIndexTemplates().size());
    }

    // Get batch sub-service

    @SuppressWarnings("unchecked")
    final Optional<ICrudService.IBatchSubservice<JsonNode>> batch_service =
        index_service_crud
            .getUnderlyingPlatformDriver(ICrudService.IBatchSubservice.class, Optional.empty())
            .map(t -> (IBatchSubservice<JsonNode>) t);

    {
      assertTrue("Batch service must exist", batch_service.isPresent());
    }

    // Get information about the crud service

    final ElasticsearchContext es_context =
        (ElasticsearchContext)
            index_service_crud
                .getUnderlyingPlatformDriver(ElasticsearchContext.class, Optional.empty())
                .get();

    {
      assertTrue("Read write index", es_context instanceof ElasticsearchContext.ReadWriteContext);
      assertTrue(
          "Temporal index",
          es_context.indexContext()
              instanceof
              ElasticsearchContext.IndexContext.ReadWriteIndexContext.FixedRwIndexContext);
      assertTrue(
          "Auto type",
          es_context.typeContext()
              instanceof ElasticsearchContext.TypeContext.ReadWriteTypeContext.FixedRwTypeContext);
    }

    // Write some docs out

    Arrays.asList(1, 2, 3, 4, 5)
        .stream()
        .map(
            i -> {
              time_setter.set(2015, i, 1, 13, 0, 0);
              return time_setter.getTime();
            })
        .map(d -> (ObjectNode) _mapper.createObjectNode().put("@timestamp", d.getTime()))
        .forEach(
            o -> {
              ObjectNode o1 = o.deepCopy();
              o1.put("val1", 10);
              ObjectNode o2 = o.deepCopy();
              o2.put("val1", "test");
              batch_service.get().storeObject(o1, false);
              batch_service.get().storeObject(o2, false);
            });

    // (give it a chance to run)
    Thread.sleep(5000L);

    final GetMappingsResponse gmr =
        es_context
            .client()
            .admin()
            .indices()
            .prepareGetMappings(template_name + "*")
            .execute()
            .actionGet();

    // Should have 5 different indexes, each with 2 types + _default_

    assertEquals(1, gmr.getMappings().keys().size());
    final Set<String> expected_keys =
        Arrays.asList("test_fixed_fixed__1cb6bdcdf44f").stream().collect(Collectors.toSet());
    final Set<String> expected_types =
        Arrays.asList("data_object").stream().collect(Collectors.toSet());

    StreamSupport.stream(gmr.getMappings().spliterator(), false)
        .forEach(
            x -> {
              assertTrue(
                  "Is one of the expected keys: "
                      + x.key
                      + " vs  "
                      + expected_keys.stream().collect(Collectors.joining(":")),
                  expected_keys.contains(x.key));
              // Size 1: data_object
              assertEquals(1, x.value.size());
              // DEBUG
              // System.out.println(" ? " + x.key);
              StreamSupport.stream(x.value.spliterator(), false)
                  .forEach(
                      Lambdas.wrap_consumer_u(
                          y -> {
                            // DEBUG
                            // System.out.println("?? " + y.key + " --- " +
                            // y.value.sourceAsMap().toString());
                            assertTrue(
                                "Is expected type: " + y.key, expected_types.contains(y.key));
                          }));
            });

    // TEST DELETION:
    test_handleDeleteOrPurge(bucket, false);
  }
  public void test_handleDeleteOrPurge(final DataBucketBean to_handle, boolean delete_not_purge)
      throws InterruptedException, ExecutionException {
    System.out.println("****** Checking delete/purge");

    final String template_name = ElasticsearchIndexUtils.getBaseIndexName(to_handle);
    final ICrudService<JsonNode> index_service_crud =
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, to_handle, Optional.empty(), Optional.empty()))
            .flatMap(IDataWriteService::getCrudService)
            .get();

    final ElasticsearchContext es_context =
        (ElasticsearchContext)
            index_service_crud
                .getUnderlyingPlatformDriver(ElasticsearchContext.class, Optional.empty())
                .get();

    // (Actually first off, check there's data and templates)
    // Data:
    {
      final GetMappingsResponse gmr =
          es_context
              .client()
              .admin()
              .indices()
              .prepareGetMappings(template_name + "*")
              .execute()
              .actionGet();
      assertTrue("There are indexes", gmr.getMappings().keys().size() > 0);
    }
    // Templates:
    {
      final GetIndexTemplatesRequest gt_pre = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr_pre =
          _crud_factory.getClient().admin().indices().getTemplates(gt_pre).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr_pre.getIndexTemplates().size());
    }

    // Then, perform request
    final BasicMessageBean result =
        _index_service
            .getDataService()
            .get()
            .handleBucketDeletionRequest(to_handle, Optional.empty(), delete_not_purge)
            .get();
    assertEquals("Deletion should succeed: " + result.message(), true, result.success());

    // Check templates gone iff deleting not purging

    if (delete_not_purge) {
      final GetIndexTemplatesRequest gt = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr =
          _crud_factory.getClient().admin().indices().getTemplates(gt).actionGet();
      assertTrue("No templates after deletion", gtr.getIndexTemplates().isEmpty());
    } else {
      final GetIndexTemplatesRequest gt2 = new GetIndexTemplatesRequest().names(template_name);
      final GetIndexTemplatesResponse gtr2 =
          _crud_factory.getClient().admin().indices().getTemplates(gt2).actionGet();
      assertEquals(1, _index_service._bucket_template_cache.size());
      assertEquals(1, gtr2.getIndexTemplates().size());
    }

    // Check all files deleted

    // Check via mappings
    {
      final GetMappingsResponse gmr =
          es_context
              .client()
              .admin()
              .indices()
              .prepareGetMappings(template_name + "*")
              .execute()
              .actionGet();
      assertEquals(0, gmr.getMappings().keys().size());
    }
    // Check via index size (recreates templates)

    final ICrudService<JsonNode> index_service_crud_2 =
        _index_service
            .getDataService()
            .flatMap(
                s ->
                    s.getWritableDataService(
                        JsonNode.class, to_handle, Optional.empty(), Optional.empty()))
            .flatMap(IDataWriteService::getCrudService)
            .get();

    assertEquals(0, index_service_crud_2.countObjects().get().intValue());
  }
  @SuppressWarnings("unchecked")
  @Test
  public void updateIncludeExclude() throws Exception {
    assertAcked(
        prepareCreate("test")
            .addMapping(
                "type",
                jsonBuilder()
                    .startObject()
                    .startObject("type")
                    .startObject("properties")
                    .startObject("normal")
                    .field("type", "long")
                    .endObject()
                    .startObject("exclude")
                    .field("type", "long")
                    .endObject()
                    .startObject("include")
                    .field("type", "long")
                    .endObject()
                    .endObject()
                    .endObject()
                    .endObject()));
    ensureGreen(); // make sure that replicas are initialized so the refresh command will work them
                   // too

    logger.info("Index doc");
    index(
        "test",
        "type",
        "1",
        JsonXContent.contentBuilder()
            .startObject()
            .field("normal", 1)
            .field("exclude", 1)
            .field("include", 1)
            .endObject());
    refresh(); // commit it for later testing.

    logger.info("Adding exclude settings");
    PutMappingResponse putResponse =
        client()
            .admin()
            .indices()
            .preparePutMapping("test")
            .setType("type")
            .setSource(
                JsonXContent.contentBuilder()
                    .startObject()
                    .startObject("type")
                    .startObject("_source")
                    .startArray("excludes")
                    .value("exclude")
                    .endArray()
                    .endObject()
                    .endObject())
            .get();

    assertTrue(putResponse.isAcknowledged());

    // changed mapping doesn't affect indexed documents (checking backward compatibility)
    GetResponse getResponse = client().prepareGet("test", "type", "1").setRealtime(false).get();
    assertThat(getResponse.getSource(), hasKey("normal"));
    assertThat(getResponse.getSource(), hasKey("exclude"));
    assertThat(getResponse.getSource(), hasKey("include"));

    logger.info("Index doc again");
    index(
        "test",
        "type",
        "1",
        JsonXContent.contentBuilder()
            .startObject()
            .field("normal", 2)
            .field("exclude", 1)
            .field("include", 2)
            .endObject());

    // but do affect newly indexed docs
    getResponse = get("test", "type", "1");
    assertThat(getResponse.getSource(), hasKey("normal"));
    assertThat(getResponse.getSource(), not(hasKey("exclude")));
    assertThat(getResponse.getSource(), hasKey("include"));

    logger.info("Changing mapping to includes");
    putResponse =
        client()
            .admin()
            .indices()
            .preparePutMapping("test")
            .setType("type")
            .setSource(
                JsonXContent.contentBuilder()
                    .startObject()
                    .startObject("type")
                    .startObject("_source")
                    .startArray("excludes")
                    .endArray()
                    .startArray("includes")
                    .value("include")
                    .endArray()
                    .endObject()
                    .endObject())
            .get();
    assertTrue(putResponse.isAcknowledged());

    GetMappingsResponse getMappingsResponse =
        client().admin().indices().prepareGetMappings("test").get();
    MappingMetaData typeMapping = getMappingsResponse.getMappings().get("test").get("type");
    assertThat(
        (Map<String, Object>) typeMapping.getSourceAsMap().get("_source"), hasKey("includes"));
    ArrayList<String> includes =
        (ArrayList<String>)
            ((Map<String, Object>) typeMapping.getSourceAsMap().get("_source")).get("includes");
    assertThat(includes, contains("include"));
    assertThat(
        (Map<String, Object>) typeMapping.getSourceAsMap().get("_source"), hasKey("excludes"));
    assertThat(
        (ArrayList<String>)
            ((Map<String, Object>) typeMapping.getSourceAsMap().get("_source")).get("excludes"),
        emptyIterable());

    logger.info("Indexing doc yet again");
    index(
        "test",
        "type",
        "1",
        JsonXContent.contentBuilder()
            .startObject()
            .field("normal", 3)
            .field("exclude", 3)
            .field("include", 3)
            .endObject());

    getResponse = get("test", "type", "1");
    assertThat(getResponse.getSource(), not(hasKey("normal")));
    assertThat(getResponse.getSource(), not(hasKey("exclude")));
    assertThat(getResponse.getSource(), hasKey("include"));

    logger.info("Adding excludes, but keep includes");
    putResponse =
        client()
            .admin()
            .indices()
            .preparePutMapping("test")
            .setType("type")
            .setSource(
                JsonXContent.contentBuilder()
                    .startObject()
                    .startObject("type")
                    .startObject("_source")
                    .startArray("excludes")
                    .value("*.excludes")
                    .endArray()
                    .endObject()
                    .endObject())
            .get();
    assertTrue(putResponse.isAcknowledged());

    getMappingsResponse = client().admin().indices().prepareGetMappings("test").get();
    typeMapping = getMappingsResponse.getMappings().get("test").get("type");
    assertThat(
        (Map<String, Object>) typeMapping.getSourceAsMap().get("_source"), hasKey("includes"));
    includes =
        (ArrayList<String>)
            ((Map<String, Object>) typeMapping.getSourceAsMap().get("_source")).get("includes");
    assertThat(includes, contains("include"));
    assertThat(
        (Map<String, Object>) typeMapping.getSourceAsMap().get("_source"), hasKey("excludes"));
    ArrayList<String> excludes =
        (ArrayList<String>)
            ((Map<String, Object>) typeMapping.getSourceAsMap().get("_source")).get("excludes");
    assertThat(excludes, contains("*.excludes"));
  }