Esempio n. 1
0
 private Settings getSettings(int nodeOrdinal, Settings others) {
   Builder builder = ImmutableSettings.settingsBuilder().put(defaultSettings);
   Settings settings = nodeSettingsSource.settings(nodeOrdinal);
   if (settings != null) {
     builder.put(settings);
   }
   if (others != null) {
     builder.put(others);
   }
   return builder.build();
 }
  public void start() throws Exception {
    try {
      final Builder settings = ImmutableSettings.settingsBuilder();
      settings.put("path.data", filesSettings.getPermanentDirectory());
      settings.put("client.transport.sniff", true);
      settings.build();

      final NodeBuilder nb = nodeBuilder().settings(settings).local(true).client(false).data(true);
      final Node node = nb.node();
      client = node.client();

      IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest("entities");

      // we just send back a response, no need to fork a listener
      indicesExistsRequest.listenerThreaded(false);
      client
          .admin()
          .indices()
          .exists(
              indicesExistsRequest,
              new ActionListener<IndicesExistsResponse>() {
                public void onResponse(IndicesExistsResponse response) {
                  if (!response.isExists()) {
                    try {
                      logger.debug("Entities indice does not exists. Creating it...");
                      CreateIndexResponse r =
                          client
                              .admin()
                              .indices()
                              .create(new CreateIndexRequest("entities"))
                              .actionGet();

                      logger.debug(
                          "Created indice with response {}",
                          r.isAcknowledged() ? "\u2713 acknowledged" : "\2718 not acknowledged");
                    } catch (Exception e) {
                      logger.error("Failed to create entities indice status ...");
                    }
                  }

                  updateMappings();
                }

                public void onFailure(Throwable e) {
                  logger.error("Failed to enquiry entities indice status ...", e);
                }
              });
    } catch (Exception e) {
      throw new InitializationException("Failed to initialize embedded elastic search", e);
    }
  }
Esempio n. 3
0
 TestCluster(
     long clusterSeed, int numNodes, String clusterName, NodeSettingsSource nodeSettingsSource) {
   this.clusterName = clusterName;
   Random random = new Random(clusterSeed);
   numSharedNodes =
       numNodes == -1 ? 2 + random.nextInt(4) : numNodes; // at least 2 nodes if randomized
   assert numSharedNodes >= 0;
   /*
    *  TODO
    *  - we might want start some master only nodes?
    *  - we could add a flag that returns a client to the master all the time?
    *  - we could add a flag that never returns a client to the master
    *  - along those lines use a dedicated node that is master eligible and let all other nodes be only data nodes
    */
   sharedNodesSeeds = new long[numSharedNodes];
   for (int i = 0; i < sharedNodesSeeds.length; i++) {
     sharedNodesSeeds[i] = random.nextLong();
   }
   logger.info(
       "Setup TestCluster [{}] with seed [{}] using [{}] nodes",
       clusterName,
       SeedUtils.formatSeed(clusterSeed),
       numSharedNodes);
   Builder builder =
       ImmutableSettings.settingsBuilder()
           /* use RAM directories in 10% of the runs */
           //        .put("index.store.type", random.nextInt(10) == 0 ?
           // MockRamIndexStoreModule.class.getName() : MockFSIndexStoreModule.class.getName())
           .put("index.store.type", MockFSIndexStoreModule.class.getName()) // no RAM dir for now!
           .put(IndexEngineModule.EngineSettings.ENGINE_TYPE, MockEngineModule.class.getName())
           .put("cluster.name", clusterName)
           // decrease the routing schedule so new nodes will be added quickly - some random value
           // between 30 and 80 ms
           .put("cluster.routing.schedule", (30 + random.nextInt(50)) + "ms")
           // default to non gateway
           .put("gateway.type", "none");
   if (isLocalTransportConfigured()) {
     builder.put(
         TransportModule.TRANSPORT_TYPE_KEY, AssertingLocalTransportModule.class.getName());
   } else {
     builder.put(Transport.TransportSettings.TRANSPORT_TCP_COMPRESS, random.nextInt(10) == 0);
   }
   this.defaultSettings = builder.build();
   this.nodeSettingsSource = nodeSettingsSource;
 }
  private int indexRandomNumbers(String analyzer, int shards) throws Exception {
    Builder builder =
        ImmutableSettings.settingsBuilder()
            .put(indexSettings())
            .put(SETTING_NUMBER_OF_REPLICAS, between(0, 1));

    if (shards > 0) {
      builder.put(SETTING_NUMBER_OF_SHARDS, shards);
    }

    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .addMapping(
            "type1",
            jsonBuilder()
                .startObject()
                .startObject("type1")
                .startObject("properties")
                .startObject("field1")
                .field("analyzer", analyzer)
                .field("type", "string")
                .endObject()
                .endObject()
                .endObject()
                .endObject())
        .setSettings(builder)
        .get();
    int numDocs = atLeast(100);
    IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
    for (int i = 0; i < numDocs; i++) {
      docs[i] =
          client()
              .prepareIndex("test", "type1", String.valueOf(i))
              .setSource("field1", English.intToEnglish(i));
    }

    indexRandom(true, docs);
    ensureGreen();
    return numDocs;
  }
  @Test
  public void testBackwardsCompatibilityNgramTokenizer()
      throws IllegalArgumentException, IllegalAccessException {
    int iters = atLeast(20);
    for (int i = 0; i < iters; i++) {
      final Index index = new Index("test");
      final String name = "ngr";
      Version v = randomVersion(random());
      if (v.onOrAfter(Version.V_0_90_2)) {
        Builder builder =
            ImmutableSettings.builder()
                .put("min_gram", 2)
                .put("max_gram", 3)
                .put("token_chars", "letter,digit");
        boolean compatVersion = false;
        if ((compatVersion = random().nextBoolean())) {
          builder.put("version", "4." + random().nextInt(3));
        }
        Settings settings = builder.build();
        Settings indexSettings =
            ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build();
        Tokenizer nGramTokenizer =
            new NGramTokenizerFactory(index, indexSettings, name, settings)
                .create(new StringReader("foo bar"));
        if (compatVersion) {
          assertThat(nGramTokenizer, instanceOf(Lucene43NGramTokenizer.class));
        } else {
          assertThat(nGramTokenizer, instanceOf(NGramTokenizer.class));
        }

      } else {
        Settings settings =
            ImmutableSettings.builder().put("min_gram", 2).put("max_gram", 3).build();
        Settings indexSettings =
            ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build();
        Tokenizer nGramTokenizer =
            new NGramTokenizerFactory(index, indexSettings, name, settings)
                .create(new StringReader("foo bar"));
        assertThat(nGramTokenizer, instanceOf(Lucene43NGramTokenizer.class));
      }
    }
  }
  private static Settings normalizeSettings(final Settings settings) {
    Builder builder = settingsBuilder();

    builder.put(settings);

    // automatically overridden
    builder.put("client.type", "transport");
    builder.put("node.client", true);
    builder.put("network.server", false);

    // ensure behavior
    builder.put("node.master", false);
    builder.put("node.data", false);

    // disable HTTP
    builder.put("http.enabled", false);

    return builder.build();
  }
  @Before
  public void setUp() throws Exception {

    System.out.println(
        "--------------------- SETUP "
            + name.getMethodName()
            + " -------------------------------------");

    MockMailbox.resetAll();

    // Instantiates a local node & client

    esSetup = new EsSetup(settingsBuilder.build());

    // Clean all, and creates some indices

    esSetup.execute(deleteAll());
  }
  @Before
  public void buildNode1() throws Exception {
    super.setUp();
    Builder builder = ImmutableSettings.builder();
    builder.put("cluster.routing.schedule", "100ms");
    builder.put("gateway.type", "fs");
    if (between(0, 5) == 0) {
      builder.put("gateway.fs.buffer_size", between(1, 100) + "kb");
    }
    if (between(0, 5) == 0) {
      builder.put("gateway.fs.chunk_size", between(1, 100) + "kb");
    }

    builder.put("index.number_of_replicas", "1");
    builder.put("index.number_of_shards", rarely() ? Integer.toString(between(2, 6)) : "1");
    storeType = rarely() ? "ram" : "fs";
    builder.put("index.store.type", storeType);
    defaultSettings = builder.build();
    buildNode("server1");
    // since we store (by default) the index snapshot under the gateway, resetting it will reset the
    // index data as well
    ((InternalNode) node("server1")).injector().getInstance(Gateway.class).reset();
    closeAllNodes();
  }
Esempio n. 9
0
 public static IndexMetaData readFrom(StreamInput in) throws IOException {
   Builder builder = new Builder(in.readUTF());
   builder.version(in.readLong());
   builder.state(State.fromId(in.readByte()));
   builder.settings(readSettingsFromStream(in));
   int mappingsSize = in.readVInt();
   for (int i = 0; i < mappingsSize; i++) {
     MappingMetaData mappingMd = MappingMetaData.readFrom(in);
     builder.putMapping(mappingMd);
   }
   int aliasesSize = in.readVInt();
   for (int i = 0; i < aliasesSize; i++) {
     AliasMetaData aliasMd = AliasMetaData.Builder.readFrom(in);
     builder.putAlias(aliasMd);
   }
   int customSize = in.readVInt();
   for (int i = 0; i < customSize; i++) {
     String type = in.readUTF();
     Custom customIndexMetaData = lookupFactorySafe(type).readFrom(in);
     builder.putCustom(type, customIndexMetaData);
   }
   return builder.build();
 }
  @Test
  public void testRandomExceptions() throws IOException, InterruptedException, ExecutionException {
    final int numShards = between(1, 5);
    String mapping =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type")
            .startObject("properties")
            .startObject("test")
            .field("type", "string")
            .field("index", "not_analyzed")
            .endObject()
            .endObject()
            .endObject()
            .endObject()
            .string();
    final double exceptionRate;
    final double exceptionOnOpenRate;
    if (frequently()) {
      if (randomBoolean()) {
        if (randomBoolean()) {
          exceptionOnOpenRate = 1.0 / between(5, 100);
          exceptionRate = 0.0d;
        } else {
          exceptionRate = 1.0 / between(5, 100);
          exceptionOnOpenRate = 0.0d;
        }
      } else {
        exceptionOnOpenRate = 1.0 / between(5, 100);
        exceptionRate = 1.0 / between(5, 100);
      }
    } else {
      // rarely no exception
      exceptionRate = 0d;
      exceptionOnOpenRate = 0d;
    }

    Builder settings =
        settingsBuilder()
            .put("index.number_of_shards", numShards)
            .put("index.number_of_replicas", randomIntBetween(0, 1))
            .put(MockDirectoryHelper.RANDOM_IO_EXCEPTION_RATE, exceptionRate)
            .put(MockDirectoryHelper.RANDOM_IO_EXCEPTION_RATE_ON_OPEN, exceptionOnOpenRate)
            .put(MockDirectoryHelper.CHECK_INDEX_ON_CLOSE, true);
    logger.info("creating index: [test] using settings: [{}]", settings.build().getAsMap());
    client()
        .admin()
        .indices()
        .prepareCreate("test")
        .setSettings(settings)
        .addMapping("type", mapping)
        .execute()
        .actionGet();
    ClusterHealthResponse clusterHealthResponse =
        client()
            .admin()
            .cluster()
            .health(
                Requests.clusterHealthRequest()
                    .waitForYellowStatus()
                    .timeout(TimeValue.timeValueSeconds(5)))
            .get(); // it's OK to timeout here
    final int numDocs;
    final boolean expectAllShardsFailed;
    if (clusterHealthResponse.isTimedOut()) {
      /* some seeds just won't let you create the index at all and we enter a ping-pong mode
       * trying one node after another etc. that is ok but we need to make sure we don't wait
       * forever when indexing documents so we set numDocs = 1 and expecte all shards to fail
       * when we search below.*/
      logger.info("ClusterHealth timed out - only index one doc and expect searches to fail");
      numDocs = 1;
      expectAllShardsFailed = true;
    } else {
      numDocs = between(10, 100);
      expectAllShardsFailed = false;
    }
    long numCreated = 0;
    boolean[] added = new boolean[numDocs];
    for (int i = 0; i < numDocs; i++) {
      try {
        IndexResponse indexResponse =
            client()
                .prepareIndex("test", "type", "" + i)
                .setTimeout(TimeValue.timeValueSeconds(1))
                .setSource("test", English.intToEnglish(i))
                .get();
        if (indexResponse.isCreated()) {
          numCreated++;
          added[i] = true;
        }
      } catch (ElasticSearchException ex) {
      }
    }
    logger.info("Start Refresh");
    RefreshResponse refreshResponse =
        client()
            .admin()
            .indices()
            .prepareRefresh("test")
            .execute()
            .get(); // don't assert on failures here
    final boolean refreshFailed =
        refreshResponse.getShardFailures().length != 0 || refreshResponse.getFailedShards() != 0;
    logger.info(
        "Refresh failed [{}] numShardsFailed: [{}], shardFailuresLength: [{}], successfulShards: [{}], totalShards: [{}] ",
        refreshFailed,
        refreshResponse.getFailedShards(),
        refreshResponse.getShardFailures().length,
        refreshResponse.getSuccessfulShards(),
        refreshResponse.getTotalShards());

    final int numSearches = atLeast(10);
    // we don't check anything here really just making sure we don't leave any open files or a
    // broken index behind.
    for (int i = 0; i < numSearches; i++) {
      try {
        int docToQuery = between(0, numDocs - 1);
        long expectedResults = added[docToQuery] ? 1 : 0;
        logger.info("Searching for [test:{}]", English.intToEnglish(docToQuery));
        SearchResponse searchResponse =
            client()
                .prepareSearch()
                .setQuery(QueryBuilders.matchQuery("test", English.intToEnglish(docToQuery)))
                .get();
        logger.info(
            "Successful shards: [{}]  numShards: [{}]",
            searchResponse.getSuccessfulShards(),
            numShards);
        if (searchResponse.getSuccessfulShards() == numShards && !refreshFailed) {
          assertThat(searchResponse.getHits().getTotalHits(), Matchers.equalTo(expectedResults));
        }
        // check match all
        searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchAllQuery()).get();
        if (searchResponse.getSuccessfulShards() == numShards && !refreshFailed) {
          assertThat(searchResponse.getHits().getTotalHits(), Matchers.equalTo(numCreated));
        }

      } catch (SearchPhaseExecutionException ex) {
        if (!expectAllShardsFailed) {
          throw ex;
        } else {
          logger.info("expected SearchPhaseException: [{}]", ex.getMessage());
        }
      }
    }
  }
Esempio n. 11
0
  /** Stuff to do before the server is started. */
  protected void preStart() throws Exception {
    if (ManagerTestUtils.getTestType() == TestType.jpa) {
      TestUtil.setProperty("apiman.hibernate.hbm2ddl.auto", "create-drop");
      TestUtil.setProperty(
          "apiman.hibernate.connection.datasource", "java:/comp/env/jdbc/ApiManagerDS");
      try {
        InitialContext ctx = new InitialContext();
        ensureCtx(ctx, "java:/comp/env");
        ensureCtx(ctx, "java:/comp/env/jdbc");
        String dbOutputPath = System.getProperty("apiman.test.h2-output-dir", null);
        if (dbOutputPath != null) {
          ds = createFileDatasource(new File(dbOutputPath));
        } else {
          ds = createInMemoryDatasource();
        }
        ctx.bind("java:/comp/env/jdbc/ApiManagerDS", ds);
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
    }
    if (ManagerTestUtils.getTestType() == TestType.es && node == null) {
      System.out.println("Creating the ES node.");
      File esHome = new File("target/es");
      String esHomeSP = System.getProperty("apiman.test.es-home", null);
      if (esHomeSP != null) {
        esHome = new File(esHomeSP);
      }
      if (esHome.isDirectory()) {
        FileUtils.deleteDirectory(esHome);
      }
      Builder settings = NodeBuilder.nodeBuilder().settings();
      settings.put("path.home", esHome.getAbsolutePath());
      settings.put("http.port", "6500-6600");
      settings.put("transport.tcp.port", "6600-6700");
      settings.put("discovery.zen.ping.multicast.enabled", false);

      String clusterName = System.getProperty("apiman.test.es-cluster-name", ES_CLUSTER_NAME);

      boolean isPersistent =
          "true".equals(System.getProperty("apiman.test.es-persistence", "false"));
      if (!isPersistent) {
        System.out.println("Creating non-persistent ES");
        settings
            .put("index.store.type", "memory")
            .put("gateway.type", "none")
            .put("index.number_of_shards", 1)
            .put("index.number_of_replicas", 1);
        node =
            NodeBuilder.nodeBuilder()
                .client(false)
                .clusterName(clusterName)
                .data(true)
                .local(true)
                .settings(settings)
                .build();
      } else {
        System.out.println("Creating *persistent* ES here: " + esHome);
        node =
            NodeBuilder.nodeBuilder()
                .client(false)
                .clusterName(clusterName)
                .data(true)
                .local(false)
                .settings(settings)
                .build();
      }

      System.out.println("Starting the ES node.");
      node.start();
      System.out.println("ES node was successfully started.");

      // TODO parameterize this
      String connectionUrl = "http://localhost:6500";
      JestClientFactory factory = new JestClientFactory();
      factory.setHttpClientConfig(
          new HttpClientConfig.Builder(connectionUrl).multiThreaded(true).build());
      client = factory.getObject();
      ES_CLIENT = client;
    }
  }
  @Test
  public void testMoreDocs() throws Exception {
    Builder builder = ImmutableSettings.builder();
    builder.put("index.analysis.analyzer.synonym.tokenizer", "whitespace");
    builder.putArray("index.analysis.analyzer.synonym.filter", "synonym", "lowercase");
    builder.put("index.analysis.filter.synonym.type", "synonym");
    builder.putArray(
        "index.analysis.filter.synonym.synonyms", "ave => ave, avenue", "street => str, street");

    XContentBuilder mapping =
        XContentFactory.jsonBuilder()
            .startObject()
            .startObject("type1")
            .startObject("properties")
            .startObject("field1")
            .field("type", "string")
            .field("index_analyzer", "whitespace")
            .field("search_analyzer", "synonym")
            .endObject()
            .endObject()
            .endObject()
            .endObject();

    assertAcked(
        client()
            .admin()
            .indices()
            .prepareCreate("test")
            .addMapping("type1", mapping)
            .setSettings(builder.put("index.number_of_shards", 1)));

    client()
        .prepareIndex("test", "type1", "1")
        .setSource("field1", "massachusetts avenue boston massachusetts")
        .execute()
        .actionGet();
    client()
        .prepareIndex("test", "type1", "2")
        .setSource("field1", "lexington avenue boston massachusetts")
        .execute()
        .actionGet();
    client()
        .prepareIndex("test", "type1", "3")
        .setSource("field1", "boston avenue lexington massachusetts")
        .execute()
        .actionGet();
    client().admin().indices().prepareRefresh("test").execute().actionGet();
    client()
        .prepareIndex("test", "type1", "4")
        .setSource("field1", "boston road lexington massachusetts")
        .execute()
        .actionGet();
    client()
        .prepareIndex("test", "type1", "5")
        .setSource("field1", "lexington street lexington massachusetts")
        .execute()
        .actionGet();
    client()
        .prepareIndex("test", "type1", "6")
        .setSource("field1", "massachusetts avenue lexington massachusetts")
        .execute()
        .actionGet();
    client()
        .prepareIndex("test", "type1", "7")
        .setSource("field1", "bosten street san franciso california")
        .execute()
        .actionGet();
    client().admin().indices().prepareRefresh("test").execute().actionGet();
    client()
        .prepareIndex("test", "type1", "8")
        .setSource("field1", "hollywood boulevard los angeles california")
        .execute()
        .actionGet();
    client()
        .prepareIndex("test", "type1", "9")
        .setSource("field1", "1st street boston massachussetts")
        .execute()
        .actionGet();
    client()
        .prepareIndex("test", "type1", "10")
        .setSource("field1", "1st street boston massachusetts")
        .execute()
        .actionGet();
    client().admin().indices().prepareRefresh("test").execute().actionGet();
    client()
        .prepareIndex("test", "type1", "11")
        .setSource("field1", "2st street boston massachusetts")
        .execute()
        .actionGet();
    client()
        .prepareIndex("test", "type1", "12")
        .setSource("field1", "3st street boston massachusetts")
        .execute()
        .actionGet();
    client().admin().indices().prepareRefresh("test").execute().actionGet();
    SearchResponse searchResponse =
        client()
            .prepareSearch()
            .setQuery(
                QueryBuilders.matchQuery("field1", "lexington avenue massachusetts")
                    .operator(MatchQueryBuilder.Operator.OR))
            .setFrom(0)
            .setSize(5)
            .setRescorer(
                RescoreBuilder.queryRescorer(
                        QueryBuilders.matchPhraseQuery("field1", "lexington avenue massachusetts")
                            .slop(3))
                    .setQueryWeight(0.6f)
                    .setRescoreQueryWeight(2.0f))
            .setRescoreWindow(20)
            .execute()
            .actionGet();

    assertThat(searchResponse.getHits().hits().length, equalTo(5));
    assertHitCount(searchResponse, 9);
    assertFirstHit(searchResponse, hasId("2"));
    assertSecondHit(searchResponse, hasId("6"));
    assertThirdHit(searchResponse, hasId("3"));

    searchResponse =
        client()
            .prepareSearch()
            .setQuery(
                QueryBuilders.matchQuery("field1", "lexington avenue massachusetts")
                    .operator(MatchQueryBuilder.Operator.OR))
            .setFrom(0)
            .setSize(5)
            .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
            .setRescorer(
                RescoreBuilder.queryRescorer(
                        QueryBuilders.matchPhraseQuery("field1", "lexington avenue massachusetts")
                            .slop(3))
                    .setQueryWeight(0.6f)
                    .setRescoreQueryWeight(2.0f))
            .setRescoreWindow(20)
            .execute()
            .actionGet();

    assertThat(searchResponse.getHits().hits().length, equalTo(5));
    assertHitCount(searchResponse, 9);
    assertFirstHit(searchResponse, hasId("2"));
    assertSecondHit(searchResponse, hasId("6"));
    assertThirdHit(searchResponse, hasId("3"));
  }
  public static void createCommunityDocIndex(
      String nameOrCommunityIdStr,
      ObjectId parentCommunityId,
      boolean bPersonalGroup,
      boolean bSystemGroup,
      boolean bClearIndex,
      boolean bParentsOnly) {
    // create elasticsearch indexes
    PropertiesManager pm = new PropertiesManager();
    boolean languageNormalization = pm.getNormalizeEncoding();
    int nPreferredReplicas = pm.getMaxIndexReplicas();

    String docMapping =
        new Gson().toJson(new DocumentPojoIndexMap.Mapping(), DocumentPojoIndexMap.Mapping.class);

    String sGroupIndex = null;
    try {
      sGroupIndex =
          new StringBuffer("doc_").append(new ObjectId(nameOrCommunityIdStr).toString()).toString();
    } catch (Exception e) {
      sGroupIndex = nameOrCommunityIdStr;
    }

    if (!bPersonalGroup) {

      String parentCommunityIdStr = null;
      if (null != parentCommunityId) {
        parentCommunityIdStr = parentCommunityId.toString();
      }

      if ((null == parentCommunityIdStr)
          || (parentCommunityIdStr.equals("4c927585d591d31d7b37097a"))) {
        // (system community is hardwired - children of this community are ignored)

        int nShards = bSystemGroup ? 10 : 5; // (system group is largest)

        // Remove the alias, in case it exists:
        // Then create an index with this name:
        Builder localSettingsGroupIndex = ImmutableSettings.settingsBuilder();
        localSettingsGroupIndex
            .put("number_of_shards", nShards)
            .put("number_of_replicas", nPreferredReplicas);
        if (languageNormalization) {
          localSettingsGroupIndex.put("index.analysis.analyzer.default.tokenizer", "standard");
          localSettingsGroupIndex.putArray(
              "index.analysis.analyzer.default.filter",
              "icu_normalizer",
              "icu_folding",
              "standard",
              "lowercase",
              "stop");
        } // TESTED

        ElasticSearchManager docIndex = null;
        try {
          docIndex =
              IndexManager.createIndex(
                  sGroupIndex,
                  DocumentPojoIndexMap.documentType_,
                  false,
                  null,
                  docMapping,
                  localSettingsGroupIndex);
        } catch (
            RuntimeException e) { // illegal arg exception, probably the language normalization?
          if (languageNormalization) { // (likely the required plugins have not been installed, just
            // regress back to normal)
            localSettingsGroupIndex = ImmutableSettings.settingsBuilder();
            localSettingsGroupIndex
                .put("number_of_shards", nShards)
                .put("number_of_replicas", nPreferredReplicas);

            docIndex =
                IndexManager.createIndex(
                    sGroupIndex,
                    DocumentPojoIndexMap.documentType_,
                    false,
                    null,
                    docMapping,
                    localSettingsGroupIndex);
          } // TESTED
          else throw e;
        } // TOTEST
        if (bClearIndex) {
          docIndex.deleteMe();
          docIndex =
              IndexManager.createIndex(
                  sGroupIndex,
                  DocumentPojoIndexMap.documentType_,
                  false,
                  null,
                  docMapping,
                  localSettingsGroupIndex);
        }
        if (null != docIndex) {
          try {
            docIndex.pingIndex(); // (wait until it's created itself)
          } catch (Exception e) {
          } // (just make sure this doesn't die horribly)

          docIndex.closeIndex();
        }
      } else if (!bParentsOnly) {
        String sParentGroupIndex =
            new StringBuffer("doc_")
                .append(new ObjectId(parentCommunityIdStr).toString())
                .toString();
        ElasticSearchManager docIndex = IndexManager.getIndex(sParentGroupIndex);

        // DEBUG (alias corruption)
        //				if (null == _aliasInfo) {
        //					ClusterStateResponse clusterState =
        // docIndex.getRawClient().admin().cluster().state(new ClusterStateRequest()).actionGet();
        //					_aliasInfo = clusterState.getState().getMetaData().getAliases();
        //				}
        //				else {
        //					if (_aliasInfo.containsKey(sGroupIndex)) { // has no aliases, we're not good
        //						return;
        //					}
        //					else {
        //						//DEBUG
        //						System.out.println("Alias " + sGroupIndex + " has no aliases (but should)");
        //						ElasticSearchManager docIndex2 = IndexManager.getIndex(sGroupIndex);
        //						docIndex2.deleteMe();
        //					}
        //				}

        docIndex.createAlias(sGroupIndex);
        docIndex.closeIndex();
        // (do nothing on delete - that will be handled at the parent index level)
      }
      // TESTED (parents, children, and personal)
    } else {
      // Just create the dummy index, no different to getting it in practice
      Builder localSettingsGroupIndex = ImmutableSettings.settingsBuilder();
      localSettingsGroupIndex
          .put("number_of_shards", 1)
          .put("number_of_replicas", 0); // (ie guaranteed to be local to each ES node)
      ElasticSearchManager dummyGroupIndex =
          IndexManager.createIndex(
              DocumentPojoIndexMap.dummyDocumentIndex_,
              DocumentPojoIndexMap.documentType_,
              false,
              null,
              docMapping,
              localSettingsGroupIndex);
      if (null == dummyGroupIndex) {
        dummyGroupIndex = IndexManager.getIndex(DocumentPojoIndexMap.dummyDocumentIndex_);
      }

      // Just create an alias, so that queries work arbitrarily:
      dummyGroupIndex.createAlias(sGroupIndex);
      // (do nothing on delete since don't have any docs in here anyway)
    }
  }
  public void InitializeIndex(
      boolean bDeleteDocs,
      boolean bDeleteEntityFeature,
      boolean bDeleteEventFeature,
      boolean bRebuildDocsIndex) {

    try { // create elasticsearch indexes

      PropertiesManager pm = new PropertiesManager();

      if (!pm.getAggregationDisabled()) {

        Builder localSettingsEvent = ImmutableSettings.settingsBuilder();
        localSettingsEvent.put("number_of_shards", 1).put("number_of_replicas", 0);
        localSettingsEvent.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
        localSettingsEvent.putArray(
            "index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");

        localSettingsEvent.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
        localSettingsEvent.putArray(
            "index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");

        Builder localSettingsGaz = ImmutableSettings.settingsBuilder();
        localSettingsGaz.put("number_of_shards", 1).put("number_of_replicas", 0);
        localSettingsGaz.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
        localSettingsGaz.putArray(
            "index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");

        // event feature
        String eventGazMapping =
            new Gson()
                .toJson(
                    new AssociationFeaturePojoIndexMap.Mapping(),
                    AssociationFeaturePojoIndexMap.Mapping.class);
        ElasticSearchManager eventIndex =
            IndexManager.createIndex(
                AssociationFeaturePojoIndexMap.indexName_,
                null,
                false,
                null,
                eventGazMapping,
                localSettingsEvent);
        if (bDeleteEventFeature) {
          eventIndex.deleteMe();
          eventIndex =
              IndexManager.createIndex(
                  AssociationFeaturePojoIndexMap.indexName_,
                  null,
                  false,
                  null,
                  eventGazMapping,
                  localSettingsEvent);
        }
        // entity feature
        String gazMapping =
            new Gson()
                .toJson(
                    new EntityFeaturePojoIndexMap.Mapping(),
                    EntityFeaturePojoIndexMap.Mapping.class);
        ElasticSearchManager entityIndex =
            IndexManager.createIndex(
                EntityFeaturePojoIndexMap.indexName_,
                null,
                false,
                null,
                gazMapping,
                localSettingsGaz);
        if (bDeleteEntityFeature) {
          entityIndex.deleteMe();
          entityIndex =
              IndexManager.createIndex(
                  EntityFeaturePojoIndexMap.indexName_,
                  null,
                  false,
                  null,
                  gazMapping,
                  localSettingsGaz);
        }
      }

      // DOCS - much more complicated than anything else

      boolean bPingMainIndexFailed =
          !ElasticSearchManager.pingIndex(DocumentPojoIndexMap.globalDocumentIndex_);
      // (ie if main doc index doesn't exist then always rebuild all indexes)

      if (bPingMainIndexFailed) { // extra level of robustness... sleep for a minute then double
        // check the index is really missing...
        try {
          Thread.sleep(60000);
        } catch (Exception e) {
        }
        bPingMainIndexFailed =
            !ElasticSearchManager.pingIndex(DocumentPojoIndexMap.globalDocumentIndex_);
      }
      bRebuildDocsIndex |= bPingMainIndexFailed;

      createCommunityDocIndex(
          DocumentPojoIndexMap.globalDocumentIndex_, null, false, true, bDeleteDocs);
      createCommunityDocIndex(
          DocumentPojoIndexMap.manyGeoDocumentIndex_, null, false, false, bDeleteDocs);

      // Some hardwired dummy communities
      createCommunityDocIndex(
          "4e3706c48d26852237078005", null, true, false, bDeleteDocs); // (admin)
      createCommunityDocIndex(
          "4e3706c48d26852237079004", null, true, false, bDeleteDocs); // (test user)
      // (create dummy index used to keep personal group aliases)

      // OK, going to have different shards for different communities:
      // Get a list of all the communities:

      BasicDBObject query = new BasicDBObject();
      BasicDBObject fieldsToDrop = new BasicDBObject("members", 0);
      fieldsToDrop.put("communityAttributes", 0);
      fieldsToDrop.put("userAttributes", 0);
      DBCursor dbc = DbManager.getSocial().getCommunity().find(query, fieldsToDrop);

      if (bRebuildDocsIndex || bDeleteDocs) {

        List<DBObject> tmparray =
            dbc.toArray(); // (brings the entire thing into memory so don't get cursor timeouts)
        int i = 0;
        System.out.println("Initializing " + dbc.size() + " indexes:");
        for (int j = 0; j < 2; ++j) {
          for (DBObject dbotmp : tmparray) {
            if ((++i % 100) == 0) {
              System.out.println("Initialized " + i + " indexes.");
            }
            BasicDBObject dbo = (BasicDBObject) dbotmp;

            // OK, going to see if there are any sources with this group id, create a new index if
            // so:
            // (Don't use CommunityPojo data model here for performance reasons....
            //  (Also, haven't gotten round to porting CommunityPojo field access to using static
            // fields))
            ObjectId communityId = (ObjectId) dbo.get("_id");
            boolean bPersonalGroup = dbo.getBoolean("isPersonalCommunity", false);
            boolean bSystemGroup = dbo.getBoolean("isSystemCommunity", false);
            ObjectId parentCommunityId = (ObjectId) dbo.get("parentId");

            createCommunityDocIndex(
                communityId.toString(),
                parentCommunityId,
                bPersonalGroup,
                bSystemGroup,
                bDeleteDocs,
                j == 0);
          } // end loop over communities
        } // end loop over communities - first time parents only
      } // (end if need to do big loop over all sources)
    } catch (Exception e) {
      e.printStackTrace();
      throw new RuntimeException(e.getMessage());
    }
  } // TESTED (not changed since by-eye test in Beta - retested after moving code into
Esempio n. 15
0
    public static IndexMetaData fromXContent(XContentParser parser) throws IOException {
      if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
        parser.nextToken();
      }
      Builder builder = new Builder(parser.currentName());

      String currentFieldName = null;
      XContentParser.Token token = parser.nextToken();
      while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
        if (token == XContentParser.Token.FIELD_NAME) {
          currentFieldName = parser.currentName();
        } else if (token == XContentParser.Token.START_OBJECT) {
          if ("settings".equals(currentFieldName)) {
            builder.settings(
                ImmutableSettings.settingsBuilder()
                    .put(SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered())));
          } else if ("mappings".equals(currentFieldName)) {
            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
              if (token == XContentParser.Token.FIELD_NAME) {
                currentFieldName = parser.currentName();
              } else if (token == XContentParser.Token.START_OBJECT) {
                String mappingType = currentFieldName;
                Map<String, Object> mappingSource =
                    MapBuilder.<String, Object>newMapBuilder()
                        .put(mappingType, parser.mapOrdered())
                        .map();
                builder.putMapping(new MappingMetaData(mappingType, mappingSource));
              }
            }
          } else if ("aliases".equals(currentFieldName)) {
            while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
              builder.putAlias(AliasMetaData.Builder.fromXContent(parser));
            }
          } else {
            // check if its a custom index metadata
            Custom.Factory<Custom> factory = lookupFactory(currentFieldName);
            if (factory == null) {
              // TODO warn
              parser.skipChildren();
            } else {
              builder.putCustom(factory.type(), factory.fromXContent(parser));
            }
          }
        } else if (token == XContentParser.Token.START_ARRAY) {
          if ("mappings".equals(currentFieldName)) {
            while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
              if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
                builder.putMapping(new MappingMetaData(new CompressedString(parser.binaryValue())));
              } else {
                Map<String, Object> mapping = parser.mapOrdered();
                if (mapping.size() == 1) {
                  String mappingType = mapping.keySet().iterator().next();
                  builder.putMapping(new MappingMetaData(mappingType, mapping));
                }
              }
            }
          }
        } else if (token.isValue()) {
          if ("state".equals(currentFieldName)) {
            builder.state(State.fromString(parser.text()));
          } else if ("version".equals(currentFieldName)) {
            builder.version(parser.longValue());
          }
        }
      }
      return builder.build();
    }
  @Test
  public void testBackwardsCompatibilityEdgeNgramTokenizer()
      throws IllegalArgumentException, IllegalAccessException {
    int iters = atLeast(20);
    final Index index = new Index("test");
    final String name = "ngr";
    for (int i = 0; i < iters; i++) {
      Version v = randomVersion(random());
      if (v.onOrAfter(Version.V_0_90_2)) {
        Builder builder =
            ImmutableSettings.builder()
                .put("min_gram", 2)
                .put("max_gram", 3)
                .put("token_chars", "letter,digit");
        boolean compatVersion = false;
        if ((compatVersion = random().nextBoolean())) {
          builder.put("version", "4." + random().nextInt(3));
          builder.put("side", "back");
        }
        Settings settings = builder.build();
        Settings indexSettings =
            ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build();
        Tokenizer edgeNGramTokenizer =
            new EdgeNGramTokenizerFactory(index, indexSettings, name, settings)
                .create(new StringReader("foo bar"));
        if (compatVersion) {
          assertThat(edgeNGramTokenizer, instanceOf(Lucene43EdgeNGramTokenizer.class));
        } else {
          assertThat(edgeNGramTokenizer, instanceOf(EdgeNGramTokenizer.class));
        }

      } else {
        Settings settings =
            ImmutableSettings.builder()
                .put("min_gram", 2)
                .put("max_gram", 3)
                .put("side", "back")
                .build();
        Settings indexSettings =
            ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build();
        Tokenizer edgeNGramTokenizer =
            new EdgeNGramTokenizerFactory(index, indexSettings, name, settings)
                .create(new StringReader("foo bar"));
        assertThat(edgeNGramTokenizer, instanceOf(Lucene43EdgeNGramTokenizer.class));
      }
    }
    Settings settings =
        ImmutableSettings.builder()
            .put("min_gram", 2)
            .put("max_gram", 3)
            .put("side", "back")
            .build();
    Settings indexSettings =
        ImmutableSettings.builder()
            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
            .build();
    try {
      new EdgeNGramTokenizerFactory(index, indexSettings, name, settings)
          .create(new StringReader("foo bar"));
      fail("should fail side:back is not supported anymore");
    } catch (ElasticSearchIllegalArgumentException ex) {
    }
  }