コード例 #1
0
  public static void createCommunityDocIndex(
      String nameOrCommunityIdStr,
      ObjectId parentCommunityId,
      boolean bPersonalGroup,
      boolean bSystemGroup,
      boolean bClearIndex,
      boolean bParentsOnly) {
    // create elasticsearch indexes
    PropertiesManager pm = new PropertiesManager();
    boolean languageNormalization = pm.getNormalizeEncoding();
    int nPreferredReplicas = pm.getMaxIndexReplicas();

    String docMapping =
        new Gson().toJson(new DocumentPojoIndexMap.Mapping(), DocumentPojoIndexMap.Mapping.class);

    String sGroupIndex = null;
    try {
      sGroupIndex =
          new StringBuffer("doc_").append(new ObjectId(nameOrCommunityIdStr).toString()).toString();
    } catch (Exception e) {
      sGroupIndex = nameOrCommunityIdStr;
    }

    if (!bPersonalGroup) {

      String parentCommunityIdStr = null;
      if (null != parentCommunityId) {
        parentCommunityIdStr = parentCommunityId.toString();
      }

      if ((null == parentCommunityIdStr)
          || (parentCommunityIdStr.equals("4c927585d591d31d7b37097a"))) {
        // (system community is hardwired - children of this community are ignored)

        int nShards = bSystemGroup ? 10 : 5; // (system group is largest)

        // Remove the alias, in case it exists:
        // Then create an index with this name:
        Builder localSettingsGroupIndex = ImmutableSettings.settingsBuilder();
        localSettingsGroupIndex
            .put("number_of_shards", nShards)
            .put("number_of_replicas", nPreferredReplicas);
        if (languageNormalization) {
          localSettingsGroupIndex.put("index.analysis.analyzer.default.tokenizer", "standard");
          localSettingsGroupIndex.putArray(
              "index.analysis.analyzer.default.filter",
              "icu_normalizer",
              "icu_folding",
              "standard",
              "lowercase",
              "stop");
        } // TESTED

        ElasticSearchManager docIndex = null;
        try {
          docIndex =
              IndexManager.createIndex(
                  sGroupIndex,
                  DocumentPojoIndexMap.documentType_,
                  false,
                  null,
                  docMapping,
                  localSettingsGroupIndex);
        } catch (
            RuntimeException e) { // illegal arg exception, probably the language normalization?
          if (languageNormalization) { // (likely the required plugins have not been installed, just
            // regress back to normal)
            localSettingsGroupIndex = ImmutableSettings.settingsBuilder();
            localSettingsGroupIndex
                .put("number_of_shards", nShards)
                .put("number_of_replicas", nPreferredReplicas);

            docIndex =
                IndexManager.createIndex(
                    sGroupIndex,
                    DocumentPojoIndexMap.documentType_,
                    false,
                    null,
                    docMapping,
                    localSettingsGroupIndex);
          } // TESTED
          else throw e;
        } // TOTEST
        if (bClearIndex) {
          docIndex.deleteMe();
          docIndex =
              IndexManager.createIndex(
                  sGroupIndex,
                  DocumentPojoIndexMap.documentType_,
                  false,
                  null,
                  docMapping,
                  localSettingsGroupIndex);
        }
        if (null != docIndex) {
          try {
            docIndex.pingIndex(); // (wait until it's created itself)
          } catch (Exception e) {
          } // (just make sure this doesn't die horribly)

          docIndex.closeIndex();
        }
      } else if (!bParentsOnly) {
        String sParentGroupIndex =
            new StringBuffer("doc_")
                .append(new ObjectId(parentCommunityIdStr).toString())
                .toString();
        ElasticSearchManager docIndex = IndexManager.getIndex(sParentGroupIndex);

        // DEBUG (alias corruption)
        //				if (null == _aliasInfo) {
        //					ClusterStateResponse clusterState =
        // docIndex.getRawClient().admin().cluster().state(new ClusterStateRequest()).actionGet();
        //					_aliasInfo = clusterState.getState().getMetaData().getAliases();
        //				}
        //				else {
        //					if (_aliasInfo.containsKey(sGroupIndex)) { // has no aliases, we're not good
        //						return;
        //					}
        //					else {
        //						//DEBUG
        //						System.out.println("Alias " + sGroupIndex + " has no aliases (but should)");
        //						ElasticSearchManager docIndex2 = IndexManager.getIndex(sGroupIndex);
        //						docIndex2.deleteMe();
        //					}
        //				}

        docIndex.createAlias(sGroupIndex);
        docIndex.closeIndex();
        // (do nothing on delete - that will be handled at the parent index level)
      }
      // TESTED (parents, children, and personal)
    } else {
      // Just create the dummy index, no different to getting it in practice
      Builder localSettingsGroupIndex = ImmutableSettings.settingsBuilder();
      localSettingsGroupIndex
          .put("number_of_shards", 1)
          .put("number_of_replicas", 0); // (ie guaranteed to be local to each ES node)
      ElasticSearchManager dummyGroupIndex =
          IndexManager.createIndex(
              DocumentPojoIndexMap.dummyDocumentIndex_,
              DocumentPojoIndexMap.documentType_,
              false,
              null,
              docMapping,
              localSettingsGroupIndex);
      if (null == dummyGroupIndex) {
        dummyGroupIndex = IndexManager.getIndex(DocumentPojoIndexMap.dummyDocumentIndex_);
      }

      // Just create an alias, so that queries work arbitrarily:
      dummyGroupIndex.createAlias(sGroupIndex);
      // (do nothing on delete since don't have any docs in here anyway)
    }
  }
コード例 #2
0
  public void InitializeIndex(
      boolean bDeleteDocs,
      boolean bDeleteEntityFeature,
      boolean bDeleteEventFeature,
      boolean bRebuildDocsIndex) {

    try { // create elasticsearch indexes

      PropertiesManager pm = new PropertiesManager();

      if (!pm.getAggregationDisabled()) {

        Builder localSettingsEvent = ImmutableSettings.settingsBuilder();
        localSettingsEvent.put("number_of_shards", 1).put("number_of_replicas", 0);
        localSettingsEvent.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
        localSettingsEvent.putArray(
            "index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");

        localSettingsEvent.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
        localSettingsEvent.putArray(
            "index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");

        Builder localSettingsGaz = ImmutableSettings.settingsBuilder();
        localSettingsGaz.put("number_of_shards", 1).put("number_of_replicas", 0);
        localSettingsGaz.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
        localSettingsGaz.putArray(
            "index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");

        // event feature
        String eventGazMapping =
            new Gson()
                .toJson(
                    new AssociationFeaturePojoIndexMap.Mapping(),
                    AssociationFeaturePojoIndexMap.Mapping.class);
        ElasticSearchManager eventIndex =
            IndexManager.createIndex(
                AssociationFeaturePojoIndexMap.indexName_,
                null,
                false,
                null,
                eventGazMapping,
                localSettingsEvent);
        if (bDeleteEventFeature) {
          eventIndex.deleteMe();
          eventIndex =
              IndexManager.createIndex(
                  AssociationFeaturePojoIndexMap.indexName_,
                  null,
                  false,
                  null,
                  eventGazMapping,
                  localSettingsEvent);
        }
        // entity feature
        String gazMapping =
            new Gson()
                .toJson(
                    new EntityFeaturePojoIndexMap.Mapping(),
                    EntityFeaturePojoIndexMap.Mapping.class);
        ElasticSearchManager entityIndex =
            IndexManager.createIndex(
                EntityFeaturePojoIndexMap.indexName_,
                null,
                false,
                null,
                gazMapping,
                localSettingsGaz);
        if (bDeleteEntityFeature) {
          entityIndex.deleteMe();
          entityIndex =
              IndexManager.createIndex(
                  EntityFeaturePojoIndexMap.indexName_,
                  null,
                  false,
                  null,
                  gazMapping,
                  localSettingsGaz);
        }
      }

      // DOCS - much more complicated than anything else

      boolean bPingMainIndexFailed =
          !ElasticSearchManager.pingIndex(DocumentPojoIndexMap.globalDocumentIndex_);
      // (ie if main doc index doesn't exist then always rebuild all indexes)

      if (bPingMainIndexFailed) { // extra level of robustness... sleep for a minute then double
        // check the index is really missing...
        try {
          Thread.sleep(60000);
        } catch (Exception e) {
        }
        bPingMainIndexFailed =
            !ElasticSearchManager.pingIndex(DocumentPojoIndexMap.globalDocumentIndex_);
      }
      bRebuildDocsIndex |= bPingMainIndexFailed;

      createCommunityDocIndex(
          DocumentPojoIndexMap.globalDocumentIndex_, null, false, true, bDeleteDocs);
      createCommunityDocIndex(
          DocumentPojoIndexMap.manyGeoDocumentIndex_, null, false, false, bDeleteDocs);

      // Some hardwired dummy communities
      createCommunityDocIndex(
          "4e3706c48d26852237078005", null, true, false, bDeleteDocs); // (admin)
      createCommunityDocIndex(
          "4e3706c48d26852237079004", null, true, false, bDeleteDocs); // (test user)
      // (create dummy index used to keep personal group aliases)

      // OK, going to have different shards for different communities:
      // Get a list of all the communities:

      BasicDBObject query = new BasicDBObject();
      BasicDBObject fieldsToDrop = new BasicDBObject("members", 0);
      fieldsToDrop.put("communityAttributes", 0);
      fieldsToDrop.put("userAttributes", 0);
      DBCursor dbc = DbManager.getSocial().getCommunity().find(query, fieldsToDrop);

      if (bRebuildDocsIndex || bDeleteDocs) {

        List<DBObject> tmparray =
            dbc.toArray(); // (brings the entire thing into memory so don't get cursor timeouts)
        int i = 0;
        System.out.println("Initializing " + dbc.size() + " indexes:");
        for (int j = 0; j < 2; ++j) {
          for (DBObject dbotmp : tmparray) {
            if ((++i % 100) == 0) {
              System.out.println("Initialized " + i + " indexes.");
            }
            BasicDBObject dbo = (BasicDBObject) dbotmp;

            // OK, going to see if there are any sources with this group id, create a new index if
            // so:
            // (Don't use CommunityPojo data model here for performance reasons....
            //  (Also, haven't gotten round to porting CommunityPojo field access to using static
            // fields))
            ObjectId communityId = (ObjectId) dbo.get("_id");
            boolean bPersonalGroup = dbo.getBoolean("isPersonalCommunity", false);
            boolean bSystemGroup = dbo.getBoolean("isSystemCommunity", false);
            ObjectId parentCommunityId = (ObjectId) dbo.get("parentId");

            createCommunityDocIndex(
                communityId.toString(),
                parentCommunityId,
                bPersonalGroup,
                bSystemGroup,
                bDeleteDocs,
                j == 0);
          } // end loop over communities
        } // end loop over communities - first time parents only
      } // (end if need to do big loop over all sources)
    } catch (Exception e) {
      e.printStackTrace();
      throw new RuntimeException(e.getMessage());
    }
  } // TESTED (not changed since by-eye test in Beta - retested after moving code into