public static void createCommunityDocIndex(
String nameOrCommunityIdStr,
ObjectId parentCommunityId,
boolean bPersonalGroup,
boolean bSystemGroup,
boolean bClearIndex,
boolean bParentsOnly) {
// create elasticsearch indexes
PropertiesManager pm = new PropertiesManager();
boolean languageNormalization = pm.getNormalizeEncoding();
int nPreferredReplicas = pm.getMaxIndexReplicas();
String docMapping =
new Gson().toJson(new DocumentPojoIndexMap.Mapping(), DocumentPojoIndexMap.Mapping.class);
String sGroupIndex = null;
try {
sGroupIndex =
new StringBuffer("doc_").append(new ObjectId(nameOrCommunityIdStr).toString()).toString();
} catch (Exception e) {
sGroupIndex = nameOrCommunityIdStr;
}
if (!bPersonalGroup) {
String parentCommunityIdStr = null;
if (null != parentCommunityId) {
parentCommunityIdStr = parentCommunityId.toString();
}
if ((null == parentCommunityIdStr)
|| (parentCommunityIdStr.equals("4c927585d591d31d7b37097a"))) {
// (system community is hardwired - children of this community are ignored)
int nShards = bSystemGroup ? 10 : 5; // (system group is largest)
// Remove the alias, in case it exists:
// Then create an index with this name:
Builder localSettingsGroupIndex = ImmutableSettings.settingsBuilder();
localSettingsGroupIndex
.put("number_of_shards", nShards)
.put("number_of_replicas", nPreferredReplicas);
if (languageNormalization) {
localSettingsGroupIndex.put("index.analysis.analyzer.default.tokenizer", "standard");
localSettingsGroupIndex.putArray(
"index.analysis.analyzer.default.filter",
"icu_normalizer",
"icu_folding",
"standard",
"lowercase",
"stop");
} // TESTED
ElasticSearchManager docIndex = null;
try {
docIndex =
IndexManager.createIndex(
sGroupIndex,
DocumentPojoIndexMap.documentType_,
false,
null,
docMapping,
localSettingsGroupIndex);
} catch (
RuntimeException e) { // illegal arg exception, probably the language normalization?
if (languageNormalization) { // (likely the required plugins have not been installed, just
// regress back to normal)
localSettingsGroupIndex = ImmutableSettings.settingsBuilder();
localSettingsGroupIndex
.put("number_of_shards", nShards)
.put("number_of_replicas", nPreferredReplicas);
docIndex =
IndexManager.createIndex(
sGroupIndex,
DocumentPojoIndexMap.documentType_,
false,
null,
docMapping,
localSettingsGroupIndex);
} // TESTED
else throw e;
} // TOTEST
if (bClearIndex) {
docIndex.deleteMe();
docIndex =
IndexManager.createIndex(
sGroupIndex,
DocumentPojoIndexMap.documentType_,
false,
null,
docMapping,
localSettingsGroupIndex);
}
if (null != docIndex) {
try {
docIndex.pingIndex(); // (wait until it's created itself)
} catch (Exception e) {
} // (just make sure this doesn't die horribly)
docIndex.closeIndex();
}
} else if (!bParentsOnly) {
String sParentGroupIndex =
new StringBuffer("doc_")
.append(new ObjectId(parentCommunityIdStr).toString())
.toString();
ElasticSearchManager docIndex = IndexManager.getIndex(sParentGroupIndex);
// DEBUG (alias corruption)
// if (null == _aliasInfo) {
// ClusterStateResponse clusterState =
// docIndex.getRawClient().admin().cluster().state(new ClusterStateRequest()).actionGet();
// _aliasInfo = clusterState.getState().getMetaData().getAliases();
// }
// else {
// if (_aliasInfo.containsKey(sGroupIndex)) { // has no aliases, we're not good
// return;
// }
// else {
// //DEBUG
// System.out.println("Alias " + sGroupIndex + " has no aliases (but should)");
// ElasticSearchManager docIndex2 = IndexManager.getIndex(sGroupIndex);
// docIndex2.deleteMe();
// }
// }
docIndex.createAlias(sGroupIndex);
docIndex.closeIndex();
// (do nothing on delete - that will be handled at the parent index level)
}
// TESTED (parents, children, and personal)
} else {
// Just create the dummy index, no different to getting it in practice
Builder localSettingsGroupIndex = ImmutableSettings.settingsBuilder();
localSettingsGroupIndex
.put("number_of_shards", 1)
.put("number_of_replicas", 0); // (ie guaranteed to be local to each ES node)
ElasticSearchManager dummyGroupIndex =
IndexManager.createIndex(
DocumentPojoIndexMap.dummyDocumentIndex_,
DocumentPojoIndexMap.documentType_,
false,
null,
docMapping,
localSettingsGroupIndex);
if (null == dummyGroupIndex) {
dummyGroupIndex = IndexManager.getIndex(DocumentPojoIndexMap.dummyDocumentIndex_);
}
// Just create an alias, so that queries work arbitrarily:
dummyGroupIndex.createAlias(sGroupIndex);
// (do nothing on delete since don't have any docs in here anyway)
}
}
public void InitializeIndex(
boolean bDeleteDocs,
boolean bDeleteEntityFeature,
boolean bDeleteEventFeature,
boolean bRebuildDocsIndex) {
try { // create elasticsearch indexes
PropertiesManager pm = new PropertiesManager();
if (!pm.getAggregationDisabled()) {
Builder localSettingsEvent = ImmutableSettings.settingsBuilder();
localSettingsEvent.put("number_of_shards", 1).put("number_of_replicas", 0);
localSettingsEvent.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
localSettingsEvent.putArray(
"index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");
localSettingsEvent.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
localSettingsEvent.putArray(
"index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");
Builder localSettingsGaz = ImmutableSettings.settingsBuilder();
localSettingsGaz.put("number_of_shards", 1).put("number_of_replicas", 0);
localSettingsGaz.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
localSettingsGaz.putArray(
"index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");
// event feature
String eventGazMapping =
new Gson()
.toJson(
new AssociationFeaturePojoIndexMap.Mapping(),
AssociationFeaturePojoIndexMap.Mapping.class);
ElasticSearchManager eventIndex =
IndexManager.createIndex(
AssociationFeaturePojoIndexMap.indexName_,
null,
false,
null,
eventGazMapping,
localSettingsEvent);
if (bDeleteEventFeature) {
eventIndex.deleteMe();
eventIndex =
IndexManager.createIndex(
AssociationFeaturePojoIndexMap.indexName_,
null,
false,
null,
eventGazMapping,
localSettingsEvent);
}
// entity feature
String gazMapping =
new Gson()
.toJson(
new EntityFeaturePojoIndexMap.Mapping(),
EntityFeaturePojoIndexMap.Mapping.class);
ElasticSearchManager entityIndex =
IndexManager.createIndex(
EntityFeaturePojoIndexMap.indexName_,
null,
false,
null,
gazMapping,
localSettingsGaz);
if (bDeleteEntityFeature) {
entityIndex.deleteMe();
entityIndex =
IndexManager.createIndex(
EntityFeaturePojoIndexMap.indexName_,
null,
false,
null,
gazMapping,
localSettingsGaz);
}
}
// DOCS - much more complicated than anything else
boolean bPingMainIndexFailed =
!ElasticSearchManager.pingIndex(DocumentPojoIndexMap.globalDocumentIndex_);
// (ie if main doc index doesn't exist then always rebuild all indexes)
if (bPingMainIndexFailed) { // extra level of robustness... sleep for a minute then double
// check the index is really missing...
try {
Thread.sleep(60000);
} catch (Exception e) {
}
bPingMainIndexFailed =
!ElasticSearchManager.pingIndex(DocumentPojoIndexMap.globalDocumentIndex_);
}
bRebuildDocsIndex |= bPingMainIndexFailed;
createCommunityDocIndex(
DocumentPojoIndexMap.globalDocumentIndex_, null, false, true, bDeleteDocs);
createCommunityDocIndex(
DocumentPojoIndexMap.manyGeoDocumentIndex_, null, false, false, bDeleteDocs);
// Some hardwired dummy communities
createCommunityDocIndex(
"4e3706c48d26852237078005", null, true, false, bDeleteDocs); // (admin)
createCommunityDocIndex(
"4e3706c48d26852237079004", null, true, false, bDeleteDocs); // (test user)
// (create dummy index used to keep personal group aliases)
// OK, going to have different shards for different communities:
// Get a list of all the communities:
BasicDBObject query = new BasicDBObject();
BasicDBObject fieldsToDrop = new BasicDBObject("members", 0);
fieldsToDrop.put("communityAttributes", 0);
fieldsToDrop.put("userAttributes", 0);
DBCursor dbc = DbManager.getSocial().getCommunity().find(query, fieldsToDrop);
if (bRebuildDocsIndex || bDeleteDocs) {
List<DBObject> tmparray =
dbc.toArray(); // (brings the entire thing into memory so don't get cursor timeouts)
int i = 0;
System.out.println("Initializing " + dbc.size() + " indexes:");
for (int j = 0; j < 2; ++j) {
for (DBObject dbotmp : tmparray) {
if ((++i % 100) == 0) {
System.out.println("Initialized " + i + " indexes.");
}
BasicDBObject dbo = (BasicDBObject) dbotmp;
// OK, going to see if there are any sources with this group id, create a new index if
// so:
// (Don't use CommunityPojo data model here for performance reasons....
// (Also, haven't gotten round to porting CommunityPojo field access to using static
// fields))
ObjectId communityId = (ObjectId) dbo.get("_id");
boolean bPersonalGroup = dbo.getBoolean("isPersonalCommunity", false);
boolean bSystemGroup = dbo.getBoolean("isSystemCommunity", false);
ObjectId parentCommunityId = (ObjectId) dbo.get("parentId");
createCommunityDocIndex(
communityId.toString(),
parentCommunityId,
bPersonalGroup,
bSystemGroup,
bDeleteDocs,
j == 0);
} // end loop over communities
} // end loop over communities - first time parents only
} // (end if need to do big loop over all sources)
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
} // TESTED (not changed since by-eye test in Beta - retested after moving code into
