private Settings getSettings(int nodeOrdinal, Settings others) {
Builder builder = ImmutableSettings.settingsBuilder().put(defaultSettings);
Settings settings = nodeSettingsSource.settings(nodeOrdinal);
if (settings != null) {
builder.put(settings);
}
if (others != null) {
builder.put(others);
}
return builder.build();
}
public void start() throws Exception {
try {
final Builder settings = ImmutableSettings.settingsBuilder();
settings.put("path.data", filesSettings.getPermanentDirectory());
settings.put("client.transport.sniff", true);
settings.build();
final NodeBuilder nb = nodeBuilder().settings(settings).local(true).client(false).data(true);
final Node node = nb.node();
client = node.client();
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest("entities");
// we just send back a response, no need to fork a listener
indicesExistsRequest.listenerThreaded(false);
client
.admin()
.indices()
.exists(
indicesExistsRequest,
new ActionListener<IndicesExistsResponse>() {
public void onResponse(IndicesExistsResponse response) {
if (!response.isExists()) {
try {
logger.debug("Entities indice does not exists. Creating it...");
CreateIndexResponse r =
client
.admin()
.indices()
.create(new CreateIndexRequest("entities"))
.actionGet();
logger.debug(
"Created indice with response {}",
r.isAcknowledged() ? "\u2713 acknowledged" : "\2718 not acknowledged");
} catch (Exception e) {
logger.error("Failed to create entities indice status ...");
}
}
updateMappings();
}
public void onFailure(Throwable e) {
logger.error("Failed to enquiry entities indice status ...", e);
}
});
} catch (Exception e) {
throw new InitializationException("Failed to initialize embedded elastic search", e);
}
}
TestCluster(
long clusterSeed, int numNodes, String clusterName, NodeSettingsSource nodeSettingsSource) {
this.clusterName = clusterName;
Random random = new Random(clusterSeed);
numSharedNodes =
numNodes == -1 ? 2 + random.nextInt(4) : numNodes; // at least 2 nodes if randomized
assert numSharedNodes >= 0;
/*
* TODO
* - we might want start some master only nodes?
* - we could add a flag that returns a client to the master all the time?
* - we could add a flag that never returns a client to the master
* - along those lines use a dedicated node that is master eligible and let all other nodes be only data nodes
*/
sharedNodesSeeds = new long[numSharedNodes];
for (int i = 0; i < sharedNodesSeeds.length; i++) {
sharedNodesSeeds[i] = random.nextLong();
}
logger.info(
"Setup TestCluster [{}] with seed [{}] using [{}] nodes",
clusterName,
SeedUtils.formatSeed(clusterSeed),
numSharedNodes);
Builder builder =
ImmutableSettings.settingsBuilder()
/* use RAM directories in 10% of the runs */
// .put("index.store.type", random.nextInt(10) == 0 ?
// MockRamIndexStoreModule.class.getName() : MockFSIndexStoreModule.class.getName())
.put("index.store.type", MockFSIndexStoreModule.class.getName()) // no RAM dir for now!
.put(IndexEngineModule.EngineSettings.ENGINE_TYPE, MockEngineModule.class.getName())
.put("cluster.name", clusterName)
// decrease the routing schedule so new nodes will be added quickly - some random value
// between 30 and 80 ms
.put("cluster.routing.schedule", (30 + random.nextInt(50)) + "ms")
// default to non gateway
.put("gateway.type", "none");
if (isLocalTransportConfigured()) {
builder.put(
TransportModule.TRANSPORT_TYPE_KEY, AssertingLocalTransportModule.class.getName());
} else {
builder.put(Transport.TransportSettings.TRANSPORT_TCP_COMPRESS, random.nextInt(10) == 0);
}
this.defaultSettings = builder.build();
this.nodeSettingsSource = nodeSettingsSource;
}
private int indexRandomNumbers(String analyzer, int shards) throws Exception {
Builder builder =
ImmutableSettings.settingsBuilder()
.put(indexSettings())
.put(SETTING_NUMBER_OF_REPLICAS, between(0, 1));
if (shards > 0) {
builder.put(SETTING_NUMBER_OF_SHARDS, shards);
}
client()
.admin()
.indices()
.prepareCreate("test")
.addMapping(
"type1",
jsonBuilder()
.startObject()
.startObject("type1")
.startObject("properties")
.startObject("field1")
.field("analyzer", analyzer)
.field("type", "string")
.endObject()
.endObject()
.endObject()
.endObject())
.setSettings(builder)
.get();
int numDocs = atLeast(100);
IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs];
for (int i = 0; i < numDocs; i++) {
docs[i] =
client()
.prepareIndex("test", "type1", String.valueOf(i))
.setSource("field1", English.intToEnglish(i));
}
indexRandom(true, docs);
ensureGreen();
return numDocs;
}
@Test
public void testBackwardsCompatibilityNgramTokenizer()
throws IllegalArgumentException, IllegalAccessException {
int iters = atLeast(20);
for (int i = 0; i < iters; i++) {
final Index index = new Index("test");
final String name = "ngr";
Version v = randomVersion(random());
if (v.onOrAfter(Version.V_0_90_2)) {
Builder builder =
ImmutableSettings.builder()
.put("min_gram", 2)
.put("max_gram", 3)
.put("token_chars", "letter,digit");
boolean compatVersion = false;
if ((compatVersion = random().nextBoolean())) {
builder.put("version", "4." + random().nextInt(3));
}
Settings settings = builder.build();
Settings indexSettings =
ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build();
Tokenizer nGramTokenizer =
new NGramTokenizerFactory(index, indexSettings, name, settings)
.create(new StringReader("foo bar"));
if (compatVersion) {
assertThat(nGramTokenizer, instanceOf(Lucene43NGramTokenizer.class));
} else {
assertThat(nGramTokenizer, instanceOf(NGramTokenizer.class));
}
} else {
Settings settings =
ImmutableSettings.builder().put("min_gram", 2).put("max_gram", 3).build();
Settings indexSettings =
ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build();
Tokenizer nGramTokenizer =
new NGramTokenizerFactory(index, indexSettings, name, settings)
.create(new StringReader("foo bar"));
assertThat(nGramTokenizer, instanceOf(Lucene43NGramTokenizer.class));
}
}
}
private static Settings normalizeSettings(final Settings settings) {
Builder builder = settingsBuilder();
builder.put(settings);
// automatically overridden
builder.put("client.type", "transport");
builder.put("node.client", true);
builder.put("network.server", false);
// ensure behavior
builder.put("node.master", false);
builder.put("node.data", false);
// disable HTTP
builder.put("http.enabled", false);
return builder.build();
}
@Before
public void setUp() throws Exception {
System.out.println(
"--------------------- SETUP "
+ name.getMethodName()
+ " -------------------------------------");
MockMailbox.resetAll();
// Instantiates a local node & client
esSetup = new EsSetup(settingsBuilder.build());
// Clean all, and creates some indices
esSetup.execute(deleteAll());
}
@Before
public void buildNode1() throws Exception {
super.setUp();
Builder builder = ImmutableSettings.builder();
builder.put("cluster.routing.schedule", "100ms");
builder.put("gateway.type", "fs");
if (between(0, 5) == 0) {
builder.put("gateway.fs.buffer_size", between(1, 100) + "kb");
}
if (between(0, 5) == 0) {
builder.put("gateway.fs.chunk_size", between(1, 100) + "kb");
}
builder.put("index.number_of_replicas", "1");
builder.put("index.number_of_shards", rarely() ? Integer.toString(between(2, 6)) : "1");
storeType = rarely() ? "ram" : "fs";
builder.put("index.store.type", storeType);
defaultSettings = builder.build();
buildNode("server1");
// since we store (by default) the index snapshot under the gateway, resetting it will reset the
// index data as well
((InternalNode) node("server1")).injector().getInstance(Gateway.class).reset();
closeAllNodes();
}
public static IndexMetaData readFrom(StreamInput in) throws IOException {
Builder builder = new Builder(in.readUTF());
builder.version(in.readLong());
builder.state(State.fromId(in.readByte()));
builder.settings(readSettingsFromStream(in));
int mappingsSize = in.readVInt();
for (int i = 0; i < mappingsSize; i++) {
MappingMetaData mappingMd = MappingMetaData.readFrom(in);
builder.putMapping(mappingMd);
}
int aliasesSize = in.readVInt();
for (int i = 0; i < aliasesSize; i++) {
AliasMetaData aliasMd = AliasMetaData.Builder.readFrom(in);
builder.putAlias(aliasMd);
}
int customSize = in.readVInt();
for (int i = 0; i < customSize; i++) {
String type = in.readUTF();
Custom customIndexMetaData = lookupFactorySafe(type).readFrom(in);
builder.putCustom(type, customIndexMetaData);
}
return builder.build();
}
@Test
public void testRandomExceptions() throws IOException, InterruptedException, ExecutionException {
final int numShards = between(1, 5);
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("test")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
final double exceptionRate;
final double exceptionOnOpenRate;
if (frequently()) {
if (randomBoolean()) {
if (randomBoolean()) {
exceptionOnOpenRate = 1.0 / between(5, 100);
exceptionRate = 0.0d;
} else {
exceptionRate = 1.0 / between(5, 100);
exceptionOnOpenRate = 0.0d;
}
} else {
exceptionOnOpenRate = 1.0 / between(5, 100);
exceptionRate = 1.0 / between(5, 100);
}
} else {
// rarely no exception
exceptionRate = 0d;
exceptionOnOpenRate = 0d;
}
Builder settings =
settingsBuilder()
.put("index.number_of_shards", numShards)
.put("index.number_of_replicas", randomIntBetween(0, 1))
.put(MockDirectoryHelper.RANDOM_IO_EXCEPTION_RATE, exceptionRate)
.put(MockDirectoryHelper.RANDOM_IO_EXCEPTION_RATE_ON_OPEN, exceptionOnOpenRate)
.put(MockDirectoryHelper.CHECK_INDEX_ON_CLOSE, true);
logger.info("creating index: [test] using settings: [{}]", settings.build().getAsMap());
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(settings)
.addMapping("type", mapping)
.execute()
.actionGet();
ClusterHealthResponse clusterHealthResponse =
client()
.admin()
.cluster()
.health(
Requests.clusterHealthRequest()
.waitForYellowStatus()
.timeout(TimeValue.timeValueSeconds(5)))
.get(); // it's OK to timeout here
final int numDocs;
final boolean expectAllShardsFailed;
if (clusterHealthResponse.isTimedOut()) {
/* some seeds just won't let you create the index at all and we enter a ping-pong mode
* trying one node after another etc. that is ok but we need to make sure we don't wait
* forever when indexing documents so we set numDocs = 1 and expecte all shards to fail
* when we search below.*/
logger.info("ClusterHealth timed out - only index one doc and expect searches to fail");
numDocs = 1;
expectAllShardsFailed = true;
} else {
numDocs = between(10, 100);
expectAllShardsFailed = false;
}
long numCreated = 0;
boolean[] added = new boolean[numDocs];
for (int i = 0; i < numDocs; i++) {
try {
IndexResponse indexResponse =
client()
.prepareIndex("test", "type", "" + i)
.setTimeout(TimeValue.timeValueSeconds(1))
.setSource("test", English.intToEnglish(i))
.get();
if (indexResponse.isCreated()) {
numCreated++;
added[i] = true;
}
} catch (ElasticSearchException ex) {
}
}
logger.info("Start Refresh");
RefreshResponse refreshResponse =
client()
.admin()
.indices()
.prepareRefresh("test")
.execute()
.get(); // don't assert on failures here
final boolean refreshFailed =
refreshResponse.getShardFailures().length != 0 || refreshResponse.getFailedShards() != 0;
logger.info(
"Refresh failed [{}] numShardsFailed: [{}], shardFailuresLength: [{}], successfulShards: [{}], totalShards: [{}] ",
refreshFailed,
refreshResponse.getFailedShards(),
refreshResponse.getShardFailures().length,
refreshResponse.getSuccessfulShards(),
refreshResponse.getTotalShards());
final int numSearches = atLeast(10);
// we don't check anything here really just making sure we don't leave any open files or a
// broken index behind.
for (int i = 0; i < numSearches; i++) {
try {
int docToQuery = between(0, numDocs - 1);
long expectedResults = added[docToQuery] ? 1 : 0;
logger.info("Searching for [test:{}]", English.intToEnglish(docToQuery));
SearchResponse searchResponse =
client()
.prepareSearch()
.setQuery(QueryBuilders.matchQuery("test", English.intToEnglish(docToQuery)))
.get();
logger.info(
"Successful shards: [{}] numShards: [{}]",
searchResponse.getSuccessfulShards(),
numShards);
if (searchResponse.getSuccessfulShards() == numShards && !refreshFailed) {
assertThat(searchResponse.getHits().getTotalHits(), Matchers.equalTo(expectedResults));
}
// check match all
searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchAllQuery()).get();
if (searchResponse.getSuccessfulShards() == numShards && !refreshFailed) {
assertThat(searchResponse.getHits().getTotalHits(), Matchers.equalTo(numCreated));
}
} catch (SearchPhaseExecutionException ex) {
if (!expectAllShardsFailed) {
throw ex;
} else {
logger.info("expected SearchPhaseException: [{}]", ex.getMessage());
}
}
}
}
/** Stuff to do before the server is started. */
protected void preStart() throws Exception {
if (ManagerTestUtils.getTestType() == TestType.jpa) {
TestUtil.setProperty("apiman.hibernate.hbm2ddl.auto", "create-drop");
TestUtil.setProperty(
"apiman.hibernate.connection.datasource", "java:/comp/env/jdbc/ApiManagerDS");
try {
InitialContext ctx = new InitialContext();
ensureCtx(ctx, "java:/comp/env");
ensureCtx(ctx, "java:/comp/env/jdbc");
String dbOutputPath = System.getProperty("apiman.test.h2-output-dir", null);
if (dbOutputPath != null) {
ds = createFileDatasource(new File(dbOutputPath));
} else {
ds = createInMemoryDatasource();
}
ctx.bind("java:/comp/env/jdbc/ApiManagerDS", ds);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
if (ManagerTestUtils.getTestType() == TestType.es && node == null) {
System.out.println("Creating the ES node.");
File esHome = new File("target/es");
String esHomeSP = System.getProperty("apiman.test.es-home", null);
if (esHomeSP != null) {
esHome = new File(esHomeSP);
}
if (esHome.isDirectory()) {
FileUtils.deleteDirectory(esHome);
}
Builder settings = NodeBuilder.nodeBuilder().settings();
settings.put("path.home", esHome.getAbsolutePath());
settings.put("http.port", "6500-6600");
settings.put("transport.tcp.port", "6600-6700");
settings.put("discovery.zen.ping.multicast.enabled", false);
String clusterName = System.getProperty("apiman.test.es-cluster-name", ES_CLUSTER_NAME);
boolean isPersistent =
"true".equals(System.getProperty("apiman.test.es-persistence", "false"));
if (!isPersistent) {
System.out.println("Creating non-persistent ES");
settings
.put("index.store.type", "memory")
.put("gateway.type", "none")
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 1);
node =
NodeBuilder.nodeBuilder()
.client(false)
.clusterName(clusterName)
.data(true)
.local(true)
.settings(settings)
.build();
} else {
System.out.println("Creating *persistent* ES here: " + esHome);
node =
NodeBuilder.nodeBuilder()
.client(false)
.clusterName(clusterName)
.data(true)
.local(false)
.settings(settings)
.build();
}
System.out.println("Starting the ES node.");
node.start();
System.out.println("ES node was successfully started.");
// TODO parameterize this
String connectionUrl = "http://localhost:6500";
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(
new HttpClientConfig.Builder(connectionUrl).multiThreaded(true).build());
client = factory.getObject();
ES_CLIENT = client;
}
}
@Test
public void testMoreDocs() throws Exception {
Builder builder = ImmutableSettings.builder();
builder.put("index.analysis.analyzer.synonym.tokenizer", "whitespace");
builder.putArray("index.analysis.analyzer.synonym.filter", "synonym", "lowercase");
builder.put("index.analysis.filter.synonym.type", "synonym");
builder.putArray(
"index.analysis.filter.synonym.synonyms", "ave => ave, avenue", "street => str, street");
XContentBuilder mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type1")
.startObject("properties")
.startObject("field1")
.field("type", "string")
.field("index_analyzer", "whitespace")
.field("search_analyzer", "synonym")
.endObject()
.endObject()
.endObject()
.endObject();
assertAcked(
client()
.admin()
.indices()
.prepareCreate("test")
.addMapping("type1", mapping)
.setSettings(builder.put("index.number_of_shards", 1)));
client()
.prepareIndex("test", "type1", "1")
.setSource("field1", "massachusetts avenue boston massachusetts")
.execute()
.actionGet();
client()
.prepareIndex("test", "type1", "2")
.setSource("field1", "lexington avenue boston massachusetts")
.execute()
.actionGet();
client()
.prepareIndex("test", "type1", "3")
.setSource("field1", "boston avenue lexington massachusetts")
.execute()
.actionGet();
client().admin().indices().prepareRefresh("test").execute().actionGet();
client()
.prepareIndex("test", "type1", "4")
.setSource("field1", "boston road lexington massachusetts")
.execute()
.actionGet();
client()
.prepareIndex("test", "type1", "5")
.setSource("field1", "lexington street lexington massachusetts")
.execute()
.actionGet();
client()
.prepareIndex("test", "type1", "6")
.setSource("field1", "massachusetts avenue lexington massachusetts")
.execute()
.actionGet();
client()
.prepareIndex("test", "type1", "7")
.setSource("field1", "bosten street san franciso california")
.execute()
.actionGet();
client().admin().indices().prepareRefresh("test").execute().actionGet();
client()
.prepareIndex("test", "type1", "8")
.setSource("field1", "hollywood boulevard los angeles california")
.execute()
.actionGet();
client()
.prepareIndex("test", "type1", "9")
.setSource("field1", "1st street boston massachussetts")
.execute()
.actionGet();
client()
.prepareIndex("test", "type1", "10")
.setSource("field1", "1st street boston massachusetts")
.execute()
.actionGet();
client().admin().indices().prepareRefresh("test").execute().actionGet();
client()
.prepareIndex("test", "type1", "11")
.setSource("field1", "2st street boston massachusetts")
.execute()
.actionGet();
client()
.prepareIndex("test", "type1", "12")
.setSource("field1", "3st street boston massachusetts")
.execute()
.actionGet();
client().admin().indices().prepareRefresh("test").execute().actionGet();
SearchResponse searchResponse =
client()
.prepareSearch()
.setQuery(
QueryBuilders.matchQuery("field1", "lexington avenue massachusetts")
.operator(MatchQueryBuilder.Operator.OR))
.setFrom(0)
.setSize(5)
.setRescorer(
RescoreBuilder.queryRescorer(
QueryBuilders.matchPhraseQuery("field1", "lexington avenue massachusetts")
.slop(3))
.setQueryWeight(0.6f)
.setRescoreQueryWeight(2.0f))
.setRescoreWindow(20)
.execute()
.actionGet();
assertThat(searchResponse.getHits().hits().length, equalTo(5));
assertHitCount(searchResponse, 9);
assertFirstHit(searchResponse, hasId("2"));
assertSecondHit(searchResponse, hasId("6"));
assertThirdHit(searchResponse, hasId("3"));
searchResponse =
client()
.prepareSearch()
.setQuery(
QueryBuilders.matchQuery("field1", "lexington avenue massachusetts")
.operator(MatchQueryBuilder.Operator.OR))
.setFrom(0)
.setSize(5)
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.setRescorer(
RescoreBuilder.queryRescorer(
QueryBuilders.matchPhraseQuery("field1", "lexington avenue massachusetts")
.slop(3))
.setQueryWeight(0.6f)
.setRescoreQueryWeight(2.0f))
.setRescoreWindow(20)
.execute()
.actionGet();
assertThat(searchResponse.getHits().hits().length, equalTo(5));
assertHitCount(searchResponse, 9);
assertFirstHit(searchResponse, hasId("2"));
assertSecondHit(searchResponse, hasId("6"));
assertThirdHit(searchResponse, hasId("3"));
}
public static void createCommunityDocIndex(
String nameOrCommunityIdStr,
ObjectId parentCommunityId,
boolean bPersonalGroup,
boolean bSystemGroup,
boolean bClearIndex,
boolean bParentsOnly) {
// create elasticsearch indexes
PropertiesManager pm = new PropertiesManager();
boolean languageNormalization = pm.getNormalizeEncoding();
int nPreferredReplicas = pm.getMaxIndexReplicas();
String docMapping =
new Gson().toJson(new DocumentPojoIndexMap.Mapping(), DocumentPojoIndexMap.Mapping.class);
String sGroupIndex = null;
try {
sGroupIndex =
new StringBuffer("doc_").append(new ObjectId(nameOrCommunityIdStr).toString()).toString();
} catch (Exception e) {
sGroupIndex = nameOrCommunityIdStr;
}
if (!bPersonalGroup) {
String parentCommunityIdStr = null;
if (null != parentCommunityId) {
parentCommunityIdStr = parentCommunityId.toString();
}
if ((null == parentCommunityIdStr)
|| (parentCommunityIdStr.equals("4c927585d591d31d7b37097a"))) {
// (system community is hardwired - children of this community are ignored)
int nShards = bSystemGroup ? 10 : 5; // (system group is largest)
// Remove the alias, in case it exists:
// Then create an index with this name:
Builder localSettingsGroupIndex = ImmutableSettings.settingsBuilder();
localSettingsGroupIndex
.put("number_of_shards", nShards)
.put("number_of_replicas", nPreferredReplicas);
if (languageNormalization) {
localSettingsGroupIndex.put("index.analysis.analyzer.default.tokenizer", "standard");
localSettingsGroupIndex.putArray(
"index.analysis.analyzer.default.filter",
"icu_normalizer",
"icu_folding",
"standard",
"lowercase",
"stop");
} // TESTED
ElasticSearchManager docIndex = null;
try {
docIndex =
IndexManager.createIndex(
sGroupIndex,
DocumentPojoIndexMap.documentType_,
false,
null,
docMapping,
localSettingsGroupIndex);
} catch (
RuntimeException e) { // illegal arg exception, probably the language normalization?
if (languageNormalization) { // (likely the required plugins have not been installed, just
// regress back to normal)
localSettingsGroupIndex = ImmutableSettings.settingsBuilder();
localSettingsGroupIndex
.put("number_of_shards", nShards)
.put("number_of_replicas", nPreferredReplicas);
docIndex =
IndexManager.createIndex(
sGroupIndex,
DocumentPojoIndexMap.documentType_,
false,
null,
docMapping,
localSettingsGroupIndex);
} // TESTED
else throw e;
} // TOTEST
if (bClearIndex) {
docIndex.deleteMe();
docIndex =
IndexManager.createIndex(
sGroupIndex,
DocumentPojoIndexMap.documentType_,
false,
null,
docMapping,
localSettingsGroupIndex);
}
if (null != docIndex) {
try {
docIndex.pingIndex(); // (wait until it's created itself)
} catch (Exception e) {
} // (just make sure this doesn't die horribly)
docIndex.closeIndex();
}
} else if (!bParentsOnly) {
String sParentGroupIndex =
new StringBuffer("doc_")
.append(new ObjectId(parentCommunityIdStr).toString())
.toString();
ElasticSearchManager docIndex = IndexManager.getIndex(sParentGroupIndex);
// DEBUG (alias corruption)
// if (null == _aliasInfo) {
// ClusterStateResponse clusterState =
// docIndex.getRawClient().admin().cluster().state(new ClusterStateRequest()).actionGet();
// _aliasInfo = clusterState.getState().getMetaData().getAliases();
// }
// else {
// if (_aliasInfo.containsKey(sGroupIndex)) { // has no aliases, we're not good
// return;
// }
// else {
// //DEBUG
// System.out.println("Alias " + sGroupIndex + " has no aliases (but should)");
// ElasticSearchManager docIndex2 = IndexManager.getIndex(sGroupIndex);
// docIndex2.deleteMe();
// }
// }
docIndex.createAlias(sGroupIndex);
docIndex.closeIndex();
// (do nothing on delete - that will be handled at the parent index level)
}
// TESTED (parents, children, and personal)
} else {
// Just create the dummy index, no different to getting it in practice
Builder localSettingsGroupIndex = ImmutableSettings.settingsBuilder();
localSettingsGroupIndex
.put("number_of_shards", 1)
.put("number_of_replicas", 0); // (ie guaranteed to be local to each ES node)
ElasticSearchManager dummyGroupIndex =
IndexManager.createIndex(
DocumentPojoIndexMap.dummyDocumentIndex_,
DocumentPojoIndexMap.documentType_,
false,
null,
docMapping,
localSettingsGroupIndex);
if (null == dummyGroupIndex) {
dummyGroupIndex = IndexManager.getIndex(DocumentPojoIndexMap.dummyDocumentIndex_);
}
// Just create an alias, so that queries work arbitrarily:
dummyGroupIndex.createAlias(sGroupIndex);
// (do nothing on delete since don't have any docs in here anyway)
}
}
public void InitializeIndex(
boolean bDeleteDocs,
boolean bDeleteEntityFeature,
boolean bDeleteEventFeature,
boolean bRebuildDocsIndex) {
try { // create elasticsearch indexes
PropertiesManager pm = new PropertiesManager();
if (!pm.getAggregationDisabled()) {
Builder localSettingsEvent = ImmutableSettings.settingsBuilder();
localSettingsEvent.put("number_of_shards", 1).put("number_of_replicas", 0);
localSettingsEvent.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
localSettingsEvent.putArray(
"index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");
localSettingsEvent.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
localSettingsEvent.putArray(
"index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");
Builder localSettingsGaz = ImmutableSettings.settingsBuilder();
localSettingsGaz.put("number_of_shards", 1).put("number_of_replicas", 0);
localSettingsGaz.put("index.analysis.analyzer.suggestAnalyzer.tokenizer", "standard");
localSettingsGaz.putArray(
"index.analysis.analyzer.suggestAnalyzer.filter", "standard", "lowercase");
// event feature
String eventGazMapping =
new Gson()
.toJson(
new AssociationFeaturePojoIndexMap.Mapping(),
AssociationFeaturePojoIndexMap.Mapping.class);
ElasticSearchManager eventIndex =
IndexManager.createIndex(
AssociationFeaturePojoIndexMap.indexName_,
null,
false,
null,
eventGazMapping,
localSettingsEvent);
if (bDeleteEventFeature) {
eventIndex.deleteMe();
eventIndex =
IndexManager.createIndex(
AssociationFeaturePojoIndexMap.indexName_,
null,
false,
null,
eventGazMapping,
localSettingsEvent);
}
// entity feature
String gazMapping =
new Gson()
.toJson(
new EntityFeaturePojoIndexMap.Mapping(),
EntityFeaturePojoIndexMap.Mapping.class);
ElasticSearchManager entityIndex =
IndexManager.createIndex(
EntityFeaturePojoIndexMap.indexName_,
null,
false,
null,
gazMapping,
localSettingsGaz);
if (bDeleteEntityFeature) {
entityIndex.deleteMe();
entityIndex =
IndexManager.createIndex(
EntityFeaturePojoIndexMap.indexName_,
null,
false,
null,
gazMapping,
localSettingsGaz);
}
}
// DOCS - much more complicated than anything else
boolean bPingMainIndexFailed =
!ElasticSearchManager.pingIndex(DocumentPojoIndexMap.globalDocumentIndex_);
// (ie if main doc index doesn't exist then always rebuild all indexes)
if (bPingMainIndexFailed) { // extra level of robustness... sleep for a minute then double
// check the index is really missing...
try {
Thread.sleep(60000);
} catch (Exception e) {
}
bPingMainIndexFailed =
!ElasticSearchManager.pingIndex(DocumentPojoIndexMap.globalDocumentIndex_);
}
bRebuildDocsIndex |= bPingMainIndexFailed;
createCommunityDocIndex(
DocumentPojoIndexMap.globalDocumentIndex_, null, false, true, bDeleteDocs);
createCommunityDocIndex(
DocumentPojoIndexMap.manyGeoDocumentIndex_, null, false, false, bDeleteDocs);
// Some hardwired dummy communities
createCommunityDocIndex(
"4e3706c48d26852237078005", null, true, false, bDeleteDocs); // (admin)
createCommunityDocIndex(
"4e3706c48d26852237079004", null, true, false, bDeleteDocs); // (test user)
// (create dummy index used to keep personal group aliases)
// OK, going to have different shards for different communities:
// Get a list of all the communities:
BasicDBObject query = new BasicDBObject();
BasicDBObject fieldsToDrop = new BasicDBObject("members", 0);
fieldsToDrop.put("communityAttributes", 0);
fieldsToDrop.put("userAttributes", 0);
DBCursor dbc = DbManager.getSocial().getCommunity().find(query, fieldsToDrop);
if (bRebuildDocsIndex || bDeleteDocs) {
List<DBObject> tmparray =
dbc.toArray(); // (brings the entire thing into memory so don't get cursor timeouts)
int i = 0;
System.out.println("Initializing " + dbc.size() + " indexes:");
for (int j = 0; j < 2; ++j) {
for (DBObject dbotmp : tmparray) {
if ((++i % 100) == 0) {
System.out.println("Initialized " + i + " indexes.");
}
BasicDBObject dbo = (BasicDBObject) dbotmp;
// OK, going to see if there are any sources with this group id, create a new index if
// so:
// (Don't use CommunityPojo data model here for performance reasons....
// (Also, haven't gotten round to porting CommunityPojo field access to using static
// fields))
ObjectId communityId = (ObjectId) dbo.get("_id");
boolean bPersonalGroup = dbo.getBoolean("isPersonalCommunity", false);
boolean bSystemGroup = dbo.getBoolean("isSystemCommunity", false);
ObjectId parentCommunityId = (ObjectId) dbo.get("parentId");
createCommunityDocIndex(
communityId.toString(),
parentCommunityId,
bPersonalGroup,
bSystemGroup,
bDeleteDocs,
j == 0);
} // end loop over communities
} // end loop over communities - first time parents only
} // (end if need to do big loop over all sources)
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e.getMessage());
}
} // TESTED (not changed since by-eye test in Beta - retested after moving code into
public static IndexMetaData fromXContent(XContentParser parser) throws IOException {
if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
parser.nextToken();
}
Builder builder = new Builder(parser.currentName());
String currentFieldName = null;
XContentParser.Token token = parser.nextToken();
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if ("settings".equals(currentFieldName)) {
builder.settings(
ImmutableSettings.settingsBuilder()
.put(SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered())));
} else if ("mappings".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
String mappingType = currentFieldName;
Map<String, Object> mappingSource =
MapBuilder.<String, Object>newMapBuilder()
.put(mappingType, parser.mapOrdered())
.map();
builder.putMapping(new MappingMetaData(mappingType, mappingSource));
}
}
} else if ("aliases".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
builder.putAlias(AliasMetaData.Builder.fromXContent(parser));
}
} else {
// check if its a custom index metadata
Custom.Factory<Custom> factory = lookupFactory(currentFieldName);
if (factory == null) {
// TODO warn
parser.skipChildren();
} else {
builder.putCustom(factory.type(), factory.fromXContent(parser));
}
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("mappings".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
builder.putMapping(new MappingMetaData(new CompressedString(parser.binaryValue())));
} else {
Map<String, Object> mapping = parser.mapOrdered();
if (mapping.size() == 1) {
String mappingType = mapping.keySet().iterator().next();
builder.putMapping(new MappingMetaData(mappingType, mapping));
}
}
}
}
} else if (token.isValue()) {
if ("state".equals(currentFieldName)) {
builder.state(State.fromString(parser.text()));
} else if ("version".equals(currentFieldName)) {
builder.version(parser.longValue());
}
}
}
return builder.build();
}
@Test
public void testBackwardsCompatibilityEdgeNgramTokenizer()
throws IllegalArgumentException, IllegalAccessException {
int iters = atLeast(20);
final Index index = new Index("test");
final String name = "ngr";
for (int i = 0; i < iters; i++) {
Version v = randomVersion(random());
if (v.onOrAfter(Version.V_0_90_2)) {
Builder builder =
ImmutableSettings.builder()
.put("min_gram", 2)
.put("max_gram", 3)
.put("token_chars", "letter,digit");
boolean compatVersion = false;
if ((compatVersion = random().nextBoolean())) {
builder.put("version", "4." + random().nextInt(3));
builder.put("side", "back");
}
Settings settings = builder.build();
Settings indexSettings =
ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build();
Tokenizer edgeNGramTokenizer =
new EdgeNGramTokenizerFactory(index, indexSettings, name, settings)
.create(new StringReader("foo bar"));
if (compatVersion) {
assertThat(edgeNGramTokenizer, instanceOf(Lucene43EdgeNGramTokenizer.class));
} else {
assertThat(edgeNGramTokenizer, instanceOf(EdgeNGramTokenizer.class));
}
} else {
Settings settings =
ImmutableSettings.builder()
.put("min_gram", 2)
.put("max_gram", 3)
.put("side", "back")
.build();
Settings indexSettings =
ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build();
Tokenizer edgeNGramTokenizer =
new EdgeNGramTokenizerFactory(index, indexSettings, name, settings)
.create(new StringReader("foo bar"));
assertThat(edgeNGramTokenizer, instanceOf(Lucene43EdgeNGramTokenizer.class));
}
}
Settings settings =
ImmutableSettings.builder()
.put("min_gram", 2)
.put("max_gram", 3)
.put("side", "back")
.build();
Settings indexSettings =
ImmutableSettings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build();
try {
new EdgeNGramTokenizerFactory(index, indexSettings, name, settings)
.create(new StringReader("foo bar"));
fail("should fail side:back is not supported anymore");
} catch (ElasticSearchIllegalArgumentException ex) {
}
}