@Test
public void testBackwardsCompatibilityEdgeNgramTokenFilter()
throws IllegalArgumentException, IllegalAccessException {
int iters = atLeast(20);
for (int i = 0; i < iters; i++) {
final Index index = new Index("test");
final String name = "ngr";
Version v = randomVersion(random());
if (v.onOrAfter(Version.V_0_90_2)) {
Builder builder = ImmutableSettings.builder().put("min_gram", 2).put("max_gram", 3);
boolean compatVersion = false;
if ((compatVersion = random().nextBoolean())) {
builder.put("version", "4." + random().nextInt(3));
}
boolean reverse = random().nextBoolean();
if (reverse) {
builder.put("side", "back");
}
Settings settings = builder.build();
Settings indexSettings =
ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build();
TokenStream edgeNGramTokenFilter =
new EdgeNGramTokenFilterFactory(index, indexSettings, name, settings)
.create(new MockTokenizer(new StringReader("foo bar")));
if (compatVersion) {
assertThat(edgeNGramTokenFilter, instanceOf(EdgeNGramTokenFilter.class));
} else if (reverse && !compatVersion) {
assertThat(edgeNGramTokenFilter, instanceOf(ReverseStringFilter.class));
} else {
assertThat(edgeNGramTokenFilter, instanceOf(EdgeNGramTokenFilter.class));
}
} else {
Builder builder = ImmutableSettings.builder().put("min_gram", 2).put("max_gram", 3);
boolean reverse = random().nextBoolean();
if (reverse) {
builder.put("side", "back");
}
Settings settings = builder.build();
Settings indexSettings =
ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build();
TokenStream edgeNGramTokenFilter =
new EdgeNGramTokenFilterFactory(index, indexSettings, name, settings)
.create(new MockTokenizer(new StringReader("foo bar")));
assertThat(edgeNGramTokenFilter, instanceOf(EdgeNGramTokenFilter.class));
}
}
}
private Settings getSettings(int nodeOrdinal, Settings others) {
Builder builder = ImmutableSettings.settingsBuilder().put(defaultSettings);
Settings settings = nodeSettingsSource.settings(nodeOrdinal);
if (settings != null) {
builder.put(settings);
}
if (others != null) {
builder.put(others);
}
return builder.build();
}
public void start() throws Exception {
try {
final Builder settings = ImmutableSettings.settingsBuilder();
settings.put("path.data", filesSettings.getPermanentDirectory());
settings.put("client.transport.sniff", true);
settings.build();
final NodeBuilder nb = nodeBuilder().settings(settings).local(true).client(false).data(true);
final Node node = nb.node();
client = node.client();
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest("entities");
// we just send back a response, no need to fork a listener
indicesExistsRequest.listenerThreaded(false);
client
.admin()
.indices()
.exists(
indicesExistsRequest,
new ActionListener<IndicesExistsResponse>() {
public void onResponse(IndicesExistsResponse response) {
if (!response.isExists()) {
try {
logger.debug("Entities indice does not exists. Creating it...");
CreateIndexResponse r =
client
.admin()
.indices()
.create(new CreateIndexRequest("entities"))
.actionGet();
logger.debug(
"Created indice with response {}",
r.isAcknowledged() ? "\u2713 acknowledged" : "\2718 not acknowledged");
} catch (Exception e) {
logger.error("Failed to create entities indice status ...");
}
}
updateMappings();
}
public void onFailure(Throwable e) {
logger.error("Failed to enquiry entities indice status ...", e);
}
});
} catch (Exception e) {
throw new InitializationException("Failed to initialize embedded elastic search", e);
}
}
TestCluster(
long clusterSeed, int numNodes, String clusterName, NodeSettingsSource nodeSettingsSource) {
this.clusterName = clusterName;
Random random = new Random(clusterSeed);
numSharedNodes =
numNodes == -1 ? 2 + random.nextInt(4) : numNodes; // at least 2 nodes if randomized
assert numSharedNodes >= 0;
/*
* TODO
* - we might want start some master only nodes?
* - we could add a flag that returns a client to the master all the time?
* - we could add a flag that never returns a client to the master
* - along those lines use a dedicated node that is master eligible and let all other nodes be only data nodes
*/
sharedNodesSeeds = new long[numSharedNodes];
for (int i = 0; i < sharedNodesSeeds.length; i++) {
sharedNodesSeeds[i] = random.nextLong();
}
logger.info(
"Setup TestCluster [{}] with seed [{}] using [{}] nodes",
clusterName,
SeedUtils.formatSeed(clusterSeed),
numSharedNodes);
Builder builder =
ImmutableSettings.settingsBuilder()
/* use RAM directories in 10% of the runs */
// .put("index.store.type", random.nextInt(10) == 0 ?
// MockRamIndexStoreModule.class.getName() : MockFSIndexStoreModule.class.getName())
.put("index.store.type", MockFSIndexStoreModule.class.getName()) // no RAM dir for now!
.put(IndexEngineModule.EngineSettings.ENGINE_TYPE, MockEngineModule.class.getName())
.put("cluster.name", clusterName)
// decrease the routing schedule so new nodes will be added quickly - some random value
// between 30 and 80 ms
.put("cluster.routing.schedule", (30 + random.nextInt(50)) + "ms")
// default to non gateway
.put("gateway.type", "none");
if (isLocalTransportConfigured()) {
builder.put(
TransportModule.TRANSPORT_TYPE_KEY, AssertingLocalTransportModule.class.getName());
} else {
builder.put(Transport.TransportSettings.TRANSPORT_TCP_COMPRESS, random.nextInt(10) == 0);
}
this.defaultSettings = builder.build();
this.nodeSettingsSource = nodeSettingsSource;
}
@Before
public void setUp() throws Exception {
System.out.println(
"--------------------- SETUP "
+ name.getMethodName()
+ " -------------------------------------");
MockMailbox.resetAll();
// Instantiates a local node & client
esSetup = new EsSetup(settingsBuilder.build());
// Clean all, and creates some indices
esSetup.execute(deleteAll());
}
private static Settings normalizeSettings(final Settings settings) {
Builder builder = settingsBuilder();
builder.put(settings);
// automatically overridden
builder.put("client.type", "transport");
builder.put("node.client", true);
builder.put("network.server", false);
// ensure behavior
builder.put("node.master", false);
builder.put("node.data", false);
// disable HTTP
builder.put("http.enabled", false);
return builder.build();
}
public static IndexMetaData readFrom(StreamInput in) throws IOException {
Builder builder = new Builder(in.readUTF());
builder.version(in.readLong());
builder.state(State.fromId(in.readByte()));
builder.settings(readSettingsFromStream(in));
int mappingsSize = in.readVInt();
for (int i = 0; i < mappingsSize; i++) {
MappingMetaData mappingMd = MappingMetaData.readFrom(in);
builder.putMapping(mappingMd);
}
int aliasesSize = in.readVInt();
for (int i = 0; i < aliasesSize; i++) {
AliasMetaData aliasMd = AliasMetaData.Builder.readFrom(in);
builder.putAlias(aliasMd);
}
int customSize = in.readVInt();
for (int i = 0; i < customSize; i++) {
String type = in.readUTF();
Custom customIndexMetaData = lookupFactorySafe(type).readFrom(in);
builder.putCustom(type, customIndexMetaData);
}
return builder.build();
}
@Before
public void buildNode1() throws Exception {
super.setUp();
Builder builder = ImmutableSettings.builder();
builder.put("cluster.routing.schedule", "100ms");
builder.put("gateway.type", "fs");
if (between(0, 5) == 0) {
builder.put("gateway.fs.buffer_size", between(1, 100) + "kb");
}
if (between(0, 5) == 0) {
builder.put("gateway.fs.chunk_size", between(1, 100) + "kb");
}
builder.put("index.number_of_replicas", "1");
builder.put("index.number_of_shards", rarely() ? Integer.toString(between(2, 6)) : "1");
storeType = rarely() ? "ram" : "fs";
builder.put("index.store.type", storeType);
defaultSettings = builder.build();
buildNode("server1");
// since we store (by default) the index snapshot under the gateway, resetting it will reset the
// index data as well
((InternalNode) node("server1")).injector().getInstance(Gateway.class).reset();
closeAllNodes();
}
@Test
public void testRandomExceptions() throws IOException, InterruptedException, ExecutionException {
final int numShards = between(1, 5);
String mapping =
XContentFactory.jsonBuilder()
.startObject()
.startObject("type")
.startObject("properties")
.startObject("test")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
.endObject()
.endObject()
.endObject()
.string();
final double exceptionRate;
final double exceptionOnOpenRate;
if (frequently()) {
if (randomBoolean()) {
if (randomBoolean()) {
exceptionOnOpenRate = 1.0 / between(5, 100);
exceptionRate = 0.0d;
} else {
exceptionRate = 1.0 / between(5, 100);
exceptionOnOpenRate = 0.0d;
}
} else {
exceptionOnOpenRate = 1.0 / between(5, 100);
exceptionRate = 1.0 / between(5, 100);
}
} else {
// rarely no exception
exceptionRate = 0d;
exceptionOnOpenRate = 0d;
}
Builder settings =
settingsBuilder()
.put("index.number_of_shards", numShards)
.put("index.number_of_replicas", randomIntBetween(0, 1))
.put(MockDirectoryHelper.RANDOM_IO_EXCEPTION_RATE, exceptionRate)
.put(MockDirectoryHelper.RANDOM_IO_EXCEPTION_RATE_ON_OPEN, exceptionOnOpenRate)
.put(MockDirectoryHelper.CHECK_INDEX_ON_CLOSE, true);
logger.info("creating index: [test] using settings: [{}]", settings.build().getAsMap());
client()
.admin()
.indices()
.prepareCreate("test")
.setSettings(settings)
.addMapping("type", mapping)
.execute()
.actionGet();
ClusterHealthResponse clusterHealthResponse =
client()
.admin()
.cluster()
.health(
Requests.clusterHealthRequest()
.waitForYellowStatus()
.timeout(TimeValue.timeValueSeconds(5)))
.get(); // it's OK to timeout here
final int numDocs;
final boolean expectAllShardsFailed;
if (clusterHealthResponse.isTimedOut()) {
/* some seeds just won't let you create the index at all and we enter a ping-pong mode
* trying one node after another etc. that is ok but we need to make sure we don't wait
* forever when indexing documents so we set numDocs = 1 and expecte all shards to fail
* when we search below.*/
logger.info("ClusterHealth timed out - only index one doc and expect searches to fail");
numDocs = 1;
expectAllShardsFailed = true;
} else {
numDocs = between(10, 100);
expectAllShardsFailed = false;
}
long numCreated = 0;
boolean[] added = new boolean[numDocs];
for (int i = 0; i < numDocs; i++) {
try {
IndexResponse indexResponse =
client()
.prepareIndex("test", "type", "" + i)
.setTimeout(TimeValue.timeValueSeconds(1))
.setSource("test", English.intToEnglish(i))
.get();
if (indexResponse.isCreated()) {
numCreated++;
added[i] = true;
}
} catch (ElasticSearchException ex) {
}
}
logger.info("Start Refresh");
RefreshResponse refreshResponse =
client()
.admin()
.indices()
.prepareRefresh("test")
.execute()
.get(); // don't assert on failures here
final boolean refreshFailed =
refreshResponse.getShardFailures().length != 0 || refreshResponse.getFailedShards() != 0;
logger.info(
"Refresh failed [{}] numShardsFailed: [{}], shardFailuresLength: [{}], successfulShards: [{}], totalShards: [{}] ",
refreshFailed,
refreshResponse.getFailedShards(),
refreshResponse.getShardFailures().length,
refreshResponse.getSuccessfulShards(),
refreshResponse.getTotalShards());
final int numSearches = atLeast(10);
// we don't check anything here really just making sure we don't leave any open files or a
// broken index behind.
for (int i = 0; i < numSearches; i++) {
try {
int docToQuery = between(0, numDocs - 1);
long expectedResults = added[docToQuery] ? 1 : 0;
logger.info("Searching for [test:{}]", English.intToEnglish(docToQuery));
SearchResponse searchResponse =
client()
.prepareSearch()
.setQuery(QueryBuilders.matchQuery("test", English.intToEnglish(docToQuery)))
.get();
logger.info(
"Successful shards: [{}] numShards: [{}]",
searchResponse.getSuccessfulShards(),
numShards);
if (searchResponse.getSuccessfulShards() == numShards && !refreshFailed) {
assertThat(searchResponse.getHits().getTotalHits(), Matchers.equalTo(expectedResults));
}
// check match all
searchResponse = client().prepareSearch().setQuery(QueryBuilders.matchAllQuery()).get();
if (searchResponse.getSuccessfulShards() == numShards && !refreshFailed) {
assertThat(searchResponse.getHits().getTotalHits(), Matchers.equalTo(numCreated));
}
} catch (SearchPhaseExecutionException ex) {
if (!expectAllShardsFailed) {
throw ex;
} else {
logger.info("expected SearchPhaseException: [{}]", ex.getMessage());
}
}
}
}
public static IndexMetaData fromXContent(XContentParser parser) throws IOException {
if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
parser.nextToken();
}
Builder builder = new Builder(parser.currentName());
String currentFieldName = null;
XContentParser.Token token = parser.nextToken();
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
if ("settings".equals(currentFieldName)) {
builder.settings(
ImmutableSettings.settingsBuilder()
.put(SettingsLoader.Helper.loadNestedFromMap(parser.mapOrdered())));
} else if ("mappings".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.START_OBJECT) {
String mappingType = currentFieldName;
Map<String, Object> mappingSource =
MapBuilder.<String, Object>newMapBuilder()
.put(mappingType, parser.mapOrdered())
.map();
builder.putMapping(new MappingMetaData(mappingType, mappingSource));
}
}
} else if ("aliases".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
builder.putAlias(AliasMetaData.Builder.fromXContent(parser));
}
} else {
// check if its a custom index metadata
Custom.Factory<Custom> factory = lookupFactory(currentFieldName);
if (factory == null) {
// TODO warn
parser.skipChildren();
} else {
builder.putCustom(factory.type(), factory.fromXContent(parser));
}
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("mappings".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
builder.putMapping(new MappingMetaData(new CompressedString(parser.binaryValue())));
} else {
Map<String, Object> mapping = parser.mapOrdered();
if (mapping.size() == 1) {
String mappingType = mapping.keySet().iterator().next();
builder.putMapping(new MappingMetaData(mappingType, mapping));
}
}
}
}
} else if (token.isValue()) {
if ("state".equals(currentFieldName)) {
builder.state(State.fromString(parser.text()));
} else if ("version".equals(currentFieldName)) {
builder.version(parser.longValue());
}
}
}
return builder.build();
}
@Test
public void testBackwardsCompatibilityEdgeNgramTokenizer()
throws IllegalArgumentException, IllegalAccessException {
int iters = atLeast(20);
final Index index = new Index("test");
final String name = "ngr";
for (int i = 0; i < iters; i++) {
Version v = randomVersion(random());
if (v.onOrAfter(Version.V_0_90_2)) {
Builder builder =
ImmutableSettings.builder()
.put("min_gram", 2)
.put("max_gram", 3)
.put("token_chars", "letter,digit");
boolean compatVersion = false;
if ((compatVersion = random().nextBoolean())) {
builder.put("version", "4." + random().nextInt(3));
builder.put("side", "back");
}
Settings settings = builder.build();
Settings indexSettings =
ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build();
Tokenizer edgeNGramTokenizer =
new EdgeNGramTokenizerFactory(index, indexSettings, name, settings)
.create(new StringReader("foo bar"));
if (compatVersion) {
assertThat(edgeNGramTokenizer, instanceOf(Lucene43EdgeNGramTokenizer.class));
} else {
assertThat(edgeNGramTokenizer, instanceOf(EdgeNGramTokenizer.class));
}
} else {
Settings settings =
ImmutableSettings.builder()
.put("min_gram", 2)
.put("max_gram", 3)
.put("side", "back")
.build();
Settings indexSettings =
ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v.id).build();
Tokenizer edgeNGramTokenizer =
new EdgeNGramTokenizerFactory(index, indexSettings, name, settings)
.create(new StringReader("foo bar"));
assertThat(edgeNGramTokenizer, instanceOf(Lucene43EdgeNGramTokenizer.class));
}
}
Settings settings =
ImmutableSettings.builder()
.put("min_gram", 2)
.put("max_gram", 3)
.put("side", "back")
.build();
Settings indexSettings =
ImmutableSettings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build();
try {
new EdgeNGramTokenizerFactory(index, indexSettings, name, settings)
.create(new StringReader("foo bar"));
fail("should fail side:back is not supported anymore");
} catch (ElasticSearchIllegalArgumentException ex) {
}
}