@Override
public synchronized IndexShard createShard(int sShardId) throws ElasticSearchException {
ShardId shardId = new ShardId(index, sShardId);
if (shardsInjectors.containsKey(shardId.id())) {
throw new IndexShardAlreadyExistsException(shardId + " already exists");
}
indicesLifecycle.beforeIndexShardCreated(shardId);
logger.debug("creating shard_id [{}]", shardId.id());
ModulesBuilder modules = new ModulesBuilder();
modules.add(new ShardsPluginsModule(indexSettings, pluginsService));
modules.add(new IndexShardModule(shardId));
modules.add(new StoreModule(indexSettings, injector.getInstance(IndexStore.class)));
modules.add(new DeletionPolicyModule(indexSettings));
modules.add(new MergePolicyModule(indexSettings));
modules.add(new MergeSchedulerModule(indexSettings));
modules.add(new TranslogModule(indexSettings));
modules.add(new EngineModule(indexSettings));
modules.add(new IndexShardGatewayModule(injector.getInstance(IndexGateway.class)));
Injector shardInjector = modules.createChildInjector(injector);
shardsInjectors =
newMapBuilder(shardsInjectors).put(shardId.id(), shardInjector).immutableMap();
IndexShard indexShard = shardInjector.getInstance(IndexShard.class);
indicesLifecycle.afterIndexShardCreated(indexShard);
shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
return indexShard;
}
public AnalysisService createAnalysisService() throws IOException {
InputStream empty_dict = getClass().getResourceAsStream("empty_user_dict.txt");
InputStream dict = getClass().getResourceAsStream("user_dict.txt");
Path home = createTempDir();
Path config = home.resolve("config");
Files.createDirectory(config);
Files.copy(empty_dict, config.resolve("empty_user_dict.txt"));
Files.copy(dict, config.resolve("user_dict.txt"));
String json = "/org/elasticsearch/index/analysis/kuromoji_analysis.json";
Settings settings =
Settings.settingsBuilder()
.put(Environment.PATH_HOME_SETTING.getKey(), home)
.loadFromStream(json, getClass().getResourceAsStream(json))
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build();
final SettingsModule settingsModule = new SettingsModule(settings);
settingsModule.registerSetting(InternalSettingsPlugin.VERSION_CREATED);
Index index = new Index("test", "_na_");
AnalysisModule analysisModule = new AnalysisModule(new Environment(settings));
new AnalysisKuromojiPlugin().onModule(analysisModule);
Injector parentInjector =
new ModulesBuilder()
.add(settingsModule, new EnvironmentModule(new Environment(settings)), analysisModule)
.createInjector();
return parentInjector
.getInstance(AnalysisRegistry.class)
.build(IndexSettingsModule.newIndexSettings(index, settings));
}
@Inject
public InternalIndexService(
Injector injector,
Index index,
@IndexSettings Settings indexSettings,
NodeEnvironment nodeEnv,
ThreadPool threadPool,
AnalysisService analysisService,
MapperService mapperService,
IndexQueryParserService queryParserService,
SimilarityService similarityService,
IndexCache indexCache,
IndexEngine indexEngine,
IndexGateway indexGateway,
IndexStore indexStore) {
super(index, indexSettings);
this.injector = injector;
this.nodeEnv = nodeEnv;
this.threadPool = threadPool;
this.indexSettings = indexSettings;
this.analysisService = analysisService;
this.mapperService = mapperService;
this.queryParserService = queryParserService;
this.similarityService = similarityService;
this.indexCache = indexCache;
this.indexEngine = indexEngine;
this.indexGateway = indexGateway;
this.indexStore = indexStore;
this.pluginsService = injector.getInstance(PluginsService.class);
this.indicesLifecycle = (InternalIndicesLifecycle) injector.getInstance(IndicesLifecycle.class);
this.indicesLifecycle.addListener(cleanCacheOnIndicesLifecycleListener);
}
@Test
public void testNativeScript() throws InterruptedException {
Settings settings =
Settings.settingsBuilder()
.put("script.native.my.type", MyNativeScriptFactory.class.getName())
.put("name", "testNativeScript")
.put("path.home", createTempDir())
.build();
Injector injector =
new ModulesBuilder()
.add(
new EnvironmentModule(new Environment(settings)),
new ThreadPoolModule(new ThreadPool(settings)),
new SettingsModule(settings),
new ScriptModule(settings))
.createInjector();
ScriptService scriptService = injector.getInstance(ScriptService.class);
ExecutableScript executable =
scriptService.executable(
new Script("my", ScriptType.INLINE, NativeScriptEngineService.NAME, null),
ScriptContext.Standard.SEARCH);
assertThat(executable.run().toString(), equalTo("test"));
terminate(injector.getInstance(ThreadPool.class));
}
@Override
public Node stop() {
if (!lifecycle.moveToStopped()) {
return this;
}
ESLogger logger = Loggers.getLogger(Node.class, settings.get("name"));
logger.info("{{}}[{}]: stopping ...", Version.full(), JvmInfo.jvmInfo().pid());
if (settings.getAsBoolean("http.enabled", true)) {
injector.getInstance(HttpServer.class).stop();
}
injector.getInstance(RoutingService.class).stop();
injector.getInstance(ClusterService.class).stop();
injector.getInstance(DiscoveryService.class).stop();
injector.getInstance(MonitorService.class).stop();
injector.getInstance(GatewayService.class).stop();
injector.getInstance(SearchService.class).stop();
injector.getInstance(RiversManager.class).stop();
injector.getInstance(IndicesClusterStateService.class).stop();
injector.getInstance(IndicesService.class).stop();
injector.getInstance(RestController.class).stop();
injector.getInstance(TransportService.class).stop();
injector.getInstance(JmxService.class).close();
for (Class<? extends LifecycleComponent> plugin : pluginsService.services()) {
injector.getInstance(plugin).stop();
}
logger.info("{{}}[{}]: stopped", Version.full(), JvmInfo.jvmInfo().pid());
return this;
}
@Test
public void testArirangAnalysis() {
Index index = new Index("test");
Settings settings =
settingsBuilder()
.put("path.home", "/Users/hwjeong/temp/tmp")
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build();
Injector parentInjector =
new ModulesBuilder()
.add(
new SettingsModule(EMPTY_SETTINGS),
new EnvironmentModule(new Environment(settings)))
.createInjector();
Injector injector =
new ModulesBuilder()
.add(
new IndexSettingsModule(index, settings),
new IndexNameModule(index),
new AnalysisModule(
EMPTY_SETTINGS, parentInjector.getInstance(IndicesAnalysisService.class))
.addProcessor(new ArirangAnalysisBinderProcessor()))
.createChildInjector(parentInjector);
AnalysisService analysisService = injector.getInstance(AnalysisService.class);
TokenizerFactory tokenizerFactory = analysisService.tokenizer("arirang_tokenizer");
MatcherAssert.assertThat(tokenizerFactory, instanceOf(ArirangTokenizerFactory.class));
}
public AnalysisService createAnalysisService() {
Settings settings =
Settings.settingsBuilder()
.put("path.home", createTempDir())
.loadFromClasspath("org/elasticsearch/index/analysis/kuromoji_analysis.json")
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build();
Index index = new Index("test");
Injector parentInjector =
new ModulesBuilder()
.add(
new SettingsModule(settings),
new EnvironmentModule(new Environment(settings)),
new IndicesAnalysisModule())
.createInjector();
AnalysisModule analysisModule =
new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class));
new AnalysisKuromojiPlugin().onModule(analysisModule);
Injector injector =
new ModulesBuilder()
.add(
new IndexSettingsModule(index, settings),
new IndexNameModule(index),
analysisModule)
.createChildInjector(parentInjector);
return injector.getInstance(AnalysisService.class);
}
@Test
public void testLookup() throws Exception {
Injector injector = new ModulesBuilder().add(new ScalarFunctionModule()).createInjector();
Functions functions = injector.getInstance(Functions.class);
assertEquals(ident, functions.get(ident).info().ident());
}
/** Builds a new instance of the transport client. */
public TransportClient build() {
Settings settings = InternalSettingsPreparer.prepareSettings(this.settings);
settings =
settingsBuilder()
.put(
NettyTransport.PING_SCHEDULE,
"5s") // enable by default the transport schedule ping interval
.put(settings)
.put("network.server", false)
.put("node.client", true)
.put(CLIENT_TYPE_SETTING, CLIENT_TYPE)
.build();
PluginsService pluginsService = new PluginsService(settings, null, null, pluginClasses);
this.settings = pluginsService.updatedSettings();
Version version = Version.CURRENT;
final ThreadPool threadPool = new ThreadPool(settings);
final NetworkService networkService = new NetworkService(settings);
final SettingsFilter settingsFilter = new SettingsFilter(settings);
NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
boolean success = false;
try {
ModulesBuilder modules = new ModulesBuilder();
modules.add(new Version.Module(version));
// plugin modules must be added here, before others or we can get crazy injection errors...
for (Module pluginModule : pluginsService.nodeModules()) {
modules.add(pluginModule);
}
modules.add(new PluginsModule(pluginsService));
modules.add(new SettingsModule(this.settings, settingsFilter));
modules.add(new NetworkModule(networkService, this.settings, true, namedWriteableRegistry));
modules.add(new ClusterNameModule(this.settings));
modules.add(new ThreadPoolModule(threadPool));
modules.add(
new SearchModule(settings, namedWriteableRegistry) {
@Override
protected void configure() {
// noop
}
});
modules.add(new ActionModule(true));
modules.add(new CircuitBreakerModule(this.settings));
pluginsService.processModules(modules);
Injector injector = modules.createInjector();
injector.getInstance(TransportService.class).start();
TransportClient transportClient = new TransportClient(injector);
success = true;
return transportClient;
} finally {
if (!success) {
ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
}
}
}
private TransportClient(Injector injector) {
super(
injector.getInstance(Settings.class),
injector.getInstance(ThreadPool.class),
injector.getInstance(Headers.class));
this.injector = injector;
nodesService = injector.getInstance(TransportClientNodesService.class);
proxy = injector.getInstance(TransportProxyClient.class);
}
@AfterClass
public static void afterClass() throws Exception {
injector.getInstance(ClusterService.class).close();
terminate(injector.getInstance(ThreadPool.class));
injector = null;
index = null;
aggParsers = null;
currentTypes = null;
namedWriteableRegistry = null;
}
@Test
public void testCustomInjection() {
Settings settings = ImmutableSettings.builder().put("name", "testCustomInjection").build();
IndexQueryParserModule queryParserModule = new IndexQueryParserModule(settings);
queryParserModule.addQueryParser("my", PluginJsonQueryParser.class);
queryParserModule.addFilterParser("my", PluginJsonFilterParser.class);
Index index = new Index("test");
Injector injector =
new ModulesBuilder()
.add(
new CodecModule(settings),
new CacheRecyclerModule(settings),
new SettingsModule(settings),
new ThreadPoolModule(settings),
new IndicesQueriesModule(),
new ScriptModule(settings),
new IndexSettingsModule(index, settings),
new IndexCacheModule(settings),
new AnalysisModule(settings),
new IndexEngineModule(settings),
new SimilarityModule(settings),
queryParserModule,
new IndexNameModule(index),
new FunctionScoreModule(),
new AbstractModule() {
@Override
protected void configure() {
bind(ClusterService.class).toProvider(Providers.of((ClusterService) null));
bind(CircuitBreakerService.class).to(NoneCircuitBreakerService.class);
}
})
.createInjector();
IndexQueryParserService indexQueryParserService =
injector.getInstance(IndexQueryParserService.class);
PluginJsonQueryParser myJsonQueryParser =
(PluginJsonQueryParser) indexQueryParserService.queryParser("my");
assertThat(myJsonQueryParser.names()[0], equalTo("my"));
PluginJsonFilterParser myJsonFilterParser =
(PluginJsonFilterParser) indexQueryParserService.filterParser("my");
assertThat(myJsonFilterParser.names()[0], equalTo("my"));
injector.getInstance(ThreadPool.class).shutdownNow();
}
/** Builds a new instance of the transport client. */
public TransportClient build() {
final PluginsService pluginsService = newPluginService(providedSettings);
final Settings settings = pluginsService.updatedSettings();
Version version = Version.CURRENT;
final ThreadPool threadPool = new ThreadPool(settings);
final NetworkService networkService = new NetworkService(settings);
NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry();
boolean success = false;
try {
ModulesBuilder modules = new ModulesBuilder();
modules.add(new Version.Module(version));
// plugin modules must be added here, before others or we can get crazy injection errors...
for (Module pluginModule : pluginsService.nodeModules()) {
modules.add(pluginModule);
}
modules.add(new PluginsModule(pluginsService));
modules.add(new SettingsModule(settings));
modules.add(new NetworkModule(networkService, settings, true, namedWriteableRegistry));
modules.add(new ClusterNameModule(settings));
modules.add(new ThreadPoolModule(threadPool));
modules.add(
new SearchModule(settings, namedWriteableRegistry) {
@Override
protected void configure() {
// noop
}
});
modules.add(new ActionModule(false, true));
modules.add(new CircuitBreakerModule(settings));
pluginsService.processModules(modules);
Injector injector = modules.createInjector();
final TransportService transportService = injector.getInstance(TransportService.class);
transportService.start();
transportService.acceptIncomingRequests();
TransportClient transportClient = new TransportClient(injector);
success = true;
return transportClient;
} finally {
if (!success) {
ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
}
}
}
/** Setup for the whole base test class. */
@BeforeClass
public static void init() throws IOException {
index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
injector = BaseAggregationTestCase.buildInjector(index);
namedWriteableRegistry = injector.getInstance(NamedWriteableRegistry.class);
aggParsers = injector.getInstance(AggregatorParsers.class);
// create some random type with some default field, those types will stick around for all of the
// subclasses
currentTypes = new String[randomIntBetween(0, 5)];
for (int i = 0; i < currentTypes.length; i++) {
String type = randomAsciiOfLengthBetween(1, 10);
currentTypes[i] = type;
}
queriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
parseFieldMatcher = ParseFieldMatcher.STRICT;
}
private List<KnapsackState> get(String name) throws IOException {
ImmutableList.Builder<KnapsackState> builder = ImmutableList.builder();
try {
logger.debug("get knapsack states: {}", name);
final Client client = injector.getInstance(Client.class);
createIndexIfNotExist(client);
GetResponse getResponse =
client.prepareGet(INDEX_NAME, MAPPING_NAME, name).execute().actionGet();
if (!getResponse.isExists()) {
return builder.build();
}
XContentParser parser = xContent(JSON).createParser(getResponse.getSourceAsBytes());
while (parser.nextToken() != START_ARRAY) {
// forward
}
while (parser.nextToken() != END_ARRAY) {
KnapsackState state = new KnapsackState();
builder.add(state.fromXContent(parser));
}
return builder.build();
} catch (Throwable t) {
logger.error("get settings failed", t);
return null;
}
}
private void remove(final String name) {
try {
logger.debug("remove: {}", name);
final Client client = injector.getInstance(Client.class);
createIndexIfNotExist(client);
client.prepareDelete(INDEX_NAME, MAPPING_NAME, name).setRefresh(true).execute().actionGet();
} catch (Throwable t) {
logger.error("remove failed", t);
}
}
private AnalysisService createAnalysisService(Settings settings) {
Index index = new Index("test");
Injector parentInjector =
new ModulesBuilder()
.add(
new SettingsModule(settings),
new EnvironmentModule(new Environment(settings)),
new IndicesAnalysisModule())
.createInjector();
AnalysisModule analysisModule =
new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class));
new AnalysisGermanPlugin().onModule(analysisModule);
Injector injector =
new ModulesBuilder()
.add(
new IndexSettingsModule(index, settings),
new IndexNameModule(index),
analysisModule)
.createChildInjector(parentInjector);
return injector.getInstance(AnalysisService.class);
}
@Test
public void testMappingCharFilter() throws Exception {
Index index = new Index("test");
Settings settings =
settingsBuilder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put("index.analysis.char_filter.my_mapping.type", "mapping")
.putArray("index.analysis.char_filter.my_mapping.mappings", "ph=>f", "qu=>q")
.put("index.analysis.analyzer.custom_with_char_filter.tokenizer", "standard")
.putArray("index.analysis.analyzer.custom_with_char_filter.char_filter", "my_mapping")
.put("path.home", createTempDir().toString())
.build();
Injector parentInjector =
new ModulesBuilder()
.add(
new SettingsModule(settings),
new EnvironmentModule(new Environment(settings)),
new IndicesAnalysisModule())
.createInjector();
Injector injector =
new ModulesBuilder()
.add(
new IndexSettingsModule(index, settings),
new IndexNameModule(index),
new AnalysisModule(
settings, parentInjector.getInstance(IndicesAnalysisService.class)))
.createChildInjector(parentInjector);
AnalysisService analysisService = injector.getInstance(AnalysisService.class);
NamedAnalyzer analyzer1 = analysisService.analyzer("custom_with_char_filter");
assertTokenStreamContents(
analyzer1.tokenStream("test", "jeff quit phish"), new String[] {"jeff", "qit", "fish"});
// Repeat one more time to make sure that char filter is reinitialized correctly
assertTokenStreamContents(
analyzer1.tokenStream("test", "jeff quit phish"), new String[] {"jeff", "qit", "fish"});
}
public static AnalysisService createAnalysisServiceFromSettings(Settings settings) {
Index index = new Index("test");
if (settings.get(IndexMetaData.SETTING_VERSION_CREATED) == null) {
settings =
Settings.builder()
.put(settings)
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.build();
}
IndicesModule indicesModule =
new IndicesModule(settings) {
@Override
public void configure() {
// skip services
bindHunspellExtension();
}
};
Injector parentInjector =
new ModulesBuilder()
.add(
new SettingsModule(settings),
new EnvironmentModule(new Environment(settings)),
indicesModule)
.createInjector();
AnalysisModule analysisModule =
new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class));
Injector injector =
new ModulesBuilder()
.add(
new IndexSettingsModule(index, settings),
new IndexNameModule(index),
analysisModule)
.createChildInjector(parentInjector);
return injector.getInstance(AnalysisService.class);
}
private void put(final String name, final XContentBuilder builder) {
try {
logger.debug("put knapsack state: {} -> {}", name, builder.string());
final Client client = injector.getInstance(Client.class);
createIndexIfNotExist(client);
client
.prepareIndex(INDEX_NAME, MAPPING_NAME, name)
.setSource(builder)
.setRefresh(true)
.execute()
.actionGet();
} catch (Throwable t) {
logger.error("update settings failed", t);
}
}
public InternalNode(Settings pSettings, boolean loadConfigSettings)
throws ElasticSearchException {
Tuple<Settings, Environment> tuple =
InternalSettingsPerparer.prepareSettings(pSettings, loadConfigSettings);
ESLogger logger = Loggers.getLogger(Node.class, tuple.v1().get("name"));
logger.info("{{}}[{}]: initializing ...", Version.full(), JvmInfo.jvmInfo().pid());
this.pluginsService = new PluginsService(tuple.v1(), tuple.v2());
this.settings = pluginsService.updatedSettings();
this.environment = tuple.v2();
ModulesBuilder modules = new ModulesBuilder();
modules.add(new PluginsModule(settings, pluginsService));
modules.add(new SettingsModule(settings));
modules.add(new NodeModule(this));
modules.add(new NetworkModule());
modules.add(new NodeCacheModule(settings));
modules.add(new ScriptModule());
modules.add(new JmxModule(settings));
modules.add(new EnvironmentModule(environment));
modules.add(new NodeEnvironmentModule());
modules.add(new ClusterNameModule(settings));
modules.add(new ThreadPoolModule(settings));
modules.add(new TimerModule());
modules.add(new DiscoveryModule(settings));
modules.add(new ClusterModule(settings));
modules.add(new RestModule(settings));
modules.add(new TransportModule(settings));
if (settings.getAsBoolean("http.enabled", true)) {
modules.add(new HttpServerModule(settings));
}
modules.add(new RiversModule(settings));
modules.add(new IndicesModule(settings));
modules.add(new SearchModule());
modules.add(new TransportActionModule());
modules.add(new MonitorModule(settings));
modules.add(new GatewayModule(settings));
modules.add(new NodeClientModule());
injector = modules.createInjector();
client = injector.getInstance(Client.class);
logger.info("{{}}[{}]: initialized", Version.full(), JvmInfo.jvmInfo().pid());
}
/** Closes the client. */
@Override
public void close() {
injector.getInstance(TransportClientNodesService.class).close();
injector.getInstance(TransportService.class).close();
try {
injector.getInstance(MonitorService.class).close();
} catch (Exception e) {
// ignore, might not be bounded
}
for (Class<? extends LifecycleComponent> plugin :
injector.getInstance(PluginsService.class).nodeServices()) {
injector.getInstance(plugin).close();
}
try {
ThreadPool.terminate(injector.getInstance(ThreadPool.class), 10, TimeUnit.SECONDS);
} catch (Exception e) {
// ignore
}
injector.getInstance(PageCacheRecycler.class).close();
}
/** Setup for the whole base test class. */
@BeforeClass
public static void init() throws IOException {
// we have to prefer CURRENT since with the range of versions we support
// it's rather unlikely to get the current actually.
Version version =
randomBoolean()
? Version.CURRENT
: VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.CURRENT);
Settings settings =
Settings.builder()
.put("node.name", AbstractQueryTestCase.class.toString())
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.put(ScriptService.SCRIPT_AUTO_RELOAD_ENABLED_SETTING.getKey(), false)
.build();
index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
Settings indexSettings =
Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build();
final ThreadPool threadPool = new ThreadPool(settings);
final ClusterService clusterService = createClusterService(threadPool);
setState(
clusterService,
new ClusterState.Builder(clusterService.state())
.metaData(
new MetaData.Builder()
.put(
new IndexMetaData.Builder(index.getName())
.settings(indexSettings)
.numberOfShards(1)
.numberOfReplicas(0))));
ScriptModule scriptModule = newTestScriptModule();
List<Setting<?>> scriptSettings = scriptModule.getSettings();
scriptSettings.add(InternalSettingsPlugin.VERSION_CREATED);
SettingsModule settingsModule =
new SettingsModule(settings, scriptSettings, Collections.emptyList());
IndicesModule indicesModule =
new IndicesModule(Collections.emptyList()) {
@Override
protected void configure() {
bindMapperExtension();
}
};
SearchModule searchModule =
new SearchModule(settings, false, emptyList()) {
@Override
protected void configureSearch() {
// Skip me
}
};
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
entries.addAll(indicesModule.getNamedWriteables());
entries.addAll(searchModule.getNamedWriteables());
namedWriteableRegistry = new NamedWriteableRegistry(entries);
injector =
new ModulesBuilder()
.add(
(b) -> {
b.bind(Environment.class).toInstance(new Environment(settings));
b.bind(ThreadPool.class).toInstance(threadPool);
b.bind(ScriptService.class).toInstance(scriptModule.getScriptService());
},
settingsModule,
indicesModule,
searchModule,
new IndexSettingsModule(index, settings),
new AbstractModule() {
@Override
protected void configure() {
bind(ClusterService.class).toInstance(clusterService);
bind(CircuitBreakerService.class).toInstance(new NoneCircuitBreakerService());
bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
}
})
.createInjector();
aggParsers = injector.getInstance(SearchRequestParsers.class).aggParsers;
// create some random type with some default field, those types will
// stick around for all of the subclasses
currentTypes = new String[randomIntBetween(0, 5)];
for (int i = 0; i < currentTypes.length; i++) {
String type = randomAsciiOfLengthBetween(1, 10);
currentTypes[i] = type;
}
queriesRegistry = injector.getInstance(IndicesQueriesRegistry.class);
parseFieldMatcher = ParseFieldMatcher.STRICT;
}
private void testSimpleConfiguration(Settings settings) {
Index index = new Index("test");
Injector parentInjector =
new ModulesBuilder()
.add(
new SettingsModule(settings),
new EnvironmentModule(new Environment(settings)),
new IndicesAnalysisModule())
.createInjector();
Injector injector =
new ModulesBuilder()
.add(
new IndexSettingsModule(index, settings),
new IndexNameModule(index),
new AnalysisModule(
settings, parentInjector.getInstance(IndicesAnalysisService.class)))
.createChildInjector(parentInjector);
AnalysisService analysisService = injector.getInstance(AnalysisService.class);
Analyzer analyzer = analysisService.analyzer("custom1").analyzer();
assertThat(analyzer, instanceOf(CustomAnalyzer.class));
CustomAnalyzer custom1 = (CustomAnalyzer) analyzer;
assertThat(custom1.tokenizerFactory(), instanceOf(StandardTokenizerFactory.class));
assertThat(custom1.tokenFilters().length, equalTo(2));
StopTokenFilterFactory stop1 = (StopTokenFilterFactory) custom1.tokenFilters()[0];
assertThat(stop1.stopWords().size(), equalTo(1));
assertThat((Iterable<String>) stop1.stopWords(), hasItem("test-stop"));
analyzer = analysisService.analyzer("custom2").analyzer();
assertThat(analyzer, instanceOf(CustomAnalyzer.class));
CustomAnalyzer custom2 = (CustomAnalyzer) analyzer;
// HtmlStripCharFilterFactory html = (HtmlStripCharFilterFactory)
// custom2.charFilters()[0];
// assertThat(html.readAheadLimit(), equalTo(HTMLStripCharFilter.DEFAULT_READ_AHEAD));
//
// html = (HtmlStripCharFilterFactory) custom2.charFilters()[1];
// assertThat(html.readAheadLimit(), equalTo(1024));
// verify characters mapping
analyzer = analysisService.analyzer("custom5").analyzer();
assertThat(analyzer, instanceOf(CustomAnalyzer.class));
CustomAnalyzer custom5 = (CustomAnalyzer) analyzer;
assertThat(custom5.tokenFilters()[0], instanceOf(MappingCharFilterFactory.class));
// verify aliases
analyzer = analysisService.analyzer("alias1").analyzer();
assertThat(analyzer, instanceOf(StandardAnalyzer.class));
// check phonetic
analyzer = analysisService.analyzer("custom3").analyzer();
assertThat(analyzer, instanceOf(CustomAnalyzer.class));
CustomAnalyzer custom3 = (CustomAnalyzer) analyzer;
assertThat(custom3.tokenFilters()[0], instanceOf(PhoneticTokenFilterFactory.class));
// check custom class name (my)
analyzer = analysisService.analyzer("custom4").analyzer();
assertThat(analyzer, instanceOf(CustomAnalyzer.class));
CustomAnalyzer custom4 = (CustomAnalyzer) analyzer;
assertThat(custom4.tokenFilters()[0], instanceOf(MyFilterTokenFilterFactory.class));
// // verify Czech stemmer
// analyzer = analysisService.analyzer("czechAnalyzerWithStemmer").analyzer();
// assertThat(analyzer, instanceOf(CustomAnalyzer.class));
// CustomAnalyzer czechstemmeranalyzer = (CustomAnalyzer) analyzer;
// assertThat(czechstemmeranalyzer.tokenizerFactory(),
// instanceOf(StandardTokenizerFactory.class));
// assertThat(czechstemmeranalyzer.tokenFilters().length, equalTo(4));
// assertThat(czechstemmeranalyzer.tokenFilters()[3],
// instanceOf(CzechStemTokenFilterFactory.class));
//
// // check dictionary decompounder
// analyzer = analysisService.analyzer("decompoundingAnalyzer").analyzer();
// assertThat(analyzer, instanceOf(CustomAnalyzer.class));
// CustomAnalyzer dictionaryDecompounderAnalyze = (CustomAnalyzer) analyzer;
// assertThat(dictionaryDecompounderAnalyze.tokenizerFactory(),
// instanceOf(StandardTokenizerFactory.class));
// assertThat(dictionaryDecompounderAnalyze.tokenFilters().length, equalTo(1));
// assertThat(dictionaryDecompounderAnalyze.tokenFilters()[0],
// instanceOf(DictionaryCompoundWordTokenFilterFactory.class));
Set<String> wordList =
Analysis.getWordSet(null, settings, "index.analysis.filter.dict_dec.word_list");
MatcherAssert.assertThat(wordList.size(), equalTo(6));
MatcherAssert.assertThat(
wordList, hasItems("donau", "dampf", "schiff", "spargel", "creme", "suppe"));
}
private void deleteShard(
int shardId, boolean delete, boolean snapshotGateway, boolean deleteGateway)
throws ElasticSearchException {
Injector shardInjector;
IndexShard indexShard;
synchronized (this) {
Map<Integer, Injector> tmpShardInjectors = newHashMap(shardsInjectors);
shardInjector = tmpShardInjectors.remove(shardId);
if (shardInjector == null) {
if (!delete) {
return;
}
throw new IndexShardMissingException(new ShardId(index, shardId));
}
shardsInjectors = ImmutableMap.copyOf(tmpShardInjectors);
if (delete) {
logger.debug("deleting shard_id [{}]", shardId);
}
Map<Integer, IndexShard> tmpShardsMap = newHashMap(shards);
indexShard = tmpShardsMap.remove(shardId);
shards = ImmutableMap.copyOf(tmpShardsMap);
}
ShardId sId = new ShardId(index, shardId);
indicesLifecycle.beforeIndexShardClosed(sId, indexShard, delete);
for (Class<? extends CloseableIndexComponent> closeable : pluginsService.shardServices()) {
try {
shardInjector.getInstance(closeable).close(delete);
} catch (Exception e) {
logger.debug("failed to clean plugin shard service [{}]", e, closeable);
}
}
try {
// now we can close the translog service, we need to close it before the we close the shard
shardInjector.getInstance(TranslogService.class).close();
} catch (Exception e) {
// ignore
}
// close shard actions
if (indexShard != null) {
shardInjector.getInstance(IndexShardManagement.class).close();
}
// this logic is tricky, we want to close the engine so we rollback the changes done to it
// and close the shard so no operations are allowed to it
if (indexShard != null) {
indexShard.close();
}
try {
shardInjector.getInstance(Engine.class).close();
} catch (Exception e) {
// ignore
}
try {
// now, we can snapshot to the gateway, it will be only the translog
if (snapshotGateway) {
shardInjector.getInstance(IndexShardGatewayService.class).snapshotOnClose();
}
} catch (Exception e) {
// ignore
}
try {
shardInjector.getInstance(IndexShardGatewayService.class).close(deleteGateway);
} catch (Exception e) {
// ignore
}
try {
// now we can close the translog
shardInjector.getInstance(Translog.class).close(delete);
} catch (Exception e) {
// ignore
}
// call this before we close the store, so we can release resources for it
indicesLifecycle.afterIndexShardClosed(sId, delete);
// if we delete or have no gateway or the store is not persistent, clean the store...
Store store = shardInjector.getInstance(Store.class);
if (delete || indexGateway.type().equals(NoneGateway.TYPE) || !indexStore.persistent()) {
try {
store.fullDelete();
} catch (IOException e) {
logger.warn("failed to clean store on shard deletion", e);
}
}
// and close it
try {
store.close();
} catch (IOException e) {
logger.warn("failed to close store on shard deletion", e);
}
Injectors.close(injector);
// delete the shard location if needed
if (delete || indexGateway.type().equals(NoneGateway.TYPE)) {
FileSystemUtils.deleteRecursively(nodeEnv.shardLocation(sId));
}
}
@Before
public void configure() {
Injector injector =
new ModulesBuilder()
.add(
new CircuitBreakerModule(),
new OperatorModule(),
new TestModule(),
new SysNodeExpressionModule())
.createInjector();
Injector shard0Injector = injector.createChildInjector(new TestShardModule(0));
Injector shard1Injector = injector.createChildInjector(new TestShardModule(1));
functions = injector.getInstance(Functions.class);
IndicesService indicesService = injector.getInstance(IndicesService.class);
indexService = injector.getInstance(IndexService.class);
when(indexService.shardInjectorSafe(0)).thenReturn(shard0Injector);
when(indexService.shardInjectorSafe(1)).thenReturn(shard1Injector);
when(indexService.shardSafe(0)).thenReturn(shard0Injector.getInstance(IndexShard.class));
when(indexService.shardSafe(1)).thenReturn(shard1Injector.getInstance(IndexShard.class));
when(indicesService.indexServiceSafe(TEST_TABLE_NAME)).thenReturn(indexService);
NodeSettingsService nodeSettingsService = mock(NodeSettingsService.class);
jobContextService =
new JobContextService(ImmutableSettings.EMPTY, testThreadPool, mock(StatsTables.class));
ClusterService clusterService = injector.getInstance(ClusterService.class);
operation =
new MapSideDataCollectOperation(
clusterService,
ImmutableSettings.EMPTY,
mock(TransportActionProvider.class, Answers.RETURNS_DEEP_STUBS.get()),
injector.getInstance(BulkRetryCoordinatorPool.class),
functions,
injector.getInstance(ReferenceResolver.class),
injector.getInstance(NodeSysExpression.class),
indicesService,
testThreadPool,
new CollectServiceResolver(
discoveryService,
new SystemCollectService(
discoveryService,
functions,
new StatsTables(ImmutableSettings.EMPTY, nodeSettingsService))),
mock(InformationSchemaCollectService.class),
mock(UnassignedShardsCollectService.class));
}
// During concurrent close() calls we want to make sure that all of them return after the node has
// completed it's shutdown cycle.
// If not, the hook that is added in Bootstrap#setup() will be useless: close() might not be
// executed, in case another (for example api) call
// to close() has already set some lifecycles to stopped. In this case the process will be
// terminated even if the first call to close() has not finished yet.
public synchronized void close() {
if (lifecycle.started()) {
stop();
}
if (!lifecycle.moveToClosed()) {
return;
}
ESLogger logger = Loggers.getLogger(Node.class, settings.get("name"));
logger.info("closing ...");
StopWatch stopWatch = new StopWatch("node_close");
stopWatch.start("tribe");
injector.getInstance(TribeService.class).close();
stopWatch.stop().start("http");
if (settings.getAsBoolean("http.enabled", true)) {
injector.getInstance(HttpServer.class).close();
}
stopWatch.stop().start("rivers");
injector.getInstance(RiversManager.class).close();
stopWatch.stop().start("snapshot_service");
injector.getInstance(SnapshotsService.class).close();
stopWatch.stop().start("client");
Releasables.close(injector.getInstance(Client.class));
stopWatch.stop().start("indices_cluster");
injector.getInstance(IndicesClusterStateService.class).close();
stopWatch.stop().start("indices");
injector.getInstance(IndicesFilterCache.class).close();
injector.getInstance(IndicesFieldDataCache.class).close();
injector.getInstance(IndexingMemoryController.class).close();
injector.getInstance(IndicesTTLService.class).close();
injector.getInstance(IndicesService.class).close();
stopWatch.stop().start("routing");
injector.getInstance(RoutingService.class).close();
stopWatch.stop().start("cluster");
injector.getInstance(ClusterService.class).close();
stopWatch.stop().start("discovery");
injector.getInstance(DiscoveryService.class).close();
stopWatch.stop().start("monitor");
injector.getInstance(MonitorService.class).close();
stopWatch.stop().start("gateway");
injector.getInstance(GatewayService.class).close();
stopWatch.stop().start("search");
injector.getInstance(SearchService.class).close();
stopWatch.stop().start("rest");
injector.getInstance(RestController.class).close();
stopWatch.stop().start("transport");
injector.getInstance(TransportService.class).close();
stopWatch.stop().start("percolator_service");
injector.getInstance(PercolatorService.class).close();
for (Class<? extends LifecycleComponent> plugin : pluginsService.services()) {
stopWatch.stop().start("plugin(" + plugin.getName() + ")");
injector.getInstance(plugin).close();
}
stopWatch.stop().start("script");
injector.getInstance(ScriptService.class).close();
stopWatch.stop().start("thread_pool");
// TODO this should really use ThreadPool.terminate()
injector.getInstance(ThreadPool.class).shutdown();
try {
injector.getInstance(ThreadPool.class).awaitTermination(10, TimeUnit.SECONDS);
} catch (InterruptedException e) {
// ignore
}
stopWatch.stop().start("thread_pool_force_shutdown");
try {
injector.getInstance(ThreadPool.class).shutdownNow();
} catch (Exception e) {
// ignore
}
stopWatch.stop();
if (logger.isTraceEnabled()) {
logger.trace("Close times for each service:\n{}", stopWatch.prettyPrint());
}
injector.getInstance(NodeEnvironment.class).close();
injector.getInstance(PageCacheRecycler.class).close();
Injectors.close(injector);
CachedStreams.clear();
logger.info("closed");
}
@Override
public Node stop() {
if (!lifecycle.moveToStopped()) {
return this;
}
ESLogger logger = Loggers.getLogger(Node.class, settings.get("name"));
logger.info("stopping ...");
injector.getInstance(TribeService.class).stop();
injector.getInstance(ResourceWatcherService.class).stop();
if (settings.getAsBoolean("http.enabled", true)) {
injector.getInstance(HttpServer.class).stop();
}
injector.getInstance(MappingUpdatedAction.class).stop();
injector.getInstance(RiversManager.class).stop();
injector.getInstance(SnapshotsService.class).stop();
// stop any changes happening as a result of cluster state changes
injector.getInstance(IndicesClusterStateService.class).stop();
// we close indices first, so operations won't be allowed on it
injector.getInstance(IndexingMemoryController.class).stop();
injector.getInstance(IndicesTTLService.class).stop();
injector.getInstance(IndicesService.class).stop();
// sleep a bit to let operations finish with indices service
// try {
// Thread.sleep(500);
// } catch (InterruptedException e) {
// // ignore
// }
injector.getInstance(RoutingService.class).stop();
injector.getInstance(ClusterService.class).stop();
injector.getInstance(DiscoveryService.class).stop();
injector.getInstance(MonitorService.class).stop();
injector.getInstance(GatewayService.class).stop();
injector.getInstance(SearchService.class).stop();
injector.getInstance(RestController.class).stop();
injector.getInstance(TransportService.class).stop();
for (Class<? extends LifecycleComponent> plugin : pluginsService.services()) {
injector.getInstance(plugin).stop();
}
logger.info("stopped");
return this;
}
public Node start() {
if (!lifecycle.moveToStarted()) {
return this;
}
ESLogger logger = Loggers.getLogger(Node.class, settings.get("name"));
logger.info("starting ...");
// hack around dependency injection problem (for now...)
injector
.getInstance(Discovery.class)
.setAllocationService(injector.getInstance(AllocationService.class));
for (Class<? extends LifecycleComponent> plugin : pluginsService.services()) {
injector.getInstance(plugin).start();
}
injector.getInstance(MappingUpdatedAction.class).start();
injector.getInstance(IndicesService.class).start();
injector.getInstance(IndexingMemoryController.class).start();
injector.getInstance(IndicesClusterStateService.class).start();
injector.getInstance(IndicesTTLService.class).start();
injector.getInstance(RiversManager.class).start();
injector.getInstance(SnapshotsService.class).start();
injector.getInstance(TransportService.class).start();
injector.getInstance(ClusterService.class).start();
injector.getInstance(RoutingService.class).start();
injector.getInstance(SearchService.class).start();
injector.getInstance(MonitorService.class).start();
injector.getInstance(RestController.class).start();
DiscoveryService discoService = injector.getInstance(DiscoveryService.class).start();
discoService.waitForInitialState();
// gateway should start after disco, so it can try and recovery from gateway on "start"
injector.getInstance(GatewayService.class).start();
if (settings.getAsBoolean("http.enabled", true)) {
injector.getInstance(HttpServer.class).start();
}
injector.getInstance(ResourceWatcherService.class).start();
injector.getInstance(TribeService.class).start();
logger.info("started");
return this;
}
public InternalNode(Settings preparedSettings, boolean loadConfigSettings)
throws ElasticsearchException {
final Settings pSettings =
settingsBuilder()
.put(preparedSettings)
.put(Client.CLIENT_TYPE_SETTING, CLIENT_TYPE)
.build();
Tuple<Settings, Environment> tuple =
InternalSettingsPreparer.prepareSettings(pSettings, loadConfigSettings);
tuple = new Tuple<>(TribeService.processSettings(tuple.v1()), tuple.v2());
// The only place we can actually fake the version a node is running on:
Version version = pSettings.getAsVersion("tests.mock.version", Version.CURRENT);
ESLogger logger = Loggers.getLogger(Node.class, tuple.v1().get("name"));
logger.info(
"version[{}], pid[{}], build[{}/{}]",
version,
JvmInfo.jvmInfo().pid(),
Build.CURRENT.hashShort(),
Build.CURRENT.timestamp());
logger.info("initializing ...");
if (logger.isDebugEnabled()) {
Environment env = tuple.v2();
logger.debug(
"using home [{}], config [{}], data [{}], logs [{}], work [{}], plugins [{}]",
env.homeFile(),
env.configFile(),
Arrays.toString(env.dataFiles()),
env.logsFile(),
env.workFile(),
env.pluginsFile());
}
this.pluginsService = new PluginsService(tuple.v1(), tuple.v2());
this.settings = pluginsService.updatedSettings();
// create the environment based on the finalized (processed) view of the settings
this.environment = new Environment(this.settings());
CompressorFactory.configure(settings);
final NodeEnvironment nodeEnvironment;
try {
nodeEnvironment = new NodeEnvironment(this.settings, this.environment);
} catch (IOException ex) {
throw new ElasticsearchIllegalStateException("Failed to created node environment", ex);
}
boolean success = false;
try {
ModulesBuilder modules = new ModulesBuilder();
modules.add(new Version.Module(version));
modules.add(new PageCacheRecyclerModule(settings));
modules.add(new CircuitBreakerModule(settings));
modules.add(new BigArraysModule(settings));
modules.add(new PluginsModule(settings, pluginsService));
modules.add(new SettingsModule(settings));
modules.add(new NodeModule(this));
modules.add(new NetworkModule());
modules.add(new ScriptModule(settings));
modules.add(new EnvironmentModule(environment));
modules.add(new NodeEnvironmentModule(nodeEnvironment));
modules.add(new ClusterNameModule(settings));
modules.add(new ThreadPoolModule(settings));
modules.add(new DiscoveryModule(settings));
modules.add(new ClusterModule(settings));
modules.add(new RestModule(settings));
modules.add(new TransportModule(settings));
if (settings.getAsBoolean(HTTP_ENABLED, true)) {
modules.add(new HttpServerModule(settings));
}
modules.add(new RiversModule(settings));
modules.add(new IndicesModule(settings));
modules.add(new SearchModule());
modules.add(new ActionModule(false));
modules.add(new MonitorModule(settings));
modules.add(new GatewayModule(settings));
modules.add(new NodeClientModule());
modules.add(new ShapeModule());
modules.add(new PercolatorModule());
modules.add(new ResourceWatcherModule());
modules.add(new RepositoriesModule());
modules.add(new TribeModule());
modules.add(new BenchmarkModule(settings));
injector = modules.createInjector();
client = injector.getInstance(Client.class);
success = true;
} finally {
if (!success) {
nodeEnvironment.close();
}
}
logger.info("initialized");
}