public static void assertSettings(Settings left, Settings right, boolean compareClusterName) {
ImmutableSet<Map.Entry<String, String>> entries0 = left.getAsMap().entrySet();
Map<String, String> entries1 = right.getAsMap();
assertThat(entries0.size(), equalTo(entries1.size()));
for (Map.Entry<String, String> entry : entries0) {
if (entry.getKey().equals(ClusterName.SETTING) && compareClusterName == false) {
continue;
}
assertThat(entries1, hasEntry(entry.getKey(), entry.getValue()));
}
}
public static void assertSettings(
Settings left, Settings right, boolean checkClusterUniqueSettings) {
Set<Map.Entry<String, String>> entries0 = left.getAsMap().entrySet();
Map<String, String> entries1 = right.getAsMap();
assertThat(entries0.size(), equalTo(entries1.size()));
for (Map.Entry<String, String> entry : entries0) {
if (clusterUniqueSettings.contains(entry.getKey()) && checkClusterUniqueSettings == false) {
continue;
}
assertThat(entries1, hasEntry(entry.getKey(), entry.getValue()));
}
}
@Test
public void testLoadFromDelimitedString() {
Settings settings =
settingsBuilder().loadFromDelimitedString("key1=value1;key2=value2", ';').build();
assertThat(settings.get("key1"), equalTo("value1"));
assertThat(settings.get("key2"), equalTo("value2"));
assertThat(settings.getAsMap().size(), equalTo(2));
assertThat(settings.toDelimitedString(';'), equalTo("key1=value1;key2=value2;"));
settings = settingsBuilder().loadFromDelimitedString("key1=value1;key2=value2;", ';').build();
assertThat(settings.get("key1"), equalTo("value1"));
assertThat(settings.get("key2"), equalTo("value2"));
assertThat(settings.getAsMap().size(), equalTo(2));
assertThat(settings.toDelimitedString(';'), equalTo("key1=value1;key2=value2;"));
}
public static Settings processSettings(Settings settings) {
if (settings.get(TRIBE_NAME) != null) {
// if its a node client started by this service as tribe, remove any tribe group setting
// to avoid recursive configuration
Settings.Builder sb = Settings.builder().put(settings);
for (String s : settings.getAsMap().keySet()) {
if (s.startsWith("tribe.") && !s.equals(TRIBE_NAME)) {
sb.remove(s);
}
}
return sb.build();
}
Map<String, Settings> nodesSettings = settings.getGroups("tribe", true);
if (nodesSettings.isEmpty()) {
return settings;
}
// its a tribe configured node..., force settings
Settings.Builder sb = Settings.builder().put(settings);
sb.put(Node.NODE_CLIENT_SETTING.getKey(), true); // this node should just act as a node client
sb.put(
DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(),
"local"); // a tribe node should not use zen discovery
sb.put(
DiscoveryService.INITIAL_STATE_TIMEOUT_SETTING.getKey(),
0); // nothing is going to be discovered, since no master will be elected
if (sb.get("cluster.name") == null) {
sb.put(
"cluster.name",
"tribe_"
+ Strings
.randomBase64UUID()); // make sure it won't join other tribe nodes in the same JVM
}
sb.put(TransportMasterNodeReadAction.FORCE_LOCAL_SETTING, true);
return sb.build();
}
private void assertMMNinClusterSetting(InternalTestCluster cluster, int masterNodes) {
final int minMasterNodes = masterNodes / 2 + 1;
for (final String node : cluster.getNodeNames()) {
Settings stateSettings =
cluster
.client(node)
.admin()
.cluster()
.prepareState()
.setLocal(true)
.get()
.getState()
.getMetaData()
.settings();
assertThat(
"dynamic setting for node ["
+ node
+ "] has the wrong min_master_node setting : ["
+ stateSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey())
+ "]",
stateSettings.getAsMap(),
hasEntry(
DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(),
Integer.toString(minMasterNodes)));
}
}
@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
if (settings.getAsMap().isEmpty()) {
validationException = addValidationError("no settings to update", validationException);
}
return validationException;
}
public void updateSettings(
final Settings pSettings, final String[] indices, final Listener listener) {
ImmutableSettings.Builder updatedSettingsBuilder = ImmutableSettings.settingsBuilder();
for (Map.Entry<String, String> entry : pSettings.getAsMap().entrySet()) {
if (!entry.getKey().startsWith("index.")) {
updatedSettingsBuilder.put("index." + entry.getKey(), entry.getValue());
} else {
updatedSettingsBuilder.put(entry.getKey(), entry.getValue());
}
}
final Settings settings = updatedSettingsBuilder.build();
clusterService.submitStateUpdateTask(
"update-settings",
new ClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) {
try {
boolean changed = false;
String[] actualIndices = currentState.metaData().concreteIndices(indices);
RoutingTable.Builder routingTableBuilder =
newRoutingTableBuilder().routingTable(currentState.routingTable());
MetaData.Builder metaDataBuilder =
MetaData.newMetaDataBuilder().metaData(currentState.metaData());
int updatedNumberOfReplicas =
settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, -1);
if (updatedNumberOfReplicas != -1) {
routingTableBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
metaDataBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
changed = true;
}
if (!changed) {
listener.onFailure(
new ElasticSearchIllegalArgumentException("No settings applied"));
return currentState;
}
logger.info(
"Updating number_of_replicas to [{}] for indices {}",
updatedNumberOfReplicas,
actualIndices);
listener.onSuccess();
return ClusterState.builder()
.state(currentState)
.metaData(metaDataBuilder)
.routingTable(routingTableBuilder)
.build();
} catch (Exception e) {
listener.onFailure(e);
return currentState;
}
}
});
}
private Node buildNode(String id) throws IOException {
Settings nodeSettings = settingsBuilder().put(getNodeSettings()).put("name", id).build();
logger.info("settings={}", nodeSettings.getAsMap());
// ES 2.1 renders NodeBuilder as useless
Set<Class<? extends Plugin>> plugins = new HashSet<>();
plugins.add(KnapsackPlugin.class);
Environment environment = new Environment(nodeSettings);
Node node = new MockNode(environment, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
logger.info("clients={}", clients);
return node;
}
@Inject
public HdfsGateway(
Settings settings,
ThreadPool threadPool,
ClusterService clusterService,
ClusterName clusterName)
throws IOException {
super(settings, threadPool, clusterService);
this.closeFileSystem = componentSettings.getAsBoolean("close_fs", true);
String uri = componentSettings.get("uri");
if (uri == null) {
throw new ElasticSearchIllegalArgumentException(
"hdfs gateway requires the 'uri' setting to be set");
}
String path = componentSettings.get("path");
if (path == null) {
throw new ElasticSearchIllegalArgumentException(
"hdfs gateway requires the 'path' path setting to be set");
}
Path hPath = new Path(new Path(path), clusterName.value());
int concurrentStreams = componentSettings.getAsInt("concurrent_streams", 5);
this.concurrentStreamPool =
EsExecutors.newScalingExecutorService(
1,
concurrentStreams,
5,
TimeUnit.SECONDS,
EsExecutors.daemonThreadFactory(settings, "[s3_stream]"));
logger.debug(
"Using uri [{}], path [{}], concurrent_streams [{}]", uri, hPath, concurrentStreams);
Configuration conf = new Configuration();
Settings hdfsSettings = settings.getByPrefix("hdfs.conf.");
for (Map.Entry<String, String> entry : hdfsSettings.getAsMap().entrySet()) {
conf.set(entry.getKey(), entry.getValue());
}
fileSystem = FileSystem.get(URI.create(uri), conf);
initialize(
new HdfsBlobStore(settings, fileSystem, concurrentStreamPool, hPath), clusterName, null);
}
private void assertMMNinNodeSetting(String node, InternalTestCluster cluster, int masterNodes) {
final int minMasterNodes = masterNodes / 2 + 1;
Settings nodeSettings =
cluster
.client(node)
.admin()
.cluster()
.prepareNodesInfo(node)
.get()
.getNodes()
.get(0)
.getSettings();
assertThat(
"node setting of node ["
+ node
+ "] has the wrong min_master_node setting: ["
+ nodeSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey())
+ "]",
nodeSettings.getAsMap(),
hasEntry(
DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.toString(minMasterNodes)));
}
@Override
public String toString() {
return settings.getAsMap().toString();
}
@SuppressForbidden(reason = "needs java.io.File api to start a process")
synchronized void startInternal(
Client client, Settings settings, String nodeName, String clusterName)
throws IOException, InterruptedException {
if (process != null) {
throw new IllegalStateException("Already started");
}
List<String> params = new ArrayList<>();
if (!Constants.WINDOWS) {
params.add("bin/elasticsearch");
} else {
params.add("bin/elasticsearch.bat");
}
params.add("-Des.cluster.name=" + clusterName);
params.add("-Des.node.name=" + nodeName);
Settings.Builder externaNodeSettingsBuilder = Settings.builder();
for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
switch (entry.getKey()) {
case "cluster.name":
case "node.name":
case "path.home":
case "node.mode":
case "node.local":
case NetworkModule.TRANSPORT_TYPE_KEY:
case "discovery.type":
case NetworkModule.TRANSPORT_SERVICE_TYPE_KEY:
case "config.ignore_system_properties":
continue;
default:
externaNodeSettingsBuilder.put(entry.getKey(), entry.getValue());
}
}
this.externalNodeSettings = externaNodeSettingsBuilder.put(REQUIRED_SETTINGS).build();
for (Map.Entry<String, String> entry : externalNodeSettings.getAsMap().entrySet()) {
params.add("-Des." + entry.getKey() + "=" + entry.getValue());
}
params.add("-Des.path.home=" + PathUtils.get(".").toAbsolutePath());
params.add("-Des.path.conf=" + path + "/config");
ProcessBuilder builder = new ProcessBuilder(params);
builder.directory(path.toFile());
builder.inheritIO();
boolean success = false;
try {
logger.info("starting external node [{}] with: {}", nodeName, builder.command());
process = builder.start();
this.nodeInfo = null;
if (waitForNode(client, nodeName)) {
nodeInfo = nodeInfo(client, nodeName);
assert nodeInfo != null;
logger.info(
"external node {} found, version [{}], build {}",
nodeInfo.getNode(),
nodeInfo.getVersion(),
nodeInfo.getBuild());
} else {
throw new IllegalStateException("Node [" + nodeName + "] didn't join the cluster");
}
success = true;
} finally {
if (!success) {
stop();
}
}
}
private void setClusterExcludeFilters(Settings settings) {
clusterExcludeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, settings.getAsMap());
}
private void setClusterRequireFilters(Settings settings) {
clusterRequireFilters = DiscoveryNodeFilters.buildFromKeyValue(AND, settings.getAsMap());
}
@Inject
public TomcatHttpServerTransport(
final Settings settings,
final Environment environment,
final NetworkService networkService,
final ClusterName clusterName,
final Client client,
final SecurityService securityService) {
super(settings);
this.settings = settings;
this.securityService = securityService;
/*
* TODO check if keep alive is managed by tomcat copy custom headers to
* response check that user under tomcat/ea is running is not a
* privilieged iuser tomcat props apply: tomcat.XXX
*/
// _aliases test with more than one index mapped to an alias
/*
*
* http.max_initial_line_length not respected http.reset_cookies not
* respected workerCount http.cors.enabled http.cors.allow-origin
* http.cors.max-age http.cors.allow-methods http.cors.allow-headers
*
*
*
* http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/
* modules-network.html
*
* http://stackoverflow.com/questions/8038718/serializing-generic-java-
* object-to-json-using-jackson
* http://tomcatspnegoad.sourceforge.net/realms.html
*
* SSL options
*
*
*
* Realm options/login waffle, spnego ... security.kerberos.provider:
* waffle
*
* Hardening EA - dynamic script disable
*/
/*
*
*
*
*
*
*/
useSSL =
componentSettings.getAsBoolean(
"ssl.enabled", settings.getAsBoolean("security.ssl.enabled", false));
useClientAuth =
componentSettings.getAsBoolean(
"ssl.clientauth.enabled",
settings.getAsBoolean("security.ssl.clientauth.enabled", false));
kerberosMode =
componentSettings.get("kerberos.mode", settings.get("security.kerberos.mode", "none"));
port = componentSettings.get("port", settings.get("http.port", "8080"));
bindHost =
componentSettings.get(
"bind_host", settings.get("http.bind_host", settings.get("http.host")));
publishHost =
componentSettings.get(
"publish_host", settings.get("http.publish_host", settings.get("http.host")));
this.networkService = networkService;
ByteSizeValue maxContentLength =
componentSettings.getAsBytesSize(
"max_content_length",
settings.getAsBytesSize(
"http.max_content_length", new ByteSizeValue(100, ByteSizeUnit.MB)));
maxChunkSize =
componentSettings.getAsBytesSize(
"max_chunk_size",
settings.getAsBytesSize("http.max_chunk_size", new ByteSizeValue(8, ByteSizeUnit.KB)));
maxHeaderSize =
componentSettings.getAsBytesSize(
"max_header_size",
settings.getAsBytesSize("http.max_header_size", new ByteSizeValue(8, ByteSizeUnit.KB)));
blockingServer =
settings.getAsBoolean(
"http.blocking_server",
settings.getAsBoolean(TCP_BLOCKING_SERVER, settings.getAsBoolean(TCP_BLOCKING, false)));
tcpNoDelay =
componentSettings.getAsBoolean("tcp_no_delay", settings.getAsBoolean(TCP_NO_DELAY, true));
tcpKeepAlive =
componentSettings.getAsBoolean(
"tcp_keep_alive", settings.getAsBoolean(TCP_KEEP_ALIVE, true));
reuseAddress =
componentSettings.getAsBoolean(
"reuse_address",
settings.getAsBoolean(TCP_REUSE_ADDRESS, NetworkUtils.defaultReuseAddress()));
tcpSendBufferSize =
componentSettings.getAsBytesSize(
"tcp_send_buffer_size",
settings.getAsBytesSize(TCP_SEND_BUFFER_SIZE, TCP_DEFAULT_SEND_BUFFER_SIZE));
tcpReceiveBufferSize =
componentSettings.getAsBytesSize(
"tcp_receive_buffer_size",
settings.getAsBytesSize(TCP_RECEIVE_BUFFER_SIZE, TCP_DEFAULT_RECEIVE_BUFFER_SIZE));
compression = settings.getAsBoolean("http.compression", false);
compressionLevel = settings.getAsInt("http.compression_level", 6);
// validate max content length
if (maxContentLength.bytes() > Integer.MAX_VALUE) {
logger.warn(
"maxContentLength[" + maxContentLength + "] set to high value, resetting it to [100mb]");
maxContentLength = new ByteSizeValue(100, ByteSizeUnit.MB);
}
this.maxContentLength = maxContentLength;
logger.debug("port: " + port);
logger.debug("bindHost: " + bindHost);
logger.debug("publishHost: " + publishHost);
logger.debug("componentsettings: " + componentSettings.getAsMap());
logger.debug("settings: " + settings.getAsMap());
}
@SuppressForbidden(reason = "C2 integration (File API)")
@Override
protected void doStart() throws ElasticsearchException {
try {
Settings.Builder builder = Settings.builder();
Path pluginConfigPath = environment.configFile().resolve(ClusteringPlugin.PLUGIN_NAME);
if (!Files.isDirectory(pluginConfigPath)) {
Path srcConfig = Paths.get("src/main/config");
if (Files.isDirectory(srcConfig)) {
// Allow running from within the IDE.
pluginConfigPath = srcConfig;
} else {
throw new ElasticsearchException("Config folder missing: " + pluginConfigPath);
}
} else {
logger.info("Configuration files at: {}", pluginConfigPath.toAbsolutePath());
}
for (String configName :
new String[] {"config.yml", "config.yaml", "config.json", "config.properties"}) {
try {
Path resolved = pluginConfigPath.resolve(configName);
if (resolved != null && Files.exists(resolved)) {
builder.loadFromPath(resolved);
}
} catch (NoClassDefFoundError e) {
logger.warn("Could not parse: {}", e, configName);
}
}
Settings c2Settings = builder.build();
// Parse suite descriptors with loggers turned off (shut them up a bit).
final Path suitePath = pluginConfigPath.resolve(c2Settings.get(DEFAULT_SUITE_PROPERTY_NAME));
if (!Files.isRegularFile(suitePath)) {
throw new ElasticsearchException(
"Could not find algorithm suite: " + suitePath.toAbsolutePath().normalize());
}
final ResourceLookup suiteLookup =
new ResourceLookup(new DirLocator(suitePath.getParent().toFile()));
final IResource suiteResource = suiteLookup.getFirst(suitePath.getFileName().toString());
final List<String> failed = Lists.newArrayList();
final ProcessingComponentSuite suite =
LoggerUtils.quietCall(
new Callable<ProcessingComponentSuite>() {
public ProcessingComponentSuite call() throws Exception {
ProcessingComponentSuite suite =
ProcessingComponentSuite.deserialize(suiteResource, suiteLookup);
for (ProcessingComponentDescriptor desc : suite.removeUnavailableComponents()) {
failed.add(desc.getId());
if (isNoClassDefFound(desc.getInitializationFailure())) {
logger.debug("Algorithm not available on classpath: {}", desc.getId());
} else {
logger.warn(
"Algorithm initialization failed: {}",
desc.getInitializationFailure(),
desc.getId());
}
}
return suite;
}
},
Logger.getLogger(ProcessingComponentDescriptor.class),
Logger.getLogger(ReflectionUtils.class));
algorithms = Lists.newArrayList();
for (ProcessingComponentDescriptor descriptor : suite.getAlgorithms()) {
algorithms.add(descriptor.getId());
}
algorithms = Collections.unmodifiableList(algorithms);
if (!algorithms.isEmpty()) {
logger.info("Available clustering components: {}", Joiner.on(", ").join(algorithms));
}
if (!failed.isEmpty()) {
logger.info("Unavailable clustering components: {}", Joiner.on(", ").join(failed));
}
final Path resourcesPath =
pluginConfigPath
.resolve(c2Settings.get(DEFAULT_RESOURCES_PROPERTY_NAME, "."))
.toAbsolutePath()
.normalize();
logger.info("Lexical resources dir: {}", resourcesPath);
final ResourceLookup resourceLookup =
new ResourceLookup(
new DirLocator(resourcesPath.toFile()),
new ClassLoaderLocator(ControllerSingleton.class.getClassLoader()));
// Change the default resource lookup to include the configured location.
Map<String, Object> c2SettingsAsMap = Maps.newHashMap();
DefaultLexicalDataFactoryDescriptor.attributeBuilder(c2SettingsAsMap)
.resourceLookup(resourceLookup);
c2SettingsAsMap.putAll(c2Settings.getAsMap());
// Set up the license for Lingo3G, if it's available.
Path lingo3gLicense = scanForLingo3GLicense(environment, pluginConfigPath);
if (lingo3gLicense != null && Files.isReadable(lingo3gLicense)) {
c2SettingsAsMap.put("license", new FileResource(lingo3gLicense.toFile()));
} else if (algorithms.contains("lingo3g")) {
logger.warn(
"Lingo3G is on classpath, but no licenses have been found. Check out the documentation.");
}
// Create component pool.
Integer poolSize = c2Settings.getAsInt(DEFAULT_COMPONENT_SIZE_PROPERTY_NAME, 0);
if (poolSize > 0) {
controller = ControllerFactory.createPooling(poolSize);
} else {
controller = ControllerFactory.createPooling();
}
controller.init(c2SettingsAsMap, suite.getComponentConfigurations());
} catch (Exception e) {
throw new ElasticsearchException("Could not start Carrot2 controller.", e);
}
if (algorithms == null || algorithms.isEmpty()) {
throw new ElasticsearchException(
"No registered/ available clustering algorithms? Check the logs, it's odd.");
}
}