public static void assertSettings(Settings left, Settings right, boolean compareClusterName) {
   ImmutableSet<Map.Entry<String, String>> entries0 = left.getAsMap().entrySet();
   Map<String, String> entries1 = right.getAsMap();
   assertThat(entries0.size(), equalTo(entries1.size()));
   for (Map.Entry<String, String> entry : entries0) {
     if (entry.getKey().equals(ClusterName.SETTING) && compareClusterName == false) {
       continue;
     }
     assertThat(entries1, hasEntry(entry.getKey(), entry.getValue()));
   }
 }
 public static void assertSettings(
     Settings left, Settings right, boolean checkClusterUniqueSettings) {
   Set<Map.Entry<String, String>> entries0 = left.getAsMap().entrySet();
   Map<String, String> entries1 = right.getAsMap();
   assertThat(entries0.size(), equalTo(entries1.size()));
   for (Map.Entry<String, String> entry : entries0) {
     if (clusterUniqueSettings.contains(entry.getKey()) && checkClusterUniqueSettings == false) {
       continue;
     }
     assertThat(entries1, hasEntry(entry.getKey(), entry.getValue()));
   }
 }
  @Test
  public void testLoadFromDelimitedString() {
    Settings settings =
        settingsBuilder().loadFromDelimitedString("key1=value1;key2=value2", ';').build();
    assertThat(settings.get("key1"), equalTo("value1"));
    assertThat(settings.get("key2"), equalTo("value2"));
    assertThat(settings.getAsMap().size(), equalTo(2));
    assertThat(settings.toDelimitedString(';'), equalTo("key1=value1;key2=value2;"));

    settings = settingsBuilder().loadFromDelimitedString("key1=value1;key2=value2;", ';').build();
    assertThat(settings.get("key1"), equalTo("value1"));
    assertThat(settings.get("key2"), equalTo("value2"));
    assertThat(settings.getAsMap().size(), equalTo(2));
    assertThat(settings.toDelimitedString(';'), equalTo("key1=value1;key2=value2;"));
  }
Exemplo n.º 4
0
 public static Settings processSettings(Settings settings) {
   if (settings.get(TRIBE_NAME) != null) {
     // if its a node client started by this service as tribe, remove any tribe group setting
     // to avoid recursive configuration
     Settings.Builder sb = Settings.builder().put(settings);
     for (String s : settings.getAsMap().keySet()) {
       if (s.startsWith("tribe.") && !s.equals(TRIBE_NAME)) {
         sb.remove(s);
       }
     }
     return sb.build();
   }
   Map<String, Settings> nodesSettings = settings.getGroups("tribe", true);
   if (nodesSettings.isEmpty()) {
     return settings;
   }
   // its a tribe configured node..., force settings
   Settings.Builder sb = Settings.builder().put(settings);
   sb.put(Node.NODE_CLIENT_SETTING.getKey(), true); // this node should just act as a node client
   sb.put(
       DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(),
       "local"); // a tribe node should not use zen discovery
   sb.put(
       DiscoveryService.INITIAL_STATE_TIMEOUT_SETTING.getKey(),
       0); // nothing is going to be discovered, since no master will be elected
   if (sb.get("cluster.name") == null) {
     sb.put(
         "cluster.name",
         "tribe_"
             + Strings
                 .randomBase64UUID()); // make sure it won't join other tribe nodes in the same JVM
   }
   sb.put(TransportMasterNodeReadAction.FORCE_LOCAL_SETTING, true);
   return sb.build();
 }
  private void assertMMNinClusterSetting(InternalTestCluster cluster, int masterNodes) {
    final int minMasterNodes = masterNodes / 2 + 1;
    for (final String node : cluster.getNodeNames()) {
      Settings stateSettings =
          cluster
              .client(node)
              .admin()
              .cluster()
              .prepareState()
              .setLocal(true)
              .get()
              .getState()
              .getMetaData()
              .settings();

      assertThat(
          "dynamic setting for node ["
              + node
              + "] has the wrong min_master_node setting : ["
              + stateSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey())
              + "]",
          stateSettings.getAsMap(),
          hasEntry(
              DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(),
              Integer.toString(minMasterNodes)));
    }
  }
 @Override
 public ActionRequestValidationException validate() {
   ActionRequestValidationException validationException = null;
   if (settings.getAsMap().isEmpty()) {
     validationException = addValidationError("no settings to update", validationException);
   }
   return validationException;
 }
  public void updateSettings(
      final Settings pSettings, final String[] indices, final Listener listener) {
    ImmutableSettings.Builder updatedSettingsBuilder = ImmutableSettings.settingsBuilder();
    for (Map.Entry<String, String> entry : pSettings.getAsMap().entrySet()) {
      if (!entry.getKey().startsWith("index.")) {
        updatedSettingsBuilder.put("index." + entry.getKey(), entry.getValue());
      } else {
        updatedSettingsBuilder.put(entry.getKey(), entry.getValue());
      }
    }
    final Settings settings = updatedSettingsBuilder.build();
    clusterService.submitStateUpdateTask(
        "update-settings",
        new ClusterStateUpdateTask() {
          @Override
          public ClusterState execute(ClusterState currentState) {
            try {
              boolean changed = false;
              String[] actualIndices = currentState.metaData().concreteIndices(indices);
              RoutingTable.Builder routingTableBuilder =
                  newRoutingTableBuilder().routingTable(currentState.routingTable());
              MetaData.Builder metaDataBuilder =
                  MetaData.newMetaDataBuilder().metaData(currentState.metaData());

              int updatedNumberOfReplicas =
                  settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, -1);
              if (updatedNumberOfReplicas != -1) {
                routingTableBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
                metaDataBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
                changed = true;
              }

              if (!changed) {
                listener.onFailure(
                    new ElasticSearchIllegalArgumentException("No settings applied"));
                return currentState;
              }

              logger.info(
                  "Updating number_of_replicas to [{}] for indices {}",
                  updatedNumberOfReplicas,
                  actualIndices);

              listener.onSuccess();

              return ClusterState.builder()
                  .state(currentState)
                  .metaData(metaDataBuilder)
                  .routingTable(routingTableBuilder)
                  .build();
            } catch (Exception e) {
              listener.onFailure(e);
              return currentState;
            }
          }
        });
  }
 private Node buildNode(String id) throws IOException {
   Settings nodeSettings = settingsBuilder().put(getNodeSettings()).put("name", id).build();
   logger.info("settings={}", nodeSettings.getAsMap());
   // ES 2.1 renders NodeBuilder as useless
   Set<Class<? extends Plugin>> plugins = new HashSet<>();
   plugins.add(KnapsackPlugin.class);
   Environment environment = new Environment(nodeSettings);
   Node node = new MockNode(environment, plugins);
   AbstractClient client = (AbstractClient) node.client();
   nodes.put(id, node);
   clients.put(id, client);
   logger.info("clients={}", clients);
   return node;
 }
Exemplo n.º 9
0
  @Inject
  public HdfsGateway(
      Settings settings,
      ThreadPool threadPool,
      ClusterService clusterService,
      ClusterName clusterName)
      throws IOException {
    super(settings, threadPool, clusterService);

    this.closeFileSystem = componentSettings.getAsBoolean("close_fs", true);
    String uri = componentSettings.get("uri");
    if (uri == null) {
      throw new ElasticSearchIllegalArgumentException(
          "hdfs gateway requires the 'uri' setting to be set");
    }
    String path = componentSettings.get("path");
    if (path == null) {
      throw new ElasticSearchIllegalArgumentException(
          "hdfs gateway requires the 'path' path setting to be set");
    }
    Path hPath = new Path(new Path(path), clusterName.value());

    int concurrentStreams = componentSettings.getAsInt("concurrent_streams", 5);
    this.concurrentStreamPool =
        EsExecutors.newScalingExecutorService(
            1,
            concurrentStreams,
            5,
            TimeUnit.SECONDS,
            EsExecutors.daemonThreadFactory(settings, "[s3_stream]"));

    logger.debug(
        "Using uri [{}], path [{}], concurrent_streams [{}]", uri, hPath, concurrentStreams);

    Configuration conf = new Configuration();
    Settings hdfsSettings = settings.getByPrefix("hdfs.conf.");
    for (Map.Entry<String, String> entry : hdfsSettings.getAsMap().entrySet()) {
      conf.set(entry.getKey(), entry.getValue());
    }

    fileSystem = FileSystem.get(URI.create(uri), conf);

    initialize(
        new HdfsBlobStore(settings, fileSystem, concurrentStreamPool, hPath), clusterName, null);
  }
 private void assertMMNinNodeSetting(String node, InternalTestCluster cluster, int masterNodes) {
   final int minMasterNodes = masterNodes / 2 + 1;
   Settings nodeSettings =
       cluster
           .client(node)
           .admin()
           .cluster()
           .prepareNodesInfo(node)
           .get()
           .getNodes()
           .get(0)
           .getSettings();
   assertThat(
       "node setting of node ["
           + node
           + "] has the wrong min_master_node setting: ["
           + nodeSettings.get(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey())
           + "]",
       nodeSettings.getAsMap(),
       hasEntry(
           DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.toString(minMasterNodes)));
 }
 @Override
 public String toString() {
   return settings.getAsMap().toString();
 }
Exemplo n.º 12
0
  @SuppressForbidden(reason = "needs java.io.File api to start a process")
  synchronized void startInternal(
      Client client, Settings settings, String nodeName, String clusterName)
      throws IOException, InterruptedException {
    if (process != null) {
      throw new IllegalStateException("Already started");
    }
    List<String> params = new ArrayList<>();

    if (!Constants.WINDOWS) {
      params.add("bin/elasticsearch");
    } else {
      params.add("bin/elasticsearch.bat");
    }
    params.add("-Des.cluster.name=" + clusterName);
    params.add("-Des.node.name=" + nodeName);
    Settings.Builder externaNodeSettingsBuilder = Settings.builder();
    for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
      switch (entry.getKey()) {
        case "cluster.name":
        case "node.name":
        case "path.home":
        case "node.mode":
        case "node.local":
        case NetworkModule.TRANSPORT_TYPE_KEY:
        case "discovery.type":
        case NetworkModule.TRANSPORT_SERVICE_TYPE_KEY:
        case "config.ignore_system_properties":
          continue;
        default:
          externaNodeSettingsBuilder.put(entry.getKey(), entry.getValue());
      }
    }
    this.externalNodeSettings = externaNodeSettingsBuilder.put(REQUIRED_SETTINGS).build();
    for (Map.Entry<String, String> entry : externalNodeSettings.getAsMap().entrySet()) {
      params.add("-Des." + entry.getKey() + "=" + entry.getValue());
    }

    params.add("-Des.path.home=" + PathUtils.get(".").toAbsolutePath());
    params.add("-Des.path.conf=" + path + "/config");

    ProcessBuilder builder = new ProcessBuilder(params);
    builder.directory(path.toFile());
    builder.inheritIO();
    boolean success = false;
    try {
      logger.info("starting external node [{}] with: {}", nodeName, builder.command());
      process = builder.start();
      this.nodeInfo = null;
      if (waitForNode(client, nodeName)) {
        nodeInfo = nodeInfo(client, nodeName);
        assert nodeInfo != null;
        logger.info(
            "external node {} found, version [{}], build {}",
            nodeInfo.getNode(),
            nodeInfo.getVersion(),
            nodeInfo.getBuild());
      } else {
        throw new IllegalStateException("Node [" + nodeName + "] didn't join the cluster");
      }
      success = true;
    } finally {
      if (!success) {
        stop();
      }
    }
  }
 private void setClusterExcludeFilters(Settings settings) {
   clusterExcludeFilters = DiscoveryNodeFilters.buildFromKeyValue(OR, settings.getAsMap());
 }
 private void setClusterRequireFilters(Settings settings) {
   clusterRequireFilters = DiscoveryNodeFilters.buildFromKeyValue(AND, settings.getAsMap());
 }
  @Inject
  public TomcatHttpServerTransport(
      final Settings settings,
      final Environment environment,
      final NetworkService networkService,
      final ClusterName clusterName,
      final Client client,
      final SecurityService securityService) {
    super(settings);

    this.settings = settings;
    this.securityService = securityService;

    /*
     * TODO check if keep alive is managed by tomcat copy custom headers to
     * response check that user under tomcat/ea is running is not a
     * privilieged iuser tomcat props apply: tomcat.XXX
     */

    // _aliases test with more than one index mapped to an alias

    /*
     *
     * http.max_initial_line_length not respected http.reset_cookies not
     * respected workerCount http.cors.enabled http.cors.allow-origin
     * http.cors.max-age http.cors.allow-methods http.cors.allow-headers
     *
     *
     *
     * http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/
     * modules-network.html
     *
     * http://stackoverflow.com/questions/8038718/serializing-generic-java-
     * object-to-json-using-jackson
     * http://tomcatspnegoad.sourceforge.net/realms.html
     *
     * SSL options
     *
     *
     *
     * Realm options/login waffle, spnego ... security.kerberos.provider:
     * waffle
     *
     * Hardening EA - dynamic script disable
     */

    /*
    *
    *

    *
    *
    *

    */

    useSSL =
        componentSettings.getAsBoolean(
            "ssl.enabled", settings.getAsBoolean("security.ssl.enabled", false));

    useClientAuth =
        componentSettings.getAsBoolean(
            "ssl.clientauth.enabled",
            settings.getAsBoolean("security.ssl.clientauth.enabled", false));

    kerberosMode =
        componentSettings.get("kerberos.mode", settings.get("security.kerberos.mode", "none"));

    port = componentSettings.get("port", settings.get("http.port", "8080"));
    bindHost =
        componentSettings.get(
            "bind_host", settings.get("http.bind_host", settings.get("http.host")));
    publishHost =
        componentSettings.get(
            "publish_host", settings.get("http.publish_host", settings.get("http.host")));
    this.networkService = networkService;

    ByteSizeValue maxContentLength =
        componentSettings.getAsBytesSize(
            "max_content_length",
            settings.getAsBytesSize(
                "http.max_content_length", new ByteSizeValue(100, ByteSizeUnit.MB)));
    maxChunkSize =
        componentSettings.getAsBytesSize(
            "max_chunk_size",
            settings.getAsBytesSize("http.max_chunk_size", new ByteSizeValue(8, ByteSizeUnit.KB)));
    maxHeaderSize =
        componentSettings.getAsBytesSize(
            "max_header_size",
            settings.getAsBytesSize("http.max_header_size", new ByteSizeValue(8, ByteSizeUnit.KB)));

    blockingServer =
        settings.getAsBoolean(
            "http.blocking_server",
            settings.getAsBoolean(TCP_BLOCKING_SERVER, settings.getAsBoolean(TCP_BLOCKING, false)));

    tcpNoDelay =
        componentSettings.getAsBoolean("tcp_no_delay", settings.getAsBoolean(TCP_NO_DELAY, true));
    tcpKeepAlive =
        componentSettings.getAsBoolean(
            "tcp_keep_alive", settings.getAsBoolean(TCP_KEEP_ALIVE, true));
    reuseAddress =
        componentSettings.getAsBoolean(
            "reuse_address",
            settings.getAsBoolean(TCP_REUSE_ADDRESS, NetworkUtils.defaultReuseAddress()));
    tcpSendBufferSize =
        componentSettings.getAsBytesSize(
            "tcp_send_buffer_size",
            settings.getAsBytesSize(TCP_SEND_BUFFER_SIZE, TCP_DEFAULT_SEND_BUFFER_SIZE));
    tcpReceiveBufferSize =
        componentSettings.getAsBytesSize(
            "tcp_receive_buffer_size",
            settings.getAsBytesSize(TCP_RECEIVE_BUFFER_SIZE, TCP_DEFAULT_RECEIVE_BUFFER_SIZE));

    compression = settings.getAsBoolean("http.compression", false);
    compressionLevel = settings.getAsInt("http.compression_level", 6);

    // validate max content length
    if (maxContentLength.bytes() > Integer.MAX_VALUE) {
      logger.warn(
          "maxContentLength[" + maxContentLength + "] set to high value, resetting it to [100mb]");
      maxContentLength = new ByteSizeValue(100, ByteSizeUnit.MB);
    }
    this.maxContentLength = maxContentLength;

    logger.debug("port: " + port);
    logger.debug("bindHost: " + bindHost);
    logger.debug("publishHost: " + publishHost);

    logger.debug("componentsettings: " + componentSettings.getAsMap());
    logger.debug("settings: " + settings.getAsMap());
  }
  @SuppressForbidden(reason = "C2 integration (File API)")
  @Override
  protected void doStart() throws ElasticsearchException {
    try {
      Settings.Builder builder = Settings.builder();
      Path pluginConfigPath = environment.configFile().resolve(ClusteringPlugin.PLUGIN_NAME);

      if (!Files.isDirectory(pluginConfigPath)) {
        Path srcConfig = Paths.get("src/main/config");
        if (Files.isDirectory(srcConfig)) {
          // Allow running from within the IDE.
          pluginConfigPath = srcConfig;
        } else {
          throw new ElasticsearchException("Config folder missing: " + pluginConfigPath);
        }
      } else {
        logger.info("Configuration files at: {}", pluginConfigPath.toAbsolutePath());
      }

      for (String configName :
          new String[] {"config.yml", "config.yaml", "config.json", "config.properties"}) {
        try {
          Path resolved = pluginConfigPath.resolve(configName);
          if (resolved != null && Files.exists(resolved)) {
            builder.loadFromPath(resolved);
          }
        } catch (NoClassDefFoundError e) {
          logger.warn("Could not parse: {}", e, configName);
        }
      }
      Settings c2Settings = builder.build();

      // Parse suite descriptors with loggers turned off (shut them up a bit).
      final Path suitePath = pluginConfigPath.resolve(c2Settings.get(DEFAULT_SUITE_PROPERTY_NAME));
      if (!Files.isRegularFile(suitePath)) {
        throw new ElasticsearchException(
            "Could not find algorithm suite: " + suitePath.toAbsolutePath().normalize());
      }

      final ResourceLookup suiteLookup =
          new ResourceLookup(new DirLocator(suitePath.getParent().toFile()));
      final IResource suiteResource = suiteLookup.getFirst(suitePath.getFileName().toString());

      final List<String> failed = Lists.newArrayList();
      final ProcessingComponentSuite suite =
          LoggerUtils.quietCall(
              new Callable<ProcessingComponentSuite>() {
                public ProcessingComponentSuite call() throws Exception {
                  ProcessingComponentSuite suite =
                      ProcessingComponentSuite.deserialize(suiteResource, suiteLookup);
                  for (ProcessingComponentDescriptor desc : suite.removeUnavailableComponents()) {
                    failed.add(desc.getId());
                    if (isNoClassDefFound(desc.getInitializationFailure())) {
                      logger.debug("Algorithm not available on classpath: {}", desc.getId());
                    } else {
                      logger.warn(
                          "Algorithm initialization failed: {}",
                          desc.getInitializationFailure(),
                          desc.getId());
                    }
                  }
                  return suite;
                }
              },
              Logger.getLogger(ProcessingComponentDescriptor.class),
              Logger.getLogger(ReflectionUtils.class));

      algorithms = Lists.newArrayList();
      for (ProcessingComponentDescriptor descriptor : suite.getAlgorithms()) {
        algorithms.add(descriptor.getId());
      }
      algorithms = Collections.unmodifiableList(algorithms);

      if (!algorithms.isEmpty()) {
        logger.info("Available clustering components: {}", Joiner.on(", ").join(algorithms));
      }
      if (!failed.isEmpty()) {
        logger.info("Unavailable clustering components: {}", Joiner.on(", ").join(failed));
      }

      final Path resourcesPath =
          pluginConfigPath
              .resolve(c2Settings.get(DEFAULT_RESOURCES_PROPERTY_NAME, "."))
              .toAbsolutePath()
              .normalize();

      logger.info("Lexical resources dir: {}", resourcesPath);

      final ResourceLookup resourceLookup =
          new ResourceLookup(
              new DirLocator(resourcesPath.toFile()),
              new ClassLoaderLocator(ControllerSingleton.class.getClassLoader()));

      // Change the default resource lookup to include the configured location.
      Map<String, Object> c2SettingsAsMap = Maps.newHashMap();
      DefaultLexicalDataFactoryDescriptor.attributeBuilder(c2SettingsAsMap)
          .resourceLookup(resourceLookup);
      c2SettingsAsMap.putAll(c2Settings.getAsMap());

      // Set up the license for Lingo3G, if it's available.
      Path lingo3gLicense = scanForLingo3GLicense(environment, pluginConfigPath);
      if (lingo3gLicense != null && Files.isReadable(lingo3gLicense)) {
        c2SettingsAsMap.put("license", new FileResource(lingo3gLicense.toFile()));
      } else if (algorithms.contains("lingo3g")) {
        logger.warn(
            "Lingo3G is on classpath, but no licenses have been found. Check out the documentation.");
      }

      // Create component pool.
      Integer poolSize = c2Settings.getAsInt(DEFAULT_COMPONENT_SIZE_PROPERTY_NAME, 0);
      if (poolSize > 0) {
        controller = ControllerFactory.createPooling(poolSize);
      } else {
        controller = ControllerFactory.createPooling();
      }
      controller.init(c2SettingsAsMap, suite.getComponentConfigurations());
    } catch (Exception e) {
      throw new ElasticsearchException("Could not start Carrot2 controller.", e);
    }

    if (algorithms == null || algorithms.isEmpty()) {
      throw new ElasticsearchException(
          "No registered/ available clustering algorithms? Check the logs, it's odd.");
    }
  }