Ejemplo n.º 1
0
 @Inject
 public NodeInfo(NodeConfig config) {
   this(
       config.getEnvironment(),
       config.getPool(),
       config.getNodeId(),
       config.getNodeInternalIp(),
       config.getNodeInternalHostname(),
       config.getNodeBindIp(),
       config.getNodeExternalAddress(),
       config.getLocation(),
       config.getBinarySpec(),
       config.getConfigSpec());
 }
Ejemplo n.º 2
0
 public CoreContainer(NodeConfig config, Properties properties, boolean asyncSolrCoreLoad) {
   this(
       config,
       properties,
       new CorePropertiesLocator(config.getCoreRootDirectory()),
       asyncSolrCoreLoad);
 }
Ejemplo n.º 3
0
  /**
   * Iterate through registered additional node types to find type which = prefix of dns, and then
   * return NodeConfig(found_type, dns). For example: given: dns = "builder2.dev.com"
   * $builder_host_name = "builder1.dev.com" => base_node_domain = ".dev.com" Result = new
   * NodeConfig(BUILDER, "builder2.dev.com")
   *
   * <p>Example 2: given: dns = "builder2.dev.com" $builder_host_name = "builder1.example.com" =>
   * base_node_domain = ".example.com" != ".dev.com" Result = IllegalArgumentException("Illegal DNS
   * name 'builder2.dev.com' of additional node....)
   *
   * @throws IllegalArgumentException if dns doesn't comply with convention
   *     '<supported_node_type><number>(base_node_domain)' where supported prefix
   */
  public NodeConfig recognizeNodeConfigFromDns(String dns)
      throws IllegalArgumentException, IllegalStateException {
    for (Map.Entry<NodeConfig.NodeType, String> entry :
        ADDITIONAL_NODES_CODENVY_PROPERTIES.entrySet()) {
      NodeConfig.NodeType type = entry.getKey();

      NodeConfig baseNode = NodeConfig.extractConfigFrom(config, type);
      if (baseNode == null) {
        throw new IllegalStateException(
            format("Host name of base node of type '%s' wasn't found.", type));
      }

      String typeString = type.toString().toLowerCase();
      String baseNodeDomain = ConfigManager.getBaseNodeDomain(baseNode).toLowerCase();
      String regex = format("^%s\\d+%s$", typeString, baseNodeDomain);

      if (dns != null && dns.toLowerCase().matches(regex)) {
        return new NodeConfig(type, dns, null);
      }
    }

    throw new IllegalArgumentException(
        format(
            "Illegal DNS name '%s' of additional node. "
                + "Correct name template is '<prefix><number><base_node_domain>' where supported prefix is one from"
                + " the list '%s'",
            dns, ADDITIONAL_NODES_CODENVY_PROPERTIES.keySet().toString().toLowerCase()));
  }
Ejemplo n.º 4
0
 public CoreContainer(
     NodeConfig config, Properties properties, CoresLocator locator, boolean asyncSolrCoreLoad) {
   this.loader = config.getSolrResourceLoader();
   this.solrHome = loader.getInstancePath().toString();
   this.cfg = checkNotNull(config);
   this.coresLocator = locator;
   this.containerProperties = new Properties(properties);
   this.asyncSolrCoreLoad = asyncSolrCoreLoad;
 }
Ejemplo n.º 5
0
  /**
   * Erase url of removing node from the list of additional nodes of type = removingNode.getType()
   * of the configuration of puppet master, and return this list as row with comma-separated values.
   * For example: given: $additional_builders =
   * "http://builder2.example.com:8080/builder/internal/builder,http://builder3.example.com:8080/builder/internal/builder"
   * removingNode = new NodeConfig(BUILDER, "builder3.example.com") Result =
   * "http://builder2.example.com:8080/builder/internal/builder"
   *
   * @throws IllegalArgumentException if there is no removing node in the list of additional nodes
   * @throws IllegalStateException if additional nodes property isn't found in Codenvy config
   */
  public String getValueWithoutNode(NodeConfig removingNode) throws IllegalArgumentException {
    String additionalNodesProperty = getPropertyNameBy(removingNode.getType());
    List<String> nodesUrls = config.getAllValues(additionalNodesProperty);
    if (nodesUrls == null) {
      throw new IllegalStateException(
          format(
              "Additional nodes property '%s' isn't found in Codenvy config",
              additionalNodesProperty));
    }

    String nodeUrl = getAdditionalNodeUrl(removingNode);
    if (!nodesUrls.contains(nodeUrl)) {
      throw new IllegalArgumentException(
          format("There is no node '%s' in the list of additional nodes", removingNode.getHost()));
    }

    nodesUrls.remove(nodeUrl);

    return on(',').skipNulls().join(nodesUrls);
  }
Ejemplo n.º 6
0
  /**
   * Construct url of adding node, add it to the list of additional nodes of type =
   * addingNode.getType() of the configuration of puppet master, and return this list as row with
   * comma-separated values. For example: given: $additional_builders =
   * "http://builder2.example.com:8080/builder/internal/builder" addingNode = new
   * NodeConfig(BUILDER, "builder3.example.com") Result =
   * "http://builder2.example.com:8080/builder/internal/builder,http://builder3.example.com:8080/builder/internal/builder"
   *
   * @throws IllegalArgumentException if there is adding node in the list of additional nodes
   * @throws IllegalStateException if additional nodes property isn't found in Codenvy config
   */
  public String getValueWithNode(NodeConfig addingNode)
      throws IllegalArgumentException, IllegalStateException {
    String additionalNodesProperty = getPropertyNameBy(addingNode.getType());
    List<String> nodesUrls = config.getAllValues(additionalNodesProperty);
    if (nodesUrls == null) {
      throw new IllegalStateException(
          format(
              "Additional nodes property '%s' isn't found in Codenvy config",
              additionalNodesProperty));
    }

    String nodeUrl = getAdditionalNodeUrl(addingNode);
    if (nodesUrls.contains(nodeUrl)) {
      throw new IllegalArgumentException(
          format("Node '%s' has been already used", addingNode.getHost()));
    }

    nodesUrls.add(nodeUrl);

    return on(',').skipNulls().join(nodesUrls);
  }
Ejemplo n.º 7
0
 protected Node(
     NodeConfig nodeConfig,
     Balancer balancerConfig,
     XnioIoThread ioThread,
     ByteBufferPool bufferPool,
     ModClusterContainer container) {
   this.id = idGen.incrementAndGet();
   this.jvmRoute = nodeConfig.getJvmRoute();
   this.nodeConfig = nodeConfig;
   this.ioThread = ioThread;
   this.bufferPool = bufferPool;
   this.balancerConfig = balancerConfig;
   this.container = container;
   this.connectionPoolManager = new NodeConnectionPoolManager();
   this.connectionPool =
       new ProxyConnectionPool(
           connectionPoolManager,
           nodeConfig.getConnectionURI(),
           container.getXnioSsl(),
           container.getClient(),
           container.getClientOptions());
 }
Ejemplo n.º 8
0
 @Inject
 public NodeInfo(NodeConfig config) {
   this(
       config.getEnvironment(),
       config.getPool(),
       config.getNodeId(),
       config.getNodeIp(),
       config.getLocation(),
       config.getBinarySpec(),
       config.getConfigSpec());
 }
Ejemplo n.º 9
0
 protected boolean checkAvailable(final boolean existingSession) {
   if (allAreClear(state, ERROR | REMOVED)) {
     // Check the state of the queue on the connection pool
     final ProxyConnectionPool.AvailabilityType availability = connectionPool.available();
     if (availability == ProxyConnectionPool.AvailabilityType.AVAILABLE) {
       return true;
     } else if (availability == ProxyConnectionPool.AvailabilityType.FULL) {
       if (existingSession) {
         return true;
       } else if (!existingSession && nodeConfig.isQueueNewRequests()) {
         return true;
       }
     }
   }
   return false;
 }
Ejemplo n.º 10
0
 public static void main(String[] args) {
   ApplicationContext ctx = new ClassPathXmlApplicationContext("application.xml");
   NodeConfig p = (NodeConfig) ctx.getBean("node");
   System.out.println(p.getId());
 }
Ejemplo n.º 11
0
 /**
  * Creates a new core, publishing the core state to the cluster
  *
  * @param coreName the core name
  * @param parameters the core parameters
  * @return the newly created core
  */
 public SolrCore create(String coreName, Map<String, String> parameters) {
   return create(coreName, cfg.getCoreRootDirectory().resolve(coreName), parameters);
 }
Ejemplo n.º 12
0
  /** Load the cores defined for this CoreContainer */
  public void load() {
    log.info("Loading cores into CoreContainer [instanceDir={}]", loader.getInstancePath());

    // add the sharedLib to the shared resource loader before initializing cfg based plugins
    String libDir = cfg.getSharedLibDirectory();
    if (libDir != null) {
      Path libPath = loader.getInstancePath().resolve(libDir);
      try {
        loader.addToClassLoader(SolrResourceLoader.getURLs(libPath));
        loader.reloadLuceneSPI();
      } catch (IOException e) {
        log.warn("Couldn't add files from {} to classpath: {}", libPath, e.getMessage());
      }
    }

    shardHandlerFactory =
        ShardHandlerFactory.newInstance(cfg.getShardHandlerFactoryPluginInfo(), loader);

    updateShardHandler = new UpdateShardHandler(cfg.getUpdateShardHandlerConfig());

    solrCores.allocateLazyCores(cfg.getTransientCacheSize(), loader);

    logging = LogWatcher.newRegisteredLogWatcher(cfg.getLogWatcherConfig(), loader);

    hostName = cfg.getNodeName();

    zkSys.initZooKeeper(this, solrHome, cfg.getCloudConfig());
    if (isZooKeeperAware())
      pkiAuthenticationPlugin =
          new PKIAuthenticationPlugin(this, zkSys.getZkController().getNodeName());

    ZkStateReader.ConfigData securityConfig =
        isZooKeeperAware()
            ? getZkController().getZkStateReader().getSecurityProps(false)
            : new ZkStateReader.ConfigData(EMPTY_MAP, -1);
    initializeAuthorizationPlugin((Map<String, Object>) securityConfig.data.get("authorization"));
    initializeAuthenticationPlugin((Map<String, Object>) securityConfig.data.get("authentication"));

    this.backupRepoFactory = new BackupRepositoryFactory(cfg.getBackupRepositoryPlugins());

    containerHandlers.put(ZK_PATH, new ZookeeperInfoHandler(this));
    securityConfHandler = new SecurityConfHandler(this);
    collectionsHandler = createHandler(cfg.getCollectionsHandlerClass(), CollectionsHandler.class);
    containerHandlers.put(COLLECTIONS_HANDLER_PATH, collectionsHandler);
    infoHandler = createHandler(cfg.getInfoHandlerClass(), InfoHandler.class);
    containerHandlers.put(INFO_HANDLER_PATH, infoHandler);
    coreAdminHandler = createHandler(cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class);
    containerHandlers.put(CORES_HANDLER_PATH, coreAdminHandler);
    configSetsHandler = createHandler(cfg.getConfigSetsHandlerClass(), ConfigSetsHandler.class);
    containerHandlers.put(CONFIGSETS_HANDLER_PATH, configSetsHandler);
    containerHandlers.put(AUTHZ_PATH, securityConfHandler);
    containerHandlers.put(AUTHC_PATH, securityConfHandler);
    if (pkiAuthenticationPlugin != null)
      containerHandlers.put(
          PKIAuthenticationPlugin.PATH, pkiAuthenticationPlugin.getRequestHandler());

    coreConfigService = ConfigSetService.createConfigSetService(cfg, loader, zkSys.zkController);

    containerProperties.putAll(cfg.getSolrProperties());

    // setup executor to load cores in parallel
    ExecutorService coreLoadExecutor =
        ExecutorUtil.newMDCAwareFixedThreadPool(
            cfg.getCoreLoadThreadCount(
                isZooKeeperAware()
                    ? DEFAULT_CORE_LOAD_THREADS_IN_CLOUD
                    : DEFAULT_CORE_LOAD_THREADS),
            new DefaultSolrThreadFactory("coreLoadExecutor"));
    final List<Future<SolrCore>> futures = new ArrayList<>();
    try {
      List<CoreDescriptor> cds = coresLocator.discover(this);
      if (isZooKeeperAware()) {
        // sort the cores if it is in SolrCloud. In standalone node the order does not matter
        CoreSorter coreComparator = new CoreSorter().init(this);
        cds = new ArrayList<>(cds); // make a copy
        Collections.sort(cds, coreComparator::compare);
      }
      checkForDuplicateCoreNames(cds);

      for (final CoreDescriptor cd : cds) {
        if (cd.isTransient() || !cd.isLoadOnStartup()) {
          solrCores.putDynamicDescriptor(cd.getName(), cd);
        } else if (asyncSolrCoreLoad) {
          solrCores.markCoreAsLoading(cd);
        }
        if (cd.isLoadOnStartup()) {
          futures.add(
              coreLoadExecutor.submit(
                  () -> {
                    SolrCore core;
                    try {
                      if (zkSys.getZkController() != null) {
                        zkSys.getZkController().throwErrorIfReplicaReplaced(cd);
                      }

                      core = create(cd, false);
                    } finally {
                      if (asyncSolrCoreLoad) {
                        solrCores.markCoreAsNotLoading(cd);
                      }
                    }
                    try {
                      zkSys.registerInZk(core, true);
                    } catch (RuntimeException e) {
                      SolrException.log(log, "Error registering SolrCore", e);
                    }
                    return core;
                  }));
        }
      }

      // Start the background thread
      backgroundCloser = new CloserThread(this, solrCores, cfg);
      backgroundCloser.start();

    } finally {
      if (asyncSolrCoreLoad && futures != null) {

        coreContainerWorkExecutor.submit(
            (Runnable)
                () -> {
                  try {
                    for (Future<SolrCore> future : futures) {
                      try {
                        future.get();
                      } catch (InterruptedException e) {
                        Thread.currentThread().interrupt();
                      } catch (ExecutionException e) {
                        log.error("Error waiting for SolrCore to be created", e);
                      }
                    }
                  } finally {
                    ExecutorUtil.shutdownAndAwaitTermination(coreLoadExecutor);
                  }
                });
      } else {
        ExecutorUtil.shutdownAndAwaitTermination(coreLoadExecutor);
      }
    }

    if (isZooKeeperAware()) {
      zkSys.getZkController().checkOverseerDesignate();
    }
  }
Ejemplo n.º 13
0
 public CoreContainer(NodeConfig config, Properties properties) {
   this(config, properties, new CorePropertiesLocator(config.getCoreRootDirectory()));
 }
Ejemplo n.º 14
0
 /**
  * Gets the alternate path for multicore handling: This is used in case there is a registered
  * unnamed core (aka name is "") to declare an alternate way of accessing named cores. This can
  * also be used in a pseudo single-core environment so admins can prepare a new version before
  * swapping.
  */
 public String getManagementPath() {
   return cfg.getManagementPath();
 }
Ejemplo n.º 15
0
 public Path getCoreRootDirectory() {
   return cfg.getCoreRootDirectory();
 }
Ejemplo n.º 16
0
 /**
  * @return link like "http://builder3.example.com:8080/builder/internal/builder", or
  *     "http://runner3.example.com:8080/runner/internal/runner" For example: given: node = new
  *     NodeConfig(BUILDER, "builder2.example.com") Result =
  *     "http://builder2.example.com:8080/builder/internal/builder"
  */
 protected String getAdditionalNodeUrl(NodeConfig node) {
   return format(
       ADDITIONAL_NODE_URL_TEMPLATE, node.getHost(), node.getType().toString().toLowerCase());
 }