private String getRemotCoreUrl(String collectionName, String origCorename) {
    ClusterState clusterState = cores.getZkController().getClusterState();
    Collection<Slice> slices = clusterState.getActiveSlices(collectionName);
    boolean byCoreName = false;

    if (slices == null) {
      slices = new ArrayList<>();
      // look by core name
      byCoreName = true;
      getSlicesForCollections(clusterState, slices, true);
      if (slices.isEmpty()) {
        getSlicesForCollections(clusterState, slices, false);
      }
    }

    if (slices.isEmpty()) {
      return null;
    }

    if (collectionsList == null) collectionsList = new ArrayList<>();

    collectionsList.add(collectionName);
    String coreUrl =
        getCoreUrl(collectionName, origCorename, clusterState, slices, byCoreName, true);

    if (coreUrl == null) {
      coreUrl = getCoreUrl(collectionName, origCorename, clusterState, slices, byCoreName, false);
    }

    return coreUrl;
  }
 private void extractRemotePath(String corename, String origCorename, int idx)
     throws UnsupportedEncodingException, KeeperException, InterruptedException {
   if (core == null && idx > 0) {
     coreUrl = getRemotCoreUrl(corename, origCorename);
     // don't proxy for internal update requests
     invalidStates = checkStateIsValid(queryParams.get(CloudSolrClient.STATE_VERSION));
     if (coreUrl != null
         && queryParams.get(DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM) == null) {
       path = path.substring(idx);
       if (invalidStates != null) {
         // it does not make sense to send the request to a remote node
         throw new SolrException(
             SolrException.ErrorCode.INVALID_STATE,
             new String(Utils.toJSON(invalidStates), org.apache.lucene.util.IOUtils.UTF_8));
       }
       action = REMOTEQUERY;
     } else {
       if (!retry) {
         // we couldn't find a core to work with, try reloading aliases
         // TODO: it would be nice if admin ui elements skipped this...
         ZkStateReader reader = cores.getZkController().getZkStateReader();
         reader.updateAliases();
         action = RETRY;
       }
     }
   }
 }
 private SolrCore checkProps(ZkNodeProps zkProps) {
   String corename;
   SolrCore core = null;
   if (cores.getZkController().getNodeName().equals(zkProps.getStr(NODE_NAME_PROP))) {
     corename = zkProps.getStr(CORE_NAME_PROP);
     core = cores.getCore(corename);
   }
   return core;
 }
 /** Make the zookeeper session on a particular jetty expire */
 public void expireZkSession(JettySolrRunner jetty) {
   CoreContainer cores = jetty.getCoreContainer();
   if (cores != null) {
     SolrZkClient zkClient = cores.getZkController().getZkClient();
     zkClient.getSolrZooKeeper().closeCnxn();
     long sessionId = zkClient.getSolrZooKeeper().getSessionId();
     zkServer.expire(sessionId);
     log.info("Expired zookeeper session {} from node {}", sessionId, jetty.getBaseUrl());
   }
 }
 /**
  * Returns an String with the name of a core.
  *
  * <p>This method searches the core with fromIndex name in the core's container. If fromIndex
  * isn't name of collection or alias it's returns fromIndex without changes. If fromIndex is name
  * of alias but if the alias points to multiple collections it's throw
  * SolrException.ErrorCode.BAD_REQUEST because multiple shards not yet supported.
  *
  * @param fromIndex name of the index
  * @param container the core container for searching the core with fromIndex name or alias
  * @return the string with name of core
  */
 public static String getCoreName(final String fromIndex, CoreContainer container) {
   if (container.isZooKeeperAware()) {
     ZkController zkController = container.getZkController();
     final String resolved =
         zkController.getClusterState().hasCollection(fromIndex)
             ? fromIndex
             : resolveAlias(fromIndex, zkController);
     if (resolved == null) {
       throw new SolrException(
           SolrException.ErrorCode.BAD_REQUEST,
           "SolrCloud join: Collection '" + fromIndex + "' not found!");
     }
     return findLocalReplicaForFromIndex(zkController, resolved);
   }
   return fromIndex;
 }
  private String getCoreUrl(
      String collectionName,
      String origCorename,
      ClusterState clusterState,
      Collection<Slice> slices,
      boolean byCoreName,
      boolean activeReplicas) {
    String coreUrl;
    Set<String> liveNodes = clusterState.getLiveNodes();
    List<Slice> randomizedSlices = new ArrayList<>(slices.size());
    randomizedSlices.addAll(slices);
    Collections.shuffle(randomizedSlices, random);

    for (Slice slice : randomizedSlices) {
      List<Replica> randomizedReplicas = new ArrayList<>();
      randomizedReplicas.addAll(slice.getReplicas());
      Collections.shuffle(randomizedReplicas, random);

      for (Replica replica : randomizedReplicas) {
        if (!activeReplicas
            || (liveNodes.contains(replica.getNodeName())
                && replica.getState() == Replica.State.ACTIVE)) {

          if (byCoreName && !collectionName.equals(replica.getStr(CORE_NAME_PROP))) {
            // if it's by core name, make sure they match
            continue;
          }
          if (replica.getStr(BASE_URL_PROP).equals(cores.getZkController().getBaseUrl())) {
            // don't count a local core
            continue;
          }

          if (origCorename != null) {
            coreUrl = replica.getStr(BASE_URL_PROP) + "/" + origCorename;
          } else {
            coreUrl = replica.getCoreUrl();
            if (coreUrl.endsWith("/")) {
              coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
            }
          }

          return coreUrl;
        }
      }
    }
    return null;
  }
  private SolrCore getCoreByCollection(String collectionName) {
    ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();

    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection collection = clusterState.getCollectionOrNull(collectionName);
    if (collection == null) {
      return null;
    }
    Map<String, Slice> slices = collection.getActiveSlicesMap();
    if (slices == null) {
      return null;
    }
    Set<String> liveNodes = clusterState.getLiveNodes();
    // look for a core on this node
    Set<Map.Entry<String, Slice>> entries = slices.entrySet();
    SolrCore core = null;

    // Hitting the leaders is useful when it's an update request.
    // For queries it doesn't matter and hence we don't distinguish here.
    for (Map.Entry<String, Slice> entry : entries) {
      // first see if we have the leader
      Replica leaderProps = collection.getLeader(entry.getKey());
      if (leaderProps != null
          && liveNodes.contains(leaderProps.getNodeName())
          && leaderProps.getState() == Replica.State.ACTIVE) {
        core = checkProps(leaderProps);
        if (core != null) {
          return core;
        }
      }

      // check everyone then
      Map<String, Replica> shards = entry.getValue().getReplicasMap();
      Set<Map.Entry<String, Replica>> shardEntries = shards.entrySet();
      for (Map.Entry<String, Replica> shardEntry : shardEntries) {
        Replica zkProps = shardEntry.getValue();
        if (liveNodes.contains(zkProps.getNodeName())
            && zkProps.getState() == Replica.State.ACTIVE) {
          core = checkProps(zkProps);
          if (core != null) {
            return core;
          }
        }
      }
    }
    return null;
  }
 private Map<String, Integer> checkStateIsValid(String stateVer) {
   Map<String, Integer> result = null;
   String[] pairs;
   if (stateVer != null && !stateVer.isEmpty() && cores.isZooKeeperAware()) {
     // many have multiple collections separated by |
     pairs = StringUtils.split(stateVer, '|');
     for (String pair : pairs) {
       String[] pcs = StringUtils.split(pair, ':');
       if (pcs.length == 2 && !pcs[0].isEmpty() && !pcs[1].isEmpty()) {
         Integer status =
             cores
                 .getZkController()
                 .getZkStateReader()
                 .compareStateVersions(pcs[0], Integer.parseInt(pcs[1]));
         if (status != null) {
           if (result == null) result = new HashMap<>();
           result.put(pcs[0], status);
         }
       }
     }
   }
   return result;
 }
  private void init() throws Exception {
    // The states of client that is invalid in this request
    Aliases aliases = null;
    String corename = "";
    String origCorename = null;
    // set a request timer which can be reused by requests if needed
    req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimerTree());
    // put the core container in request attribute
    req.setAttribute("org.apache.solr.CoreContainer", cores);
    path = req.getServletPath();
    if (req.getPathInfo() != null) {
      // this lets you handle /update/commit when /update is a servlet
      path += req.getPathInfo();
    }
    // check for management path
    String alternate = cores.getManagementPath();
    if (alternate != null && path.startsWith(alternate)) {
      path = path.substring(0, alternate.length());
    }
    // unused feature ?
    int idx = path.indexOf(':');
    if (idx > 0) {
      // save the portion after the ':' for a 'handler' path parameter
      path = path.substring(0, idx);
    }

    boolean usingAliases = false;

    // Check for container handlers
    handler = cores.getRequestHandler(path);
    if (handler != null) {
      solrReq = SolrRequestParsers.DEFAULT.parse(null, path, req);
      solrReq.getContext().put(CoreContainer.class.getName(), cores);
      requestType = RequestType.ADMIN;
      action = ADMIN;
      return;
    } else {
      // otherwise, we should find a core from the path
      idx = path.indexOf("/", 1);
      if (idx > 1) {
        // try to get the corename as a request parameter first
        corename = path.substring(1, idx);

        // look at aliases
        if (cores.isZooKeeperAware()) {
          origCorename = corename;
          ZkStateReader reader = cores.getZkController().getZkStateReader();
          aliases = reader.getAliases();
          if (aliases != null && aliases.collectionAliasSize() > 0) {
            usingAliases = true;
            String alias = aliases.getCollectionAlias(corename);
            if (alias != null) {
              collectionsList = StrUtils.splitSmart(alias, ",", true);
              corename = collectionsList.get(0);
            }
          }
        }

        core = cores.getCore(corename);
        if (core != null) {
          path = path.substring(idx);
        } else if (cores.isCoreLoading(
            corename)) { // extra mem barriers, so don't look at this before trying to get core
          throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "SolrCore is loading");
        } else {
          // the core may have just finished loading
          core = cores.getCore(corename);
          if (core != null) {
            path = path.substring(idx);
          }
        }
      }
      if (core == null) {
        if (!cores.isZooKeeperAware()) {
          core = cores.getCore("");
        }
      }
    }

    if (core == null && cores.isZooKeeperAware()) {
      // we couldn't find the core - lets make sure a collection was not specified instead
      core = getCoreByCollection(corename);
      if (core != null) {
        // we found a core, update the path
        path = path.substring(idx);
        if (collectionsList == null) collectionsList = new ArrayList<>();
        collectionsList.add(corename);
      }

      // if we couldn't find it locally, look on other nodes
      extractRemotePath(corename, origCorename, idx);
      if (action != null) return;
    }

    // With a valid core...
    if (core != null) {
      MDCLoggingContext.setCore(core);
      config = core.getSolrConfig();
      // get or create/cache the parser for the core
      SolrRequestParsers parser = config.getRequestParsers();

      // Determine the handler from the url path if not set
      // (we might already have selected the cores handler)
      extractHandlerFromURLPath(parser);
      if (action != null) return;

      // With a valid handler and a valid core...
      if (handler != null) {
        // if not a /select, create the request
        if (solrReq == null) {
          solrReq = parser.parse(core, path, req);
        }

        if (usingAliases) {
          processAliases(aliases, collectionsList);
        }

        action = PROCESS;
        return; // we are done with a valid handler
      }
    }
    log.debug("no handler or core retrieved for " + path + ", follow through...");

    action = PASSTHROUGH;
  }
Exemple #10
0
  public void handleRequest(RequestGetter requestGetter) {
    MDCLoggingContext.reset();
    MDCLoggingContext.setNode(cores);

    String path = requestGetter.getPath();
    solrParams = requestGetter.getSolrParams();
    SolrRequestHandler handler = null;
    String corename = "";
    String origCorename = null;
    try {
      // set a request timer which can be reused by requests if needed
      // req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimer());
      // put the core container in request attribute
      // req.setAttribute("org.apache.solr.CoreContainer", cores);
      // check for management path
      String alternate = cores.getManagementPath();
      if (alternate != null && path.startsWith(alternate)) {
        path = path.substring(0, alternate.length());
      }
      // unused feature ?
      int idx = path.indexOf(':');
      if (idx > 0) {
        // save the portion after the ':' for a 'handler' path parameter
        path = path.substring(0, idx);
      }

      boolean usingAliases = false;
      List<String> collectionsList = null;

      // Check for container handlers
      handler = cores.getRequestHandler(path);
      if (handler != null) {
        solrReq = parseSolrQueryRequest(SolrRequestParsers.DEFAULT, requestGetter);
        handleAdminRequest(handler, solrReq);
        return;
      } else {
        // otherwise, we should find a core from the path
        idx = path.indexOf("/", 1);
        if (idx > 1) {
          // try to get the corename as a request parameter first
          corename = path.substring(1, idx);

          // look at aliases
          if (cores.isZooKeeperAware()) {
            origCorename = corename;
            ZkStateReader reader = cores.getZkController().getZkStateReader();
            aliases = reader.getAliases();
            if (aliases != null && aliases.collectionAliasSize() > 0) {
              usingAliases = true;
              String alias = aliases.getCollectionAlias(corename);
              if (alias != null) {
                collectionsList = StrUtils.splitSmart(alias, ",", true);
                corename = collectionsList.get(0);
              }
            }
          }

          core = cores.getCore(corename);

          if (core != null) {
            path = path.substring(idx);
          }
        }

        // add collection name
        if (core == null && StringUtils.isNotBlank(requestGetter.getCollection())) {
          corename = requestGetter.getCollection();
          core = cores.getCore(corename);
        }

        if (core == null) {
          if (!cores.isZooKeeperAware()) {
            core = cores.getCore("");
          }
        }
      }

      if (core == null && cores.isZooKeeperAware()) {
        // we couldn't find the core - lets make sure a collection was not specified instead
        core = getCoreByCollection(cores, corename);

        if (core != null) {
          // we found a core, update the path
          path = path.substring(idx);
        }

        // try the default core
        if (core == null) {
          core = cores.getCore("");
          if (core != null) {}
        }
      }

      // With a valid core...
      if (core != null) {
        MDCLoggingContext.setCore(core);
        final SolrConfig config = core.getSolrConfig();
        // get or create/cache the parser for the core
        SolrRequestParsers parser = config.getRequestParsers();

        // Determine the handler from the url path if not set
        // (we might already have selected the cores handler)
        if (handler == null && path.length() > 1) { // don't match "" or "/" as valid path
          handler = core.getRequestHandler(path);

          if (handler == null) {
            // may be a restlet path
            // Handle /schema/* paths via Restlet
            if (path.equals("/schema") || path.startsWith("/schema/")) {
              throw new SolrException(
                  SolrException.ErrorCode.BAD_REQUEST, "unsupport /schema/**, use http solr");
            }
          }
          // no handler yet but allowed to handle select; let's check
          if (handler == null && parser.isHandleSelect()) {
            if ("/select".equals(path) || "/select/".equals(path)) {
              solrReq = parseSolrQueryRequest(parser, requestGetter);

              invalidStates =
                  checkStateIsValid(cores, solrReq.getParams().get(CloudSolrClient.STATE_VERSION));
              String qt = solrReq.getParams().get(CommonParams.QT);
              handler = core.getRequestHandler(qt);
              if (handler == null) {
                throw new SolrException(
                    SolrException.ErrorCode.BAD_REQUEST, "unknown handler: " + qt);
              }
              if (qt != null
                  && qt.startsWith("/")
                  && (handler instanceof ContentStreamHandlerBase)) {
                // For security reasons it's a bad idea to allow a leading '/', ex:
                // /select?qt=/update see SOLR-3161
                // There was no restriction from Solr 1.4 thru 3.5 and it's not supported for update
                // handlers.
                throw new SolrException(
                    SolrException.ErrorCode.BAD_REQUEST,
                    "Invalid Request Handler ('qt').  Do not use /select to access: " + qt);
              }
            }
          }
        }

        // With a valid handler and a valid core...
        if (handler != null) {
          // if not a /select, create the request
          if (solrReq == null) {
            solrReq = parseSolrQueryRequest(parser, requestGetter);
          }

          if (usingAliases) {
            processAliases(solrReq, aliases, collectionsList);
          }

          SolrQueryResponse solrRsp = new SolrQueryResponse();
          SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, solrRsp));
          this.execute(handler, solrReq, solrRsp);
          QueryResponseWriter responseWriter = core.getQueryResponseWriter(solrReq);
          if (invalidStates != null)
            solrReq.getContext().put(CloudSolrClient.STATE_VERSION, invalidStates);
          writeResponse(solrRsp, responseWriter, solrReq);

          return; // we are done with a valid handler
        }
      }
      logger.debug("no handler or core retrieved for {}, follow through...", path);
      throw new SolrException(
          SolrException.ErrorCode.BAD_REQUEST, "no handler or core retrieved for " + path);
    } catch (Throwable ex) {
      sendError(core, solrReq, ex);
      // walk the the entire cause chain to search for an Error
      Throwable t = ex;
      while (t != null) {
        if (t instanceof Error) {
          if (t != ex) {
            logger.error(
                "An Error was wrapped in another exception - please report complete stacktrace on SOLR-6161",
                ex);
          }
          throw (Error) t;
        }
        t = t.getCause();
      }
      return;
    } finally {
      try {
        if (solrReq != null) {
          logger.debug("Closing out SolrRequest: {}", solrReq);
          solrReq.close();
        }
      } finally {
        try {
          if (core != null) {
            core.close();
          }
        } finally {
          SolrRequestInfo.clearRequestInfo();
        }
      }
      MDCLoggingContext.clear();
    }
  }
  @VisibleForTesting
  protected FilterConfig getInitFilterConfig(
      Map<String, Object> pluginConfig, boolean skipKerberosChecking) {
    Map<String, String> params = new HashMap();
    params.put("type", "kerberos");
    putParam(params, "kerberos.name.rules", NAME_RULES_PARAM, "DEFAULT");
    putParam(params, "token.valid", TOKEN_VALID_PARAM, "30");
    putParam(params, "cookie.path", COOKIE_PATH_PARAM, "/");
    if (!skipKerberosChecking) {
      putParam(params, "kerberos.principal", PRINCIPAL_PARAM, null);
      putParam(params, "kerberos.keytab", KEYTAB_PARAM, null);
    } else {
      putParamOptional(params, "kerberos.principal", PRINCIPAL_PARAM);
      putParamOptional(params, "kerberos.keytab", KEYTAB_PARAM);
    }

    String delegationTokenStr = System.getProperty(DELEGATION_TOKEN_ENABLED, null);
    boolean delegationTokenEnabled =
        (delegationTokenStr == null) ? false : Boolean.parseBoolean(delegationTokenStr);
    ZkController controller = coreContainer.getZkController();

    if (delegationTokenEnabled) {
      putParam(
          params,
          "delegation-token.token-kind",
          DELEGATION_TOKEN_KIND,
          DELEGATION_TOKEN_TYPE_DEFAULT);
      if (coreContainer.isZooKeeperAware()) {
        putParam(params, "signer.secret.provider", DELEGATION_TOKEN_SECRET_PROVIDER, "zookeeper");
        if ("zookeeper".equals(params.get("signer.secret.provider"))) {
          String zkHost = controller.getZkServerAddress();
          putParam(params, "token.validity", DELEGATION_TOKEN_VALIDITY, "36000");
          params.put("zk-dt-secret-manager.enable", "true");
          // Note - Curator complains if the znodeWorkingPath starts with /
          String chrootPath = zkHost.substring(zkHost.indexOf("/"));
          String relativePath = chrootPath.startsWith("/") ? chrootPath.substring(1) : chrootPath;
          putParam(
              params,
              "zk-dt-secret-manager.znodeWorkingPath",
              DELEGATION_TOKEN_SECRET_MANAGER_ZNODE_WORKING_PATH,
              relativePath + SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH + "/zkdtsm");
          putParam(
              params,
              "signer.secret.provider.zookeeper.path",
              DELEGATION_TOKEN_SECRET_PROVIDER_ZK_PATH,
              "/token");
          // ensure krb5 is setup properly before running curator
          getHttpClientBuilder(SolrHttpClientBuilder.create());
        }
      } else {
        log.info(
            "CoreContainer is not ZooKeeperAware, not setting ZK-related delegation token properties");
      }
    }

    // Special handling for the "cookie.domain" based on whether port should be
    // appended to the domain. Useful for situations where multiple solr nodes are
    // on the same host.
    String usePortStr = System.getProperty(COOKIE_PORT_AWARE_PARAM, null);
    boolean needPortAwareCookies = (usePortStr == null) ? false : Boolean.parseBoolean(usePortStr);

    if (!needPortAwareCookies || !coreContainer.isZooKeeperAware()) {
      putParam(params, "cookie.domain", COOKIE_DOMAIN_PARAM, null);
    } else { // we need port aware cookies and we are in SolrCloud mode.
      String host = System.getProperty(COOKIE_DOMAIN_PARAM, null);
      if (host == null) {
        throw new SolrException(
            ErrorCode.SERVER_ERROR, "Missing required parameter '" + COOKIE_DOMAIN_PARAM + "'.");
      }
      int port = controller.getHostPort();
      params.put("cookie.domain", host + ":" + port);
    }

    // check impersonator config
    for (Enumeration e = System.getProperties().propertyNames(); e.hasMoreElements(); ) {
      String key = e.nextElement().toString();
      if (key.startsWith(IMPERSONATOR_PREFIX)) {
        if (!delegationTokenEnabled) {
          throw new SolrException(
              ErrorCode.SERVER_ERROR,
              "Impersonator configuration requires delegation tokens to be enabled: " + key);
        }
        params.put(key, System.getProperty(key));
      }
    }
    final ServletContext servletContext = new AttributeOnlyServletContext();
    if (controller != null) {
      servletContext.setAttribute(DELEGATION_TOKEN_ZK_CLIENT, controller.getZkClient());
    }
    if (delegationTokenEnabled) {
      kerberosFilter = new DelegationTokenKerberosFilter();
      // pass an attribute-enabled context in order to pass the zkClient
      // and because the filter may pass a curator instance.
    } else {
      kerberosFilter = new KerberosFilter();
    }
    log.info("Params: " + params);

    FilterConfig conf =
        new FilterConfig() {
          @Override
          public ServletContext getServletContext() {
            return servletContext;
          }

          @Override
          public Enumeration<String> getInitParameterNames() {
            return new IteratorEnumeration(params.keySet().iterator());
          }

          @Override
          public String getInitParameter(String param) {
            return params.get(param);
          }

          @Override
          public String getFilterName() {
            return "KerberosFilter";
          }
        };

    return conf;
  }