/**
* Returns an String with the name of a core.
*
* <p>This method searches the core with fromIndex name in the core's container. If fromIndex
* isn't name of collection or alias it's returns fromIndex without changes. If fromIndex is name
* of alias but if the alias points to multiple collections it's throw
* SolrException.ErrorCode.BAD_REQUEST because multiple shards not yet supported.
*
* @param fromIndex name of the index
* @param container the core container for searching the core with fromIndex name or alias
* @return the string with name of core
*/
public static String getCoreName(final String fromIndex, CoreContainer container) {
if (container.isZooKeeperAware()) {
ZkController zkController = container.getZkController();
final String resolved =
zkController.getClusterState().hasCollection(fromIndex)
? fromIndex
: resolveAlias(fromIndex, zkController);
if (resolved == null) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"SolrCloud join: Collection '" + fromIndex + "' not found!");
}
return findLocalReplicaForFromIndex(zkController, resolved);
}
return fromIndex;
}
private Map<String, Integer> checkStateIsValid(String stateVer) {
Map<String, Integer> result = null;
String[] pairs;
if (stateVer != null && !stateVer.isEmpty() && cores.isZooKeeperAware()) {
// many have multiple collections separated by |
pairs = StringUtils.split(stateVer, '|');
for (String pair : pairs) {
String[] pcs = StringUtils.split(pair, ':');
if (pcs.length == 2 && !pcs[0].isEmpty() && !pcs[1].isEmpty()) {
Integer status =
cores
.getZkController()
.getZkStateReader()
.compareStateVersions(pcs[0], Integer.parseInt(pcs[1]));
if (status != null) {
if (result == null) result = new HashMap<>();
result.put(pcs[0], status);
}
}
}
}
return result;
}
private void init() throws Exception {
// The states of client that is invalid in this request
Aliases aliases = null;
String corename = "";
String origCorename = null;
// set a request timer which can be reused by requests if needed
req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimerTree());
// put the core container in request attribute
req.setAttribute("org.apache.solr.CoreContainer", cores);
path = req.getServletPath();
if (req.getPathInfo() != null) {
// this lets you handle /update/commit when /update is a servlet
path += req.getPathInfo();
}
// check for management path
String alternate = cores.getManagementPath();
if (alternate != null && path.startsWith(alternate)) {
path = path.substring(0, alternate.length());
}
// unused feature ?
int idx = path.indexOf(':');
if (idx > 0) {
// save the portion after the ':' for a 'handler' path parameter
path = path.substring(0, idx);
}
boolean usingAliases = false;
// Check for container handlers
handler = cores.getRequestHandler(path);
if (handler != null) {
solrReq = SolrRequestParsers.DEFAULT.parse(null, path, req);
solrReq.getContext().put(CoreContainer.class.getName(), cores);
requestType = RequestType.ADMIN;
action = ADMIN;
return;
} else {
// otherwise, we should find a core from the path
idx = path.indexOf("/", 1);
if (idx > 1) {
// try to get the corename as a request parameter first
corename = path.substring(1, idx);
// look at aliases
if (cores.isZooKeeperAware()) {
origCorename = corename;
ZkStateReader reader = cores.getZkController().getZkStateReader();
aliases = reader.getAliases();
if (aliases != null && aliases.collectionAliasSize() > 0) {
usingAliases = true;
String alias = aliases.getCollectionAlias(corename);
if (alias != null) {
collectionsList = StrUtils.splitSmart(alias, ",", true);
corename = collectionsList.get(0);
}
}
}
core = cores.getCore(corename);
if (core != null) {
path = path.substring(idx);
} else if (cores.isCoreLoading(
corename)) { // extra mem barriers, so don't look at this before trying to get core
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "SolrCore is loading");
} else {
// the core may have just finished loading
core = cores.getCore(corename);
if (core != null) {
path = path.substring(idx);
}
}
}
if (core == null) {
if (!cores.isZooKeeperAware()) {
core = cores.getCore("");
}
}
}
if (core == null && cores.isZooKeeperAware()) {
// we couldn't find the core - lets make sure a collection was not specified instead
core = getCoreByCollection(corename);
if (core != null) {
// we found a core, update the path
path = path.substring(idx);
if (collectionsList == null) collectionsList = new ArrayList<>();
collectionsList.add(corename);
}
// if we couldn't find it locally, look on other nodes
extractRemotePath(corename, origCorename, idx);
if (action != null) return;
}
// With a valid core...
if (core != null) {
MDCLoggingContext.setCore(core);
config = core.getSolrConfig();
// get or create/cache the parser for the core
SolrRequestParsers parser = config.getRequestParsers();
// Determine the handler from the url path if not set
// (we might already have selected the cores handler)
extractHandlerFromURLPath(parser);
if (action != null) return;
// With a valid handler and a valid core...
if (handler != null) {
// if not a /select, create the request
if (solrReq == null) {
solrReq = parser.parse(core, path, req);
}
if (usingAliases) {
processAliases(aliases, collectionsList);
}
action = PROCESS;
return; // we are done with a valid handler
}
}
log.debug("no handler or core retrieved for " + path + ", follow through...");
action = PASSTHROUGH;
}
public void inform(SolrCore core) {
/* The stream factory will always contain the zkUrl for the given collection
* Adds default streams with their corresponding function names. These
* defaults can be overridden or added to in the solrConfig in the stream
* RequestHandler def. Example config override
* <lst name="streamFunctions">
* <str name="group">org.apache.solr.client.solrj.io.stream.ReducerStream</str>
* <str name="count">org.apache.solr.client.solrj.io.stream.RecordCountStream</str>
* </lst>
* */
String defaultCollection = null;
String defaultZkhost = null;
CoreContainer coreContainer = core.getCoreDescriptor().getCoreContainer();
if (coreContainer.isZooKeeperAware()) {
defaultCollection = core.getCoreDescriptor().getCollectionName();
defaultZkhost =
core.getCoreDescriptor().getCoreContainer().getZkController().getZkServerAddress();
streamFactory.withCollectionZkHost(defaultCollection, defaultZkhost);
}
streamFactory
// streams
.withFunctionName("search", CloudSolrStream.class)
.withFunctionName("merge", MergeStream.class)
.withFunctionName("unique", UniqueStream.class)
.withFunctionName("top", RankStream.class)
.withFunctionName("group", GroupOperation.class)
.withFunctionName("reduce", ReducerStream.class)
.withFunctionName("parallel", ParallelStream.class)
.withFunctionName("rollup", RollupStream.class)
.withFunctionName("stats", StatsStream.class)
.withFunctionName("innerJoin", InnerJoinStream.class)
.withFunctionName("leftOuterJoin", LeftOuterJoinStream.class)
.withFunctionName("hashJoin", HashJoinStream.class)
.withFunctionName("outerHashJoin", OuterHashJoinStream.class)
.withFunctionName("facet", FacetStream.class)
// metrics
.withFunctionName("min", MinMetric.class)
.withFunctionName("max", MaxMetric.class)
.withFunctionName("avg", MeanMetric.class)
.withFunctionName("sum", SumMetric.class)
.withFunctionName("count", CountMetric.class);
// This pulls all the overrides and additions from the config
Object functionMappingsObj = initArgs.get("streamFunctions");
if (null != functionMappingsObj) {
NamedList<?> functionMappings = (NamedList<?>) functionMappingsObj;
for (Entry<String, ?> functionMapping : functionMappings) {
Class<?> clazz =
core.getResourceLoader()
.findClass((String) functionMapping.getValue(), Expressible.class);
streamFactory.withFunctionName(functionMapping.getKey(), clazz);
}
}
core.addCloseHook(
new CloseHook() {
@Override
public void preClose(SolrCore core) {
// To change body of implemented methods use File | Settings | File Templates.
}
@Override
public void postClose(SolrCore core) {
clientCache.close();
}
});
}
public void inform(SolrCore core) {
/* The stream factory will always contain the zkUrl for the given collection
* Adds default streams with their corresponding function names. These
* defaults can be overridden or added to in the solrConfig in the stream
* RequestHandler def. Example config override
* <lst name="streamFunctions">
* <str name="group">org.apache.solr.client.solrj.io.stream.ReducerStream</str>
* <str name="count">org.apache.solr.client.solrj.io.stream.RecordCountStream</str>
* </lst>
* */
String defaultCollection;
String defaultZkhost;
CoreContainer coreContainer = core.getCoreDescriptor().getCoreContainer();
this.coreName = core.getName();
if (coreContainer.isZooKeeperAware()) {
defaultCollection = core.getCoreDescriptor().getCollectionName();
defaultZkhost =
core.getCoreDescriptor().getCoreContainer().getZkController().getZkServerAddress();
streamFactory.withCollectionZkHost(defaultCollection, defaultZkhost);
streamFactory.withDefaultZkHost(defaultZkhost);
modelCache = new ModelCache(250, defaultZkhost, clientCache);
}
streamFactory
// source streams
.withFunctionName("search", CloudSolrStream.class)
.withFunctionName("facet", FacetStream.class)
.withFunctionName("update", UpdateStream.class)
.withFunctionName("jdbc", JDBCStream.class)
.withFunctionName("topic", TopicStream.class)
.withFunctionName("commit", CommitStream.class)
.withFunctionName("random", RandomStream.class)
// decorator streams
.withFunctionName("merge", MergeStream.class)
.withFunctionName("unique", UniqueStream.class)
.withFunctionName("top", RankStream.class)
.withFunctionName("group", GroupOperation.class)
.withFunctionName("reduce", ReducerStream.class)
.withFunctionName("parallel", ParallelStream.class)
.withFunctionName("rollup", RollupStream.class)
.withFunctionName("stats", StatsStream.class)
.withFunctionName("innerJoin", InnerJoinStream.class)
.withFunctionName("leftOuterJoin", LeftOuterJoinStream.class)
.withFunctionName("hashJoin", HashJoinStream.class)
.withFunctionName("outerHashJoin", OuterHashJoinStream.class)
.withFunctionName("intersect", IntersectStream.class)
.withFunctionName("complement", ComplementStream.class)
.withFunctionName("sort", SortStream.class)
.withFunctionName("train", TextLogitStream.class)
.withFunctionName("features", FeaturesSelectionStream.class)
.withFunctionName("daemon", DaemonStream.class)
.withFunctionName("shortestPath", ShortestPathStream.class)
.withFunctionName("gatherNodes", GatherNodesStream.class)
.withFunctionName("nodes", GatherNodesStream.class)
.withFunctionName("select", SelectStream.class)
.withFunctionName("scoreNodes", ScoreNodesStream.class)
.withFunctionName("model", ModelStream.class)
.withFunctionName("classify", ClassifyStream.class)
.withFunctionName("fetch", FetchStream.class)
.withFunctionName("executor", ExecutorStream.class)
.withFunctionName("null", NullStream.class)
.withFunctionName("priority", PriorityStream.class)
// metrics
.withFunctionName("min", MinMetric.class)
.withFunctionName("max", MaxMetric.class)
.withFunctionName("avg", MeanMetric.class)
.withFunctionName("sum", SumMetric.class)
.withFunctionName("count", CountMetric.class)
// tuple manipulation operations
.withFunctionName("replace", ReplaceOperation.class)
.withFunctionName("concat", ConcatOperation.class)
// stream reduction operations
.withFunctionName("group", GroupOperation.class)
.withFunctionName("distinct", DistinctOperation.class)
.withFunctionName("having", HavingStream.class)
.withFunctionName("and", AndOperation.class)
.withFunctionName("or", OrOperation.class)
.withFunctionName("not", NotOperation.class)
.withFunctionName("gt", GreaterThanOperation.class)
.withFunctionName("lt", LessThanOperation.class)
.withFunctionName("eq", EqualsOperation.class)
.withFunctionName("lteq", LessThanEqualToOperation.class)
.withFunctionName("gteq", GreaterThanEqualToOperation.class);
// This pulls all the overrides and additions from the config
List<PluginInfo> pluginInfos = core.getSolrConfig().getPluginInfos(Expressible.class.getName());
for (PluginInfo pluginInfo : pluginInfos) {
Class<? extends Expressible> clazz =
core.getResourceLoader().findClass(pluginInfo.className, Expressible.class);
streamFactory.withFunctionName(pluginInfo.name, clazz);
}
core.addCloseHook(
new CloseHook() {
@Override
public void preClose(SolrCore core) {
// To change body of implemented methods use File | Settings | File Templates.
}
@Override
public void postClose(SolrCore core) {
clientCache.close();
}
});
}
public void handleRequest(RequestGetter requestGetter) {
MDCLoggingContext.reset();
MDCLoggingContext.setNode(cores);
String path = requestGetter.getPath();
solrParams = requestGetter.getSolrParams();
SolrRequestHandler handler = null;
String corename = "";
String origCorename = null;
try {
// set a request timer which can be reused by requests if needed
// req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimer());
// put the core container in request attribute
// req.setAttribute("org.apache.solr.CoreContainer", cores);
// check for management path
String alternate = cores.getManagementPath();
if (alternate != null && path.startsWith(alternate)) {
path = path.substring(0, alternate.length());
}
// unused feature ?
int idx = path.indexOf(':');
if (idx > 0) {
// save the portion after the ':' for a 'handler' path parameter
path = path.substring(0, idx);
}
boolean usingAliases = false;
List<String> collectionsList = null;
// Check for container handlers
handler = cores.getRequestHandler(path);
if (handler != null) {
solrReq = parseSolrQueryRequest(SolrRequestParsers.DEFAULT, requestGetter);
handleAdminRequest(handler, solrReq);
return;
} else {
// otherwise, we should find a core from the path
idx = path.indexOf("/", 1);
if (idx > 1) {
// try to get the corename as a request parameter first
corename = path.substring(1, idx);
// look at aliases
if (cores.isZooKeeperAware()) {
origCorename = corename;
ZkStateReader reader = cores.getZkController().getZkStateReader();
aliases = reader.getAliases();
if (aliases != null && aliases.collectionAliasSize() > 0) {
usingAliases = true;
String alias = aliases.getCollectionAlias(corename);
if (alias != null) {
collectionsList = StrUtils.splitSmart(alias, ",", true);
corename = collectionsList.get(0);
}
}
}
core = cores.getCore(corename);
if (core != null) {
path = path.substring(idx);
}
}
// add collection name
if (core == null && StringUtils.isNotBlank(requestGetter.getCollection())) {
corename = requestGetter.getCollection();
core = cores.getCore(corename);
}
if (core == null) {
if (!cores.isZooKeeperAware()) {
core = cores.getCore("");
}
}
}
if (core == null && cores.isZooKeeperAware()) {
// we couldn't find the core - lets make sure a collection was not specified instead
core = getCoreByCollection(cores, corename);
if (core != null) {
// we found a core, update the path
path = path.substring(idx);
}
// try the default core
if (core == null) {
core = cores.getCore("");
if (core != null) {}
}
}
// With a valid core...
if (core != null) {
MDCLoggingContext.setCore(core);
final SolrConfig config = core.getSolrConfig();
// get or create/cache the parser for the core
SolrRequestParsers parser = config.getRequestParsers();
// Determine the handler from the url path if not set
// (we might already have selected the cores handler)
if (handler == null && path.length() > 1) { // don't match "" or "/" as valid path
handler = core.getRequestHandler(path);
if (handler == null) {
// may be a restlet path
// Handle /schema/* paths via Restlet
if (path.equals("/schema") || path.startsWith("/schema/")) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "unsupport /schema/**, use http solr");
}
}
// no handler yet but allowed to handle select; let's check
if (handler == null && parser.isHandleSelect()) {
if ("/select".equals(path) || "/select/".equals(path)) {
solrReq = parseSolrQueryRequest(parser, requestGetter);
invalidStates =
checkStateIsValid(cores, solrReq.getParams().get(CloudSolrClient.STATE_VERSION));
String qt = solrReq.getParams().get(CommonParams.QT);
handler = core.getRequestHandler(qt);
if (handler == null) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "unknown handler: " + qt);
}
if (qt != null
&& qt.startsWith("/")
&& (handler instanceof ContentStreamHandlerBase)) {
// For security reasons it's a bad idea to allow a leading '/', ex:
// /select?qt=/update see SOLR-3161
// There was no restriction from Solr 1.4 thru 3.5 and it's not supported for update
// handlers.
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Invalid Request Handler ('qt'). Do not use /select to access: " + qt);
}
}
}
}
// With a valid handler and a valid core...
if (handler != null) {
// if not a /select, create the request
if (solrReq == null) {
solrReq = parseSolrQueryRequest(parser, requestGetter);
}
if (usingAliases) {
processAliases(solrReq, aliases, collectionsList);
}
SolrQueryResponse solrRsp = new SolrQueryResponse();
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, solrRsp));
this.execute(handler, solrReq, solrRsp);
QueryResponseWriter responseWriter = core.getQueryResponseWriter(solrReq);
if (invalidStates != null)
solrReq.getContext().put(CloudSolrClient.STATE_VERSION, invalidStates);
writeResponse(solrRsp, responseWriter, solrReq);
return; // we are done with a valid handler
}
}
logger.debug("no handler or core retrieved for {}, follow through...", path);
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "no handler or core retrieved for " + path);
} catch (Throwable ex) {
sendError(core, solrReq, ex);
// walk the the entire cause chain to search for an Error
Throwable t = ex;
while (t != null) {
if (t instanceof Error) {
if (t != ex) {
logger.error(
"An Error was wrapped in another exception - please report complete stacktrace on SOLR-6161",
ex);
}
throw (Error) t;
}
t = t.getCause();
}
return;
} finally {
try {
if (solrReq != null) {
logger.debug("Closing out SolrRequest: {}", solrReq);
solrReq.close();
}
} finally {
try {
if (core != null) {
core.close();
}
} finally {
SolrRequestInfo.clearRequestInfo();
}
}
MDCLoggingContext.clear();
}
}
@VisibleForTesting
protected FilterConfig getInitFilterConfig(
Map<String, Object> pluginConfig, boolean skipKerberosChecking) {
Map<String, String> params = new HashMap();
params.put("type", "kerberos");
putParam(params, "kerberos.name.rules", NAME_RULES_PARAM, "DEFAULT");
putParam(params, "token.valid", TOKEN_VALID_PARAM, "30");
putParam(params, "cookie.path", COOKIE_PATH_PARAM, "/");
if (!skipKerberosChecking) {
putParam(params, "kerberos.principal", PRINCIPAL_PARAM, null);
putParam(params, "kerberos.keytab", KEYTAB_PARAM, null);
} else {
putParamOptional(params, "kerberos.principal", PRINCIPAL_PARAM);
putParamOptional(params, "kerberos.keytab", KEYTAB_PARAM);
}
String delegationTokenStr = System.getProperty(DELEGATION_TOKEN_ENABLED, null);
boolean delegationTokenEnabled =
(delegationTokenStr == null) ? false : Boolean.parseBoolean(delegationTokenStr);
ZkController controller = coreContainer.getZkController();
if (delegationTokenEnabled) {
putParam(
params,
"delegation-token.token-kind",
DELEGATION_TOKEN_KIND,
DELEGATION_TOKEN_TYPE_DEFAULT);
if (coreContainer.isZooKeeperAware()) {
putParam(params, "signer.secret.provider", DELEGATION_TOKEN_SECRET_PROVIDER, "zookeeper");
if ("zookeeper".equals(params.get("signer.secret.provider"))) {
String zkHost = controller.getZkServerAddress();
putParam(params, "token.validity", DELEGATION_TOKEN_VALIDITY, "36000");
params.put("zk-dt-secret-manager.enable", "true");
// Note - Curator complains if the znodeWorkingPath starts with /
String chrootPath = zkHost.substring(zkHost.indexOf("/"));
String relativePath = chrootPath.startsWith("/") ? chrootPath.substring(1) : chrootPath;
putParam(
params,
"zk-dt-secret-manager.znodeWorkingPath",
DELEGATION_TOKEN_SECRET_MANAGER_ZNODE_WORKING_PATH,
relativePath + SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH + "/zkdtsm");
putParam(
params,
"signer.secret.provider.zookeeper.path",
DELEGATION_TOKEN_SECRET_PROVIDER_ZK_PATH,
"/token");
// ensure krb5 is setup properly before running curator
getHttpClientBuilder(SolrHttpClientBuilder.create());
}
} else {
log.info(
"CoreContainer is not ZooKeeperAware, not setting ZK-related delegation token properties");
}
}
// Special handling for the "cookie.domain" based on whether port should be
// appended to the domain. Useful for situations where multiple solr nodes are
// on the same host.
String usePortStr = System.getProperty(COOKIE_PORT_AWARE_PARAM, null);
boolean needPortAwareCookies = (usePortStr == null) ? false : Boolean.parseBoolean(usePortStr);
if (!needPortAwareCookies || !coreContainer.isZooKeeperAware()) {
putParam(params, "cookie.domain", COOKIE_DOMAIN_PARAM, null);
} else { // we need port aware cookies and we are in SolrCloud mode.
String host = System.getProperty(COOKIE_DOMAIN_PARAM, null);
if (host == null) {
throw new SolrException(
ErrorCode.SERVER_ERROR, "Missing required parameter '" + COOKIE_DOMAIN_PARAM + "'.");
}
int port = controller.getHostPort();
params.put("cookie.domain", host + ":" + port);
}
// check impersonator config
for (Enumeration e = System.getProperties().propertyNames(); e.hasMoreElements(); ) {
String key = e.nextElement().toString();
if (key.startsWith(IMPERSONATOR_PREFIX)) {
if (!delegationTokenEnabled) {
throw new SolrException(
ErrorCode.SERVER_ERROR,
"Impersonator configuration requires delegation tokens to be enabled: " + key);
}
params.put(key, System.getProperty(key));
}
}
final ServletContext servletContext = new AttributeOnlyServletContext();
if (controller != null) {
servletContext.setAttribute(DELEGATION_TOKEN_ZK_CLIENT, controller.getZkClient());
}
if (delegationTokenEnabled) {
kerberosFilter = new DelegationTokenKerberosFilter();
// pass an attribute-enabled context in order to pass the zkClient
// and because the filter may pass a curator instance.
} else {
kerberosFilter = new KerberosFilter();
}
log.info("Params: " + params);
FilterConfig conf =
new FilterConfig() {
@Override
public ServletContext getServletContext() {
return servletContext;
}
@Override
public Enumeration<String> getInitParameterNames() {
return new IteratorEnumeration(params.keySet().iterator());
}
@Override
public String getInitParameter(String param) {
return params.get(param);
}
@Override
public String getFilterName() {
return "KerberosFilter";
}
};
return conf;
}
