protected List<String> buildShardList(CloudSolrClient cloudSolrServer) {
ZkStateReader zkStateReader = cloudSolrServer.getZkStateReader();
ClusterState clusterState = zkStateReader.getClusterState();
String[] collections = null;
if (clusterState.hasCollection(collection)) {
collections = new String[] {collection};
} else {
// might be a collection alias?
Aliases aliases = zkStateReader.getAliases();
String aliasedCollections = aliases.getCollectionAlias(collection);
if (aliasedCollections == null)
throw new IllegalArgumentException("Collection " + collection + " not found!");
collections = aliasedCollections.split(",");
}
Set<String> liveNodes = clusterState.getLiveNodes();
Random random = new Random(5150);
List<String> shards = new ArrayList<String>();
for (String coll : collections) {
for (Slice slice : clusterState.getSlices(coll)) {
List<String> replicas = new ArrayList<String>();
for (Replica r : slice.getReplicas()) {
ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(r);
if (liveNodes.contains(replicaCoreProps.getNodeName()))
replicas.add(replicaCoreProps.getCoreUrl());
}
int numReplicas = replicas.size();
if (numReplicas == 0)
throw new IllegalStateException(
"Shard " + slice.getName() + " does not have any active replicas!");
String replicaUrl =
(numReplicas == 1) ? replicas.get(0) : replicas.get(random.nextInt(replicas.size()));
shards.add(replicaUrl);
}
}
return shards;
}
private Set<String> getCollectionList(ClusterState clusterState, String collection) {
// Extract each comma separated collection name and store in a List.
List<String> rawCollectionsList = StrUtils.splitSmart(collection, ",", true);
Set<String> collectionsList = new HashSet<>();
// validate collections
for (String collectionName : rawCollectionsList) {
if (!clusterState.getCollections().contains(collectionName)) {
Aliases aliases = zkStateReader.getAliases();
String alias = aliases.getCollectionAlias(collectionName);
if (alias != null) {
List<String> aliasList = StrUtils.splitSmart(alias, ",", true);
collectionsList.addAll(aliasList);
continue;
}
throw new SolrException(ErrorCode.BAD_REQUEST, "Collection not found: " + collectionName);
}
collectionsList.add(collectionName);
}
return collectionsList;
}
private void init() throws Exception {
// The states of client that is invalid in this request
Aliases aliases = null;
String corename = "";
String origCorename = null;
// set a request timer which can be reused by requests if needed
req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimerTree());
// put the core container in request attribute
req.setAttribute("org.apache.solr.CoreContainer", cores);
path = req.getServletPath();
if (req.getPathInfo() != null) {
// this lets you handle /update/commit when /update is a servlet
path += req.getPathInfo();
}
// check for management path
String alternate = cores.getManagementPath();
if (alternate != null && path.startsWith(alternate)) {
path = path.substring(0, alternate.length());
}
// unused feature ?
int idx = path.indexOf(':');
if (idx > 0) {
// save the portion after the ':' for a 'handler' path parameter
path = path.substring(0, idx);
}
boolean usingAliases = false;
// Check for container handlers
handler = cores.getRequestHandler(path);
if (handler != null) {
solrReq = SolrRequestParsers.DEFAULT.parse(null, path, req);
solrReq.getContext().put(CoreContainer.class.getName(), cores);
requestType = RequestType.ADMIN;
action = ADMIN;
return;
} else {
// otherwise, we should find a core from the path
idx = path.indexOf("/", 1);
if (idx > 1) {
// try to get the corename as a request parameter first
corename = path.substring(1, idx);
// look at aliases
if (cores.isZooKeeperAware()) {
origCorename = corename;
ZkStateReader reader = cores.getZkController().getZkStateReader();
aliases = reader.getAliases();
if (aliases != null && aliases.collectionAliasSize() > 0) {
usingAliases = true;
String alias = aliases.getCollectionAlias(corename);
if (alias != null) {
collectionsList = StrUtils.splitSmart(alias, ",", true);
corename = collectionsList.get(0);
}
}
}
core = cores.getCore(corename);
if (core != null) {
path = path.substring(idx);
} else if (cores.isCoreLoading(
corename)) { // extra mem barriers, so don't look at this before trying to get core
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "SolrCore is loading");
} else {
// the core may have just finished loading
core = cores.getCore(corename);
if (core != null) {
path = path.substring(idx);
}
}
}
if (core == null) {
if (!cores.isZooKeeperAware()) {
core = cores.getCore("");
}
}
}
if (core == null && cores.isZooKeeperAware()) {
// we couldn't find the core - lets make sure a collection was not specified instead
core = getCoreByCollection(corename);
if (core != null) {
// we found a core, update the path
path = path.substring(idx);
if (collectionsList == null) collectionsList = new ArrayList<>();
collectionsList.add(corename);
}
// if we couldn't find it locally, look on other nodes
extractRemotePath(corename, origCorename, idx);
if (action != null) return;
}
// With a valid core...
if (core != null) {
MDCLoggingContext.setCore(core);
config = core.getSolrConfig();
// get or create/cache the parser for the core
SolrRequestParsers parser = config.getRequestParsers();
// Determine the handler from the url path if not set
// (we might already have selected the cores handler)
extractHandlerFromURLPath(parser);
if (action != null) return;
// With a valid handler and a valid core...
if (handler != null) {
// if not a /select, create the request
if (solrReq == null) {
solrReq = parser.parse(core, path, req);
}
if (usingAliases) {
processAliases(aliases, collectionsList);
}
action = PROCESS;
return; // we are done with a valid handler
}
}
log.debug("no handler or core retrieved for " + path + ", follow through...");
action = PASSTHROUGH;
}
public void handleRequest(RequestGetter requestGetter) {
MDCLoggingContext.reset();
MDCLoggingContext.setNode(cores);
String path = requestGetter.getPath();
solrParams = requestGetter.getSolrParams();
SolrRequestHandler handler = null;
String corename = "";
String origCorename = null;
try {
// set a request timer which can be reused by requests if needed
// req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimer());
// put the core container in request attribute
// req.setAttribute("org.apache.solr.CoreContainer", cores);
// check for management path
String alternate = cores.getManagementPath();
if (alternate != null && path.startsWith(alternate)) {
path = path.substring(0, alternate.length());
}
// unused feature ?
int idx = path.indexOf(':');
if (idx > 0) {
// save the portion after the ':' for a 'handler' path parameter
path = path.substring(0, idx);
}
boolean usingAliases = false;
List<String> collectionsList = null;
// Check for container handlers
handler = cores.getRequestHandler(path);
if (handler != null) {
solrReq = parseSolrQueryRequest(SolrRequestParsers.DEFAULT, requestGetter);
handleAdminRequest(handler, solrReq);
return;
} else {
// otherwise, we should find a core from the path
idx = path.indexOf("/", 1);
if (idx > 1) {
// try to get the corename as a request parameter first
corename = path.substring(1, idx);
// look at aliases
if (cores.isZooKeeperAware()) {
origCorename = corename;
ZkStateReader reader = cores.getZkController().getZkStateReader();
aliases = reader.getAliases();
if (aliases != null && aliases.collectionAliasSize() > 0) {
usingAliases = true;
String alias = aliases.getCollectionAlias(corename);
if (alias != null) {
collectionsList = StrUtils.splitSmart(alias, ",", true);
corename = collectionsList.get(0);
}
}
}
core = cores.getCore(corename);
if (core != null) {
path = path.substring(idx);
}
}
// add collection name
if (core == null && StringUtils.isNotBlank(requestGetter.getCollection())) {
corename = requestGetter.getCollection();
core = cores.getCore(corename);
}
if (core == null) {
if (!cores.isZooKeeperAware()) {
core = cores.getCore("");
}
}
}
if (core == null && cores.isZooKeeperAware()) {
// we couldn't find the core - lets make sure a collection was not specified instead
core = getCoreByCollection(cores, corename);
if (core != null) {
// we found a core, update the path
path = path.substring(idx);
}
// try the default core
if (core == null) {
core = cores.getCore("");
if (core != null) {}
}
}
// With a valid core...
if (core != null) {
MDCLoggingContext.setCore(core);
final SolrConfig config = core.getSolrConfig();
// get or create/cache the parser for the core
SolrRequestParsers parser = config.getRequestParsers();
// Determine the handler from the url path if not set
// (we might already have selected the cores handler)
if (handler == null && path.length() > 1) { // don't match "" or "/" as valid path
handler = core.getRequestHandler(path);
if (handler == null) {
// may be a restlet path
// Handle /schema/* paths via Restlet
if (path.equals("/schema") || path.startsWith("/schema/")) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "unsupport /schema/**, use http solr");
}
}
// no handler yet but allowed to handle select; let's check
if (handler == null && parser.isHandleSelect()) {
if ("/select".equals(path) || "/select/".equals(path)) {
solrReq = parseSolrQueryRequest(parser, requestGetter);
invalidStates =
checkStateIsValid(cores, solrReq.getParams().get(CloudSolrClient.STATE_VERSION));
String qt = solrReq.getParams().get(CommonParams.QT);
handler = core.getRequestHandler(qt);
if (handler == null) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "unknown handler: " + qt);
}
if (qt != null
&& qt.startsWith("/")
&& (handler instanceof ContentStreamHandlerBase)) {
// For security reasons it's a bad idea to allow a leading '/', ex:
// /select?qt=/update see SOLR-3161
// There was no restriction from Solr 1.4 thru 3.5 and it's not supported for update
// handlers.
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Invalid Request Handler ('qt'). Do not use /select to access: " + qt);
}
}
}
}
// With a valid handler and a valid core...
if (handler != null) {
// if not a /select, create the request
if (solrReq == null) {
solrReq = parseSolrQueryRequest(parser, requestGetter);
}
if (usingAliases) {
processAliases(solrReq, aliases, collectionsList);
}
SolrQueryResponse solrRsp = new SolrQueryResponse();
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, solrRsp));
this.execute(handler, solrReq, solrRsp);
QueryResponseWriter responseWriter = core.getQueryResponseWriter(solrReq);
if (invalidStates != null)
solrReq.getContext().put(CloudSolrClient.STATE_VERSION, invalidStates);
writeResponse(solrRsp, responseWriter, solrReq);
return; // we are done with a valid handler
}
}
logger.debug("no handler or core retrieved for {}, follow through...", path);
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "no handler or core retrieved for " + path);
} catch (Throwable ex) {
sendError(core, solrReq, ex);
// walk the the entire cause chain to search for an Error
Throwable t = ex;
while (t != null) {
if (t instanceof Error) {
if (t != ex) {
logger.error(
"An Error was wrapped in another exception - please report complete stacktrace on SOLR-6161",
ex);
}
throw (Error) t;
}
t = t.getCause();
}
return;
} finally {
try {
if (solrReq != null) {
logger.debug("Closing out SolrRequest: {}", solrReq);
solrReq.close();
}
} finally {
try {
if (core != null) {
core.close();
}
} finally {
SolrRequestInfo.clearRequestInfo();
}
}
MDCLoggingContext.clear();
}
}
private NamedList<Object> directUpdate(AbstractUpdateRequest request, ClusterState clusterState)
throws SolrServerException {
UpdateRequest updateRequest = (UpdateRequest) request;
ModifiableSolrParams params = (ModifiableSolrParams) request.getParams();
ModifiableSolrParams routableParams = new ModifiableSolrParams();
ModifiableSolrParams nonRoutableParams = new ModifiableSolrParams();
if (params != null) {
nonRoutableParams.add(params);
routableParams.add(params);
for (String param : NON_ROUTABLE_PARAMS) {
routableParams.remove(param);
}
}
String collection = nonRoutableParams.get(UpdateParams.COLLECTION, defaultCollection);
if (collection == null) {
throw new SolrServerException(
"No collection param specified on request and no default collection has been set.");
}
// Check to see if the collection is an alias.
Aliases aliases = zkStateReader.getAliases();
if (aliases != null) {
Map<String, String> collectionAliases = aliases.getCollectionAliasMap();
if (collectionAliases != null && collectionAliases.containsKey(collection)) {
collection = collectionAliases.get(collection);
}
}
DocCollection col = getDocCollection(clusterState, collection);
DocRouter router = col.getRouter();
if (router instanceof ImplicitDocRouter) {
// short circuit as optimization
return null;
}
// Create the URL map, which is keyed on slice name.
// The value is a list of URLs for each replica in the slice.
// The first value in the list is the leader for the slice.
Map<String, List<String>> urlMap = buildUrlMap(col);
if (urlMap == null) {
// we could not find a leader yet - use unoptimized general path
return null;
}
NamedList<Throwable> exceptions = new NamedList<>();
NamedList<NamedList> shardResponses = new NamedList<>();
Map<String, LBHttpSolrServer.Req> routes =
updateRequest.getRoutes(router, col, urlMap, routableParams, this.idField);
if (routes == null) {
return null;
}
long start = System.nanoTime();
if (parallelUpdates) {
final Map<String, Future<NamedList<?>>> responseFutures = new HashMap<>(routes.size());
for (final Map.Entry<String, LBHttpSolrServer.Req> entry : routes.entrySet()) {
final String url = entry.getKey();
final LBHttpSolrServer.Req lbRequest = entry.getValue();
responseFutures.put(
url,
threadPool.submit(
new Callable<NamedList<?>>() {
@Override
public NamedList<?> call() throws Exception {
return lbServer.request(lbRequest).getResponse();
}
}));
}
for (final Map.Entry<String, Future<NamedList<?>>> entry : responseFutures.entrySet()) {
final String url = entry.getKey();
final Future<NamedList<?>> responseFuture = entry.getValue();
try {
shardResponses.add(url, responseFuture.get());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new RuntimeException(e);
} catch (ExecutionException e) {
exceptions.add(url, e.getCause());
}
}
if (exceptions.size() > 0) {
throw new RouteException(ErrorCode.SERVER_ERROR, exceptions, routes);
}
} else {
for (Map.Entry<String, LBHttpSolrServer.Req> entry : routes.entrySet()) {
String url = entry.getKey();
LBHttpSolrServer.Req lbRequest = entry.getValue();
try {
NamedList<Object> rsp = lbServer.request(lbRequest).getResponse();
shardResponses.add(url, rsp);
} catch (Exception e) {
throw new SolrServerException(e);
}
}
}
UpdateRequest nonRoutableRequest = null;
List<String> deleteQuery = updateRequest.getDeleteQuery();
if (deleteQuery != null && deleteQuery.size() > 0) {
UpdateRequest deleteQueryRequest = new UpdateRequest();
deleteQueryRequest.setDeleteQuery(deleteQuery);
nonRoutableRequest = deleteQueryRequest;
}
Set<String> paramNames = nonRoutableParams.getParameterNames();
Set<String> intersection = new HashSet<>(paramNames);
intersection.retainAll(NON_ROUTABLE_PARAMS);
if (nonRoutableRequest != null || intersection.size() > 0) {
if (nonRoutableRequest == null) {
nonRoutableRequest = new UpdateRequest();
}
nonRoutableRequest.setParams(nonRoutableParams);
List<String> urlList = new ArrayList<>();
urlList.addAll(routes.keySet());
Collections.shuffle(urlList, rand);
LBHttpSolrServer.Req req = new LBHttpSolrServer.Req(nonRoutableRequest, urlList);
try {
LBHttpSolrServer.Rsp rsp = lbServer.request(req);
shardResponses.add(urlList.get(0), rsp.getResponse());
} catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR, urlList.get(0), e);
}
}
long end = System.nanoTime();
RouteResponse rr = condenseResponse(shardResponses, (long) ((end - start) / 1000000));
rr.setRouteResponses(shardResponses);
rr.setRoutes(routes);
return rr;
}