public void inform(SolrCore core) { List<PluginInfo> children = info.getChildren("highlighting"); if (children.isEmpty()) { PluginInfo pluginInfo = core.getSolrConfig() .getPluginInfo( SolrHighlighter.class.getName()); // TODO deprecated configuration remove later if (pluginInfo != null) { highlighter = core.createInitInstance( pluginInfo, SolrHighlighter.class, null, DefaultSolrHighlighter.class.getName()); highlighter.initalize(core.getSolrConfig()); } else { DefaultSolrHighlighter defHighlighter = new DefaultSolrHighlighter(core); defHighlighter.init(PluginInfo.EMPTY_INFO); highlighter = defHighlighter; } } else { highlighter = core.createInitInstance( children.get(0), SolrHighlighter.class, null, DefaultSolrHighlighter.class.getName()); } }
private static Runnable getListener(SolrCore core, ZkSolrResourceLoader zkSolrResourceLoader) { final String coreName = core.getName(); final CoreContainer cc = core.getCoreDescriptor().getCoreContainer(); final String overlayPath = zkSolrResourceLoader.getConfigSetZkPath() + "/" + ConfigOverlay.RESOURCE_NAME; final String solrConfigPath = zkSolrResourceLoader.getConfigSetZkPath() + "/" + core.getSolrConfig().getName(); String schemaRes = null; if (core.getLatestSchema().isMutable() && core.getLatestSchema() instanceof ManagedIndexSchema) { ManagedIndexSchema mis = (ManagedIndexSchema) core.getLatestSchema(); schemaRes = mis.getResourceName(); } final String managedSchmaResourcePath = schemaRes == null ? null : zkSolrResourceLoader.getConfigSetZkPath() + "/" + schemaRes; return new Runnable() { @Override public void run() { log.info("config update listener called for core {}", coreName); SolrZkClient zkClient = cc.getZkController().getZkClient(); int solrConfigversion, overlayVersion, managedSchemaVersion = 0; SolrConfig cfg = null; try (SolrCore core = cc.getCore(coreName)) { if (core.isClosed()) return; cfg = core.getSolrConfig(); solrConfigversion = core.getSolrConfig().getOverlay().getZnodeVersion(); overlayVersion = core.getSolrConfig().getZnodeVersion(); if (managedSchmaResourcePath != null) { managedSchemaVersion = ((ManagedIndexSchema) core.getLatestSchema()).getSchemaZkVersion(); } } if (cfg != null) { cfg.refreshRequestParams(); } if (checkStale(zkClient, overlayPath, solrConfigversion) || checkStale(zkClient, solrConfigPath, overlayVersion) || checkStale(zkClient, managedSchmaResourcePath, managedSchemaVersion)) { log.info("core reload {}", coreName); cc.reload(coreName); } } }; }
/** Initialize using an explicit SolrCore */ public DirectSolrConnection(SolrCore c) { core = c; parser = new SolrRequestParsers(c.getSolrConfig()); }
private void init() throws Exception { // The states of client that is invalid in this request Aliases aliases = null; String corename = ""; String origCorename = null; // set a request timer which can be reused by requests if needed req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimerTree()); // put the core container in request attribute req.setAttribute("org.apache.solr.CoreContainer", cores); path = req.getServletPath(); if (req.getPathInfo() != null) { // this lets you handle /update/commit when /update is a servlet path += req.getPathInfo(); } // check for management path String alternate = cores.getManagementPath(); if (alternate != null && path.startsWith(alternate)) { path = path.substring(0, alternate.length()); } // unused feature ? int idx = path.indexOf(':'); if (idx > 0) { // save the portion after the ':' for a 'handler' path parameter path = path.substring(0, idx); } boolean usingAliases = false; // Check for container handlers handler = cores.getRequestHandler(path); if (handler != null) { solrReq = SolrRequestParsers.DEFAULT.parse(null, path, req); solrReq.getContext().put(CoreContainer.class.getName(), cores); requestType = RequestType.ADMIN; action = ADMIN; return; } else { // otherwise, we should find a core from the path idx = path.indexOf("/", 1); if (idx > 1) { // try to get the corename as a request parameter first corename = path.substring(1, idx); // look at aliases if (cores.isZooKeeperAware()) { origCorename = corename; ZkStateReader reader = cores.getZkController().getZkStateReader(); aliases = reader.getAliases(); if (aliases != null && aliases.collectionAliasSize() > 0) { usingAliases = true; String alias = aliases.getCollectionAlias(corename); if (alias != null) { collectionsList = StrUtils.splitSmart(alias, ",", true); corename = collectionsList.get(0); } } } core = cores.getCore(corename); if (core != null) { path = path.substring(idx); } else if (cores.isCoreLoading( corename)) { // extra mem barriers, so don't look at this before trying to get core throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "SolrCore is loading"); } else { // the core may have just finished loading core = cores.getCore(corename); if (core != null) { path = path.substring(idx); } } } if (core == null) { if (!cores.isZooKeeperAware()) { core = cores.getCore(""); } } } if (core == null && cores.isZooKeeperAware()) { // we couldn't find the core - lets make sure a collection was not specified instead core = getCoreByCollection(corename); if (core != null) { // we found a core, update the path path = path.substring(idx); if (collectionsList == null) collectionsList = new ArrayList<>(); collectionsList.add(corename); } // if we couldn't find it locally, look on other nodes extractRemotePath(corename, origCorename, idx); if (action != null) return; } // With a valid core... if (core != null) { MDCLoggingContext.setCore(core); config = core.getSolrConfig(); // get or create/cache the parser for the core SolrRequestParsers parser = config.getRequestParsers(); // Determine the handler from the url path if not set // (we might already have selected the cores handler) extractHandlerFromURLPath(parser); if (action != null) return; // With a valid handler and a valid core... if (handler != null) { // if not a /select, create the request if (solrReq == null) { solrReq = parser.parse(core, path, req); } if (usingAliases) { processAliases(aliases, collectionsList); } action = PROCESS; return; // we are done with a valid handler } } log.debug("no handler or core retrieved for " + path + ", follow through..."); action = PASSTHROUGH; }
public void inform(SolrCore core) { /* The stream factory will always contain the zkUrl for the given collection * Adds default streams with their corresponding function names. These * defaults can be overridden or added to in the solrConfig in the stream * RequestHandler def. Example config override * <lst name="streamFunctions"> * <str name="group">org.apache.solr.client.solrj.io.stream.ReducerStream</str> * <str name="count">org.apache.solr.client.solrj.io.stream.RecordCountStream</str> * </lst> * */ String defaultCollection; String defaultZkhost; CoreContainer coreContainer = core.getCoreDescriptor().getCoreContainer(); this.coreName = core.getName(); if (coreContainer.isZooKeeperAware()) { defaultCollection = core.getCoreDescriptor().getCollectionName(); defaultZkhost = core.getCoreDescriptor().getCoreContainer().getZkController().getZkServerAddress(); streamFactory.withCollectionZkHost(defaultCollection, defaultZkhost); streamFactory.withDefaultZkHost(defaultZkhost); modelCache = new ModelCache(250, defaultZkhost, clientCache); } streamFactory // source streams .withFunctionName("search", CloudSolrStream.class) .withFunctionName("facet", FacetStream.class) .withFunctionName("update", UpdateStream.class) .withFunctionName("jdbc", JDBCStream.class) .withFunctionName("topic", TopicStream.class) .withFunctionName("commit", CommitStream.class) .withFunctionName("random", RandomStream.class) // decorator streams .withFunctionName("merge", MergeStream.class) .withFunctionName("unique", UniqueStream.class) .withFunctionName("top", RankStream.class) .withFunctionName("group", GroupOperation.class) .withFunctionName("reduce", ReducerStream.class) .withFunctionName("parallel", ParallelStream.class) .withFunctionName("rollup", RollupStream.class) .withFunctionName("stats", StatsStream.class) .withFunctionName("innerJoin", InnerJoinStream.class) .withFunctionName("leftOuterJoin", LeftOuterJoinStream.class) .withFunctionName("hashJoin", HashJoinStream.class) .withFunctionName("outerHashJoin", OuterHashJoinStream.class) .withFunctionName("intersect", IntersectStream.class) .withFunctionName("complement", ComplementStream.class) .withFunctionName("sort", SortStream.class) .withFunctionName("train", TextLogitStream.class) .withFunctionName("features", FeaturesSelectionStream.class) .withFunctionName("daemon", DaemonStream.class) .withFunctionName("shortestPath", ShortestPathStream.class) .withFunctionName("gatherNodes", GatherNodesStream.class) .withFunctionName("nodes", GatherNodesStream.class) .withFunctionName("select", SelectStream.class) .withFunctionName("scoreNodes", ScoreNodesStream.class) .withFunctionName("model", ModelStream.class) .withFunctionName("classify", ClassifyStream.class) .withFunctionName("fetch", FetchStream.class) .withFunctionName("executor", ExecutorStream.class) .withFunctionName("null", NullStream.class) .withFunctionName("priority", PriorityStream.class) // metrics .withFunctionName("min", MinMetric.class) .withFunctionName("max", MaxMetric.class) .withFunctionName("avg", MeanMetric.class) .withFunctionName("sum", SumMetric.class) .withFunctionName("count", CountMetric.class) // tuple manipulation operations .withFunctionName("replace", ReplaceOperation.class) .withFunctionName("concat", ConcatOperation.class) // stream reduction operations .withFunctionName("group", GroupOperation.class) .withFunctionName("distinct", DistinctOperation.class) .withFunctionName("having", HavingStream.class) .withFunctionName("and", AndOperation.class) .withFunctionName("or", OrOperation.class) .withFunctionName("not", NotOperation.class) .withFunctionName("gt", GreaterThanOperation.class) .withFunctionName("lt", LessThanOperation.class) .withFunctionName("eq", EqualsOperation.class) .withFunctionName("lteq", LessThanEqualToOperation.class) .withFunctionName("gteq", GreaterThanEqualToOperation.class); // This pulls all the overrides and additions from the config List<PluginInfo> pluginInfos = core.getSolrConfig().getPluginInfos(Expressible.class.getName()); for (PluginInfo pluginInfo : pluginInfos) { Class<? extends Expressible> clazz = core.getResourceLoader().findClass(pluginInfo.className, Expressible.class); streamFactory.withFunctionName(pluginInfo.name, clazz); } core.addCloseHook( new CloseHook() { @Override public void preClose(SolrCore core) { // To change body of implemented methods use File | Settings | File Templates. } @Override public void postClose(SolrCore core) { clientCache.close(); } }); }
public void handleRequest(RequestGetter requestGetter) { MDCLoggingContext.reset(); MDCLoggingContext.setNode(cores); String path = requestGetter.getPath(); solrParams = requestGetter.getSolrParams(); SolrRequestHandler handler = null; String corename = ""; String origCorename = null; try { // set a request timer which can be reused by requests if needed // req.setAttribute(SolrRequestParsers.REQUEST_TIMER_SERVLET_ATTRIBUTE, new RTimer()); // put the core container in request attribute // req.setAttribute("org.apache.solr.CoreContainer", cores); // check for management path String alternate = cores.getManagementPath(); if (alternate != null && path.startsWith(alternate)) { path = path.substring(0, alternate.length()); } // unused feature ? int idx = path.indexOf(':'); if (idx > 0) { // save the portion after the ':' for a 'handler' path parameter path = path.substring(0, idx); } boolean usingAliases = false; List<String> collectionsList = null; // Check for container handlers handler = cores.getRequestHandler(path); if (handler != null) { solrReq = parseSolrQueryRequest(SolrRequestParsers.DEFAULT, requestGetter); handleAdminRequest(handler, solrReq); return; } else { // otherwise, we should find a core from the path idx = path.indexOf("/", 1); if (idx > 1) { // try to get the corename as a request parameter first corename = path.substring(1, idx); // look at aliases if (cores.isZooKeeperAware()) { origCorename = corename; ZkStateReader reader = cores.getZkController().getZkStateReader(); aliases = reader.getAliases(); if (aliases != null && aliases.collectionAliasSize() > 0) { usingAliases = true; String alias = aliases.getCollectionAlias(corename); if (alias != null) { collectionsList = StrUtils.splitSmart(alias, ",", true); corename = collectionsList.get(0); } } } core = cores.getCore(corename); if (core != null) { path = path.substring(idx); } } // add collection name if (core == null && StringUtils.isNotBlank(requestGetter.getCollection())) { corename = requestGetter.getCollection(); core = cores.getCore(corename); } if (core == null) { if (!cores.isZooKeeperAware()) { core = cores.getCore(""); } } } if (core == null && cores.isZooKeeperAware()) { // we couldn't find the core - lets make sure a collection was not specified instead core = getCoreByCollection(cores, corename); if (core != null) { // we found a core, update the path path = path.substring(idx); } // try the default core if (core == null) { core = cores.getCore(""); if (core != null) {} } } // With a valid core... if (core != null) { MDCLoggingContext.setCore(core); final SolrConfig config = core.getSolrConfig(); // get or create/cache the parser for the core SolrRequestParsers parser = config.getRequestParsers(); // Determine the handler from the url path if not set // (we might already have selected the cores handler) if (handler == null && path.length() > 1) { // don't match "" or "/" as valid path handler = core.getRequestHandler(path); if (handler == null) { // may be a restlet path // Handle /schema/* paths via Restlet if (path.equals("/schema") || path.startsWith("/schema/")) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "unsupport /schema/**, use http solr"); } } // no handler yet but allowed to handle select; let's check if (handler == null && parser.isHandleSelect()) { if ("/select".equals(path) || "/select/".equals(path)) { solrReq = parseSolrQueryRequest(parser, requestGetter); invalidStates = checkStateIsValid(cores, solrReq.getParams().get(CloudSolrClient.STATE_VERSION)); String qt = solrReq.getParams().get(CommonParams.QT); handler = core.getRequestHandler(qt); if (handler == null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "unknown handler: " + qt); } if (qt != null && qt.startsWith("/") && (handler instanceof ContentStreamHandlerBase)) { // For security reasons it's a bad idea to allow a leading '/', ex: // /select?qt=/update see SOLR-3161 // There was no restriction from Solr 1.4 thru 3.5 and it's not supported for update // handlers. throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Invalid Request Handler ('qt'). Do not use /select to access: " + qt); } } } } // With a valid handler and a valid core... if (handler != null) { // if not a /select, create the request if (solrReq == null) { solrReq = parseSolrQueryRequest(parser, requestGetter); } if (usingAliases) { processAliases(solrReq, aliases, collectionsList); } SolrQueryResponse solrRsp = new SolrQueryResponse(); SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, solrRsp)); this.execute(handler, solrReq, solrRsp); QueryResponseWriter responseWriter = core.getQueryResponseWriter(solrReq); if (invalidStates != null) solrReq.getContext().put(CloudSolrClient.STATE_VERSION, invalidStates); writeResponse(solrRsp, responseWriter, solrReq); return; // we are done with a valid handler } } logger.debug("no handler or core retrieved for {}, follow through...", path); throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "no handler or core retrieved for " + path); } catch (Throwable ex) { sendError(core, solrReq, ex); // walk the the entire cause chain to search for an Error Throwable t = ex; while (t != null) { if (t instanceof Error) { if (t != ex) { logger.error( "An Error was wrapped in another exception - please report complete stacktrace on SOLR-6161", ex); } throw (Error) t; } t = t.getCause(); } return; } finally { try { if (solrReq != null) { logger.debug("Closing out SolrRequest: {}", solrReq); solrReq.close(); } } finally { try { if (core != null) { core.close(); } } finally { SolrRequestInfo.clearRequestInfo(); } } MDCLoggingContext.clear(); } }
private boolean doRestore() throws Exception { Path backupPath = Paths.get(backupLocation).resolve(backupName); SimpleDateFormat dateFormat = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT); String restoreIndexName = "restore." + dateFormat.format(new Date()); String restoreIndexPath = core.getDataDir() + restoreIndexName; Directory restoreIndexDir = null; Directory indexDir = null; try (Directory backupDir = FSDirectory.open(backupPath)) { final Version version = IndexFetcher.checkOldestVersion(SegmentInfos.readLatestCommit(backupDir)); restoreIndexDir = core.getDirectoryFactory() .get( restoreIndexPath, DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); // Prefer local copy. indexDir = core.getDirectoryFactory() .get( core.getIndexDir(), DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType); // Move all files from backupDir to restoreIndexDir for (String filename : backupDir.listAll()) { checkInterrupted(); log.info("Copying file {} to restore directory ", filename); try (IndexInput indexInput = backupDir.openInput(filename, IOContext.READONCE)) { Long checksum = null; try { checksum = CodecUtil.retrieveChecksum(indexInput); } catch (Exception e) { log.warn("Could not read checksum from index file: " + filename, e); } long length = indexInput.length(); IndexFetcher.CompareResult compareResult = IndexFetcher.compareFile(indexDir, version, filename, length, checksum); if (!compareResult.equal || (!compareResult.checkSummed && (filename.endsWith(".si") || filename.endsWith(".liv") || filename.startsWith("segments_")))) { restoreIndexDir.copyFrom(backupDir, filename, filename, IOContext.READONCE); } else { // prefer local copy restoreIndexDir.copyFrom(indexDir, filename, filename, IOContext.READONCE); } } catch (Exception e) { throw new SolrException( SolrException.ErrorCode.UNKNOWN, "Exception while restoring the backup index", e); } } log.debug("Switching directories"); IndexFetcher.modifyIndexProps(core, restoreIndexName); boolean success; try { core.getUpdateHandler().newIndexWriter(false); openNewSearcher(); success = true; log.info("Successfully restored to the backup index"); } catch (Exception e) { // Rollback to the old index directory. Delete the restore index directory and mark the // restore as failed. log.warn("Could not switch to restored index. Rolling back to the current index"); Directory dir = null; try { dir = core.getDirectoryFactory() .get( core.getDataDir(), DirectoryFactory.DirContext.META_DATA, core.getSolrConfig().indexConfig.lockType); dir.deleteFile(IndexFetcher.INDEX_PROPERTIES); } finally { if (dir != null) { core.getDirectoryFactory().release(dir); } } core.getDirectoryFactory().doneWithDirectory(restoreIndexDir); core.getDirectoryFactory().remove(restoreIndexDir); core.getUpdateHandler().newIndexWriter(false); openNewSearcher(); throw new SolrException( SolrException.ErrorCode.UNKNOWN, "Exception while restoring the backup index", e); } if (success) { core.getDirectoryFactory().doneWithDirectory(indexDir); core.getDirectoryFactory().remove(indexDir); } return true; } finally { if (restoreIndexDir != null) { core.getDirectoryFactory().release(restoreIndexDir); } if (indexDir != null) { core.getDirectoryFactory().release(indexDir); } } }