private void handleGET() { if (parts.size() == 1) { resp.add("solrConfig", req.getCore().getSolrConfig().toMap()); } else { if (ConfigOverlay.NAME.equals(parts.get(1))) { resp.add(ConfigOverlay.NAME, req.getCore().getSolrConfig().getOverlay().toMap()); } else if (RequestParams.NAME.equals(parts.get(1))) { if (parts.size() == 3) { RequestParams params = req.getCore().getSolrConfig().getRequestParams(); MapSolrParams p = params.getParams(parts.get(2)); Map m = new LinkedHashMap<>(); m.put(ConfigOverlay.ZNODEVER, params.getZnodeVersion()); if (p != null) { m.put(RequestParams.NAME, ZkNodeProps.makeMap(parts.get(2), p.getMap())); } resp.add(SolrQueryResponse.NAME, m); } else { resp.add( SolrQueryResponse.NAME, req.getCore().getSolrConfig().getRequestParams().toMap()); } } else { Map<String, Object> m = req.getCore().getSolrConfig().toMap(); resp.add("solrConfig", ZkNodeProps.makeMap(parts.get(1), m.get(parts.get(1)))); } } }
@Override public void processAdd(AddUpdateCommand cmd) throws IOException { // TODO: check for id field? int hash = 0; if (zkEnabled) { zkCheck(); hash = hash(cmd); nodes = setupRequest(hash); } else { isLeader = getNonZkLeaderAssumption(req); } boolean dropCmd = false; if (!forwardToLeader) { dropCmd = versionAdd(cmd); } if (dropCmd) { // TODO: do we need to add anything to the response? return; } ModifiableSolrParams params = null; if (nodes != null) { params = new ModifiableSolrParams(filterParams(req.getParams())); params.set( DISTRIB_UPDATE_PARAM, (isLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString())); if (isLeader) { params.set( "distrib.from", ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName())); } params.set( "distrib.from", ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName())); cmdDistrib.distribAdd(cmd, nodes, params); } // TODO: what to do when no idField? if (returnVersions && rsp != null && idField != null) { if (addsResponse == null) { addsResponse = new NamedList<String>(); rsp.add("adds", addsResponse); } if (scratch == null) scratch = new CharsRef(); idField.getType().indexedToReadable(cmd.getIndexedId(), scratch); addsResponse.add(scratch.toString(), cmd.getVersion()); } // TODO: keep track of errors? needs to be done at a higher level though since // an id may fail before it gets to this processor. // Given that, it may also make sense to move the version reporting out of this // processor too. }
@Override public void processCommit(CommitUpdateCommand cmd) throws IOException { if (zkEnabled) { zkCheck(); } if (vinfo != null) { vinfo.lockForUpdate(); } try { if (ulog == null || ulog.getState() == UpdateLog.State.ACTIVE || (cmd.getFlags() & UpdateCommand.REPLAY) != 0) { super.processCommit(cmd); } else { log.info( "Ignoring commit while not ACTIVE - state: " + ulog.getState() + " replay:" + (cmd.getFlags() & UpdateCommand.REPLAY)); } } finally { if (vinfo != null) { vinfo.unlockForUpdate(); } } // TODO: we should consider this? commit everyone in the current collection if (zkEnabled) { ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams())); if (!req.getParams().getBool(COMMIT_END_POINT, false)) { params.set(COMMIT_END_POINT, true); String nodeName = req.getCore().getCoreDescriptor().getCoreContainer().getZkController().getNodeName(); String shardZkNodeName = nodeName + "_" + req.getCore().getName(); List<Node> nodes = getCollectionUrls( req, req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName(), shardZkNodeName); if (nodes != null) { cmdDistrib.distribCommit(cmd, nodes, params); finish(); } } } }
private List<Node> getCollectionUrls( SolrQueryRequest req, String collection, String shardZkNodeName) { ClusterState clusterState = req.getCore().getCoreDescriptor().getCoreContainer().getZkController().getClusterState(); List<Node> urls = new ArrayList<Node>(); Map<String, Slice> slices = clusterState.getSlices(collection); if (slices == null) { throw new ZooKeeperException( ErrorCode.BAD_REQUEST, "Could not find collection in zk: " + clusterState); } for (Map.Entry<String, Slice> sliceEntry : slices.entrySet()) { Slice replicas = slices.get(sliceEntry.getKey()); Map<String, Replica> shardMap = replicas.getReplicasMap(); for (Entry<String, Replica> entry : shardMap.entrySet()) { ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(entry.getValue()); if (clusterState.liveNodesContain(nodeProps.getNodeName()) && !entry.getKey().equals(shardZkNodeName)) { urls.add(new StdNode(nodeProps)); } } } if (urls.size() == 0) { return null; } return urls; }
protected void execute(SolrQueryResponse rsp) { // a custom filter could add more stuff to the request before passing it on. // for example: sreq.getContext().put( "HttpServletRequest", req ); // used for logging query stats in SolrCore.execute() solrReq.getContext().put("webapp", req.getContextPath()); solrReq.getCore().execute(handler, solrReq, rsp); }
// used for deleteByQuery to get the list of nodes this leader should forward to private List<Node> setupRequest() { List<Node> nodes = null; String shardId = cloudDesc.getShardId(); try { ZkCoreNodeProps leaderProps = new ZkCoreNodeProps(zkController.getZkStateReader().getLeaderProps(collection, shardId)); String leaderNodeName = leaderProps.getCoreNodeName(); String coreName = req.getCore().getName(); String coreNodeName = zkController.getNodeName() + "_" + coreName; isLeader = coreNodeName.equals(leaderNodeName); // TODO: what if we are no longer the leader? forwardToLeader = false; List<ZkCoreNodeProps> replicaProps = zkController .getZkStateReader() .getReplicaProps(collection, shardId, zkController.getNodeName(), coreName); if (replicaProps != null) { nodes = new ArrayList<Node>(replicaProps.size()); for (ZkCoreNodeProps props : replicaProps) { nodes.add(new StdNode(props)); } } } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); } return nodes; }
public void processGetVersions(ResponseBuilder rb) throws IOException { SolrQueryRequest req = rb.req; SolrQueryResponse rsp = rb.rsp; SolrParams params = req.getParams(); if (!params.getBool(COMPONENT_NAME, true)) { return; } int nVersions = params.getInt("getVersions", -1); if (nVersions == -1) return; String sync = params.get("sync"); if (sync != null) { processSync(rb, nVersions, sync); return; } UpdateLog ulog = req.getCore().getUpdateHandler().getUpdateLog(); if (ulog == null) return; UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates(); try { rb.rsp.add("versions", recentUpdates.getVersions(nVersions)); } finally { recentUpdates.close(); // cache this somehow? } }
private void handleCommands(List<CommandOperation> ops, ConfigOverlay overlay) throws IOException { for (CommandOperation op : ops) { switch (op.name) { case SET_PROPERTY: overlay = applySetProp(op, overlay); break; case UNSET_PROPERTY: overlay = applyUnset(op, overlay); break; case SET_USER_PROPERTY: overlay = applySetUserProp(op, overlay); break; case UNSET_USER_PROPERTY: overlay = applyUnsetUserProp(op, overlay); break; case UPDATE_REQHANDLER: case CREATE_REQHANDLER: overlay = applyRequestHandler(op, overlay); break; case DELETE_REQHANDLER: overlay = applyDeleteHandler(op, overlay); break; } } List errs = CommandOperation.captureErrors(ops); if (!errs.isEmpty()) { resp.add(CommandOperation.ERR_MSGS, errs); return; } SolrResourceLoader loader = req.getCore().getResourceLoader(); if (loader instanceof ZkSolrResourceLoader) { ZkController.persistConfigResourceToZooKeeper( loader, overlay.getZnodeVersion(), ConfigOverlay.RESOURCE_NAME, overlay.toByteArray(), true); } else { SolrResourceLoader.persistConfLocally( loader, ConfigOverlay.RESOURCE_NAME, overlay.toByteArray()); req.getCore().getCoreDescriptor().getCoreContainer().reload(req.getCore().getName()); } }
@Override public void processDelete(DeleteUpdateCommand cmd) throws IOException { if (!cmd.isDeleteById()) { doDeleteByQuery(cmd); return; } int hash = 0; if (zkEnabled) { zkCheck(); hash = hash(cmd); nodes = setupRequest(hash); } else { isLeader = getNonZkLeaderAssumption(req); } boolean dropCmd = false; if (!forwardToLeader) { dropCmd = versionDelete(cmd); } if (dropCmd) { // TODO: do we need to add anything to the response? return; } ModifiableSolrParams params = null; if (nodes != null) { params = new ModifiableSolrParams(filterParams(req.getParams())); params.set( DISTRIB_UPDATE_PARAM, (isLeader ? DistribPhase.FROMLEADER.toString() : DistribPhase.TOLEADER.toString())); if (isLeader) { params.set( "distrib.from", ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName())); } cmdDistrib.distribDelete(cmd, nodes, params); } // cmd.getIndexId == null when delete by query // TODO: what to do when no idField? if (returnVersions && rsp != null && cmd.getIndexedId() != null && idField != null) { if (deleteResponse == null) { deleteResponse = new NamedList<String>(); rsp.add("deletes", deleteResponse); } if (scratch == null) scratch = new CharsRef(); idField.getType().indexedToReadable(cmd.getIndexedId(), scratch); deleteResponse.add( scratch.toString(), cmd .getVersion()); // we're returning the version of the delete.. not the version of the // doc we deleted. } }
// make sure that log isn't needlessly replayed after a clean close @Test public void testCleanShutdown() throws Exception { DirectUpdateHandler2.commitOnClose = true; final Semaphore logReplay = new Semaphore(0); final Semaphore logReplayFinish = new Semaphore(0); UpdateLog.testing_logReplayHook = new Runnable() { @Override public void run() { try { assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS)); } catch (Exception e) { throw new RuntimeException(e); } } }; UpdateLog.testing_logReplayFinishHook = new Runnable() { @Override public void run() { logReplayFinish.release(); } }; SolrQueryRequest req = req(); UpdateHandler uhandler = req.getCore().getUpdateHandler(); UpdateLog ulog = uhandler.getUpdateLog(); try { clearIndex(); assertU(commit()); assertU(adoc("id", "E1", "val_i", "1")); assertU(adoc("id", "E2", "val_i", "1")); // set to a high enough number so this test won't hang on a bug logReplay.release(10); h.close(); createCore(); // make sure the docs got committed assertJQ(req("q", "*:*"), "/response/numFound==2"); // make sure no replay happened assertEquals(10, logReplay.availablePermits()); } finally { DirectUpdateHandler2.commitOnClose = true; UpdateLog.testing_logReplayHook = null; UpdateLog.testing_logReplayFinishHook = null; req().close(); } }
@Override public QParser createParser( String qStr, SolrParams localParams, SolrParams params, SolrQueryRequest req) { Log.info("createParser"); ModifiableSolrParams modparams = new ModifiableSolrParams(params); String modQ = filter(qStr); modparams.set("q", modQ); return req.getCore().getQueryPlugin(parserImpl).createParser(modQ, localParams, modparams, req); }
// Skip encoding for updating the index void createIndex2(int nDocs, String... fields) throws IOException { Set<String> fieldSet = new HashSet<String>(Arrays.asList(fields)); SolrQueryRequest req = lrf.makeRequest(); SolrQueryResponse rsp = new SolrQueryResponse(); UpdateRequestProcessorChain processorChain = req.getCore().getUpdateProcessingChain(null); UpdateRequestProcessor processor = processorChain.createProcessor(req, rsp); boolean foomany_s = fieldSet.contains("foomany_s"); boolean foo1_s = fieldSet.contains("foo1_s"); boolean foo2_s = fieldSet.contains("foo2_s"); boolean foo4_s = fieldSet.contains("foo4_s"); boolean foo8_s = fieldSet.contains("foo8_s"); boolean t10_100_ws = fieldSet.contains("t10_100_ws"); for (int i = 0; i < nDocs; i++) { SolrInputDocument doc = new SolrInputDocument(); doc.addField("id", Float.toString(i)); if (foomany_s) { doc.addField("foomany_s", t(r.nextInt(nDocs * 10))); } if (foo1_s) { doc.addField("foo1_s", t(0)); } if (foo2_s) { doc.addField("foo2_s", r.nextInt(2)); } if (foo4_s) { doc.addField("foo4_s", r.nextInt(4)); } if (foo8_s) { doc.addField("foo8_s", r.nextInt(8)); } if (t10_100_ws) { StringBuilder sb = new StringBuilder(9 * 100); for (int j = 0; j < 100; j++) { sb.append(' '); sb.append(t(r.nextInt(10))); } doc.addField("t10_100_ws", sb.toString()); } AddUpdateCommand cmd = new AddUpdateCommand(); cmd.solrDoc = doc; processor.processAdd(cmd); } processor.finish(); req.close(); assertU(commit()); req = lrf.makeRequest(); assertEquals(nDocs, req.getSearcher().maxDoc()); req.close(); }
private void handlePOST() throws IOException { Iterable<ContentStream> streams = req.getContentStreams(); if (streams == null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "missing content stream"); } ArrayList<CommandOperation> ops = new ArrayList<>(); for (ContentStream stream : streams) ops.addAll(CommandOperation.parse(stream.getReader())); List<Map> errList = CommandOperation.captureErrors(ops); if (!errList.isEmpty()) { resp.add(CommandOperation.ERR_MSGS, errList); return; } try { for (; ; ) { ArrayList<CommandOperation> opsCopy = new ArrayList<>(ops.size()); for (CommandOperation op : ops) opsCopy.add(op.getCopy()); try { if (parts.size() > 1 && RequestParams.NAME.equals(parts.get(1))) { RequestParams params = RequestParams.getFreshRequestParams( req.getCore().getResourceLoader(), req.getCore().getSolrConfig().getRequestParams()); handleParams(opsCopy, params); } else { ConfigOverlay overlay = SolrConfig.getConfigOverlay(req.getCore().getResourceLoader()); handleCommands(opsCopy, overlay); } break; // succeeded . so no need to go over the loop again } catch (ZkController.ResourceModifiedInZkException e) { // retry log.info( "Race condition, the node is modified in ZK by someone else " + e.getMessage()); } } } catch (Exception e) { resp.setException(e); resp.add(CommandOperation.ERR_MSGS, singletonList(SchemaManager.getErrorStr(e))); } }
public void processGetUpdates(ResponseBuilder rb) throws IOException { SolrQueryRequest req = rb.req; SolrQueryResponse rsp = rb.rsp; SolrParams params = req.getParams(); if (!params.getBool(COMPONENT_NAME, true)) { return; } String versionsStr = params.get("getUpdates"); if (versionsStr == null) return; UpdateLog ulog = req.getCore().getUpdateHandler().getUpdateLog(); if (ulog == null) return; List<String> versions = StrUtils.splitSmart(versionsStr, ",", true); List<Object> updates = new ArrayList<Object>(versions.size()); long minVersion = Long.MAX_VALUE; // TODO: get this from cache instead of rebuilding? UpdateLog.RecentUpdates recentUpdates = ulog.getRecentUpdates(); try { for (String versionStr : versions) { long version = Long.parseLong(versionStr); try { Object o = recentUpdates.lookup(version); if (o == null) continue; if (version > 0) { minVersion = Math.min(minVersion, version); } // TODO: do any kind of validation here? updates.add(o); } catch (SolrException e) { log.warn("Exception reading log for updates", e); } catch (ClassCastException e) { log.warn("Exception reading log for updates", e); } } // Must return all delete-by-query commands that occur after the first add requested // since they may apply. updates.addAll(recentUpdates.getDeleteByQuery(minVersion)); rb.rsp.add("updates", updates); } finally { recentUpdates.close(); // cache this somehow? } }
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { SolrParams params = req.getParams(); params = adjustParams(params); req.setParams(params); if (params.get("action") != null) { handleAdmin(req, rsp, params); return; } TupleStream tupleStream; try { tupleStream = this.streamFactory.constructStream(params.get("expr")); } catch (Exception e) { // Catch exceptions that occur while the stream is being created. This will include streaming // expression parse rules. SolrException.log(logger, e); rsp.add("result-set", new DummyErrorStream(e)); return; } int worker = params.getInt("workerID", 0); int numWorkers = params.getInt("numWorkers", 1); StreamContext context = new StreamContext(); context.workerID = worker; context.numWorkers = numWorkers; context.setSolrClientCache(clientCache); context.setModelCache(modelCache); context.put("core", this.coreName); context.put("solr-core", req.getCore()); tupleStream.setStreamContext(context); // if asking for explanation then go get it if (params.getBool("explain", false)) { rsp.add("explanation", tupleStream.toExplanation(this.streamFactory)); } if (tupleStream instanceof DaemonStream) { DaemonStream daemonStream = (DaemonStream) tupleStream; if (daemons.containsKey(daemonStream.getId())) { daemons.remove(daemonStream.getId()).close(); } daemonStream.setDaemons(daemons); daemonStream.open(); // This will start the deamonStream daemons.put(daemonStream.getId(), daemonStream); rsp.add( "result-set", new DaemonResponseStream("Deamon:" + daemonStream.getId() + " started on " + coreName)); } else { rsp.add("result-set", new TimerStream(new ExceptionStream(tupleStream))); } }
@Override public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { FileFloatSource.resetCache(); log.debug("readerCache has been reset."); UpdateRequestProcessor processor = req.getCore().getUpdateProcessingChain(null).createProcessor(req, rsp); try { RequestHandlerUtils.handleCommit(processor, req.getParams(), true); } finally { processor.finish(); } }
public DistributedUpdateProcessor( SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) { super(next); this.rsp = rsp; this.next = next; this.idField = req.getSchema().getUniqueKeyField(); // version init this.updateHandler = req.getCore().getUpdateHandler(); this.ulog = updateHandler.getUpdateLog(); this.vinfo = ulog == null ? null : ulog.getVersionInfo(); versionsStored = this.vinfo != null && this.vinfo.getVersionField() != null; returnVersions = req.getParams().getBool(UpdateParams.VERSIONS, false); // TODO: better way to get the response, or pass back info to it? SolrRequestInfo reqInfo = returnVersions ? SolrRequestInfo.getRequestInfo() : null; this.req = req; CoreDescriptor coreDesc = req.getCore().getCoreDescriptor(); this.zkEnabled = coreDesc.getCoreContainer().isZooKeeperAware(); zkController = req.getCore().getCoreDescriptor().getCoreContainer().getZkController(); if (zkEnabled) { numNodes = zkController.getZkStateReader().getClusterState().getLiveNodes().size(); cmdDistrib = new SolrCmdDistributor( numNodes, coreDesc.getCoreContainer().getZkController().getCmdDistribExecutor()); } // this.rsp = reqInfo != null ? reqInfo.getRsp() : null; cloudDesc = coreDesc.getCloudDescriptor(); if (cloudDesc != null) { collection = cloudDesc.getCollectionName(); } }
/** * Get Transformer from request context, or from TransformerProvider. This allows either * getContentType(...) or write(...) to instantiate the Transformer, depending on which one is * called first, then the other one reuses the same Transformer */ Transformer getTransformer(String xslt, SolrQueryRequest request) throws IOException { // not the cleanest way to achieve this // no need to synchronize access to context, right? // Nothing else happens with it at the same time final Map<Object, Object> ctx = request.getContext(); Transformer result = (Transformer) ctx.get(CONTEXT_TRANSFORMER_KEY); if (result == null) { SolrConfig solrConfig = request.getCore().getSolrConfig(); result = TransformerProvider.instance.getTransformer(solrConfig, xslt, xsltCacheLifetimeSeconds); result.setErrorListener(xmllog); ctx.put(CONTEXT_TRANSFORMER_KEY, result); } return result; }
RegexpBoostProcessor( SolrParams parameters, SolrQueryRequest request, SolrQueryResponse response, UpdateRequestProcessor nextProcessor, final Map<Object, Object> sharedObjectCache) { super(nextProcessor); this.initParameters(parameters); if (this.boostFilename == null) { log.warn("Null boost filename. Disabling processor."); setEnabled(false); } if (!isEnabled()) { return; } try { synchronized (sharedObjectCache) { List<BoostEntry> cachedBoostEntries = (List<BoostEntry>) sharedObjectCache.get(BOOST_ENTRIES_CACHE_KEY); if (cachedBoostEntries == null) { log.debug("No pre-cached boost entry list found, initializing new"); InputStream is = request.getCore().getResourceLoader().openResource(boostFilename); cachedBoostEntries = initBoostEntries(is); sharedObjectCache.put(BOOST_ENTRIES_CACHE_KEY, cachedBoostEntries); } else { if (log.isDebugEnabled()) { log.debug( "Using cached boost entry list with " + cachedBoostEntries.size() + " elements."); } } this.boostEntries = cachedBoostEntries; } } catch (IOException ioe) { log.warn("IOException while initializing boost entries from file " + this.boostFilename, ioe); } }
public ExtractingDocumentLoader( SolrQueryRequest req, UpdateRequestProcessor processor, TikaConfig config, ParseContextConfig parseContextConfig, SolrContentHandlerFactory factory) { this.params = req.getParams(); this.core = req.getCore(); this.config = config; this.parseContextConfig = parseContextConfig; this.processor = processor; templateAdd = new AddUpdateCommand(req); templateAdd.overwrite = params.getBool(UpdateParams.OVERWRITE, true); templateAdd.commitWithin = params.getInt(UpdateParams.COMMIT_WITHIN, -1); // this is lightweight autoDetectParser = new AutoDetectParser(config); this.factory = factory; ignoreTikaException = params.getBool(ExtractingParams.IGNORE_TIKA_EXCEPTION, false); }
private void doDefensiveChecks(String shardId, DistribPhase phase) { String from = req.getParams().get("distrib.from"); boolean logReplay = req.getParams().getBool(LOG_REPLAY, false); boolean localIsLeader = req.getCore().getCoreDescriptor().getCloudDescriptor().isLeader(); if (!logReplay && DistribPhase.FROMLEADER == phase && localIsLeader && from != null) { // from will be null on log replay log.error( "Request says it is coming from leader, but we are the leader: " + req.getParamString()); throw new SolrException( ErrorCode.SERVICE_UNAVAILABLE, "Request says it is coming from leader, but we are the leader"); } if (isLeader && !localIsLeader) { log.error("ClusterState says we are the leader, but locally we don't think so"); throw new SolrException( ErrorCode.SERVICE_UNAVAILABLE, "ClusterState says we are the leader, but locally we don't think so"); } }
@Override public void load( SolrQueryRequest req, SolrQueryResponse rsp, ContentStream stream, UpdateRequestProcessor processor) throws Exception { Parser parser = null; String streamType = req.getParams().get(ExtractingParams.STREAM_TYPE, null); if (streamType != null) { // Cache? Parsers are lightweight to construct and thread-safe, so I'm told MediaType mt = MediaType.parse(streamType.trim().toLowerCase(Locale.ROOT)); parser = new DefaultParser(config.getMediaTypeRegistry()).getParsers().get(mt); } else { parser = autoDetectParser; } if (parser != null) { Metadata metadata = new Metadata(); // If you specify the resource name (the filename, roughly) with this parameter, // then Tika can make use of it in guessing the appropriate MIME type: String resourceName = req.getParams().get(ExtractingParams.RESOURCE_NAME, null); if (resourceName != null) { metadata.add(TikaMetadataKeys.RESOURCE_NAME_KEY, resourceName); } // Provide stream's content type as hint for auto detection if (stream.getContentType() != null) { metadata.add(HttpHeaders.CONTENT_TYPE, stream.getContentType()); } InputStream inputStream = null; try { inputStream = stream.getStream(); metadata.add(ExtractingMetadataConstants.STREAM_NAME, stream.getName()); metadata.add(ExtractingMetadataConstants.STREAM_SOURCE_INFO, stream.getSourceInfo()); metadata.add(ExtractingMetadataConstants.STREAM_SIZE, String.valueOf(stream.getSize())); metadata.add(ExtractingMetadataConstants.STREAM_CONTENT_TYPE, stream.getContentType()); // HtmlParser and TXTParser regard Metadata.CONTENT_ENCODING in metadata String charset = ContentStreamBase.getCharsetFromContentType(stream.getContentType()); if (charset != null) { metadata.add(HttpHeaders.CONTENT_ENCODING, charset); } String xpathExpr = params.get(ExtractingParams.XPATH_EXPRESSION); boolean extractOnly = params.getBool(ExtractingParams.EXTRACT_ONLY, false); SolrContentHandler handler = factory.createSolrContentHandler(metadata, params, req.getSchema()); ContentHandler parsingHandler = handler; StringWriter writer = null; BaseMarkupSerializer serializer = null; if (extractOnly == true) { String extractFormat = params.get(ExtractingParams.EXTRACT_FORMAT, "xml"); writer = new StringWriter(); if (extractFormat.equals(TEXT_FORMAT)) { serializer = new TextSerializer(); serializer.setOutputCharStream(writer); serializer.setOutputFormat(new OutputFormat("Text", "UTF-8", true)); } else { serializer = new XMLSerializer(writer, new OutputFormat("XML", "UTF-8", true)); } if (xpathExpr != null) { Matcher matcher = PARSER.parse(xpathExpr); serializer .startDocument(); // The MatchingContentHandler does not invoke startDocument. See // http://tika.markmail.org/message/kknu3hw7argwiqin parsingHandler = new MatchingContentHandler(serializer, matcher); } else { parsingHandler = serializer; } } else if (xpathExpr != null) { Matcher matcher = PARSER.parse(xpathExpr); parsingHandler = new MatchingContentHandler(handler, matcher); } // else leave it as is try { // potentially use a wrapper handler for parsing, but we still need the SolrContentHandler // for getting the document. ParseContext context = parseContextConfig.create(); context.set(Parser.class, parser); context.set(HtmlMapper.class, MostlyPassthroughHtmlMapper.INSTANCE); // Password handling RegexRulesPasswordProvider epp = new RegexRulesPasswordProvider(); String pwMapFile = params.get(ExtractingParams.PASSWORD_MAP_FILE); if (pwMapFile != null && pwMapFile.length() > 0) { InputStream is = req.getCore().getResourceLoader().openResource(pwMapFile); if (is != null) { log.debug("Password file supplied: " + pwMapFile); epp.parse(is); } } context.set(PasswordProvider.class, epp); String resourcePassword = params.get(ExtractingParams.RESOURCE_PASSWORD); if (resourcePassword != null) { epp.setExplicitPassword(resourcePassword); log.debug("Literal password supplied for file " + resourceName); } parser.parse(inputStream, parsingHandler, metadata, context); } catch (TikaException e) { if (ignoreTikaException) log.warn( new StringBuilder("skip extracting text due to ") .append(e.getLocalizedMessage()) .append(". metadata=") .append(metadata.toString()) .toString()); else throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } if (extractOnly == false) { addDoc(handler); } else { // serializer is not null, so we need to call endDoc on it if using xpath if (xpathExpr != null) { serializer.endDocument(); } rsp.add(stream.getName(), writer.toString()); writer.close(); String[] names = metadata.names(); NamedList metadataNL = new NamedList(); for (int i = 0; i < names.length; i++) { String[] vals = metadata.getValues(names[i]); metadataNL.add(names[i], vals); } rsp.add(stream.getName() + "_metadata", metadataNL); } } catch (SAXException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); } finally { IOUtils.closeQuietly(inputStream); } } else { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Stream type of " + streamType + " didn't match any known parsers. Please supply the " + ExtractingParams.STREAM_TYPE + " parameter."); } }
@Override public void prepare(ResponseBuilder rb) throws IOException { SolrQueryRequest req = rb.req; SolrParams params = req.getParams(); // A runtime param can skip if (!params.getBool(QueryElevationParams.ENABLE, true)) { return; } boolean exclusive = params.getBool(QueryElevationParams.EXCLUSIVE, false); // A runtime parameter can alter the config value for forceElevation boolean force = params.getBool(QueryElevationParams.FORCE_ELEVATION, forceElevation); boolean markExcludes = params.getBool(QueryElevationParams.MARK_EXCLUDES, false); String boostStr = params.get(QueryElevationParams.IDS); String exStr = params.get(QueryElevationParams.EXCLUDE); Query query = rb.getQuery(); String qstr = rb.getQueryString(); if (query == null || qstr == null) { return; } ElevationObj booster = null; try { if (boostStr != null || exStr != null) { List<String> boosts = (boostStr != null) ? StrUtils.splitSmart(boostStr, ",", true) : new ArrayList<String>(0); List<String> excludes = (exStr != null) ? StrUtils.splitSmart(exStr, ",", true) : new ArrayList<String>(0); booster = new ElevationObj(qstr, boosts, excludes); } else { IndexReader reader = req.getSearcher().getIndexReader(); qstr = getAnalyzedQuery(qstr); booster = getElevationMap(reader, req.getCore()).get(qstr); } } catch (Exception ex) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error loading elevation", ex); } if (booster != null) { rb.req.getContext().put(BOOSTED, booster.ids); // Change the query to insert forced documents if (exclusive == true) { // we only want these results rb.setQuery(booster.include); } else { BooleanQuery newq = new BooleanQuery(true); newq.add(query, BooleanClause.Occur.SHOULD); newq.add(booster.include, BooleanClause.Occur.SHOULD); if (booster.exclude != null) { if (markExcludes == false) { for (TermQuery tq : booster.exclude) { newq.add(new BooleanClause(tq, BooleanClause.Occur.MUST_NOT)); } } else { // we are only going to mark items as excluded, not actually exclude them. This works // with the EditorialMarkerFactory rb.req.getContext().put(EXCLUDED, booster.excludeIds); } } rb.setQuery(newq); } ElevationComparatorSource comparator = new ElevationComparatorSource(booster); // if the sort is 'score desc' use a custom sorting method to // insert documents in their proper place SortSpec sortSpec = rb.getSortSpec(); if (sortSpec.getSort() == null) { sortSpec.setSortAndFields( new Sort( new SortField[] { new SortField("_elevate_", comparator, true), new SortField(null, SortField.Type.SCORE, false) }), Arrays.asList(new SchemaField[2])); } else { // Check if the sort is based on score SortSpec modSortSpec = this.modifySortSpec(sortSpec, force, comparator); if (null != modSortSpec) { rb.setSortSpec(modSortSpec); } } // alter the sorting in the grouping specification if there is one GroupingSpecification groupingSpec = rb.getGroupingSpec(); if (groupingSpec != null) { SortField[] groupSort = groupingSpec.getGroupSort().getSort(); Sort modGroupSort = this.modifySort(groupSort, force, comparator); if (modGroupSort != null) { groupingSpec.setGroupSort(modGroupSort); } SortField[] withinGroupSort = groupingSpec.getSortWithinGroup().getSort(); Sort modWithinGroupSort = this.modifySort(withinGroupSort, force, comparator); if (modWithinGroupSort != null) { groupingSpec.setSortWithinGroup(modWithinGroupSort); } } } // Add debugging information if (rb.isDebug()) { List<String> match = null; if (booster != null) { // Extract the elevated terms into a list match = new ArrayList<String>(booster.priority.size()); for (Object o : booster.include.clauses()) { TermQuery tq = (TermQuery) ((BooleanClause) o).getQuery(); match.add(tq.getTerm().text()); } } SimpleOrderedMap<Object> dbg = new SimpleOrderedMap<Object>(); dbg.add("q", qstr); dbg.add("match", match); if (rb.isDebugQuery()) { rb.addDebugInfo("queryBoosting", dbg); } } }
@Test public void testBufferingFlags() throws Exception { DirectUpdateHandler2.commitOnClose = false; final Semaphore logReplayFinish = new Semaphore(0); UpdateLog.testing_logReplayFinishHook = new Runnable() { @Override public void run() { logReplayFinish.release(); } }; SolrQueryRequest req = req(); UpdateHandler uhandler = req.getCore().getUpdateHandler(); UpdateLog ulog = uhandler.getUpdateLog(); try { clearIndex(); assertU(commit()); assertEquals(UpdateLog.State.ACTIVE, ulog.getState()); ulog.bufferUpdates(); // simulate updates from a leader updateJ( jsonAdd(sdoc("id", "Q1", "_version_", "101")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "Q2", "_version_", "102")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "Q3", "_version_", "103")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); req.close(); h.close(); createCore(); req = req(); uhandler = req.getCore().getUpdateHandler(); ulog = uhandler.getUpdateLog(); logReplayFinish.acquire(); // wait for replay to finish assertTrue( (ulog.getStartingOperation() & UpdateLog.FLAG_GAP) != 0); // since we died while buffering, we should see this last // // Try again to ensure that the previous log replay didn't wipe out our flags // req.close(); h.close(); createCore(); req = req(); uhandler = req.getCore().getUpdateHandler(); ulog = uhandler.getUpdateLog(); assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) != 0); // now do some normal non-buffered adds updateJ( jsonAdd(sdoc("id", "Q4", "_version_", "114")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "Q5", "_version_", "115")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "Q6", "_version_", "116")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); assertU(commit()); req.close(); h.close(); createCore(); req = req(); uhandler = req.getCore().getUpdateHandler(); ulog = uhandler.getUpdateLog(); assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) == 0); ulog.bufferUpdates(); // simulate receiving no updates ulog.applyBufferedUpdates(); updateJ( jsonAdd(sdoc("id", "Q7", "_version_", "117")), params( DISTRIB_UPDATE_PARAM, FROM_LEADER)); // do another add to make sure flags are back to normal req.close(); h.close(); createCore(); req = req(); uhandler = req.getCore().getUpdateHandler(); ulog = uhandler.getUpdateLog(); assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) == 0); // check flags on Q7 logReplayFinish.acquire(); assertEquals( UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state } finally { DirectUpdateHandler2.commitOnClose = true; UpdateLog.testing_logReplayHook = null; UpdateLog.testing_logReplayFinishHook = null; req().close(); } }
@Test @Ignore("HDFS-3107: no truncate support yet") public void testDropBuffered() throws Exception { DirectUpdateHandler2.commitOnClose = false; final Semaphore logReplay = new Semaphore(0); final Semaphore logReplayFinish = new Semaphore(0); UpdateLog.testing_logReplayHook = new Runnable() { @Override public void run() { try { assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS)); } catch (Exception e) { throw new RuntimeException(e); } } }; UpdateLog.testing_logReplayFinishHook = new Runnable() { @Override public void run() { logReplayFinish.release(); } }; SolrQueryRequest req = req(); UpdateHandler uhandler = req.getCore().getUpdateHandler(); UpdateLog ulog = uhandler.getUpdateLog(); try { clearIndex(); assertU(commit()); assertEquals(UpdateLog.State.ACTIVE, ulog.getState()); ulog.bufferUpdates(); assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates(); assertTrue(rinfoFuture == null); assertEquals(UpdateLog.State.ACTIVE, ulog.getState()); ulog.bufferUpdates(); assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); // simulate updates from a leader updateJ( jsonAdd(sdoc("id", "C1", "_version_", "101")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "C2", "_version_", "102")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "C3", "_version_", "103")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); assertTrue(ulog.dropBufferedUpdates()); ulog.bufferUpdates(); updateJ( jsonAdd(sdoc("id", "C4", "_version_", "104")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "C5", "_version_", "105")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); logReplay.release(1000); rinfoFuture = ulog.applyBufferedUpdates(); UpdateLog.RecoveryInfo rinfo = rinfoFuture.get(); assertEquals(2, rinfo.adds); assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[105,104]}"); // this time add some docs first before buffering starts (so tlog won't be at pos 0) updateJ( jsonAdd(sdoc("id", "C100", "_version_", "200")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "C101", "_version_", "201")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); ulog.bufferUpdates(); updateJ( jsonAdd(sdoc("id", "C103", "_version_", "203")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "C104", "_version_", "204")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); assertTrue(ulog.dropBufferedUpdates()); ulog.bufferUpdates(); updateJ( jsonAdd(sdoc("id", "C105", "_version_", "205")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "C106", "_version_", "206")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); rinfoFuture = ulog.applyBufferedUpdates(); rinfo = rinfoFuture.get(); assertEquals(2, rinfo.adds); assertJQ( req("q", "*:*", "sort", "_version_ asc", "fl", "id,_version_"), "/response/docs==[" + "{'id':'C4','_version_':104}" + ",{'id':'C5','_version_':105}" + ",{'id':'C100','_version_':200}" + ",{'id':'C101','_version_':201}" + ",{'id':'C105','_version_':205}" + ",{'id':'C106','_version_':206}" + "]"); assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[206,205,201,200,105,104]}"); ulog.bufferUpdates(); assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); updateJ( jsonAdd(sdoc("id", "C301", "_version_", "998")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "C302", "_version_", "999")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); assertTrue(ulog.dropBufferedUpdates()); // make sure we can overwrite with a lower version // TODO: is this functionality needed? updateJ( jsonAdd(sdoc("id", "C301", "_version_", "301")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "C302", "_version_", "302")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); assertU(commit()); assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[302,301]}"); assertJQ( req("q", "*:*", "sort", "_version_ desc", "fl", "id,_version_", "rows", "2"), "/response/docs==[" + "{'id':'C302','_version_':302}" + ",{'id':'C301','_version_':301}" + "]"); updateJ( jsonAdd(sdoc("id", "C2", "_version_", "302")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); assertEquals( UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state } finally { DirectUpdateHandler2.commitOnClose = true; UpdateLog.testing_logReplayHook = null; UpdateLog.testing_logReplayFinishHook = null; req().close(); } }
@Test public void testBuffering() throws Exception { DirectUpdateHandler2.commitOnClose = false; final Semaphore logReplay = new Semaphore(0); final Semaphore logReplayFinish = new Semaphore(0); UpdateLog.testing_logReplayHook = new Runnable() { @Override public void run() { try { assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS)); } catch (Exception e) { throw new RuntimeException(e); } } }; UpdateLog.testing_logReplayFinishHook = new Runnable() { @Override public void run() { logReplayFinish.release(); } }; SolrQueryRequest req = req(); UpdateHandler uhandler = req.getCore().getUpdateHandler(); UpdateLog ulog = uhandler.getUpdateLog(); try { clearIndex(); assertU(commit()); assertEquals(UpdateLog.State.ACTIVE, ulog.getState()); ulog.bufferUpdates(); assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates(); assertTrue(rinfoFuture == null); assertEquals(UpdateLog.State.ACTIVE, ulog.getState()); ulog.bufferUpdates(); assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); // simulate updates from a leader updateJ( jsonAdd(sdoc("id", "B1", "_version_", "1010")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "B11", "_version_", "1015")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonDelQ("id:B1 id:B11 id:B2 id:B3"), params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-1017")); updateJ( jsonAdd(sdoc("id", "B2", "_version_", "1020")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "B3", "_version_", "1030")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); deleteAndGetVersion("B1", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-2010")); assertJQ( req("qt", "/get", "getVersions", "6"), "=={'versions':[-2010,1030,1020,-1017,1015,1010]}"); assertU(commit()); assertJQ( req("qt", "/get", "getVersions", "6"), "=={'versions':[-2010,1030,1020,-1017,1015,1010]}"); // updates should be buffered, so we should not see any results yet. assertJQ(req("q", "*:*"), "/response/numFound==0"); // real-time get should also not show anything (this could change in the future, // but it's currently used for validating version numbers too, so it would // be bad for updates to be visible if we're just buffering. assertJQ(req("qt", "/get", "id", "B3"), "=={'doc':null}"); rinfoFuture = ulog.applyBufferedUpdates(); assertTrue(rinfoFuture != null); assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState()); logReplay.release(1000); UpdateLog.RecoveryInfo rinfo = rinfoFuture.get(); assertEquals(UpdateLog.State.ACTIVE, ulog.getState()); assertJQ( req("qt", "/get", "getVersions", "6"), "=={'versions':[-2010,1030,1020,-1017,1015,1010]}"); assertJQ(req("q", "*:*"), "/response/numFound==2"); // move back to recovering ulog.bufferUpdates(); assertEquals(UpdateLog.State.BUFFERING, ulog.getState()); Long ver = getVer(req("qt", "/get", "id", "B3")); assertEquals(1030L, ver.longValue()); // add a reordered doc that shouldn't overwrite one in the index updateJ( jsonAdd(sdoc("id", "B3", "_version_", "3")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); // reorder two buffered updates updateJ( jsonAdd(sdoc("id", "B4", "_version_", "1040")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); deleteAndGetVersion( "B4", params( DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-940")); // this update should not take affect updateJ( jsonAdd(sdoc("id", "B6", "_version_", "1060")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "B5", "_version_", "1050")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); updateJ( jsonAdd(sdoc("id", "B8", "_version_", "1080")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); // test that delete by query is at least buffered along with everything else so it will delete // the // currently buffered id:8 (even if it doesn't currently support versioning) updateJ( "{\"delete\": { \"query\":\"id:B2 OR id:B8\" }}", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-3000")); assertJQ( req("qt", "/get", "getVersions", "13"), "=={'versions':[-3000,1080,1050,1060,-940,1040,3,-2010,1030,1020,-1017,1015,1010]}" // the // "3" // appears because versions aren't checked while buffering ); logReplay.drainPermits(); rinfoFuture = ulog.applyBufferedUpdates(); assertTrue(rinfoFuture != null); assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState()); // apply a single update logReplay.release(1); // now add another update updateJ( jsonAdd(sdoc("id", "B7", "_version_", "1070")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER)); // a reordered update that should be dropped deleteAndGetVersion("B5", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-950")); deleteAndGetVersion("B6", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-2060")); logReplay.release(1000); UpdateLog.RecoveryInfo recInfo = rinfoFuture.get(); assertJQ( req("q", "*:*", "sort", "id asc", "fl", "id,_version_"), "/response/docs==[" + "{'id':'B3','_version_':1030}" + ",{'id':'B4','_version_':1040}" + ",{'id':'B5','_version_':1050}" + ",{'id':'B7','_version_':1070}" + "]"); assertEquals(1, recInfo.deleteByQuery); assertEquals( UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state } finally { DirectUpdateHandler2.commitOnClose = true; UpdateLog.testing_logReplayHook = null; UpdateLog.testing_logReplayFinishHook = null; req().close(); } }
public void doDeleteByQuery(DeleteUpdateCommand cmd) throws IOException { // even in non zk mode, tests simulate updates from a leader if (!zkEnabled) { isLeader = getNonZkLeaderAssumption(req); } else { zkCheck(); } // NONE: we are the first to receive this deleteByQuery // - it must be forwarded to the leader of every shard // TO: we are a leader receiving a forwarded deleteByQuery... we must: // - block all updates (use VersionInfo) // - flush *all* updates going to our replicas // - forward the DBQ to our replicas and wait for the response // - log + execute the local DBQ // FROM: we are a replica receiving a DBQ from our leader // - log + execute the local DBQ DistribPhase phase = DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM)); if (zkEnabled && DistribPhase.NONE == phase) { boolean leaderForAnyShard = false; // start off by assuming we are not a leader for any shard Map<String, Slice> slices = zkController.getClusterState().getSlices(collection); if (slices == null) { throw new SolrException( ErrorCode.BAD_REQUEST, "Cannot find collection:" + collection + " in " + zkController.getClusterState().getCollections()); } ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams())); params.set(DISTRIB_UPDATE_PARAM, DistribPhase.TOLEADER.toString()); List<Node> leaders = new ArrayList<Node>(slices.size()); for (Map.Entry<String, Slice> sliceEntry : slices.entrySet()) { String sliceName = sliceEntry.getKey(); ZkNodeProps leaderProps; try { leaderProps = zkController.getZkStateReader().getLeaderProps(collection, sliceName); } catch (InterruptedException e) { throw new SolrException( ErrorCode.SERVICE_UNAVAILABLE, "Exception finding leader for shard " + sliceName, e); } // TODO: What if leaders changed in the meantime? // should we send out slice-at-a-time and if a node returns "hey, I'm not a leader" (or we // get an error because it went down) then look up the new leader? // Am I the leader for this slice? ZkCoreNodeProps coreLeaderProps = new ZkCoreNodeProps(leaderProps); String leaderNodeName = coreLeaderProps.getCoreNodeName(); String coreName = req.getCore().getName(); String coreNodeName = zkController.getNodeName() + "_" + coreName; isLeader = coreNodeName.equals(leaderNodeName); if (isLeader) { // don't forward to ourself leaderForAnyShard = true; } else { leaders.add(new StdNode(coreLeaderProps)); } } params.remove("commit"); // this will be distributed from the local commit cmdDistrib.distribDelete(cmd, leaders, params); if (!leaderForAnyShard) { return; } // change the phase to TOLEADER so we look up and forward to our own replicas (if any) phase = DistribPhase.TOLEADER; } List<Node> replicas = null; if (zkEnabled && DistribPhase.TOLEADER == phase) { // This core should be a leader isLeader = true; replicas = setupRequest(); } else if (DistribPhase.FROMLEADER == phase) { isLeader = false; } if (vinfo == null) { super.processDelete(cmd); return; } // at this point, there is an update we need to try and apply. // we may or may not be the leader. // Find the version long versionOnUpdate = cmd.getVersion(); if (versionOnUpdate == 0) { String versionOnUpdateS = req.getParams().get(VERSION_FIELD); versionOnUpdate = versionOnUpdateS == null ? 0 : Long.parseLong(versionOnUpdateS); } versionOnUpdate = Math.abs(versionOnUpdate); // normalize to positive version boolean isReplay = (cmd.getFlags() & UpdateCommand.REPLAY) != 0; boolean leaderLogic = isLeader && !isReplay; if (!leaderLogic && versionOnUpdate == 0) { throw new SolrException(ErrorCode.BAD_REQUEST, "missing _version_ on update from leader"); } vinfo.blockUpdates(); try { if (versionsStored) { if (leaderLogic) { long version = vinfo.getNewClock(); cmd.setVersion(-version); // TODO update versions in all buckets doLocalDelete(cmd); } else { cmd.setVersion(-versionOnUpdate); if (ulog.getState() != UpdateLog.State.ACTIVE && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) { // we're not in an active state, and this update isn't from a replay, so buffer it. cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING); ulog.deleteByQuery(cmd); return; } doLocalDelete(cmd); } } // since we don't know which documents were deleted, the easiest thing to do is to invalidate // all real-time caches (i.e. UpdateLog) which involves also getting a new version of the // IndexReader // (so cache misses will see up-to-date data) } finally { vinfo.unblockUpdates(); } // forward to all replicas if (leaderLogic && replicas != null) { ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams())); params.set(VERSION_FIELD, Long.toString(cmd.getVersion())); params.set(DISTRIB_UPDATE_PARAM, DistribPhase.FROMLEADER.toString()); params.set( "update.from", ZkCoreNodeProps.getCoreUrl(zkController.getBaseUrl(), req.getCore().getName())); cmdDistrib.distribDelete(cmd, replicas, params); cmdDistrib.finish(); } if (returnVersions && rsp != null) { if (deleteByQueryResponse == null) { deleteByQueryResponse = new NamedList<String>(); rsp.add("deleteByQuery", deleteByQueryResponse); } deleteByQueryResponse.add(cmd.getQuery(), cmd.getVersion()); } }
private void handleParams(ArrayList<CommandOperation> ops, RequestParams params) { for (CommandOperation op : ops) { switch (op.name) { case CREATE: case UPDATE: { Map<String, Object> map = op.getDataMap(); if (op.hasError()) break; for (Map.Entry<String, Object> entry : map.entrySet()) { Map val = null; String key = entry.getKey(); if (key == null || key.trim().isEmpty()) { op.addError("null key "); continue; } key = key.trim(); if (!validName(key)) { op.addError( MessageFormat.format( "''{0}'' name should only have chars [a-zA-Z_-.0-9] ", key)); continue; } try { val = (Map) entry.getValue(); } catch (Exception e1) { op.addError("invalid params for key : " + key); continue; } if (val.containsKey("")) { op.addError("Empty keys are not allowed in params"); continue; } MapSolrParams old = params.getParams(key); if (op.name.equals(UPDATE)) { LinkedHashMap m = new LinkedHashMap(old.getMap()); m.putAll(val); val = m; } params = params.setParams(key, val); } break; } case "delete": { List<String> name = op.getStrs(CommandOperation.ROOT_OBJ); if (op.hasError()) break; for (String s : name) { if (params.getParams(s) == null) { op.addError( MessageFormat.format("can't delete . No such params ''{0}'' exist", s)); } params = params.setParams(s, null); } } } } List errs = CommandOperation.captureErrors(ops); if (!errs.isEmpty()) { resp.add(CommandOperation.ERR_MSGS, errs); return; } SolrResourceLoader loader = req.getCore().getResourceLoader(); if (loader instanceof ZkSolrResourceLoader) { ZkController.persistConfigResourceToZooKeeper( loader, params.getZnodeVersion(), RequestParams.RESOURCE, params.toByteArray(), true); } else { SolrResourceLoader.persistConfLocally( loader, ConfigOverlay.RESOURCE_NAME, params.toByteArray()); req.getCore().getSolrConfig().refreshRequestParams(); } }
@Test public void testSorting() throws Exception { try { init("schema12.xml"); assertU(adoc("id", "a", "title", "ipod trash trash", "str_s1", "a")); assertU(adoc("id", "b", "title", "ipod ipod trash", "str_s1", "b")); assertU(adoc("id", "c", "title", "ipod ipod ipod ", "str_s1", "c")); assertU(adoc("id", "x", "title", "boosted", "str_s1", "x")); assertU(adoc("id", "y", "title", "boosted boosted", "str_s1", "y")); assertU(adoc("id", "z", "title", "boosted boosted boosted", "str_s1", "z")); assertU(commit()); String query = "title:ipod"; Map<String, String> args = new HashMap<>(); // reusing args & requests this way is a solr-test-antipattern. PLEASE // DO NOT COPY THIS CODE args.put(CommonParams.Q, query); args.put(CommonParams.QT, "/elevate"); args.put(CommonParams.FL, "id,score"); args.put("indent", "true"); // args.put( CommonParams.FL, "id,title,score" ); SolrQueryRequest req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args)); IndexReader reader = req.getSearcher().getIndexReader(); QueryElevationComponent booster = (QueryElevationComponent) req.getCore().getSearchComponent("elevate"); assertQ( "Make sure standard sort works as expected", req, "//*[@numFound='3']", "//result/doc[1]/str[@name='id'][.='c']", "//result/doc[2]/str[@name='id'][.='b']", "//result/doc[3]/str[@name='id'][.='a']"); // Explicitly set what gets boosted booster.elevationCache.clear(); booster.setTopQueryResults(reader, query, new String[] {"x", "y", "z"}, null); req.close(); req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args)); assertQ( "All six should make it", req, "//*[@numFound='6']", "//result/doc[1]/str[@name='id'][.='x']", "//result/doc[2]/str[@name='id'][.='y']", "//result/doc[3]/str[@name='id'][.='z']", "//result/doc[4]/str[@name='id'][.='c']", "//result/doc[5]/str[@name='id'][.='b']", "//result/doc[6]/str[@name='id'][.='a']"); booster.elevationCache.clear(); // now switch the order: req.close(); req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args)); booster.setTopQueryResults(reader, query, new String[] {"a", "x"}, null); assertQ( "All four should make it", req, "//*[@numFound='4']", "//result/doc[1]/str[@name='id'][.='a']", "//result/doc[2]/str[@name='id'][.='x']", "//result/doc[3]/str[@name='id'][.='c']", "//result/doc[4]/str[@name='id'][.='b']"); // Test reverse sort args.put(CommonParams.SORT, "score asc"); req.close(); req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args)); assertQ( "All four should make it", req, "//*[@numFound='4']" // NOTE REVERSED doc[X] indices , "//result/doc[4]/str[@name='id'][.='a']", "//result/doc[3]/str[@name='id'][.='x']", "//result/doc[2]/str[@name='id'][.='c']", "//result/doc[1]/str[@name='id'][.='b']"); // Try normal sort by 'id' // default 'forceBoost' should be false assertEquals(false, booster.forceElevation); args.put(CommonParams.SORT, "str_s1 asc"); req.close(); req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args)); assertQ( null, req, "//*[@numFound='4']", "//result/doc[1]/str[@name='id'][.='a']", "//result/doc[2]/str[@name='id'][.='b']", "//result/doc[3]/str[@name='id'][.='c']", "//result/doc[4]/str[@name='id'][.='x']"); args.put(CommonParams.SORT, "id asc"); req.close(); req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args)); assertQ( null, req, "//*[@numFound='4']", "//result/doc[1]/str[@name='id'][.='a']", "//result/doc[2]/str[@name='id'][.='b']", "//result/doc[3]/str[@name='id'][.='c']", "//result/doc[4]/str[@name='id'][.='x']"); booster.forceElevation = true; args.put(CommonParams.SORT, "id asc"); req.close(); req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args)); assertQ( null, req, "//*[@numFound='4']", "//result/doc[1]/str[@name='id'][.='a']", "//result/doc[2]/str[@name='id'][.='x']", "//result/doc[3]/str[@name='id'][.='b']", "//result/doc[4]/str[@name='id'][.='c']"); // Test exclusive (not to be confused with exclusion) args.put(QueryElevationParams.EXCLUSIVE, "true"); req.close(); req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args)); booster.setTopQueryResults(reader, query, new String[] {"x", "a"}, new String[] {}); assertQ( null, req, "//*[@numFound='2']", "//result/doc[1]/str[@name='id'][.='x']", "//result/doc[2]/str[@name='id'][.='a']"); // Test exclusion booster.elevationCache.clear(); args.remove(CommonParams.SORT); args.remove(QueryElevationParams.EXCLUSIVE); req.close(); req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args)); booster.setTopQueryResults(reader, query, new String[] {"x"}, new String[] {"a"}); assertQ( null, req, "//*[@numFound='3']", "//result/doc[1]/str[@name='id'][.='x']", "//result/doc[2]/str[@name='id'][.='c']", "//result/doc[3]/str[@name='id'][.='b']"); // Test setting ids and excludes from http parameters booster.elevationCache.clear(); args.put(QueryElevationParams.IDS, "x,y,z"); args.put(QueryElevationParams.EXCLUDE, "b"); req.close(); req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args)); assertQ( "All five should make it", req, "//*[@numFound='5']", "//result/doc[1]/str[@name='id'][.='x']", "//result/doc[2]/str[@name='id'][.='y']", "//result/doc[3]/str[@name='id'][.='z']", "//result/doc[4]/str[@name='id'][.='c']", "//result/doc[5]/str[@name='id'][.='a']"); args.put(QueryElevationParams.IDS, "x,z,y"); args.put(QueryElevationParams.EXCLUDE, "b,c"); req.close(); req = new LocalSolrQueryRequest(h.getCore(), new MapSolrParams(args)); assertQ( "All four should make it", req, "//*[@numFound='4']", "//result/doc[1]/str[@name='id'][.='x']", "//result/doc[2]/str[@name='id'][.='z']", "//result/doc[3]/str[@name='id'][.='y']", "//result/doc[4]/str[@name='id'][.='a']"); req.close(); } finally { delete(); } }
private List<Node> setupRequest(int hash) { List<Node> nodes = null; // if we are in zk mode... if (zkEnabled) { // set num nodes numNodes = zkController.getClusterState().getLiveNodes().size(); String shardId = getShard( hash, collection, zkController.getClusterState()); // get the right shard based on the hash... try { ZkCoreNodeProps leaderProps = new ZkCoreNodeProps( zkController.getZkStateReader().getLeaderProps(collection, shardId)); String leaderNodeName = leaderProps.getCoreNodeName(); String coreName = req.getCore().getName(); String coreNodeName = zkController.getNodeName() + "_" + coreName; isLeader = coreNodeName.equals(leaderNodeName); DistribPhase phase = DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM)); doDefensiveChecks(shardId, phase); if (DistribPhase.FROMLEADER == phase) { // we are coming from the leader, just go local - add no urls forwardToLeader = false; } else if (isLeader) { // that means I want to forward onto my replicas... // so get the replicas... forwardToLeader = false; List<ZkCoreNodeProps> replicaProps = zkController .getZkStateReader() .getReplicaProps( collection, shardId, zkController.getNodeName(), coreName, null, ZkStateReader.DOWN); if (replicaProps != null) { nodes = new ArrayList<Node>(replicaProps.size()); // check for test param that lets us miss replicas String[] skipList = req.getParams().getParams("test.distrib.skip.servers"); Set<String> skipListSet = null; if (skipList != null) { skipListSet = new HashSet<String>(skipList.length); skipListSet.addAll(Arrays.asList(skipList)); } for (ZkCoreNodeProps props : replicaProps) { if (skipList != null) { if (!skipListSet.contains(props.getCoreUrl())) { nodes.add(new StdNode(props)); } } else { nodes.add(new StdNode(props)); } } } } else { // I need to forward onto the leader... nodes = new ArrayList<Node>(1); nodes.add( new RetryNode(leaderProps, zkController.getZkStateReader(), collection, shardId)); forwardToLeader = true; } } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); } } return nodes; }