/** * Helper method for determining the list of fields that we should try to find term vectors on. * * <p>Does simple (non-glob-supporting) parsing on the {@link TermVectorParams#FIELDS} param if * specified, otherwise it returns the concrete field values specified in {@link CommonParams#FL} * -- ignoring functions, transformers, or literals. * * <p>If "fl=*" is used, or neither param is specified, then <code>null</code> will be returned. * If the empty set is returned, it means the "fl" specified consisted entirely of things that are * not real fields (ie: functions, transformers, partial-globs, score, etc...) and not supported * by this component. */ private Set<String> getFields(ResponseBuilder rb) { SolrParams params = rb.req.getParams(); String[] fldLst = params.getParams(TermVectorParams.FIELDS); if (null == fldLst || 0 == fldLst.length || (1 == fldLst.length && 0 == fldLst[0].length())) { // no tv.fl, parse the main fl ReturnFields rf = new SolrReturnFields(params.getParams(CommonParams.FL), rb.req); if (rf.wantsAllFields()) { return null; } Set<String> fieldNames = rf.getLuceneFieldNames(); return (null != fieldNames) ? fieldNames : // return empty set indicating no fields should be used Collections.<String>emptySet(); } // otherwise us the raw fldList as is, no special parsing or globs Set<String> fieldNames = new LinkedHashSet<String>(); for (String fl : fldLst) { fieldNames.addAll(Arrays.asList(SolrPluginUtils.split(fl))); } return fieldNames; }
public static String toQueryString(SolrParams params, boolean xml) { StringBuilder sb = new StringBuilder(128); try { String amp = xml ? "&" : "&"; boolean first = true; Iterator<String> names = params.getParameterNamesIterator(); while (names.hasNext()) { String key = names.next(); String[] valarr = params.getParams(key); if (valarr == null) { sb.append(first ? "?" : amp); sb.append(key); first = false; } else { for (String val : valarr) { sb.append(first ? "?" : amp); sb.append(key); if (val != null) { sb.append('='); sb.append(URLEncoder.encode(val, "UTF-8")); } first = false; } } } } catch (IOException e) { throw new RuntimeException(e); } // can't happen return sb.toString(); }
/** * Returns a <code>NamedList</code> with each entry having the "key" of the interval as name and * the count of docs in that interval as value. All intervals added in the request are included in * the returned <code>NamedList</code> (included those with 0 count), and it's required that the * order of the intervals is deterministic and equals in all shards of a distributed request, * otherwise the collation of results will fail. */ public NamedList<Object> getFacetIntervalCounts() throws IOException, SyntaxError { NamedList<Object> res = new SimpleOrderedMap<Object>(); String[] fields = global.getParams(FacetParams.FACET_INTERVAL); if (fields == null || fields.length == 0) return res; for (String field : fields) { final ParsedParams parsed = parseParams(FacetParams.FACET_INTERVAL, field); String[] intervalStrs = parsed.required.getFieldParams(parsed.facetValue, FacetParams.FACET_INTERVAL_SET); SchemaField schemaField = searcher.getCore().getLatestSchema().getField(parsed.facetValue); if (parsed.params.getBool(GroupParams.GROUP_FACET, false)) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Interval Faceting can't be used with " + GroupParams.GROUP_FACET); } SimpleOrderedMap<Integer> fieldResults = new SimpleOrderedMap<Integer>(); res.add(parsed.key, fieldResults); IntervalFacets intervalFacets = new IntervalFacets(schemaField, searcher, parsed.docs, intervalStrs, parsed.params); for (FacetInterval interval : intervalFacets) { fieldResults.add(interval.getKey(), interval.getCount()); } } return res; }
public void init(SolrParams params) { this.params = params; String[] fields = params.getParams(TermsParams.TERMS_FIELD); if (fields != null) { for (String field : fields) { // TODO: not sure 128 is the best starting size // It use it because that is what is used for facets fieldmap.put(field, new HashMap<String, TermsResponse.Term>(128)); } } }
/** Parses request to "HeatmapFacet" instances. */ public static LinkedHashMap<String, HeatmapFacet> distribParse( SolrParams params, ResponseBuilder rb) { final LinkedHashMap<String, HeatmapFacet> heatmapFacets = new LinkedHashMap<>(); final String[] heatmapFields = params.getParams(FacetParams.FACET_HEATMAP); if (heatmapFields != null) { for (String heatmapField : heatmapFields) { HeatmapFacet facet = new HeatmapFacet(rb, heatmapField); heatmapFacets.put(facet.getKey(), facet); } } return heatmapFacets; }
@Override public void process(ResponseBuilder rb) throws IOException { SolrQueryRequest req = rb.req; if (rb.doHighlights) { SolrParams params = req.getParams(); String[] defaultHighlightFields; // TODO: get from builder by default? if (rb.getQparser() != null) { defaultHighlightFields = rb.getQparser().getDefaultHighlightFields(); } else { defaultHighlightFields = params.getParams(CommonParams.DF); } Query highlightQuery = rb.getHighlightQuery(); if (highlightQuery == null) { if (rb.getQparser() != null) { try { highlightQuery = rb.getQparser().getHighlightQuery(); rb.setHighlightQuery(highlightQuery); } catch (Exception e) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); } } else { highlightQuery = rb.getQuery(); rb.setHighlightQuery(highlightQuery); } } if (highlightQuery != null) { boolean rewrite = !(Boolean.valueOf(req.getParams().get(HighlightParams.USE_PHRASE_HIGHLIGHTER, "true")) && Boolean.valueOf( req.getParams().get(HighlightParams.HIGHLIGHT_MULTI_TERM, "true"))); highlightQuery = rewrite ? highlightQuery.rewrite(req.getSearcher().getIndexReader()) : highlightQuery; } // No highlighting if there is no query -- consider q.alt="*:* if (highlightQuery != null) { NamedList sumData = highlighter.doHighlighting( rb.getResults().docList, highlightQuery, req, defaultHighlightFields); if (sumData != null) { // TODO ???? add this directly to the response? rb.rsp.add("highlighting", sumData); } } } }
/** * Serialize Solr parameters into a String. * * @param params the parameters to serialize * @return a String serialization of the parameters */ public static String getCacheKey(SolrParams params) { StringBuilder out = new StringBuilder(); out.append('{'); Iterator<String> parameterNames = params.getParameterNamesIterator(); while (parameterNames.hasNext()) { String parameter = parameterNames.next(); out.append(parameter) .append(":[") .append(StringUtils.join(params.getParams(parameter), ", ")) .append("]\n"); } out.append('}'); return out.toString(); }
int resolveRegexpFlags(SolrParams params) { String[] flagParams = params.getParams(TermsParams.TERMS_REGEXP_FLAG); if (flagParams == null) { return 0; } int flags = 0; for (String flagParam : flagParams) { try { flags |= TermsParams.TermsRegexpFlag.valueOf(flagParam.toUpperCase(Locale.ROOT)).getValue(); } catch (IllegalArgumentException iae) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Unknown terms regex flag '" + flagParam + "'"); } } return flags; }
/** * Given some query params, executes the request against the cloudClient and then walks the pivot * facet values in the response, treating each one as a filter query to assert the pivot counts * are correct. */ private void assertPivotCountsAreCorrect(SolrParams baseParams, SolrParams pivotParams) throws SolrServerException { SolrParams initParams = SolrParams.wrapAppended(pivotParams, baseParams); log.info("Doing full run: {}", initParams); countNumFoundChecks = 0; NamedList<List<PivotField>> pivots = null; try { QueryResponse initResponse = cloudClient.query(initParams); pivots = initResponse.getFacetPivot(); assertNotNull(initParams + " has null pivots?", pivots); assertEquals( initParams + " num pivots", initParams.getParams("facet.pivot").length, pivots.size()); } catch (Exception e) { throw new RuntimeException("init query failed: " + initParams + ": " + e.getMessage(), e); } try { for (Map.Entry<String, List<PivotField>> pivot : pivots) { final String pivotKey = pivot.getKey(); // :HACK: for counting the max possible pivot depth final int maxDepth = 1 + pivotKey.length() - pivotKey.replace(",", "").length(); assertTraceOk(pivotKey, baseParams, pivot.getValue()); // NOTE: we can't make any assumptions/assertions about the number of // constraints here because of the random data - which means if pivotting is // completely broken and there are no constrains this loop could be a No-Op // but in that case we just have to trust that DistributedFacetPivotTest // will catch it. for (PivotField constraint : pivot.getValue()) { int depth = assertPivotCountsAreCorrect(pivotKey, baseParams, constraint); // we can't assert that the depth reached is the same as the depth requested // because the fq and/or mincount may have pruned the tree too much assertTrue( "went too deep: " + depth + ": " + pivotKey + " ==> " + pivot, depth <= maxDepth); } } } catch (AssertionError e) { throw new AssertionError(initParams + " ==> " + e.getMessage(), e); } finally { log.info("Ending full run (countNumFoundChecks={}): {}", countNumFoundChecks, initParams); } }
/** * Returns a list of facet counts for each of the facet queries specified in the params * * @see FacetParams#FACET_QUERY */ public NamedList<Integer> getFacetQueryCounts() throws IOException, SyntaxError { NamedList<Integer> res = new SimpleOrderedMap<>(); /* Ignore CommonParams.DF - could have init param facet.query assuming * the schema default with query param DF intented to only affect Q. * If user doesn't want schema default for facet.query, they should be * explicit. */ // SolrQueryParser qp = searcher.getSchema().getSolrQueryParser(null); String[] facetQs = global.getParams(FacetParams.FACET_QUERY); if (null != facetQs && 0 != facetQs.length) { for (String q : facetQs) { final ParsedParams parsed = parseParams(FacetParams.FACET_QUERY, q); getFacetQueryCount(parsed, res); } } return res; }
/** * Returns a list of value constraints and the associated facet counts for each facet field * specified in the params. * * @see FacetParams#FACET_FIELD * @see #getFieldMissingCount * @see #getFacetTermEnumCounts */ @SuppressWarnings("unchecked") public NamedList<Object> getFacetFieldCounts() throws IOException, SyntaxError { NamedList<Object> res = new SimpleOrderedMap<>(); String[] facetFs = global.getParams(FacetParams.FACET_FIELD); if (null == facetFs) { return res; } // Passing a negative number for FACET_THREADS implies an unlimited number of threads is // acceptable. // Also, a subtlety of directExecutor is that no matter how many times you "submit" a job, it's // really // just a method call in that it's run by the calling thread. int maxThreads = req.getParams().getInt(FacetParams.FACET_THREADS, 0); Executor executor = maxThreads == 0 ? directExecutor : facetExecutor; final Semaphore semaphore = new Semaphore((maxThreads <= 0) ? Integer.MAX_VALUE : maxThreads); List<Future<NamedList>> futures = new ArrayList<>(facetFs.length); try { // Loop over fields; submit to executor, keeping the future for (String f : facetFs) { final ParsedParams parsed = parseParams(FacetParams.FACET_FIELD, f); final SolrParams localParams = parsed.localParams; final String termList = localParams == null ? null : localParams.get(CommonParams.TERMS); final String key = parsed.key; final String facetValue = parsed.facetValue; Callable<NamedList> callable = new Callable<NamedList>() { @Override public NamedList call() throws Exception { try { NamedList<Object> result = new SimpleOrderedMap<>(); if (termList != null) { List<String> terms = StrUtils.splitSmart(termList, ",", true); result.add(key, getListedTermCounts(facetValue, parsed, terms)); } else { result.add(key, getTermCounts(facetValue, parsed)); } return result; } catch (SolrException se) { throw se; } catch (Exception e) { throw new SolrException( ErrorCode.SERVER_ERROR, "Exception during facet.field: " + facetValue, e); } finally { semaphore.release(); } } }; RunnableFuture<NamedList> runnableFuture = new FutureTask<>(callable); semaphore.acquire(); // may block and/or interrupt executor.execute(runnableFuture); // releases semaphore when done futures.add(runnableFuture); } // facetFs loop // Loop over futures to get the values. The order is the same as facetFs but shouldn't matter. for (Future<NamedList> future : futures) { res.addAll(future.get()); } assert semaphore.availablePermits() >= maxThreads; } catch (InterruptedException e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error while processing facet fields: InterruptedException", e); } catch (ExecutionException ee) { Throwable e = ee.getCause(); // unwrap if (e instanceof RuntimeException) { throw (RuntimeException) e; } throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error while processing facet fields: " + e.toString(), e); } return res; }
@Override public void process(ResponseBuilder rb) throws IOException { SolrParams params = rb.req.getParams(); if (params.getBool(TermsParams.TERMS, false)) { String lowerStr = params.get(TermsParams.TERMS_LOWER, null); String[] fields = params.getParams(TermsParams.TERMS_FIELD); if (fields != null && fields.length > 0) { NamedList terms = new SimpleOrderedMap(); rb.rsp.add("terms", terms); int limit = params.getInt(TermsParams.TERMS_LIMIT, 10); if (limit < 0) { limit = Integer.MAX_VALUE; } String upperStr = params.get(TermsParams.TERMS_UPPER); boolean upperIncl = params.getBool(TermsParams.TERMS_UPPER_INCLUSIVE, false); boolean lowerIncl = params.getBool(TermsParams.TERMS_LOWER_INCLUSIVE, true); boolean sort = !TermsParams.TERMS_SORT_INDEX.equals( params.get(TermsParams.TERMS_SORT, TermsParams.TERMS_SORT_COUNT)); int freqmin = params.getInt(TermsParams.TERMS_MINCOUNT, 1); // initialize freqmin int freqmax = params.getInt(TermsParams.TERMS_MAXCOUNT, UNLIMITED_MAX_COUNT); // initialize freqmax if (freqmax < 0) { freqmax = Integer.MAX_VALUE; } String prefix = params.get(TermsParams.TERMS_PREFIX_STR); String regexp = params.get(TermsParams.TERMS_REGEXP_STR); Pattern pattern = regexp != null ? Pattern.compile(regexp, resolveRegexpFlags(params)) : null; boolean raw = params.getBool(TermsParams.TERMS_RAW, false); for (int j = 0; j < fields.length; j++) { String field = StringHelper.intern(fields[j]); FieldType ft = raw ? null : rb.req.getSchema().getFieldTypeNoEx(field); if (ft == null) ft = new StrField(); // If no lower bound was specified, use the prefix String lower = lowerStr == null ? prefix : (raw ? lowerStr : ft.toInternal(lowerStr)); if (lower == null) lower = ""; String upper = upperStr == null ? null : (raw ? upperStr : ft.toInternal(upperStr)); Term lowerTerm = new Term(field, lower); Term upperTerm = upper == null ? null : new Term(field, upper); TermEnum termEnum = rb.req .getSearcher() .getReader() .terms(lowerTerm); // this will be positioned ready to go int i = 0; BoundedTreeSet<CountPair<String, Integer>> queue = (sort ? new BoundedTreeSet<CountPair<String, Integer>>(limit) : null); NamedList fieldTerms = new NamedList(); terms.add(field, fieldTerms); Term lowerTestTerm = termEnum.term(); // Only advance the enum if we are excluding the lower bound and the lower Term actually // matches if (lowerTestTerm != null && lowerIncl == false && lowerTestTerm.field() == field // intern'd comparison && lowerTestTerm.text().equals(lower)) { termEnum.next(); } while (i < limit || sort) { Term theTerm = termEnum.term(); // check for a different field, or the end of the index. if (theTerm == null || field != theTerm.field()) // intern'd comparison break; String indexedText = theTerm.text(); // stop if the prefix doesn't match if (prefix != null && !indexedText.startsWith(prefix)) break; if (pattern != null && !pattern.matcher(indexedText).matches()) { termEnum.next(); continue; } if (upperTerm != null) { int upperCmp = theTerm.compareTo(upperTerm); // if we are past the upper term, or equal to it (when don't include upper) then stop. if (upperCmp > 0 || (upperCmp == 0 && !upperIncl)) break; } // This is a good term in the range. Check if mincount/maxcount conditions are // satisfied. int docFreq = termEnum.docFreq(); if (docFreq >= freqmin && docFreq <= freqmax) { // add the term to the list String label = raw ? indexedText : ft.indexedToReadable(indexedText); if (sort) { queue.add(new CountPair<String, Integer>(label, docFreq)); } else { fieldTerms.add(label, docFreq); i++; } } termEnum.next(); } termEnum.close(); if (sort) { for (CountPair<String, Integer> item : queue) { if (i < limit) { fieldTerms.add(item.key, item.val); i++; } else { break; } } } } } else { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "No terms.fl parameter specified"); } } }
@Override public void process(ResponseBuilder rb) throws IOException { SolrQueryRequest req = rb.req; SolrQueryResponse rsp = rb.rsp; SolrParams params = req.getParams(); if (!params.getBool(COMPONENT_NAME, true)) { return; } String val = params.get("getVersions"); if (val != null) { processGetVersions(rb); return; } val = params.get("getUpdates"); if (val != null) { processGetUpdates(rb); return; } String id[] = params.getParams("id"); String ids[] = params.getParams("ids"); if (id == null && ids == null) { return; } String[] allIds = id == null ? new String[0] : id; if (ids != null) { List<String> lst = new ArrayList<String>(); for (String s : allIds) { lst.add(s); } for (String idList : ids) { lst.addAll(StrUtils.splitSmart(idList, ",", true)); } allIds = lst.toArray(new String[lst.size()]); } SolrCore core = req.getCore(); SchemaField idField = core.getLatestSchema().getUniqueKeyField(); FieldType fieldType = idField.getType(); SolrDocumentList docList = new SolrDocumentList(); UpdateLog ulog = core.getUpdateHandler().getUpdateLog(); RefCounted<SolrIndexSearcher> searcherHolder = null; DocTransformer transformer = rsp.getReturnFields().getTransformer(); if (transformer != null) { TransformContext context = new TransformContext(); context.req = req; transformer.setContext(context); } try { SolrIndexSearcher searcher = null; BytesRef idBytes = new BytesRef(); for (String idStr : allIds) { fieldType.readableToIndexed(idStr, idBytes); if (ulog != null) { Object o = ulog.lookup(idBytes); if (o != null) { // should currently be a List<Oper,Ver,Doc/Id> List entry = (List) o; assert entry.size() >= 3; int oper = (Integer) entry.get(0) & UpdateLog.OPERATION_MASK; switch (oper) { case UpdateLog.ADD: SolrDocument doc = toSolrDoc( (SolrInputDocument) entry.get(entry.size() - 1), core.getLatestSchema()); if (transformer != null) { transformer.transform(doc, -1); // unknown docID } docList.add(doc); break; case UpdateLog.DELETE: break; default: throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Unknown Operation! " + oper); } continue; } } // didn't find it in the update log, so it should be in the newest searcher opened if (searcher == null) { searcherHolder = core.getRealtimeSearcher(); searcher = searcherHolder.get(); } // SolrCore.verbose("RealTimeGet using searcher ", searcher); int docid = searcher.getFirstMatch(new Term(idField.getName(), idBytes)); if (docid < 0) continue; StoredDocument luceneDocument = searcher.doc(docid); SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema()); if (transformer != null) { transformer.transform(doc, docid); } docList.add(doc); } } finally { if (searcherHolder != null) { searcherHolder.decref(); } } // if the client specified a single id=foo, then use "doc":{ // otherwise use a standard doclist if (ids == null && allIds.length <= 1) { // if the doc was not found, then use a value of null. rsp.add("doc", docList.size() > 0 ? docList.get(0) : null); } else { docList.setNumFound(docList.size()); rsp.add("response", docList); } }
@Override public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { IndexSchema schema = req.getSchema(); SolrIndexSearcher searcher = req.getSearcher(); IndexReader reader = searcher.getReader(); SolrParams params = req.getParams(); int numTerms = params.getInt(NUMTERMS, DEFAULT_COUNT); // Always show the core lucene info rsp.add("index", getIndexInfo(reader, numTerms > 0)); Integer docId = params.getInt(DOC_ID); if (docId == null && params.get(ID) != null) { // Look for something with a given solr ID SchemaField uniqueKey = schema.getUniqueKeyField(); String v = uniqueKey.getType().toInternal(params.get(ID)); Term t = new Term(uniqueKey.getName(), v); docId = searcher.getFirstMatch(t); if (docId < 0) { throw new SolrException( SolrException.ErrorCode.NOT_FOUND, "Can't find document: " + params.get(ID)); } } // Read the document from the index if (docId != null) { Document doc = null; try { doc = reader.document(docId); } catch (Exception ex) { } if (doc == null) { throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "Can't find document: " + docId); } SimpleOrderedMap<Object> info = getDocumentFieldsInfo(doc, docId, reader, schema); SimpleOrderedMap<Object> docinfo = new SimpleOrderedMap<Object>(); docinfo.add("docId", docId); docinfo.add("lucene", info); docinfo.add("solr", doc); rsp.add("doc", docinfo); } else if ("schema".equals(params.get("show"))) { rsp.add("schema", getSchemaInfo(req.getSchema())); } else { // If no doc is given, show all fields and top terms Set<String> fields = null; if (params.get(CommonParams.FL) != null) { fields = new HashSet<String>(); for (String f : params.getParams(CommonParams.FL)) { fields.add(f); } } rsp.add("fields", getIndexedFieldsInfo(searcher, fields, numTerms)); } // Add some generally helpful information NamedList<Object> info = new SimpleOrderedMap<Object>(); info.add("key", getFieldFlagsKey()); info.add( "NOTE", "Document Frequency (df) is not updated when a document is marked for deletion. df values include deleted documents."); rsp.add("info", info); rsp.setHttpCaching(false); }
protected boolean handleMergeAction(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { SolrParams params = req.getParams(); String cname = params.required().get(CoreAdminParams.CORE); SolrCore core = coreContainer.getCore(cname); SolrCore[] sourceCores = null; RefCounted<SolrIndexSearcher>[] searchers = null; // stores readers created from indexDir param values IndexReader[] readersToBeClosed = null; if (core != null) { try { String[] dirNames = params.getParams(CoreAdminParams.INDEX_DIR); if (dirNames == null || dirNames.length == 0) { String[] sources = params.getParams("srcCore"); if (sources == null || sources.length == 0) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "At least one indexDir or srcCore must be specified"); sourceCores = new SolrCore[sources.length]; for (int i = 0; i < sources.length; i++) { String source = sources[i]; SolrCore srcCore = coreContainer.getCore(source); if (srcCore == null) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Core: " + source + " does not exist"); sourceCores[i] = srcCore; } } else { readersToBeClosed = new IndexReader[dirNames.length]; DirectoryFactory dirFactory = core.getDirectoryFactory(); for (int i = 0; i < dirNames.length; i++) { readersToBeClosed[i] = IndexReader.open(dirFactory.open(dirNames[i]), true); } } IndexReader[] readers = null; if (readersToBeClosed != null) { readers = readersToBeClosed; } else { readers = new IndexReader[sourceCores.length]; searchers = new RefCounted[sourceCores.length]; for (int i = 0; i < sourceCores.length; i++) { SolrCore solrCore = sourceCores[i]; // record the searchers so that we can decref searchers[i] = solrCore.getSearcher(); readers[i] = searchers[i].get().getIndexReader(); } } UpdateRequestProcessorChain processorChain = core.getUpdateProcessingChain(params.get(UpdateParams.UPDATE_CHAIN)); SolrQueryRequest wrappedReq = new LocalSolrQueryRequest(core, req.getParams()); UpdateRequestProcessor processor = processorChain.createProcessor(wrappedReq, rsp); processor.processMergeIndexes(new MergeIndexesCommand(readers)); } finally { if (searchers != null) { for (RefCounted<SolrIndexSearcher> searcher : searchers) { if (searcher != null) searcher.decref(); } } if (sourceCores != null) { for (SolrCore solrCore : sourceCores) { if (solrCore != null) solrCore.close(); } } if (readersToBeClosed != null) IOUtils.closeWhileHandlingException(readersToBeClosed); core.close(); } } return coreContainer.isPersistent(); }
@Override public void process(ResponseBuilder rb) throws IOException { SolrParams params = rb.req.getParams(); if (!params.getBool(COMPONENT_NAME, false)) { return; } NamedList<Object> termVectors = new NamedList<Object>(); rb.rsp.add(TERM_VECTORS, termVectors); IndexSchema schema = rb.req.getSchema(); SchemaField keyField = schema.getUniqueKeyField(); String uniqFieldName = null; if (keyField != null) { uniqFieldName = keyField.getName(); termVectors.add("uniqueKeyFieldName", uniqFieldName); } FieldOptions allFields = new FieldOptions(); // figure out what options we have, and try to get the appropriate vector allFields.termFreq = params.getBool(TermVectorParams.TF, false); allFields.positions = params.getBool(TermVectorParams.POSITIONS, false); allFields.offsets = params.getBool(TermVectorParams.OFFSETS, false); allFields.docFreq = params.getBool(TermVectorParams.DF, false); allFields.tfIdf = params.getBool(TermVectorParams.TF_IDF, false); // boolean cacheIdf = params.getBool(TermVectorParams.IDF, false); // short cut to all values. if (params.getBool(TermVectorParams.ALL, false)) { allFields.termFreq = true; allFields.positions = true; allFields.offsets = true; allFields.docFreq = true; allFields.tfIdf = true; } // Build up our per field mapping Map<String, FieldOptions> fieldOptions = new HashMap<String, FieldOptions>(); NamedList<List<String>> warnings = new NamedList<List<String>>(); List<String> noTV = new ArrayList<String>(); List<String> noPos = new ArrayList<String>(); List<String> noOff = new ArrayList<String>(); Set<String> fields = getFields(rb); if (null != fields) { // we have specific fields to retrieve, or no fields for (String field : fields) { // workarround SOLR-3523 if (null == field || "score".equals(field)) continue; // we don't want to issue warnings about the uniqueKey field // since it can cause lots of confusion in distributed requests // where the uniqueKey field is injected into the fl for merging final boolean fieldIsUniqueKey = field.equals(uniqFieldName); SchemaField sf = schema.getFieldOrNull(field); if (sf != null) { if (sf.storeTermVector()) { FieldOptions option = fieldOptions.get(field); if (option == null) { option = new FieldOptions(); option.fieldName = field; fieldOptions.put(field, option); } // get the per field mappings option.termFreq = params.getFieldBool(field, TermVectorParams.TF, allFields.termFreq); option.docFreq = params.getFieldBool(field, TermVectorParams.DF, allFields.docFreq); option.tfIdf = params.getFieldBool(field, TermVectorParams.TF_IDF, allFields.tfIdf); // Validate these are even an option option.positions = params.getFieldBool(field, TermVectorParams.POSITIONS, allFields.positions); if (option.positions && !sf.storeTermPositions() && !fieldIsUniqueKey) { noPos.add(field); } option.offsets = params.getFieldBool(field, TermVectorParams.OFFSETS, allFields.offsets); if (option.offsets && !sf.storeTermOffsets() && !fieldIsUniqueKey) { noOff.add(field); } } else { // field doesn't have term vectors if (!fieldIsUniqueKey) noTV.add(field); } } else { // field doesn't exist throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "undefined field: " + field); } } } // else, deal with all fields // NOTE: currently all typs of warnings are schema driven, and garunteed // to be consistent across all shards - if additional types of warnings // are added that might be differnet between shards, finishStage() needs // to be changed to account for that. boolean hasWarnings = false; if (!noTV.isEmpty()) { warnings.add("noTermVectors", noTV); hasWarnings = true; } if (!noPos.isEmpty()) { warnings.add("noPositions", noPos); hasWarnings = true; } if (!noOff.isEmpty()) { warnings.add("noOffsets", noOff); hasWarnings = true; } if (hasWarnings) { termVectors.add("warnings", warnings); } DocListAndSet listAndSet = rb.getResults(); List<Integer> docIds = getInts(params.getParams(TermVectorParams.DOC_IDS)); Iterator<Integer> iter; if (docIds != null && !docIds.isEmpty()) { iter = docIds.iterator(); } else { DocList list = listAndSet.docList; iter = list.iterator(); } SolrIndexSearcher searcher = rb.req.getSearcher(); IndexReader reader = searcher.getIndexReader(); // the TVMapper is a TermVectorMapper which can be used to optimize loading of Term Vectors // Only load the id field to get the uniqueKey of that // field final String finalUniqFieldName = uniqFieldName; final List<String> uniqValues = new ArrayList<String>(); // TODO: is this required to be single-valued? if so, we should STOP // once we find it... final StoredFieldVisitor getUniqValue = new StoredFieldVisitor() { @Override public void stringField(FieldInfo fieldInfo, String value) { uniqValues.add(value); } @Override public void intField(FieldInfo fieldInfo, int value) { uniqValues.add(Integer.toString(value)); } @Override public void longField(FieldInfo fieldInfo, long value) { uniqValues.add(Long.toString(value)); } @Override public Status needsField(FieldInfo fieldInfo) { return (fieldInfo.name.equals(finalUniqFieldName)) ? Status.YES : Status.NO; } }; TermsEnum termsEnum = null; while (iter.hasNext()) { Integer docId = iter.next(); NamedList<Object> docNL = new NamedList<Object>(); if (keyField != null) { reader.document(docId, getUniqValue); String uniqVal = null; if (uniqValues.size() != 0) { uniqVal = uniqValues.get(0); uniqValues.clear(); docNL.add("uniqueKey", uniqVal); termVectors.add(uniqVal, docNL); } } else { // support for schemas w/o a unique key, termVectors.add("doc-" + docId, docNL); } if (null != fields) { for (Map.Entry<String, FieldOptions> entry : fieldOptions.entrySet()) { final String field = entry.getKey(); final Terms vector = reader.getTermVector(docId, field); if (vector != null) { termsEnum = vector.iterator(termsEnum); mapOneVector(docNL, entry.getValue(), reader, docId, vector.iterator(termsEnum), field); } } } else { // extract all fields final Fields vectors = reader.getTermVectors(docId); for (String field : vectors) { Terms terms = vectors.terms(field); if (terms != null) { termsEnum = terms.iterator(termsEnum); mapOneVector(docNL, allFields, reader, docId, termsEnum, field); } } } } }
public int createSubRequests(ResponseBuilder rb) throws IOException { SolrParams params = rb.req.getParams(); String id1[] = params.getParams("id"); String ids[] = params.getParams("ids"); if (id1 == null && ids == null) { return ResponseBuilder.STAGE_DONE; } List<String> allIds = new ArrayList<String>(); if (id1 != null) { for (String s : id1) { allIds.add(s); } } if (ids != null) { for (String s : ids) { allIds.addAll(StrUtils.splitSmart(s, ",", true)); } } // TODO: handle collection=...? ZkController zkController = rb.req.getCore().getCoreDescriptor().getCoreContainer().getZkController(); // if shards=... then use that if (zkController != null && params.get("shards") == null) { CloudDescriptor cloudDescriptor = rb.req.getCore().getCoreDescriptor().getCloudDescriptor(); String collection = cloudDescriptor.getCollectionName(); ClusterState clusterState = zkController.getClusterState(); DocCollection coll = clusterState.getCollection(collection); Map<String, List<String>> sliceToId = new HashMap<String, List<String>>(); for (String id : allIds) { Slice slice = coll.getRouter().getTargetSlice(id, null, params, coll); List<String> idsForShard = sliceToId.get(slice.getName()); if (idsForShard == null) { idsForShard = new ArrayList<String>(2); sliceToId.put(slice.getName(), idsForShard); } idsForShard.add(id); } for (Map.Entry<String, List<String>> entry : sliceToId.entrySet()) { String shard = entry.getKey(); String shardIdList = StrUtils.join(entry.getValue(), ','); ShardRequest sreq = new ShardRequest(); sreq.purpose = 1; // sreq.shards = new String[]{shard}; // TODO: would be nice if this would work... sreq.shards = sliceToShards(rb, collection, shard); sreq.actualShards = sreq.shards; sreq.params = new ModifiableSolrParams(); sreq.params.set( ShardParams.SHARDS_QT, "/get"); // TODO: how to avoid hardcoding this and hit the same handler? sreq.params.set("distrib", false); sreq.params.set("ids", shardIdList); rb.addRequest(this, sreq); } } else { String shardIdList = StrUtils.join(allIds, ','); ShardRequest sreq = new ShardRequest(); sreq.purpose = 1; sreq.shards = null; // ALL sreq.actualShards = sreq.shards; sreq.params = new ModifiableSolrParams(); sreq.params.set( ShardParams.SHARDS_QT, "/get"); // TODO: how to avoid hardcoding this and hit the same handler? sreq.params.set("distrib", false); sreq.params.set("ids", shardIdList); rb.addRequest(this, sreq); } return ResponseBuilder.STAGE_DONE; }
@Override public void process(ResponseBuilder rb) throws IOException { SolrParams params = rb.req.getParams(); if (!params.getBool(COMPONENT_NAME, false)) { return; } NamedList termVectors = new NamedList(); rb.rsp.add(TERM_VECTORS, termVectors); FieldOptions allFields = new FieldOptions(); // figure out what options we have, and try to get the appropriate vector allFields.termFreq = params.getBool(TermVectorParams.TF, false); allFields.positions = params.getBool(TermVectorParams.POSITIONS, false); allFields.offsets = params.getBool(TermVectorParams.OFFSETS, false); allFields.docFreq = params.getBool(TermVectorParams.DF, false); allFields.tfIdf = params.getBool(TermVectorParams.TF_IDF, false); // boolean cacheIdf = params.getBool(TermVectorParams.IDF, false); // short cut to all values. boolean all = params.getBool(TermVectorParams.ALL, false); if (all == true) { allFields.termFreq = true; allFields.positions = true; allFields.offsets = true; allFields.docFreq = true; allFields.tfIdf = true; } String fldLst = params.get(TermVectorParams.FIELDS); if (fldLst == null) { fldLst = params.get(CommonParams.FL); } // use this to validate our fields IndexSchema schema = rb.req.getSchema(); // Build up our per field mapping Map<String, FieldOptions> fieldOptions = new HashMap<String, FieldOptions>(); NamedList warnings = new NamedList(); List<String> noTV = new ArrayList<String>(); List<String> noPos = new ArrayList<String>(); List<String> noOff = new ArrayList<String>(); // we have specific fields to retrieve if (fldLst != null) { String[] fields = SolrPluginUtils.split(fldLst); for (String field : fields) { SchemaField sf = schema.getFieldOrNull(field); if (sf != null) { if (sf.storeTermVector()) { FieldOptions option = fieldOptions.get(field); if (option == null) { option = new FieldOptions(); option.fieldName = field; fieldOptions.put(field, option); } // get the per field mappings option.termFreq = params.getFieldBool(field, TermVectorParams.TF, allFields.termFreq); option.docFreq = params.getFieldBool(field, TermVectorParams.DF, allFields.docFreq); option.tfIdf = params.getFieldBool(field, TermVectorParams.TF_IDF, allFields.tfIdf); // Validate these are even an option option.positions = params.getFieldBool(field, TermVectorParams.POSITIONS, allFields.positions); if (option.positions == true && sf.storeTermPositions() == false) { noPos.add(field); } option.offsets = params.getFieldBool(field, TermVectorParams.OFFSETS, allFields.offsets); if (option.offsets == true && sf.storeTermOffsets() == false) { noOff.add(field); } } else { // field doesn't have term vectors noTV.add(field); } } else { // field doesn't exist throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "undefined field: " + field); } } } // else, deal with all fields boolean hasWarnings = false; if (noTV.isEmpty() == false) { warnings.add("noTermVectors", noTV); hasWarnings = true; } if (noPos.isEmpty() == false) { warnings.add("noPositions", noPos); hasWarnings = true; } if (noOff.isEmpty() == false) { warnings.add("noOffsets", noOff); hasWarnings = true; } if (hasWarnings == true) { termVectors.add("warnings", warnings); } DocListAndSet listAndSet = rb.getResults(); List<Integer> docIds = getInts(params.getParams(TermVectorParams.DOC_IDS)); Iterator<Integer> iter; if (docIds != null && docIds.isEmpty() == false) { iter = docIds.iterator(); } else { DocList list = listAndSet.docList; iter = list.iterator(); } SolrIndexSearcher searcher = rb.req.getSearcher(); IndexReader reader = searcher.getReader(); // the TVMapper is a TermVectorMapper which can be used to optimize loading of Term Vectors SchemaField keyField = schema.getUniqueKeyField(); String uniqFieldName = null; if (keyField != null) { uniqFieldName = keyField.getName(); } // Only load the id field to get the uniqueKey of that field SetBasedFieldSelector fieldSelector = new SetBasedFieldSelector( Collections.singleton(uniqFieldName), Collections.<String>emptySet()); TVMapper mapper = new TVMapper(reader); mapper.fieldOptions = allFields; // this will only stay set if fieldOptions.isEmpty() (in other words, only if the // user didn't set any fields) while (iter.hasNext()) { Integer docId = iter.next(); NamedList docNL = new NamedList(); mapper.docNL = docNL; termVectors.add("doc-" + docId, docNL); if (keyField != null) { Document document = reader.document(docId, fieldSelector); Fieldable uniqId = document.getFieldable(uniqFieldName); String uniqVal = null; if (uniqId != null) { uniqVal = keyField.getType().storedToReadable(uniqId); } if (uniqVal != null) { docNL.add("uniqueKey", uniqVal); termVectors.add("uniqueKeyFieldName", uniqFieldName); } } if (fieldOptions.isEmpty() == false) { for (Map.Entry<String, FieldOptions> entry : fieldOptions.entrySet()) { mapper.fieldOptions = entry.getValue(); reader.getTermFreqVector(docId, entry.getKey(), mapper); } } else { // deal with all fields by using the allFieldMapper reader.getTermFreqVector(docId, mapper); } } }
@Override public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { // Don't do anything if the framework is unknown if (watcher == null) { rsp.add("error", "Logging Not Initalized"); return; } rsp.add("watcher", watcher.getName()); SolrParams params = req.getParams(); if (params.get("threshold") != null) { watcher.setThreshold(params.get("threshold")); } // Write something at each level if (params.get("test") != null) { log.trace("trace message"); log.debug("debug message"); log.info("info (with exception)", new RuntimeException("test")); log.warn("warn (with exception)", new RuntimeException("test")); log.error("error (with exception)", new RuntimeException("test")); } String[] set = params.getParams("set"); if (set != null) { for (String pair : set) { String[] split = pair.split(":"); if (split.length != 2) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Invalid format, expected level:value, got " + pair); } String category = split[0]; String level = split[1]; watcher.setLogLevel(category, level); } } String since = req.getParams().get("since"); if (since != null) { long time = -1; try { time = Long.parseLong(since); } catch (Exception ex) { throw new SolrException(ErrorCode.BAD_REQUEST, "invalid timestamp: " + since); } AtomicBoolean found = new AtomicBoolean(false); SolrDocumentList docs = watcher.getHistory(time, found); if (docs == null) { rsp.add("error", "History not enabled"); return; } else { SimpleOrderedMap<Object> info = new SimpleOrderedMap<Object>(); if (time > 0) { info.add("since", time); info.add("found", found); } else { info.add("levels", watcher.getAllLevels()); // show for the first request } info.add("last", watcher.getLastEvent()); info.add("buffer", watcher.getHistorySize()); info.add("threshold", watcher.getThreshold()); rsp.add("info", info); rsp.add("history", docs); } } else { rsp.add("levels", watcher.getAllLevels()); List<LoggerInfo> loggers = new ArrayList<LoggerInfo>(watcher.getAllLoggers()); Collections.sort(loggers); List<SimpleOrderedMap<?>> info = new ArrayList<SimpleOrderedMap<?>>(); for (LoggerInfo wrap : loggers) { info.add(wrap.getInfo()); } rsp.add("loggers", info); } rsp.setHttpCaching(false); }
/** Actually run the query */ @Override public void process(ResponseBuilder rb) throws IOException { SolrQueryRequest req = rb.req; SolrQueryResponse rsp = rb.rsp; SolrParams params = req.getParams(); if (!params.getBool(COMPONENT_NAME, true)) { return; } SolrIndexSearcher searcher = req.getSearcher(); if (rb.getQueryCommand().getOffset() < 0) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "'start' parameter cannot be negative"); } // -1 as flag if not set. long timeAllowed = (long) params.getInt(CommonParams.TIME_ALLOWED, -1); // Optional: This could also be implemented by the top-level searcher sending // a filter that lists the ids... that would be transparent to // the request handler, but would be more expensive (and would preserve score // too if desired). String ids = params.get(ShardParams.IDS); if (ids != null) { SchemaField idField = req.getSchema().getUniqueKeyField(); List<String> idArr = StrUtils.splitSmart(ids, ",", true); int[] luceneIds = new int[idArr.size()]; int docs = 0; for (int i = 0; i < idArr.size(); i++) { int id = req.getSearcher() .getFirstMatch( new Term(idField.getName(), idField.getType().toInternal(idArr.get(i)))); if (id >= 0) luceneIds[docs++] = id; } DocListAndSet res = new DocListAndSet(); res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0); if (rb.isNeedDocSet()) { List<Query> queries = new ArrayList<Query>(); queries.add(rb.getQuery()); List<Query> filters = rb.getFilters(); if (filters != null) queries.addAll(filters); res.docSet = searcher.getDocSet(queries); } rb.setResults(res); rsp.add("response", rb.getResults().docList); return; } SolrIndexSearcher.QueryCommand cmd = rb.getQueryCommand(); cmd.setTimeAllowed(timeAllowed); SolrIndexSearcher.QueryResult result = new SolrIndexSearcher.QueryResult(); // // grouping / field collapsing // boolean doGroup = params.getBool(GroupParams.GROUP, false); if (doGroup) { try { cmd.groupCommands = new ArrayList<Grouping.Command>(); String[] fields = params.getParams(GroupParams.GROUP_FIELD); String[] funcs = params.getParams(GroupParams.GROUP_FUNC); String[] queries = params.getParams(GroupParams.GROUP_QUERY); String groupSortStr = params.get(GroupParams.GROUP_SORT); Sort groupSort = groupSortStr != null ? QueryParsing.parseSort(groupSortStr, req.getSchema()) : null; int limitDefault = cmd.getLen(); // this is normally from "rows" int docsPerGroupDefault = params.getInt(GroupParams.GROUP_LIMIT, 1); // temporary: implement all group-by-field as group-by-func if (funcs == null) { funcs = fields; } else if (fields != null) { // catenate functions and fields String[] both = new String[fields.length + funcs.length]; System.arraycopy(fields, 0, both, 0, fields.length); System.arraycopy(funcs, 0, both, fields.length, funcs.length); funcs = both; } if (funcs != null) { for (String groupByStr : funcs) { QParser parser = QParser.getParser(groupByStr, "func", rb.req); Query q = parser.getQuery(); Grouping.CommandFunc gc = new Grouping.CommandFunc(); gc.groupSort = groupSort; if (q instanceof FunctionQuery) { gc.groupBy = ((FunctionQuery) q).getValueSource(); } else { gc.groupBy = new QueryValueSource(q, 0.0f); } gc.key = groupByStr; gc.groupLimit = limitDefault; gc.docsPerGroup = docsPerGroupDefault; cmd.groupCommands.add(gc); } } if (cmd.groupCommands.size() == 0) cmd.groupCommands = null; if (cmd.groupCommands != null) { if (rb.doHighlights || rb.isDebug()) { // we need a single list of the returned docs cmd.setFlags(SolrIndexSearcher.GET_DOCLIST); } searcher.search(result, cmd); rb.setResult(result); rsp.add("grouped", result.groupedResults); // TODO: get "hits" a different way to log return; } } catch (ParseException e) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); } } // normal search result searcher.search(result, cmd); rb.setResult(result); rsp.add("response", rb.getResults().docList); rsp.getToLog().add("hits", rb.getResults().docList.matches()); doFieldSortValues(rb, searcher); doPrefetch(rb); }
@Override public void process(ResponseBuilder rb) throws IOException { SolrParams params = rb.req.getParams(); if (!params.getBool(TermsParams.TERMS, false)) return; String[] fields = params.getParams(TermsParams.TERMS_FIELD); NamedList<Object> termsResult = new SimpleOrderedMap<>(); rb.rsp.add("terms", termsResult); if (fields == null || fields.length == 0) return; int limit = params.getInt(TermsParams.TERMS_LIMIT, 10); if (limit < 0) { limit = Integer.MAX_VALUE; } String lowerStr = params.get(TermsParams.TERMS_LOWER); String upperStr = params.get(TermsParams.TERMS_UPPER); boolean upperIncl = params.getBool(TermsParams.TERMS_UPPER_INCLUSIVE, false); boolean lowerIncl = params.getBool(TermsParams.TERMS_LOWER_INCLUSIVE, true); boolean sort = !TermsParams.TERMS_SORT_INDEX.equals( params.get(TermsParams.TERMS_SORT, TermsParams.TERMS_SORT_COUNT)); int freqmin = params.getInt(TermsParams.TERMS_MINCOUNT, 1); int freqmax = params.getInt(TermsParams.TERMS_MAXCOUNT, UNLIMITED_MAX_COUNT); if (freqmax < 0) { freqmax = Integer.MAX_VALUE; } String prefix = params.get(TermsParams.TERMS_PREFIX_STR); String regexp = params.get(TermsParams.TERMS_REGEXP_STR); Pattern pattern = regexp != null ? Pattern.compile(regexp, resolveRegexpFlags(params)) : null; boolean raw = params.getBool(TermsParams.TERMS_RAW, false); final AtomicReader indexReader = rb.req.getSearcher().getAtomicReader(); Fields lfields = indexReader.fields(); for (String field : fields) { NamedList<Integer> fieldTerms = new NamedList<>(); termsResult.add(field, fieldTerms); Terms terms = lfields == null ? null : lfields.terms(field); if (terms == null) { // no terms for this field continue; } FieldType ft = raw ? null : rb.req.getSchema().getFieldTypeNoEx(field); if (ft == null) ft = new StrField(); // prefix must currently be text BytesRef prefixBytes = prefix == null ? null : new BytesRef(prefix); BytesRef upperBytes = null; if (upperStr != null) { upperBytes = new BytesRef(); ft.readableToIndexed(upperStr, upperBytes); } BytesRef lowerBytes; if (lowerStr == null) { // If no lower bound was specified, use the prefix lowerBytes = prefixBytes; } else { lowerBytes = new BytesRef(); if (raw) { // TODO: how to handle binary? perhaps we don't for "raw"... or if the field exists // perhaps we detect if the FieldType is non-character and expect hex if so? lowerBytes = new BytesRef(lowerStr); } else { lowerBytes = new BytesRef(); ft.readableToIndexed(lowerStr, lowerBytes); } } TermsEnum termsEnum = terms.iterator(null); BytesRef term = null; if (lowerBytes != null) { if (termsEnum.seekCeil(lowerBytes) == TermsEnum.SeekStatus.END) { termsEnum = null; } else { term = termsEnum.term(); // Only advance the enum if we are excluding the lower bound and the lower Term actually // matches if (lowerIncl == false && term.equals(lowerBytes)) { term = termsEnum.next(); } } } else { // position termsEnum on first term term = termsEnum.next(); } int i = 0; BoundedTreeSet<CountPair<BytesRef, Integer>> queue = (sort ? new BoundedTreeSet<CountPair<BytesRef, Integer>>(limit) : null); CharsRef external = new CharsRef(); while (term != null && (i < limit || sort)) { boolean externalized = false; // did we fill in "external" yet for this term? // stop if the prefix doesn't match if (prefixBytes != null && !StringHelper.startsWith(term, prefixBytes)) break; if (pattern != null) { // indexed text or external text? // TODO: support "raw" mode? ft.indexedToReadable(term, external); externalized = true; if (!pattern.matcher(external).matches()) { term = termsEnum.next(); continue; } } if (upperBytes != null) { int upperCmp = term.compareTo(upperBytes); // if we are past the upper term, or equal to it (when don't include upper) then stop. if (upperCmp > 0 || (upperCmp == 0 && !upperIncl)) break; } // This is a good term in the range. Check if mincount/maxcount conditions are satisfied. int docFreq = termsEnum.docFreq(); if (docFreq >= freqmin && docFreq <= freqmax) { // add the term to the list if (sort) { queue.add(new CountPair<>(BytesRef.deepCopyOf(term), docFreq)); } else { // TODO: handle raw somehow if (!externalized) { ft.indexedToReadable(term, external); } fieldTerms.add(external.toString(), docFreq); i++; } } term = termsEnum.next(); } if (sort) { for (CountPair<BytesRef, Integer> item : queue) { if (i >= limit) break; ft.indexedToReadable(item.key, external); fieldTerms.add(external.toString(), item.val); i++; } } } }
public NamedList<Object> request(final SolrRequest request, ResponseParser processor) throws SolrServerException, IOException { HttpMethod method = null; InputStream is = null; SolrParams params = request.getParams(); Collection<ContentStream> streams = requestWriter.getContentStreams(request); String path = requestWriter.getPath(request); if (path == null || !path.startsWith("/")) { path = "/select"; } ResponseParser parser = request.getResponseParser(); if (parser == null) { parser = _parser; } // The parser 'wt=' and 'version=' params are used instead of the original params ModifiableSolrParams wparams = new ModifiableSolrParams(); wparams.set(CommonParams.WT, parser.getWriterType()); wparams.set(CommonParams.VERSION, parser.getVersion()); if (params == null) { params = wparams; } else { params = new DefaultSolrParams(wparams, params); } if (_invariantParams != null) { params = new DefaultSolrParams(_invariantParams, params); } int tries = _maxRetries + 1; try { while (tries-- > 0) { // Note: since we aren't do intermittent time keeping // ourselves, the potential non-timeout latency could be as // much as tries-times (plus scheduling effects) the given // timeAllowed. try { if (SolrRequest.METHOD.GET == request.getMethod()) { if (streams != null) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!"); } method = new GetMethod(_baseURL + path + ClientUtils.toQueryString(params, false)); } else if (SolrRequest.METHOD.POST == request.getMethod()) { String url = _baseURL + path; boolean isMultipart = (streams != null && streams.size() > 1); if (streams == null || isMultipart) { PostMethod post = new PostMethod(url); post.getParams().setContentCharset("UTF-8"); if (!this.useMultiPartPost && !isMultipart) { post.addRequestHeader( "Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); } List<Part> parts = new LinkedList<Part>(); Iterator<String> iter = params.getParameterNamesIterator(); while (iter.hasNext()) { String p = iter.next(); String[] vals = params.getParams(p); if (vals != null) { for (String v : vals) { if (this.useMultiPartPost || isMultipart) { parts.add(new StringPart(p, v, "UTF-8")); } else { post.addParameter(p, v); } } } } if (isMultipart) { int i = 0; for (ContentStream content : streams) { final ContentStream c = content; String charSet = null; String transferEncoding = null; parts.add( new PartBase(c.getName(), c.getContentType(), charSet, transferEncoding) { @Override protected long lengthOfData() throws IOException { return c.getSize(); } @Override protected void sendData(OutputStream out) throws IOException { InputStream in = c.getStream(); try { IOUtils.copy(in, out); } finally { in.close(); } } }); } } if (parts.size() > 0) { post.setRequestEntity( new MultipartRequestEntity( parts.toArray(new Part[parts.size()]), post.getParams())); } method = post; } // It is has one stream, it is the post body, put the params in the URL else { String pstr = ClientUtils.toQueryString(params, false); PostMethod post = new PostMethod(url + pstr); // Single stream as body // Using a loop just to get the first one final ContentStream[] contentStream = new ContentStream[1]; for (ContentStream content : streams) { contentStream[0] = content; break; } if (contentStream[0] instanceof RequestWriter.LazyContentStream) { post.setRequestEntity( new RequestEntity() { public long getContentLength() { return -1; } public String getContentType() { return contentStream[0].getContentType(); } public boolean isRepeatable() { return false; } public void writeRequest(OutputStream outputStream) throws IOException { ((RequestWriter.LazyContentStream) contentStream[0]).writeTo(outputStream); } }); } else { is = contentStream[0].getStream(); post.setRequestEntity( new InputStreamRequestEntity(is, contentStream[0].getContentType())); } method = post; } } else { throw new SolrServerException("Unsupported method: " + request.getMethod()); } } catch (NoHttpResponseException r) { // This is generally safe to retry on method.releaseConnection(); method = null; if (is != null) { is.close(); } // If out of tries then just rethrow (as normal error). if ((tries < 1)) { throw r; } // log.warn( "Caught: " + r + ". Retrying..." ); } } } catch (IOException ex) { throw new SolrServerException("error reading streams", ex); } method.setFollowRedirects(_followRedirects); method.addRequestHeader("User-Agent", AGENT); if (_allowCompression) { method.setRequestHeader(new Header("Accept-Encoding", "gzip,deflate")); } try { // Execute the method. // System.out.println( "EXECUTE:"+method.getURI() ); int statusCode = _httpClient.executeMethod(method); if (statusCode != HttpStatus.SC_OK) { StringBuilder msg = new StringBuilder(); msg.append(method.getStatusLine().getReasonPhrase()); msg.append("\n\n"); msg.append(method.getStatusText()); msg.append("\n\n"); msg.append("request: " + method.getURI()); throw new SolrException(statusCode, java.net.URLDecoder.decode(msg.toString(), "UTF-8")); } // Read the contents String charset = "UTF-8"; if (method instanceof HttpMethodBase) { charset = ((HttpMethodBase) method).getResponseCharSet(); } InputStream respBody = method.getResponseBodyAsStream(); // Jakarta Commons HTTPClient doesn't handle any // compression natively. Handle gzip or deflate // here if applicable. if (_allowCompression) { Header contentEncodingHeader = method.getResponseHeader("Content-Encoding"); if (contentEncodingHeader != null) { String contentEncoding = contentEncodingHeader.getValue(); if (contentEncoding.contains("gzip")) { // log.debug( "wrapping response in GZIPInputStream" ); respBody = new GZIPInputStream(respBody); } else if (contentEncoding.contains("deflate")) { // log.debug( "wrapping response in InflaterInputStream" ); respBody = new InflaterInputStream(respBody); } } else { Header contentTypeHeader = method.getResponseHeader("Content-Type"); if (contentTypeHeader != null) { String contentType = contentTypeHeader.getValue(); if (contentType != null) { if (contentType.startsWith("application/x-gzip-compressed")) { // log.debug( "wrapping response in GZIPInputStream" ); respBody = new GZIPInputStream(respBody); } else if (contentType.startsWith("application/x-deflate")) { // log.debug( "wrapping response in InflaterInputStream" ); respBody = new InflaterInputStream(respBody); } } } } } return processor.processResponse(respBody, charset); } catch (HttpException e) { throw new SolrServerException(e); } catch (IOException e) { throw new SolrServerException(e); } finally { method.releaseConnection(); if (is != null) { is.close(); } } }