public NamedList buildResponse() {
NamedList<Object> response = new SimpleOrderedMap<>();
// determine if we are going index or count sort
boolean sort =
!TermsParams.TERMS_SORT_INDEX.equals(
params.get(TermsParams.TERMS_SORT, TermsParams.TERMS_SORT_COUNT));
// init minimum frequency
long freqmin = 1;
String s = params.get(TermsParams.TERMS_MINCOUNT);
if (s != null) freqmin = Long.parseLong(s);
// init maximum frequency, default to max int
long freqmax = -1;
s = params.get(TermsParams.TERMS_MAXCOUNT);
if (s != null) freqmax = Long.parseLong(s);
if (freqmax < 0) {
freqmax = Long.MAX_VALUE;
}
// init limit, default to max int
long limit = 10;
s = params.get(TermsParams.TERMS_LIMIT);
if (s != null) limit = Long.parseLong(s);
if (limit < 0) {
limit = Long.MAX_VALUE;
}
// loop though each field we want terms from
for (String key : fieldmap.keySet()) {
NamedList<Number> fieldterms = new SimpleOrderedMap<>();
TermsResponse.Term[] data = null;
if (sort) {
data = getCountSorted(fieldmap.get(key));
} else {
data = getLexSorted(fieldmap.get(key));
}
// loop though each term until we hit limit
int cnt = 0;
for (TermsResponse.Term tc : data) {
if (tc.getFrequency() >= freqmin && tc.getFrequency() <= freqmax) {
fieldterms.add(tc.getTerm(), num(tc.getFrequency()));
cnt++;
}
if (cnt >= limit) {
break;
}
}
response.add(key, fieldterms);
}
return response;
}
private ShardRequest createShardQuery(SolrParams params) {
ShardRequest sreq = new ShardRequest();
sreq.purpose = ShardRequest.PURPOSE_GET_TERMS;
// base shard request on original parameters
sreq.params = new ModifiableSolrParams(params);
// don't pass through the shards param
sreq.params.remove(ShardParams.SHARDS);
// remove any limits for shards, we want them to return all possible
// responses
// we want this so we can calculate the correct counts
// dont sort by count to avoid that unnecessary overhead on the shards
sreq.params.remove(TermsParams.TERMS_MAXCOUNT);
sreq.params.remove(TermsParams.TERMS_MINCOUNT);
sreq.params.set(TermsParams.TERMS_LIMIT, -1);
sreq.params.set(TermsParams.TERMS_SORT, TermsParams.TERMS_SORT_INDEX);
// TODO: is there a better way to handle this?
String qt = params.get(CommonParams.QT);
if (qt != null) {
sreq.params.add(CommonParams.QT, qt);
}
return sreq;
}
@Override
public void prepare(ResponseBuilder rb) throws IOException {
SolrParams params = rb.req.getParams();
if (params.getBool(TermsParams.TERMS, false)) {
rb.doTerms = true;
}
// TODO: temporary... this should go in a different component.
String shards = params.get(ShardParams.SHARDS);
if (shards != null) {
if (params.get(ShardParams.SHARDS_QT) == null) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "No shards.qt parameter specified");
}
List<String> lst = StrUtils.splitSmart(shards, ",", true);
rb.shards = lst.toArray(new String[lst.size()]);
}
}
@Override
public void process(ResponseBuilder rb) throws IOException {
SolrParams params = rb.req.getParams();
if (!params.getBool(TermsParams.TERMS, false)) return;
String[] fields = params.getParams(TermsParams.TERMS_FIELD);
NamedList<Object> termsResult = new SimpleOrderedMap<>();
rb.rsp.add("terms", termsResult);
if (fields == null || fields.length == 0) return;
int limit = params.getInt(TermsParams.TERMS_LIMIT, 10);
if (limit < 0) {
limit = Integer.MAX_VALUE;
}
String lowerStr = params.get(TermsParams.TERMS_LOWER);
String upperStr = params.get(TermsParams.TERMS_UPPER);
boolean upperIncl = params.getBool(TermsParams.TERMS_UPPER_INCLUSIVE, false);
boolean lowerIncl = params.getBool(TermsParams.TERMS_LOWER_INCLUSIVE, true);
boolean sort =
!TermsParams.TERMS_SORT_INDEX.equals(
params.get(TermsParams.TERMS_SORT, TermsParams.TERMS_SORT_COUNT));
int freqmin = params.getInt(TermsParams.TERMS_MINCOUNT, 1);
int freqmax = params.getInt(TermsParams.TERMS_MAXCOUNT, UNLIMITED_MAX_COUNT);
if (freqmax < 0) {
freqmax = Integer.MAX_VALUE;
}
String prefix = params.get(TermsParams.TERMS_PREFIX_STR);
String regexp = params.get(TermsParams.TERMS_REGEXP_STR);
Pattern pattern = regexp != null ? Pattern.compile(regexp, resolveRegexpFlags(params)) : null;
boolean raw = params.getBool(TermsParams.TERMS_RAW, false);
final AtomicReader indexReader = rb.req.getSearcher().getAtomicReader();
Fields lfields = indexReader.fields();
for (String field : fields) {
NamedList<Integer> fieldTerms = new NamedList<>();
termsResult.add(field, fieldTerms);
Terms terms = lfields == null ? null : lfields.terms(field);
if (terms == null) {
// no terms for this field
continue;
}
FieldType ft = raw ? null : rb.req.getSchema().getFieldTypeNoEx(field);
if (ft == null) ft = new StrField();
// prefix must currently be text
BytesRef prefixBytes = prefix == null ? null : new BytesRef(prefix);
BytesRef upperBytes = null;
if (upperStr != null) {
upperBytes = new BytesRef();
ft.readableToIndexed(upperStr, upperBytes);
}
BytesRef lowerBytes;
if (lowerStr == null) {
// If no lower bound was specified, use the prefix
lowerBytes = prefixBytes;
} else {
lowerBytes = new BytesRef();
if (raw) {
// TODO: how to handle binary? perhaps we don't for "raw"... or if the field exists
// perhaps we detect if the FieldType is non-character and expect hex if so?
lowerBytes = new BytesRef(lowerStr);
} else {
lowerBytes = new BytesRef();
ft.readableToIndexed(lowerStr, lowerBytes);
}
}
TermsEnum termsEnum = terms.iterator(null);
BytesRef term = null;
if (lowerBytes != null) {
if (termsEnum.seekCeil(lowerBytes) == TermsEnum.SeekStatus.END) {
termsEnum = null;
} else {
term = termsEnum.term();
// Only advance the enum if we are excluding the lower bound and the lower Term actually
// matches
if (lowerIncl == false && term.equals(lowerBytes)) {
term = termsEnum.next();
}
}
} else {
// position termsEnum on first term
term = termsEnum.next();
}
int i = 0;
BoundedTreeSet<CountPair<BytesRef, Integer>> queue =
(sort ? new BoundedTreeSet<CountPair<BytesRef, Integer>>(limit) : null);
CharsRef external = new CharsRef();
while (term != null && (i < limit || sort)) {
boolean externalized = false; // did we fill in "external" yet for this term?
// stop if the prefix doesn't match
if (prefixBytes != null && !StringHelper.startsWith(term, prefixBytes)) break;
if (pattern != null) {
// indexed text or external text?
// TODO: support "raw" mode?
ft.indexedToReadable(term, external);
externalized = true;
if (!pattern.matcher(external).matches()) {
term = termsEnum.next();
continue;
}
}
if (upperBytes != null) {
int upperCmp = term.compareTo(upperBytes);
// if we are past the upper term, or equal to it (when don't include upper) then stop.
if (upperCmp > 0 || (upperCmp == 0 && !upperIncl)) break;
}
// This is a good term in the range. Check if mincount/maxcount conditions are satisfied.
int docFreq = termsEnum.docFreq();
if (docFreq >= freqmin && docFreq <= freqmax) {
// add the term to the list
if (sort) {
queue.add(new CountPair<>(BytesRef.deepCopyOf(term), docFreq));
} else {
// TODO: handle raw somehow
if (!externalized) {
ft.indexedToReadable(term, external);
}
fieldTerms.add(external.toString(), docFreq);
i++;
}
}
term = termsEnum.next();
}
if (sort) {
for (CountPair<BytesRef, Integer> item : queue) {
if (i >= limit) break;
ft.indexedToReadable(item.key, external);
fieldTerms.add(external.toString(), item.val);
i++;
}
}
}
}
@Override
public void process(ResponseBuilder rb) throws IOException {
SolrParams params = rb.req.getParams();
if (params.getBool(TermsParams.TERMS, false)) {
String lowerStr = params.get(TermsParams.TERMS_LOWER, null);
String[] fields = params.getParams(TermsParams.TERMS_FIELD);
if (fields != null && fields.length > 0) {
NamedList terms = new SimpleOrderedMap();
rb.rsp.add("terms", terms);
int limit = params.getInt(TermsParams.TERMS_LIMIT, 10);
if (limit < 0) {
limit = Integer.MAX_VALUE;
}
String upperStr = params.get(TermsParams.TERMS_UPPER);
boolean upperIncl = params.getBool(TermsParams.TERMS_UPPER_INCLUSIVE, false);
boolean lowerIncl = params.getBool(TermsParams.TERMS_LOWER_INCLUSIVE, true);
boolean sort =
!TermsParams.TERMS_SORT_INDEX.equals(
params.get(TermsParams.TERMS_SORT, TermsParams.TERMS_SORT_COUNT));
int freqmin = params.getInt(TermsParams.TERMS_MINCOUNT, 1); // initialize freqmin
int freqmax =
params.getInt(TermsParams.TERMS_MAXCOUNT, UNLIMITED_MAX_COUNT); // initialize freqmax
if (freqmax < 0) {
freqmax = Integer.MAX_VALUE;
}
String prefix = params.get(TermsParams.TERMS_PREFIX_STR);
String regexp = params.get(TermsParams.TERMS_REGEXP_STR);
Pattern pattern =
regexp != null ? Pattern.compile(regexp, resolveRegexpFlags(params)) : null;
boolean raw = params.getBool(TermsParams.TERMS_RAW, false);
for (int j = 0; j < fields.length; j++) {
String field = StringHelper.intern(fields[j]);
FieldType ft = raw ? null : rb.req.getSchema().getFieldTypeNoEx(field);
if (ft == null) ft = new StrField();
// If no lower bound was specified, use the prefix
String lower = lowerStr == null ? prefix : (raw ? lowerStr : ft.toInternal(lowerStr));
if (lower == null) lower = "";
String upper = upperStr == null ? null : (raw ? upperStr : ft.toInternal(upperStr));
Term lowerTerm = new Term(field, lower);
Term upperTerm = upper == null ? null : new Term(field, upper);
TermEnum termEnum =
rb.req
.getSearcher()
.getReader()
.terms(lowerTerm); // this will be positioned ready to go
int i = 0;
BoundedTreeSet<CountPair<String, Integer>> queue =
(sort ? new BoundedTreeSet<CountPair<String, Integer>>(limit) : null);
NamedList fieldTerms = new NamedList();
terms.add(field, fieldTerms);
Term lowerTestTerm = termEnum.term();
// Only advance the enum if we are excluding the lower bound and the lower Term actually
// matches
if (lowerTestTerm != null
&& lowerIncl == false
&& lowerTestTerm.field() == field // intern'd comparison
&& lowerTestTerm.text().equals(lower)) {
termEnum.next();
}
while (i < limit || sort) {
Term theTerm = termEnum.term();
// check for a different field, or the end of the index.
if (theTerm == null || field != theTerm.field()) // intern'd comparison
break;
String indexedText = theTerm.text();
// stop if the prefix doesn't match
if (prefix != null && !indexedText.startsWith(prefix)) break;
if (pattern != null && !pattern.matcher(indexedText).matches()) {
termEnum.next();
continue;
}
if (upperTerm != null) {
int upperCmp = theTerm.compareTo(upperTerm);
// if we are past the upper term, or equal to it (when don't include upper) then stop.
if (upperCmp > 0 || (upperCmp == 0 && !upperIncl)) break;
}
// This is a good term in the range. Check if mincount/maxcount conditions are
// satisfied.
int docFreq = termEnum.docFreq();
if (docFreq >= freqmin && docFreq <= freqmax) {
// add the term to the list
String label = raw ? indexedText : ft.indexedToReadable(indexedText);
if (sort) {
queue.add(new CountPair<String, Integer>(label, docFreq));
} else {
fieldTerms.add(label, docFreq);
i++;
}
}
termEnum.next();
}
termEnum.close();
if (sort) {
for (CountPair<String, Integer> item : queue) {
if (i < limit) {
fieldTerms.add(item.key, item.val);
i++;
} else {
break;
}
}
}
}
} else {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "No terms.fl parameter specified");
}
}
}
@Override
public void prepare(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrParams params = req.getParams();
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
SolrQueryResponse rsp = rb.rsp;
// Set field flags
String fl = params.get(CommonParams.FL);
int fieldFlags = 0;
if (fl != null) {
fieldFlags |= SolrPluginUtils.setReturnFields(fl, rsp);
}
rb.setFieldFlags(fieldFlags);
String defType = params.get(QueryParsing.DEFTYPE, QParserPlugin.DEFAULT_QTYPE);
// get it from the response builder to give a different component a chance
// to set it.
String queryString = rb.getQueryString();
if (queryString == null) {
// this is the normal way it's set.
queryString = params.get(CommonParams.Q);
rb.setQueryString(queryString);
}
try {
QParser parser = QParser.getParser(rb.getQueryString(), defType, req);
Query q = parser.getQuery();
if (q == null) {
// normalize a null query to a query that matches nothing
q = new BooleanQuery();
}
rb.setQuery(q);
rb.setSortSpec(parser.getSort(true));
rb.setQparser(parser);
String[] fqs = req.getParams().getParams(CommonParams.FQ);
if (fqs != null && fqs.length != 0) {
List<Query> filters = rb.getFilters();
if (filters == null) {
filters = new ArrayList<Query>();
rb.setFilters(filters);
}
for (String fq : fqs) {
if (fq != null && fq.trim().length() != 0) {
QParser fqp = QParser.getParser(fq, null, req);
filters.add(fqp.getQuery());
}
}
}
} catch (ParseException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
// TODO: temporary... this should go in a different component.
String shards = params.get(ShardParams.SHARDS);
if (shards != null) {
List<String> lst = StrUtils.splitSmart(shards, ",", true);
rb.shards = lst.toArray(new String[lst.size()]);
}
String shards_rows = params.get(ShardParams.SHARDS_ROWS);
if (shards_rows != null) {
rb.shards_rows = Integer.parseInt(shards_rows);
}
String shards_start = params.get(ShardParams.SHARDS_START);
if (shards_start != null) {
rb.shards_start = Integer.parseInt(shards_start);
}
}
/** Actually run the query */
@Override
public void process(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
SolrIndexSearcher searcher = req.getSearcher();
if (rb.getQueryCommand().getOffset() < 0) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "'start' parameter cannot be negative");
}
// -1 as flag if not set.
long timeAllowed = (long) params.getInt(CommonParams.TIME_ALLOWED, -1);
// Optional: This could also be implemented by the top-level searcher sending
// a filter that lists the ids... that would be transparent to
// the request handler, but would be more expensive (and would preserve score
// too if desired).
String ids = params.get(ShardParams.IDS);
if (ids != null) {
SchemaField idField = req.getSchema().getUniqueKeyField();
List<String> idArr = StrUtils.splitSmart(ids, ",", true);
int[] luceneIds = new int[idArr.size()];
int docs = 0;
for (int i = 0; i < idArr.size(); i++) {
int id =
req.getSearcher()
.getFirstMatch(
new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
if (id >= 0) luceneIds[docs++] = id;
}
DocListAndSet res = new DocListAndSet();
res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
if (rb.isNeedDocSet()) {
List<Query> queries = new ArrayList<Query>();
queries.add(rb.getQuery());
List<Query> filters = rb.getFilters();
if (filters != null) queries.addAll(filters);
res.docSet = searcher.getDocSet(queries);
}
rb.setResults(res);
rsp.add("response", rb.getResults().docList);
return;
}
SolrIndexSearcher.QueryCommand cmd = rb.getQueryCommand();
cmd.setTimeAllowed(timeAllowed);
SolrIndexSearcher.QueryResult result = new SolrIndexSearcher.QueryResult();
//
// grouping / field collapsing
//
boolean doGroup = params.getBool(GroupParams.GROUP, false);
if (doGroup) {
try {
cmd.groupCommands = new ArrayList<Grouping.Command>();
String[] fields = params.getParams(GroupParams.GROUP_FIELD);
String[] funcs = params.getParams(GroupParams.GROUP_FUNC);
String[] queries = params.getParams(GroupParams.GROUP_QUERY);
String groupSortStr = params.get(GroupParams.GROUP_SORT);
Sort groupSort =
groupSortStr != null ? QueryParsing.parseSort(groupSortStr, req.getSchema()) : null;
int limitDefault = cmd.getLen(); // this is normally from "rows"
int docsPerGroupDefault = params.getInt(GroupParams.GROUP_LIMIT, 1);
// temporary: implement all group-by-field as group-by-func
if (funcs == null) {
funcs = fields;
} else if (fields != null) {
// catenate functions and fields
String[] both = new String[fields.length + funcs.length];
System.arraycopy(fields, 0, both, 0, fields.length);
System.arraycopy(funcs, 0, both, fields.length, funcs.length);
funcs = both;
}
if (funcs != null) {
for (String groupByStr : funcs) {
QParser parser = QParser.getParser(groupByStr, "func", rb.req);
Query q = parser.getQuery();
Grouping.CommandFunc gc = new Grouping.CommandFunc();
gc.groupSort = groupSort;
if (q instanceof FunctionQuery) {
gc.groupBy = ((FunctionQuery) q).getValueSource();
} else {
gc.groupBy = new QueryValueSource(q, 0.0f);
}
gc.key = groupByStr;
gc.groupLimit = limitDefault;
gc.docsPerGroup = docsPerGroupDefault;
cmd.groupCommands.add(gc);
}
}
if (cmd.groupCommands.size() == 0) cmd.groupCommands = null;
if (cmd.groupCommands != null) {
if (rb.doHighlights || rb.isDebug()) {
// we need a single list of the returned docs
cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
}
searcher.search(result, cmd);
rb.setResult(result);
rsp.add("grouped", result.groupedResults);
// TODO: get "hits" a different way to log
return;
}
} catch (ParseException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
}
// normal search result
searcher.search(result, cmd);
rb.setResult(result);
rsp.add("response", rb.getResults().docList);
rsp.getToLog().add("hits", rb.getResults().docList.matches());
doFieldSortValues(rb, searcher);
doPrefetch(rb);
}