@Override
public void finishStage(ResponseBuilder rb) {
if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
NamedList termVectors = new NamedList<Object>();
Map.Entry<String, Object>[] arr = new NamedList.NamedListEntry[rb.resultIds.size()];
for (ShardRequest sreq : rb.finished) {
if ((sreq.purpose & ShardRequest.PURPOSE_GET_FIELDS) == 0
|| !sreq.params.getBool(COMPONENT_NAME, false)) {
continue;
}
for (ShardResponse srsp : sreq.responses) {
NamedList<Object> nl =
(NamedList<Object>) srsp.getSolrResponse().getResponse().get(TERM_VECTORS);
for (int i = 0; i < nl.size(); i++) {
String key = nl.getName(i);
ShardDoc sdoc = rb.resultIds.get(key);
if (null == sdoc) {
// metadata, only need from one node, leave in order
if (termVectors.indexOf(key, 0) < 0) {
termVectors.add(key, nl.getVal(i));
}
} else {
int idx = sdoc.positionInResponse;
arr[idx] = new NamedList.NamedListEntry<Object>(key, nl.getVal(i));
}
}
}
}
// remove nulls in case not all docs were able to be retrieved
termVectors.addAll(SolrPluginUtils.removeNulls(new NamedList<Object>(arr)));
rb.rsp.add(TERM_VECTORS, termVectors);
}
}
/**
* Helper method for determining the list of fields that we should try to find term vectors on.
*
* <p>Does simple (non-glob-supporting) parsing on the {@link TermVectorParams#FIELDS} param if
* specified, otherwise it returns the concrete field values specified in {@link CommonParams#FL}
* -- ignoring functions, transformers, or literals.
*
* <p>If "fl=*" is used, or neither param is specified, then <code>null</code> will be returned.
* If the empty set is returned, it means the "fl" specified consisted entirely of things that are
* not real fields (ie: functions, transformers, partial-globs, score, etc...) and not supported
* by this component.
*/
private Set<String> getFields(ResponseBuilder rb) {
SolrParams params = rb.req.getParams();
String[] fldLst = params.getParams(TermVectorParams.FIELDS);
if (null == fldLst || 0 == fldLst.length || (1 == fldLst.length && 0 == fldLst[0].length())) {
// no tv.fl, parse the main fl
ReturnFields rf = new SolrReturnFields(params.getParams(CommonParams.FL), rb.req);
if (rf.wantsAllFields()) {
return null;
}
Set<String> fieldNames = rf.getLuceneFieldNames();
return (null != fieldNames)
? fieldNames
:
// return empty set indicating no fields should be used
Collections.<String>emptySet();
}
// otherwise us the raw fldList as is, no special parsing or globs
Set<String> fieldNames = new LinkedHashSet<String>();
for (String fl : fldLst) {
fieldNames.addAll(Arrays.asList(SolrPluginUtils.split(fl)));
}
return fieldNames;
}
protected void doPrefetch(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
// pre-fetch returned documents
if (!req.getParams().getBool(ShardParams.IS_SHARD, false)
&& rb.getResults().docList != null
&& rb.getResults().docList.size() <= 50) {
// TODO: this may depend on the highlighter component (or other components?)
SolrPluginUtils.optimizePreFetchDocs(rb.getResults().docList, rb.getQuery(), req, rsp);
}
}
@Override
public void transform(SolrDocument doc, int docid) {
if (context != null && context.query != null) {
try {
Explanation exp = context.searcher.explain(context.query, docid);
if (style == Style.nl) {
doc.setField(name, SolrPluginUtils.explanationToNamedList(exp));
} else if (style == Style.html) {
doc.setField(name, exp.toHtml());
} else {
doc.setField(name, exp.toString());
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
@SuppressWarnings("rawtypes")
@Override
public void init(NamedList args) {
defaultConfig = new SolrParserConfigParams();
NamedList defs = (NamedList) args.get("defaults");
if (defs == null) {
defs = new NamedList();
}
if (defs.get("virtual-fields") != null) {
NamedList vf = (NamedList) defs.get("virtual-fields");
for (int i = 0; i < vf.size(); i++) {
String fName = vf.getName(i);
String fValue = (String) vf.getVal(i);
defaultConfig.virtualFields.put(fName, SolrPluginUtils.parseFieldBoosts(fValue));
}
}
}
@Override
public void process(ResponseBuilder rb) throws IOException {
SolrParams params = rb.req.getParams();
if (!params.getBool(COMPONENT_NAME, false)) {
return;
}
String name = getClusteringEngineName(rb);
boolean useResults = params.getBool(ClusteringParams.USE_SEARCH_RESULTS, false);
if (useResults == true) {
SearchClusteringEngine engine = getSearchClusteringEngine(rb);
if (engine != null) {
DocListAndSet results = rb.getResults();
Map<SolrDocument, Integer> docIds = Maps.newHashMapWithExpectedSize(results.docList.size());
SolrDocumentList solrDocList =
SolrPluginUtils.docListToSolrDocumentList(
results.docList, rb.req.getSearcher(), engine.getFieldsToLoad(rb.req), docIds);
Object clusters = engine.cluster(rb.getQuery(), solrDocList, docIds, rb.req);
rb.rsp.add("clusters", clusters);
} else {
log.warn("No engine for: " + name);
}
}
boolean useCollection = params.getBool(ClusteringParams.USE_COLLECTION, false);
if (useCollection == true) {
DocumentClusteringEngine engine = documentClusteringEngines.get(name);
if (engine != null) {
boolean useDocSet = params.getBool(ClusteringParams.USE_DOC_SET, false);
NamedList<?> nl = null;
// TODO: This likely needs to be made into a background task that runs in an executor
if (useDocSet == true) {
nl = engine.cluster(rb.getResults().docSet, params);
} else {
nl = engine.cluster(params);
}
rb.rsp.add("clusters", nl);
} else {
log.warn("No engine for " + name);
}
}
}
@Override
public void finishStage(ResponseBuilder rb) {
if (rb.doHighlights && rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
Map.Entry<String, Object>[] arr = new NamedList.NamedListEntry[rb.resultIds.size()];
// TODO: make a generic routine to do automatic merging of id keyed data
for (ShardRequest sreq : rb.finished) {
if ((sreq.purpose & ShardRequest.PURPOSE_GET_HIGHLIGHTS) == 0) continue;
for (ShardResponse srsp : sreq.responses) {
NamedList hl = (NamedList) srsp.getSolrResponse().getResponse().get("highlighting");
for (int i = 0; i < hl.size(); i++) {
String id = hl.getName(i);
ShardDoc sdoc = rb.resultIds.get(id);
int idx = sdoc.positionInResponse;
arr[idx] = new NamedList.NamedListEntry<Object>(id, hl.getVal(i));
}
}
}
// remove nulls in case not all docs were able to be retrieved
rb.rsp.add("highlighting", SolrPluginUtils.removeNulls(new SimpleOrderedMap(arr)));
}
}
public SimpleQParser(
String qstr, SolrParams localParams, SolrParams params, SolrQueryRequest req) {
super(qstr, localParams, params, req);
// Some of the parameters may come in through localParams, so combine them with params.
SolrParams defaultParams = SolrParams.wrapDefaults(localParams, params);
// This will be used to specify what fields and boosts will be used by SimpleQueryParser.
Map<String, Float> queryFields =
SolrPluginUtils.parseFieldBoosts(defaultParams.get(SimpleParams.QF));
if (queryFields.isEmpty()) {
// It qf is not specified setup up the queryFields map to use the defaultField.
String defaultField =
QueryParsing.getDefaultField(req.getSchema(), defaultParams.get(CommonParams.DF));
if (defaultField == null) {
// A query cannot be run without having a field or set of fields to run against.
throw new IllegalStateException(
"Neither "
+ SimpleParams.QF
+ ", "
+ CommonParams.DF
+ ", nor the default search field are present.");
}
queryFields.put(defaultField, 1.0F);
} else {
for (Map.Entry<String, Float> queryField : queryFields.entrySet()) {
if (queryField.getValue() == null) {
// Some fields may be specified without a boost, so default the boost to 1.0 since a
// null value
// will not be accepted by SimpleQueryParser.
queryField.setValue(1.0F);
}
}
}
// Setup the operations that are enabled for the query.
int enabledOps = 0;
String opParam = defaultParams.get(SimpleParams.QO);
if (opParam == null) {
// All operations will be enabled.
enabledOps = -1;
} else {
// Parse the specified enabled operations to be used by the query.
String[] operations = opParam.split(",");
for (String operation : operations) {
Integer enabledOp = OPERATORS.get(operation.trim().toUpperCase(Locale.ROOT));
if (enabledOp != null) {
enabledOps |= enabledOp;
}
}
}
// Create a SimpleQueryParser using the analyzer from the schema.
final IndexSchema schema = req.getSchema();
parser =
new SolrSimpleQueryParser(
req.getSchema().getQueryAnalyzer(), queryFields, enabledOps, this, schema);
// Set the default operator to be either 'AND' or 'OR' for the query.
QueryParser.Operator defaultOp =
QueryParsing.getQueryParserDefaultOperator(
req.getSchema(), defaultParams.get(QueryParsing.OP));
if (defaultOp == QueryParser.Operator.AND) {
parser.setDefaultOperator(BooleanClause.Occur.MUST);
}
}
@Override
public void prepare(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrParams params = req.getParams();
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
SolrQueryResponse rsp = rb.rsp;
// Set field flags
String fl = params.get(CommonParams.FL);
int fieldFlags = 0;
if (fl != null) {
fieldFlags |= SolrPluginUtils.setReturnFields(fl, rsp);
}
rb.setFieldFlags(fieldFlags);
String defType = params.get(QueryParsing.DEFTYPE, QParserPlugin.DEFAULT_QTYPE);
// get it from the response builder to give a different component a chance
// to set it.
String queryString = rb.getQueryString();
if (queryString == null) {
// this is the normal way it's set.
queryString = params.get(CommonParams.Q);
rb.setQueryString(queryString);
}
try {
QParser parser = QParser.getParser(rb.getQueryString(), defType, req);
Query q = parser.getQuery();
if (q == null) {
// normalize a null query to a query that matches nothing
q = new BooleanQuery();
}
rb.setQuery(q);
rb.setSortSpec(parser.getSort(true));
rb.setQparser(parser);
String[] fqs = req.getParams().getParams(CommonParams.FQ);
if (fqs != null && fqs.length != 0) {
List<Query> filters = rb.getFilters();
if (filters == null) {
filters = new ArrayList<Query>();
rb.setFilters(filters);
}
for (String fq : fqs) {
if (fq != null && fq.trim().length() != 0) {
QParser fqp = QParser.getParser(fq, null, req);
filters.add(fqp.getQuery());
}
}
}
} catch (ParseException e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
// TODO: temporary... this should go in a different component.
String shards = params.get(ShardParams.SHARDS);
if (shards != null) {
List<String> lst = StrUtils.splitSmart(shards, ",", true);
rb.shards = lst.toArray(new String[lst.size()]);
}
String shards_rows = params.get(ShardParams.SHARDS_ROWS);
if (shards_rows != null) {
rb.shards_rows = Integer.parseInt(shards_rows);
}
String shards_start = params.get(ShardParams.SHARDS_START);
if (shards_start != null) {
rb.shards_start = Integer.parseInt(shards_start);
}
}
@Override
public void process(ResponseBuilder rb) throws IOException {
SolrParams params = rb.req.getParams();
if (!params.getBool(COMPONENT_NAME, false)) {
return;
}
NamedList termVectors = new NamedList();
rb.rsp.add(TERM_VECTORS, termVectors);
FieldOptions allFields = new FieldOptions();
// figure out what options we have, and try to get the appropriate vector
allFields.termFreq = params.getBool(TermVectorParams.TF, false);
allFields.positions = params.getBool(TermVectorParams.POSITIONS, false);
allFields.offsets = params.getBool(TermVectorParams.OFFSETS, false);
allFields.docFreq = params.getBool(TermVectorParams.DF, false);
allFields.tfIdf = params.getBool(TermVectorParams.TF_IDF, false);
// boolean cacheIdf = params.getBool(TermVectorParams.IDF, false);
// short cut to all values.
boolean all = params.getBool(TermVectorParams.ALL, false);
if (all == true) {
allFields.termFreq = true;
allFields.positions = true;
allFields.offsets = true;
allFields.docFreq = true;
allFields.tfIdf = true;
}
String fldLst = params.get(TermVectorParams.FIELDS);
if (fldLst == null) {
fldLst = params.get(CommonParams.FL);
}
// use this to validate our fields
IndexSchema schema = rb.req.getSchema();
// Build up our per field mapping
Map<String, FieldOptions> fieldOptions = new HashMap<String, FieldOptions>();
NamedList warnings = new NamedList();
List<String> noTV = new ArrayList<String>();
List<String> noPos = new ArrayList<String>();
List<String> noOff = new ArrayList<String>();
// we have specific fields to retrieve
if (fldLst != null) {
String[] fields = SolrPluginUtils.split(fldLst);
for (String field : fields) {
SchemaField sf = schema.getFieldOrNull(field);
if (sf != null) {
if (sf.storeTermVector()) {
FieldOptions option = fieldOptions.get(field);
if (option == null) {
option = new FieldOptions();
option.fieldName = field;
fieldOptions.put(field, option);
}
// get the per field mappings
option.termFreq = params.getFieldBool(field, TermVectorParams.TF, allFields.termFreq);
option.docFreq = params.getFieldBool(field, TermVectorParams.DF, allFields.docFreq);
option.tfIdf = params.getFieldBool(field, TermVectorParams.TF_IDF, allFields.tfIdf);
// Validate these are even an option
option.positions =
params.getFieldBool(field, TermVectorParams.POSITIONS, allFields.positions);
if (option.positions == true && sf.storeTermPositions() == false) {
noPos.add(field);
}
option.offsets =
params.getFieldBool(field, TermVectorParams.OFFSETS, allFields.offsets);
if (option.offsets == true && sf.storeTermOffsets() == false) {
noOff.add(field);
}
} else { // field doesn't have term vectors
noTV.add(field);
}
} else {
// field doesn't exist
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "undefined field: " + field);
}
}
} // else, deal with all fields
boolean hasWarnings = false;
if (noTV.isEmpty() == false) {
warnings.add("noTermVectors", noTV);
hasWarnings = true;
}
if (noPos.isEmpty() == false) {
warnings.add("noPositions", noPos);
hasWarnings = true;
}
if (noOff.isEmpty() == false) {
warnings.add("noOffsets", noOff);
hasWarnings = true;
}
if (hasWarnings == true) {
termVectors.add("warnings", warnings);
}
DocListAndSet listAndSet = rb.getResults();
List<Integer> docIds = getInts(params.getParams(TermVectorParams.DOC_IDS));
Iterator<Integer> iter;
if (docIds != null && docIds.isEmpty() == false) {
iter = docIds.iterator();
} else {
DocList list = listAndSet.docList;
iter = list.iterator();
}
SolrIndexSearcher searcher = rb.req.getSearcher();
IndexReader reader = searcher.getReader();
// the TVMapper is a TermVectorMapper which can be used to optimize loading of Term Vectors
SchemaField keyField = schema.getUniqueKeyField();
String uniqFieldName = null;
if (keyField != null) {
uniqFieldName = keyField.getName();
}
// Only load the id field to get the uniqueKey of that field
SetBasedFieldSelector fieldSelector =
new SetBasedFieldSelector(
Collections.singleton(uniqFieldName), Collections.<String>emptySet());
TVMapper mapper = new TVMapper(reader);
mapper.fieldOptions =
allFields; // this will only stay set if fieldOptions.isEmpty() (in other words, only if the
// user didn't set any fields)
while (iter.hasNext()) {
Integer docId = iter.next();
NamedList docNL = new NamedList();
mapper.docNL = docNL;
termVectors.add("doc-" + docId, docNL);
if (keyField != null) {
Document document = reader.document(docId, fieldSelector);
Fieldable uniqId = document.getFieldable(uniqFieldName);
String uniqVal = null;
if (uniqId != null) {
uniqVal = keyField.getType().storedToReadable(uniqId);
}
if (uniqVal != null) {
docNL.add("uniqueKey", uniqVal);
termVectors.add("uniqueKeyFieldName", uniqFieldName);
}
}
if (fieldOptions.isEmpty() == false) {
for (Map.Entry<String, FieldOptions> entry : fieldOptions.entrySet()) {
mapper.fieldOptions = entry.getValue();
reader.getTermFreqVector(docId, entry.getKey(), mapper);
}
} else {
// deal with all fields by using the allFieldMapper
reader.getTermFreqVector(docId, mapper);
}
}
}