int collect(DocSet docs, int slot) throws IOException {
int count = 0;
SolrIndexSearcher searcher = fcontext.searcher;
final List<LeafReaderContext> leaves = searcher.getIndexReader().leaves();
final Iterator<LeafReaderContext> ctxIt = leaves.iterator();
LeafReaderContext ctx = null;
int segBase = 0;
int segMax;
int adjustedMax = 0;
for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) {
final int doc = docsIt.nextDoc();
if (doc >= adjustedMax) {
do {
ctx = ctxIt.next();
if (ctx == null) {
// should be impossible
throw new RuntimeException("INTERNAL FACET ERROR");
}
segBase = ctx.docBase;
segMax = ctx.reader().maxDoc();
adjustedMax = segBase + segMax;
} while (doc >= adjustedMax);
assert doc >= ctx.docBase;
setNextReader(ctx);
}
count++;
collect(doc - segBase, slot); // per-seg collectors
}
return count;
}
@Override
public long getSize() {
RefCounted<SolrIndexSearcher> refCountedIndexSearcher = this.core.getSearcher();
SolrIndexSearcher searcher = refCountedIndexSearcher.get();
DirectoryReader reader = searcher.getIndexReader();
long numDocs = reader.numDocs();
refCountedIndexSearcher.decref();
return numDocs;
}
@Override
public void build(SolrCore core, SolrIndexSearcher searcher) throws IOException {
LOG.info("build()");
if (sourceLocation == null) {
reader = searcher.getIndexReader();
dictionary = new HighFrequencyDictionary(reader, field, threshold);
} else {
try {
final String fileDelim = ",";
if (sourceLocation.contains(fileDelim)) {
String[] files = sourceLocation.split(fileDelim);
Reader[] readers = new Reader[files.length];
for (int i = 0; i < files.length; i++) {
Reader reader =
new InputStreamReader(
core.getResourceLoader().openResource(files[i]), IOUtils.CHARSET_UTF_8);
readers[i] = reader;
}
dictionary = new MultipleFileDictionary(readers);
} else {
dictionary =
new FileDictionary(
new InputStreamReader(
core.getResourceLoader().openResource(sourceLocation),
IOUtils.CHARSET_UTF_8));
}
} catch (UnsupportedEncodingException e) {
// should not happen
LOG.error("should not happen", e);
}
}
lookup.build(dictionary);
if (storeDir != null) {
File target = new File(storeDir, factory.storeFileName());
if (!lookup.store(new FileOutputStream(target))) {
if (sourceLocation == null) {
assert reader != null && field != null;
LOG.error(
"Store Lookup build from index on field: "
+ field
+ " failed reader has: "
+ reader.maxDoc()
+ " docs");
} else {
LOG.error("Store Lookup build from sourceloaction: " + sourceLocation + " failed");
}
} else {
LOG.info("Stored suggest data to: " + target.getAbsolutePath());
}
}
}
/**
* Generates a list of Highlighted query fragments for each item in a list of documents, or
* returns null if highlighting is disabled.
*
* @param docs query results
* @param query the query
* @param req the current request
* @param defaultFields default list of fields to summarize
* @return NamedList containing a NamedList for each document, which in turns contains sets
* (field, summary) pairs.
*/
@Override
@SuppressWarnings("unchecked")
public NamedList<Object> doHighlighting(
DocList docs, Query query, SolrQueryRequest req, String[] defaultFields) throws IOException {
SolrParams params = req.getParams();
if (!isHighlightingEnabled(params)) return null;
SolrIndexSearcher searcher = req.getSearcher();
IndexSchema schema = searcher.getSchema();
NamedList fragments = new SimpleOrderedMap();
String[] fieldNames = getHighlightFields(query, req, defaultFields);
Set<String> fset = new HashSet<String>();
{
// pre-fetch documents using the Searcher's doc cache
for (String f : fieldNames) {
fset.add(f);
}
// fetch unique key if one exists.
SchemaField keyField = schema.getUniqueKeyField();
if (null != keyField) fset.add(keyField.getName());
}
// get FastVectorHighlighter instance out of the processing loop
FastVectorHighlighter fvh =
new FastVectorHighlighter(
// FVH cannot process hl.usePhraseHighlighter parameter per-field basis
params.getBool(HighlightParams.USE_PHRASE_HIGHLIGHTER, true),
// FVH cannot process hl.requireFieldMatch parameter per-field basis
params.getBool(HighlightParams.FIELD_MATCH, false));
fvh.setPhraseLimit(params.getInt(HighlightParams.PHRASE_LIMIT, Integer.MAX_VALUE));
FieldQuery fieldQuery = fvh.getFieldQuery(query, searcher.getIndexReader());
// Highlight each document
DocIterator iterator = docs.iterator();
for (int i = 0; i < docs.size(); i++) {
int docId = iterator.nextDoc();
Document doc = searcher.doc(docId, fset);
NamedList docSummaries = new SimpleOrderedMap();
for (String fieldName : fieldNames) {
fieldName = fieldName.trim();
if (useFastVectorHighlighter(params, schema, fieldName))
doHighlightingByFastVectorHighlighter(
fvh, fieldQuery, req, docSummaries, docId, doc, fieldName);
else doHighlightingByHighlighter(query, req, docSummaries, docId, doc, fieldName);
}
String printId = schema.printableUniqueKey(doc);
fragments.add(printId == null ? null : printId, docSummaries);
}
return fragments;
}
@Override
public void build(SolrCore core, SolrIndexSearcher searcher) {
IndexReader reader = null;
try {
if (sourceLocation == null) {
// Load from Solr's index
reader = searcher.getIndexReader();
} else {
// Load from Lucene index at given sourceLocation
reader = this.reader;
}
// Create the dictionary
dictionary = new HighFrequencyDictionary(reader, field, threshold);
spellChecker.clearIndex();
spellChecker.indexDictionary(dictionary);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
final int off = readerContext.docBase;
final LeafReader r;
Object o = context.get("searcher");
if (o instanceof SolrIndexSearcher) {
SolrIndexSearcher is = (SolrIndexSearcher) o;
SchemaField sf = is.getSchema().getFieldOrNull(field);
if (sf != null
&& sf.hasDocValues() == false
&& sf.multiValued() == false
&& sf.getType().getNumericType() != null) {
// it's a single-valued numeric field: we must currently create insanity :(
List<LeafReaderContext> leaves = is.getIndexReader().leaves();
LeafReader insaneLeaves[] = new LeafReader[leaves.size()];
int upto = 0;
for (LeafReaderContext raw : leaves) {
insaneLeaves[upto++] = Insanity.wrapInsanity(raw.reader(), field);
}
r = SlowCompositeReaderWrapper.wrap(new MultiReader(insaneLeaves));
} else {
// reuse ordinalmap
r = ((SolrIndexSearcher) o).getLeafReader();
}
} else {
IndexReader topReader = ReaderUtil.getTopLevelContext(readerContext).reader();
r = SlowCompositeReaderWrapper.wrap(topReader);
}
// if it's e.g. tokenized/multivalued, emulate old behavior of single-valued fc
final SortedDocValues sindex =
SortedSetSelector.wrap(DocValues.getSortedSet(r, field), SortedSetSelector.Type.MIN);
final int end = sindex.getValueCount();
return new IntDocValues(this) {
@Override
public int intVal(int doc) {
return (end - sindex.getOrd(doc + off) - 1);
}
};
}
private void doHighlightingByHighlighter(
Query query,
SolrQueryRequest req,
NamedList docSummaries,
int docId,
Document doc,
String fieldName)
throws IOException {
SolrParams params = req.getParams();
String[] docTexts = doc.getValues(fieldName);
// according to Document javadoc, doc.getValues() never returns null. check empty instead of
// null
if (docTexts.length == 0) return;
SolrIndexSearcher searcher = req.getSearcher();
IndexSchema schema = searcher.getSchema();
TokenStream tstream = null;
int numFragments = getMaxSnippets(fieldName, params);
boolean mergeContiguousFragments = isMergeContiguousFragments(fieldName, params);
String[] summaries = null;
List<TextFragment> frags = new ArrayList<TextFragment>();
TermOffsetsTokenStream tots = null; // to be non-null iff we're using TermOffsets optimization
try {
TokenStream tvStream =
TokenSources.getTokenStream(searcher.getIndexReader(), docId, fieldName);
if (tvStream != null) {
tots = new TermOffsetsTokenStream(tvStream);
}
} catch (IllegalArgumentException e) {
// No problem. But we can't use TermOffsets optimization.
}
for (int j = 0; j < docTexts.length; j++) {
if (tots != null) {
// if we're using TermOffsets optimization, then get the next
// field value's TokenStream (i.e. get field j's TokenStream) from tots:
tstream = tots.getMultiValuedTokenStream(docTexts[j].length());
} else {
// fall back to analyzer
tstream = createAnalyzerTStream(schema, fieldName, docTexts[j]);
}
Highlighter highlighter;
if (Boolean.valueOf(req.getParams().get(HighlightParams.USE_PHRASE_HIGHLIGHTER, "true"))) {
// TODO: this is not always necessary - eventually we would like to avoid this wrap
// when it is not needed.
tstream = new CachingTokenFilter(tstream);
// get highlighter
highlighter = getPhraseHighlighter(query, fieldName, req, (CachingTokenFilter) tstream);
// after highlighter initialization, reset tstream since construction of highlighter already
// used it
tstream.reset();
} else {
// use "the old way"
highlighter = getHighlighter(query, fieldName, req);
}
int maxCharsToAnalyze =
params.getFieldInt(
fieldName, HighlightParams.MAX_CHARS, Highlighter.DEFAULT_MAX_CHARS_TO_ANALYZE);
if (maxCharsToAnalyze < 0) {
highlighter.setMaxDocCharsToAnalyze(docTexts[j].length());
} else {
highlighter.setMaxDocCharsToAnalyze(maxCharsToAnalyze);
}
try {
TextFragment[] bestTextFragments =
highlighter.getBestTextFragments(
tstream, docTexts[j], mergeContiguousFragments, numFragments);
for (int k = 0; k < bestTextFragments.length; k++) {
if ((bestTextFragments[k] != null) && (bestTextFragments[k].getScore() > 0)) {
frags.add(bestTextFragments[k]);
}
}
} catch (InvalidTokenOffsetsException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
// sort such that the fragments with the highest score come first
Collections.sort(
frags,
new Comparator<TextFragment>() {
public int compare(TextFragment arg0, TextFragment arg1) {
return Math.round(arg1.getScore() - arg0.getScore());
}
});
// convert fragments back into text
// TODO: we can include score and position information in output as snippet attributes
if (frags.size() > 0) {
ArrayList<String> fragTexts = new ArrayList<String>();
for (TextFragment fragment : frags) {
if ((fragment != null) && (fragment.getScore() > 0)) {
fragTexts.add(fragment.toString());
}
if (fragTexts.size() >= numFragments) break;
}
summaries = fragTexts.toArray(new String[0]);
if (summaries.length > 0) docSummaries.add(fieldName, summaries);
}
// no summeries made, copy text from alternate field
if (summaries == null || summaries.length == 0) {
alternateField(docSummaries, params, doc, fieldName);
}
}
@Override
public void process(ResponseBuilder rb) throws IOException {
SolrParams params = rb.req.getParams();
if (!params.getBool(COMPONENT_NAME, false)) {
return;
}
NamedList<Object> termVectors = new NamedList<Object>();
rb.rsp.add(TERM_VECTORS, termVectors);
IndexSchema schema = rb.req.getSchema();
SchemaField keyField = schema.getUniqueKeyField();
String uniqFieldName = null;
if (keyField != null) {
uniqFieldName = keyField.getName();
termVectors.add("uniqueKeyFieldName", uniqFieldName);
}
FieldOptions allFields = new FieldOptions();
// figure out what options we have, and try to get the appropriate vector
allFields.termFreq = params.getBool(TermVectorParams.TF, false);
allFields.positions = params.getBool(TermVectorParams.POSITIONS, false);
allFields.offsets = params.getBool(TermVectorParams.OFFSETS, false);
allFields.docFreq = params.getBool(TermVectorParams.DF, false);
allFields.tfIdf = params.getBool(TermVectorParams.TF_IDF, false);
// boolean cacheIdf = params.getBool(TermVectorParams.IDF, false);
// short cut to all values.
if (params.getBool(TermVectorParams.ALL, false)) {
allFields.termFreq = true;
allFields.positions = true;
allFields.offsets = true;
allFields.docFreq = true;
allFields.tfIdf = true;
}
// Build up our per field mapping
Map<String, FieldOptions> fieldOptions = new HashMap<String, FieldOptions>();
NamedList<List<String>> warnings = new NamedList<List<String>>();
List<String> noTV = new ArrayList<String>();
List<String> noPos = new ArrayList<String>();
List<String> noOff = new ArrayList<String>();
Set<String> fields = getFields(rb);
if (null != fields) {
// we have specific fields to retrieve, or no fields
for (String field : fields) {
// workarround SOLR-3523
if (null == field || "score".equals(field)) continue;
// we don't want to issue warnings about the uniqueKey field
// since it can cause lots of confusion in distributed requests
// where the uniqueKey field is injected into the fl for merging
final boolean fieldIsUniqueKey = field.equals(uniqFieldName);
SchemaField sf = schema.getFieldOrNull(field);
if (sf != null) {
if (sf.storeTermVector()) {
FieldOptions option = fieldOptions.get(field);
if (option == null) {
option = new FieldOptions();
option.fieldName = field;
fieldOptions.put(field, option);
}
// get the per field mappings
option.termFreq = params.getFieldBool(field, TermVectorParams.TF, allFields.termFreq);
option.docFreq = params.getFieldBool(field, TermVectorParams.DF, allFields.docFreq);
option.tfIdf = params.getFieldBool(field, TermVectorParams.TF_IDF, allFields.tfIdf);
// Validate these are even an option
option.positions =
params.getFieldBool(field, TermVectorParams.POSITIONS, allFields.positions);
if (option.positions && !sf.storeTermPositions() && !fieldIsUniqueKey) {
noPos.add(field);
}
option.offsets =
params.getFieldBool(field, TermVectorParams.OFFSETS, allFields.offsets);
if (option.offsets && !sf.storeTermOffsets() && !fieldIsUniqueKey) {
noOff.add(field);
}
} else { // field doesn't have term vectors
if (!fieldIsUniqueKey) noTV.add(field);
}
} else {
// field doesn't exist
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "undefined field: " + field);
}
}
} // else, deal with all fields
// NOTE: currently all typs of warnings are schema driven, and garunteed
// to be consistent across all shards - if additional types of warnings
// are added that might be differnet between shards, finishStage() needs
// to be changed to account for that.
boolean hasWarnings = false;
if (!noTV.isEmpty()) {
warnings.add("noTermVectors", noTV);
hasWarnings = true;
}
if (!noPos.isEmpty()) {
warnings.add("noPositions", noPos);
hasWarnings = true;
}
if (!noOff.isEmpty()) {
warnings.add("noOffsets", noOff);
hasWarnings = true;
}
if (hasWarnings) {
termVectors.add("warnings", warnings);
}
DocListAndSet listAndSet = rb.getResults();
List<Integer> docIds = getInts(params.getParams(TermVectorParams.DOC_IDS));
Iterator<Integer> iter;
if (docIds != null && !docIds.isEmpty()) {
iter = docIds.iterator();
} else {
DocList list = listAndSet.docList;
iter = list.iterator();
}
SolrIndexSearcher searcher = rb.req.getSearcher();
IndexReader reader = searcher.getIndexReader();
// the TVMapper is a TermVectorMapper which can be used to optimize loading of Term Vectors
// Only load the id field to get the uniqueKey of that
// field
final String finalUniqFieldName = uniqFieldName;
final List<String> uniqValues = new ArrayList<String>();
// TODO: is this required to be single-valued? if so, we should STOP
// once we find it...
final StoredFieldVisitor getUniqValue =
new StoredFieldVisitor() {
@Override
public void stringField(FieldInfo fieldInfo, String value) {
uniqValues.add(value);
}
@Override
public void intField(FieldInfo fieldInfo, int value) {
uniqValues.add(Integer.toString(value));
}
@Override
public void longField(FieldInfo fieldInfo, long value) {
uniqValues.add(Long.toString(value));
}
@Override
public Status needsField(FieldInfo fieldInfo) {
return (fieldInfo.name.equals(finalUniqFieldName)) ? Status.YES : Status.NO;
}
};
TermsEnum termsEnum = null;
while (iter.hasNext()) {
Integer docId = iter.next();
NamedList<Object> docNL = new NamedList<Object>();
if (keyField != null) {
reader.document(docId, getUniqValue);
String uniqVal = null;
if (uniqValues.size() != 0) {
uniqVal = uniqValues.get(0);
uniqValues.clear();
docNL.add("uniqueKey", uniqVal);
termVectors.add(uniqVal, docNL);
}
} else {
// support for schemas w/o a unique key,
termVectors.add("doc-" + docId, docNL);
}
if (null != fields) {
for (Map.Entry<String, FieldOptions> entry : fieldOptions.entrySet()) {
final String field = entry.getKey();
final Terms vector = reader.getTermVector(docId, field);
if (vector != null) {
termsEnum = vector.iterator(termsEnum);
mapOneVector(docNL, entry.getValue(), reader, docId, vector.iterator(termsEnum), field);
}
}
} else {
// extract all fields
final Fields vectors = reader.getTermVectors(docId);
for (String field : vectors) {
Terms terms = vectors.terms(field);
if (terms != null) {
termsEnum = terms.iterator(termsEnum);
mapOneVector(docNL, allFields, reader, docId, termsEnum, field);
}
}
}
}
}
private void doHighlightingByHighlighter(
Query query,
SolrQueryRequest req,
NamedList docSummaries,
int docId,
Document doc,
String fieldName)
throws IOException {
final SolrIndexSearcher searcher = req.getSearcher();
final IndexSchema schema = searcher.getSchema();
// TODO: Currently in trunk highlighting numeric fields is broken (Lucene) -
// so we disable them until fixed (see LUCENE-3080)!
// BEGIN: Hack
final SchemaField schemaField = schema.getFieldOrNull(fieldName);
if (schemaField != null
&& ((schemaField.getType() instanceof org.apache.solr.schema.TrieField)
|| (schemaField.getType() instanceof org.apache.solr.schema.TrieDateField))) return;
// END: Hack
SolrParams params = req.getParams();
IndexableField[] docFields = doc.getFields(fieldName);
List<String> listFields = new ArrayList<String>();
for (IndexableField field : docFields) {
listFields.add(field.stringValue());
}
// preserve order of values in a multiValued list
boolean preserveMulti = params.getFieldBool(fieldName, HighlightParams.PRESERVE_MULTI, false);
String[] docTexts = (String[]) listFields.toArray(new String[listFields.size()]);
// according to Document javadoc, doc.getValues() never returns null. check empty instead of
// null
if (docTexts.length == 0) return;
TokenStream tstream = null;
int numFragments = getMaxSnippets(fieldName, params);
boolean mergeContiguousFragments = isMergeContiguousFragments(fieldName, params);
String[] summaries = null;
List<TextFragment> frags = new ArrayList<TextFragment>();
TermOffsetsTokenStream tots = null; // to be non-null iff we're using TermOffsets optimization
TokenStream tvStream =
TokenSources.getTokenStreamWithOffsets(searcher.getIndexReader(), docId, fieldName);
if (tvStream != null) {
tots = new TermOffsetsTokenStream(tvStream);
}
for (int j = 0; j < docTexts.length; j++) {
if (tots != null) {
// if we're using TermOffsets optimization, then get the next
// field value's TokenStream (i.e. get field j's TokenStream) from tots:
tstream = tots.getMultiValuedTokenStream(docTexts[j].length());
} else {
// fall back to analyzer
tstream = createAnalyzerTStream(schema, fieldName, docTexts[j]);
}
int maxCharsToAnalyze =
params.getFieldInt(
fieldName, HighlightParams.MAX_CHARS, Highlighter.DEFAULT_MAX_CHARS_TO_ANALYZE);
Highlighter highlighter;
if (Boolean.valueOf(req.getParams().get(HighlightParams.USE_PHRASE_HIGHLIGHTER, "true"))) {
if (maxCharsToAnalyze < 0) {
tstream = new CachingTokenFilter(tstream);
} else {
tstream = new CachingTokenFilter(new OffsetLimitTokenFilter(tstream, maxCharsToAnalyze));
}
// get highlighter
highlighter = getPhraseHighlighter(query, fieldName, req, (CachingTokenFilter) tstream);
// after highlighter initialization, reset tstream since construction of highlighter already
// used it
tstream.reset();
} else {
// use "the old way"
highlighter = getHighlighter(query, fieldName, req);
}
if (maxCharsToAnalyze < 0) {
highlighter.setMaxDocCharsToAnalyze(docTexts[j].length());
} else {
highlighter.setMaxDocCharsToAnalyze(maxCharsToAnalyze);
}
try {
TextFragment[] bestTextFragments =
highlighter.getBestTextFragments(
tstream, docTexts[j], mergeContiguousFragments, numFragments);
for (int k = 0; k < bestTextFragments.length; k++) {
if (preserveMulti) {
if (bestTextFragments[k] != null) {
frags.add(bestTextFragments[k]);
}
} else {
if ((bestTextFragments[k] != null) && (bestTextFragments[k].getScore() > 0)) {
frags.add(bestTextFragments[k]);
}
}
}
} catch (InvalidTokenOffsetsException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
// sort such that the fragments with the highest score come first
if (!preserveMulti) {
Collections.sort(
frags,
new Comparator<TextFragment>() {
@Override
public int compare(TextFragment arg0, TextFragment arg1) {
return Math.round(arg1.getScore() - arg0.getScore());
}
});
}
// convert fragments back into text
// TODO: we can include score and position information in output as snippet attributes
if (frags.size() > 0) {
ArrayList<String> fragTexts = new ArrayList<String>();
for (TextFragment fragment : frags) {
if (preserveMulti) {
if (fragment != null) {
fragTexts.add(fragment.toString());
}
} else {
if ((fragment != null) && (fragment.getScore() > 0)) {
fragTexts.add(fragment.toString());
}
}
if (fragTexts.size() >= numFragments && !preserveMulti) break;
}
summaries = fragTexts.toArray(new String[0]);
if (summaries.length > 0) docSummaries.add(fieldName, summaries);
}
// no summeries made, copy text from alternate field
if (summaries == null || summaries.length == 0) {
alternateField(docSummaries, params, doc, fieldName);
}
}
