/**
* check if a given document, identified by url hash as document id exists
*
* @param id the url hash and document id
* @return the load date if any entry in solr exists, -1 otherwise
* @throws IOException
*/
@Override
public LoadTimeURL getLoadTimeURL(String id) throws IOException {
int responseCount = 0;
DocListSearcher docListSearcher = null;
try {
docListSearcher =
new DocListSearcher(
"{!cache=false raw f=" + CollectionSchema.id.getSolrFieldName() + "}" + id,
null,
0,
1,
CollectionSchema.id.getSolrFieldName(),
CollectionSchema.load_date_dt.getSolrFieldName());
responseCount = docListSearcher.response.size();
if (responseCount == 0) return null;
SolrIndexSearcher searcher = docListSearcher.request.getSearcher();
DocIterator iterator = docListSearcher.response.iterator();
// for (int i = 0; i < responseCount; i++) {
Document doc =
searcher.doc(iterator.nextDoc(), AbstractSolrConnector.SOLR_ID_and_LOAD_DATE_FIELDS);
if (doc == null) return null;
return AbstractSolrConnector.getLoadTimeURL(doc);
// }
} catch (Throwable e) {
ConcurrentLog.logException(e);
throw new IOException(e.getMessage());
} finally {
if (docListSearcher != null) docListSearcher.close();
}
}
public void writeResponse() throws IOException {
Boolean omitHeader = req.getParams().getBool(CommonParams.OMIT_HEADER);
if (omitHeader != null && omitHeader) rsp.getValues().remove("responseHeader");
SolrIndexSearcher searcher = req.getSearcher();
if (liveDocs == null) {
liveDocs = searcher.getLeafReader().getLiveDocs();
}
int maxDoc = searcher.maxDoc();
try {
// responseWriter.write(sw,req,rsp);
ReturnFields fields = rsp.getReturnFields(); // return everything
Set<String> fnames = fields.getLuceneFieldNames();
int docCounter = 0;
for (int i = 0; i < maxDoc; i++) {
if (liveDocs != null && !liveDocs.get(i)) {
continue;
}
Document doc = searcher.doc(i);
SolrDocument sdoc = toSolrDocument(doc, schema);
writeSolrDocument(null, sdoc, fields, docCounter++);
getWriter().write("\n");
}
} finally {
close();
writer.write('\n'); // ending with a newline looks much better from the command line
writer.close();
}
}
/**
* conversion from a SolrQueryResponse (which is a solr-internal data format) to SolrDocumentList
* (which is a solrj-format) The conversion is done inside the solrj api using the
* BinaryResponseWriter and a very complex unfolding process via
* org.apache.solr.common.util.JavaBinCodec.marshal.
*
* @param request
* @param sqr
* @return
*/
public SolrDocumentList SolrQueryResponse2SolrDocumentList(
final SolrQueryRequest req, final SolrQueryResponse rsp) {
SolrDocumentList sdl = new SolrDocumentList();
NamedList<?> nl = rsp.getValues();
ResultContext resultContext = (ResultContext) nl.get("response");
DocList response =
resultContext == null
? new DocSlice(0, 0, new int[0], new float[0], 0, 0.0f)
: resultContext.docs;
sdl.setNumFound(response == null ? 0 : response.matches());
sdl.setStart(response == null ? 0 : response.offset());
String originalName = Thread.currentThread().getName();
if (response != null) {
try {
SolrIndexSearcher searcher = req.getSearcher();
final int responseCount = response.size();
DocIterator iterator = response.iterator();
for (int i = 0; i < responseCount; i++) {
int docid = iterator.nextDoc();
Thread.currentThread()
.setName("EmbeddedSolrConnector.SolrQueryResponse2SolrDocumentList: " + docid);
Document responsedoc = searcher.doc(docid, (Set<String>) null);
SolrDocument sordoc = doc2SolrDoc(responsedoc);
sdl.add(sordoc);
}
} catch (IOException e) {
ConcurrentLog.logException(e);
}
}
Thread.currentThread().setName(originalName);
return sdl;
}
@Override
public void writeDocList(String name, DocList ids, Set<String> fields, Map otherFields)
throws IOException {
boolean includeScore = false;
if (fields != null) {
includeScore = fields.contains("score");
if (fields.size() == 0 || (fields.size() == 1 && includeScore) || fields.contains("*")) {
fields = null; // null means return all stored fields
}
}
int sz = ids.size();
writeMapOpener(includeScore ? 4 : 3);
incLevel();
writeKey("numFound", false);
writeInt(null, ids.matches());
writeMapSeparator();
writeKey("start", false);
writeInt(null, ids.offset());
if (includeScore) {
writeMapSeparator();
writeKey("maxScore", false);
writeFloat(null, ids.maxScore());
}
writeMapSeparator();
// indent();
writeKey("docs", false);
writeArrayOpener(sz);
incLevel();
boolean first = true;
SolrIndexSearcher searcher = req.getSearcher();
DocIterator iterator = ids.iterator();
for (int i = 0; i < sz; i++) {
int id = iterator.nextDoc();
Document doc = searcher.doc(id, fields);
if (first) {
first = false;
} else {
writeArraySeparator();
}
indent();
writeDoc(null, doc, fields, (includeScore ? iterator.score() : 0.0f), includeScore);
}
decLevel();
writeArrayCloser();
if (otherFields != null) {
writeMap(null, otherFields, true, false);
}
decLevel();
indent();
writeMapCloser();
}
/**
* Generates a list of Highlighted query fragments for each item in a list of documents, or
* returns null if highlighting is disabled.
*
* @param docs query results
* @param query the query
* @param req the current request
* @param defaultFields default list of fields to summarize
* @return NamedList containing a NamedList for each document, which in turns contains sets
* (field, summary) pairs.
*/
@Override
@SuppressWarnings("unchecked")
public NamedList<Object> doHighlighting(
DocList docs, Query query, SolrQueryRequest req, String[] defaultFields) throws IOException {
SolrParams params = req.getParams();
if (!isHighlightingEnabled(params)) return null;
SolrIndexSearcher searcher = req.getSearcher();
IndexSchema schema = searcher.getSchema();
NamedList fragments = new SimpleOrderedMap();
String[] fieldNames = getHighlightFields(query, req, defaultFields);
Set<String> fset = new HashSet<String>();
{
// pre-fetch documents using the Searcher's doc cache
for (String f : fieldNames) {
fset.add(f);
}
// fetch unique key if one exists.
SchemaField keyField = schema.getUniqueKeyField();
if (null != keyField) fset.add(keyField.getName());
}
// get FastVectorHighlighter instance out of the processing loop
FastVectorHighlighter fvh =
new FastVectorHighlighter(
// FVH cannot process hl.usePhraseHighlighter parameter per-field basis
params.getBool(HighlightParams.USE_PHRASE_HIGHLIGHTER, true),
// FVH cannot process hl.requireFieldMatch parameter per-field basis
params.getBool(HighlightParams.FIELD_MATCH, false));
fvh.setPhraseLimit(params.getInt(HighlightParams.PHRASE_LIMIT, Integer.MAX_VALUE));
FieldQuery fieldQuery = fvh.getFieldQuery(query, searcher.getIndexReader());
// Highlight each document
DocIterator iterator = docs.iterator();
for (int i = 0; i < docs.size(); i++) {
int docId = iterator.nextDoc();
Document doc = searcher.doc(docId, fset);
NamedList docSummaries = new SimpleOrderedMap();
for (String fieldName : fieldNames) {
fieldName = fieldName.trim();
if (useFastVectorHighlighter(params, schema, fieldName))
doHighlightingByFastVectorHighlighter(
fvh, fieldQuery, req, docSummaries, docId, doc, fieldName);
else doHighlightingByHighlighter(query, req, docSummaries, docId, doc, fieldName);
}
String printId = schema.printableUniqueKey(doc);
fragments.add(printId == null ? null : printId, docSummaries);
}
return fragments;
}
public static SolrInputDocument getInputDocument(SolrCore core, BytesRef idBytes)
throws IOException {
SolrInputDocument sid = null;
RefCounted<SolrIndexSearcher> searcherHolder = null;
try {
SolrIndexSearcher searcher = null;
UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
if (ulog != null) {
Object o = ulog.lookup(idBytes);
if (o != null) {
// should currently be a List<Oper,Ver,Doc/Id>
List entry = (List) o;
assert entry.size() >= 3;
int oper = (Integer) entry.get(0) & UpdateLog.OPERATION_MASK;
switch (oper) {
case UpdateLog.ADD:
sid = (SolrInputDocument) entry.get(entry.size() - 1);
break;
case UpdateLog.DELETE:
return null;
default:
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, "Unknown Operation! " + oper);
}
}
}
if (sid == null) {
// didn't find it in the update log, so it should be in the newest searcher opened
if (searcher == null) {
searcherHolder = core.getRealtimeSearcher();
searcher = searcherHolder.get();
}
// SolrCore.verbose("RealTimeGet using searcher ", searcher);
SchemaField idField = core.getLatestSchema().getUniqueKeyField();
int docid = searcher.getFirstMatch(new Term(idField.getName(), idBytes));
if (docid < 0) return null;
StoredDocument luceneDocument = searcher.doc(docid);
sid = toSolrInputDocument(luceneDocument, core.getLatestSchema());
}
} finally {
if (searcherHolder != null) {
searcherHolder.decref();
}
}
return sid;
}
protected void processIds(
ResponseBuilder rb, DocList dl, IndexSchema schema, SolrIndexSearcher searcher)
throws IOException {
StringBuilder sb = new StringBuilder();
Set<String> fields = Collections.singleton(schema.getUniqueKeyField().getName());
for (DocIterator iter = dl.iterator(); iter.hasNext(); ) {
sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))).append(',');
}
if (sb.length() > 0) {
rb.rsp.addToLog("responseLog", sb.substring(0, sb.length() - 1));
}
}
private void mockDocument(int docId, double price, double discount, boolean isCloseout)
throws IOException {
Document doc = PowerMockito.mock(Document.class);
when(searcher.doc(eq(docId), any(Set.class))).thenReturn(doc);
when(searcher.getSchema()).thenReturn(schema);
IndexableField priceField = mock(IndexableField.class);
when(doc.getField(FIELD_PRICE)).thenReturn(priceField);
when(priceField.numericValue()).thenReturn(new Double(price));
IndexableField discountField = mock(IndexableField.class);
when(doc.getField(FIELD_DISCOUNT)).thenReturn(discountField);
when(discountField.numericValue()).thenReturn(new Double(discount));
IndexableField closeoutField = mock(IndexableField.class);
when(doc.getField(FIELD_CLOSEOUT)).thenReturn(closeoutField);
when(closeoutField.stringValue()).thenReturn(isCloseout ? "T" : "F");
}
@SuppressWarnings("unchecked")
private static SimpleOrderedMap<Object> getIndexedFieldsInfo(
final SolrIndexSearcher searcher, final Set<String> fields, final int numTerms)
throws Exception {
IndexReader reader = searcher.getReader();
IndexSchema schema = searcher.getSchema();
// Walk the term enum and keep a priority queue for each map in our set
Map<String, TopTermQueue> ttinfo = null;
if (numTerms > 0) {
ttinfo = getTopTerms(reader, fields, numTerms, null);
}
SimpleOrderedMap<Object> finfo = new SimpleOrderedMap<Object>();
Collection<String> fieldNames = reader.getFieldNames(IndexReader.FieldOption.ALL);
for (String fieldName : fieldNames) {
if (fields != null && !fields.contains(fieldName)) {
continue; // if a field is specified, only them
}
SimpleOrderedMap<Object> f = new SimpleOrderedMap<Object>();
SchemaField sfield = schema.getFieldOrNull(fieldName);
FieldType ftype = (sfield == null) ? null : sfield.getType();
f.add("type", (ftype == null) ? null : ftype.getTypeName());
f.add("schema", getFieldFlags(sfield));
if (sfield != null
&& schema.isDynamicField(sfield.getName())
&& schema.getDynamicPattern(sfield.getName()) != null) {
f.add("dynamicBase", schema.getDynamicPattern(sfield.getName()));
}
// If numTerms==0, the call is just asking for a quick field list
if (ttinfo != null && sfield != null && sfield.indexed()) {
Query q = new TermRangeQuery(fieldName, null, null, false, false);
TopDocs top = searcher.search(q, 1);
if (top.totalHits > 0) {
// Find a document with this field
try {
Document doc = searcher.doc(top.scoreDocs[0].doc);
Fieldable fld = doc.getFieldable(fieldName);
if (fld != null) {
f.add("index", getFieldFlags(fld));
} else {
// it is a non-stored field...
f.add("index", "(unstored field)");
}
} catch (Exception ex) {
log.warn("error reading field: " + fieldName);
}
}
f.add("docs", top.totalHits);
TopTermQueue topTerms = ttinfo.get(fieldName);
if (topTerms != null) {
f.add("distinct", topTerms.distinctTerms);
// Include top terms
f.add("topTerms", topTerms.toNamedList(searcher.getSchema()));
// Add a histogram
f.add("histogram", topTerms.histogram.toNamedList());
}
}
// Add the field
finfo.add(fieldName, f);
}
return finfo;
}
@Override
public void process(ResponseBuilder rb) throws IOException {
SolrQueryRequest req = rb.req;
SolrQueryResponse rsp = rb.rsp;
SolrParams params = req.getParams();
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
String val = params.get("getVersions");
if (val != null) {
processGetVersions(rb);
return;
}
val = params.get("getUpdates");
if (val != null) {
processGetUpdates(rb);
return;
}
String id[] = params.getParams("id");
String ids[] = params.getParams("ids");
if (id == null && ids == null) {
return;
}
String[] allIds = id == null ? new String[0] : id;
if (ids != null) {
List<String> lst = new ArrayList<String>();
for (String s : allIds) {
lst.add(s);
}
for (String idList : ids) {
lst.addAll(StrUtils.splitSmart(idList, ",", true));
}
allIds = lst.toArray(new String[lst.size()]);
}
SolrCore core = req.getCore();
SchemaField idField = core.getLatestSchema().getUniqueKeyField();
FieldType fieldType = idField.getType();
SolrDocumentList docList = new SolrDocumentList();
UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
RefCounted<SolrIndexSearcher> searcherHolder = null;
DocTransformer transformer = rsp.getReturnFields().getTransformer();
if (transformer != null) {
TransformContext context = new TransformContext();
context.req = req;
transformer.setContext(context);
}
try {
SolrIndexSearcher searcher = null;
BytesRef idBytes = new BytesRef();
for (String idStr : allIds) {
fieldType.readableToIndexed(idStr, idBytes);
if (ulog != null) {
Object o = ulog.lookup(idBytes);
if (o != null) {
// should currently be a List<Oper,Ver,Doc/Id>
List entry = (List) o;
assert entry.size() >= 3;
int oper = (Integer) entry.get(0) & UpdateLog.OPERATION_MASK;
switch (oper) {
case UpdateLog.ADD:
SolrDocument doc =
toSolrDoc(
(SolrInputDocument) entry.get(entry.size() - 1), core.getLatestSchema());
if (transformer != null) {
transformer.transform(doc, -1); // unknown docID
}
docList.add(doc);
break;
case UpdateLog.DELETE:
break;
default:
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR, "Unknown Operation! " + oper);
}
continue;
}
}
// didn't find it in the update log, so it should be in the newest searcher opened
if (searcher == null) {
searcherHolder = core.getRealtimeSearcher();
searcher = searcherHolder.get();
}
// SolrCore.verbose("RealTimeGet using searcher ", searcher);
int docid = searcher.getFirstMatch(new Term(idField.getName(), idBytes));
if (docid < 0) continue;
StoredDocument luceneDocument = searcher.doc(docid);
SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema());
if (transformer != null) {
transformer.transform(doc, docid);
}
docList.add(doc);
}
} finally {
if (searcherHolder != null) {
searcherHolder.decref();
}
}
// if the client specified a single id=foo, then use "doc":{
// otherwise use a standard doclist
if (ids == null && allIds.length <= 1) {
// if the doc was not found, then use a value of null.
rsp.add("doc", docList.size() > 0 ? docList.get(0) : null);
} else {
docList.setNumFound(docList.size());
rsp.add("response", docList);
}
}