@Test
public void testNaNInf() throws IOException {
SolrQueryRequest req = req("dummy");
SolrQueryResponse rsp = new SolrQueryResponse();
QueryResponseWriter w = new PythonResponseWriter();
StringWriter buf = new StringWriter();
rsp.add("data1", Float.NaN);
rsp.add("data2", Double.NEGATIVE_INFINITY);
rsp.add("data3", Float.POSITIVE_INFINITY);
w.write(buf, req, rsp);
assertEquals(
buf.toString(), "{'data1':float('NaN'),'data2':-float('Inf'),'data3':float('Inf')}");
w = new RubyResponseWriter();
buf = new StringWriter();
w.write(buf, req, rsp);
assertEquals(buf.toString(), "{'data1'=>(0.0/0.0),'data2'=>-(1.0/0.0),'data3'=>(1.0/0.0)}");
w = new JSONResponseWriter();
buf = new StringWriter();
w.write(buf, req, rsp);
assertEquals(
buf.toString(), "{\"data1\":\"NaN\",\"data2\":\"-Infinity\",\"data3\":\"Infinity\"}");
req.close();
}
public void testJSON() throws IOException {
SolrQueryRequest req = req("wt", "json", "json.nl", "arrarr");
SolrQueryResponse rsp = new SolrQueryResponse();
JSONResponseWriter w = new JSONResponseWriter();
StringWriter buf = new StringWriter();
NamedList nl = new NamedList();
nl.add("data1", "hello");
nl.add(null, 42);
rsp.add("nl", nl);
w.write(buf, req, rsp);
assertEquals(buf.toString(), "{\"nl\":[[\"data1\",\"hello\"],[null,42]]}");
req.close();
}
public SimpleFacets(SolrQueryRequest req, DocSet docs, SolrParams params, ResponseBuilder rb) {
this.req = req;
this.searcher = req.getSearcher();
this.docsOrig = docs;
this.global = params;
this.rb = rb;
}
void doPerf(String writerName, SolrQueryRequest req, int encIter, int decIter) throws Exception {
SolrQueryResponse rsp = getResponse(req);
QueryResponseWriter w = h.getCore().getQueryResponseWriter(writerName);
ByteArrayOutputStream out = null;
System.gc();
long start = System.currentTimeMillis();
for (int i = 0; i < encIter; i++) {
if (w instanceof BinaryQueryResponseWriter) {
BinaryQueryResponseWriter binWriter = (BinaryQueryResponseWriter) w;
out = new ByteArrayOutputStream();
binWriter.write(out, req, rsp);
out.close();
} else {
out = new ByteArrayOutputStream();
// to be fair, from my previous tests, much of the performance will be sucked up
// by java's UTF-8 encoding/decoding, not the actual writing
Writer writer = new OutputStreamWriter(out, "UTF-8");
w.write(writer, req, rsp);
writer.close();
}
}
long encodeTime = Math.max(System.currentTimeMillis() - start, 1);
byte[] arr = out.toByteArray();
start = System.currentTimeMillis();
writerName = writerName.intern();
for (int i = 0; i < decIter; i++) {
ResponseParser rp = null;
if (writerName == "xml") {
rp = new XMLResponseParser();
} else if (writerName == "javabin") {
rp = new BinaryResponseParser();
} else {
break;
}
ByteArrayInputStream in = new ByteArrayInputStream(arr);
rp.processResponse(in, "UTF-8");
}
long decodeTime = Math.max(System.currentTimeMillis() - start, 1);
log.info(
"writer "
+ writerName
+ ", size="
+ out.size()
+ ", encodeRate="
+ (encodeTime == 1 ? "N/A" : "" + (encIter * 1000L / encodeTime))
+ ", decodeRate="
+ (decodeTime == 1 ? "N/A" : "" + (decIter * 1000L / decodeTime)));
req.close();
SolrRequestInfo.clearRequestInfo();
}
protected DocSet computeDocSet(DocSet baseDocSet, List<String> excludeTagList)
throws SyntaxError, IOException {
Map<?, ?> tagMap = (Map<?, ?>) req.getContext().get("tags");
// rb can be null if facets are being calculated from a RequestHandler e.g. MoreLikeThisHandler
if (tagMap == null || rb == null) {
return baseDocSet;
}
IdentityHashMap<Query, Boolean> excludeSet = new IdentityHashMap<>();
for (String excludeTag : excludeTagList) {
Object olst = tagMap.get(excludeTag);
// tagMap has entries of List<String,List<QParser>>, but subject to change in the future
if (!(olst instanceof Collection)) continue;
for (Object o : (Collection<?>) olst) {
if (!(o instanceof QParser)) continue;
QParser qp = (QParser) o;
excludeSet.put(qp.getQuery(), Boolean.TRUE);
}
}
if (excludeSet.size() == 0) return baseDocSet;
List<Query> qlist = new ArrayList<>();
// add the base query
if (!excludeSet.containsKey(rb.getQuery())) {
qlist.add(rb.getQuery());
}
// add the filters
if (rb.getFilters() != null) {
for (Query q : rb.getFilters()) {
if (!excludeSet.containsKey(q)) {
qlist.add(q);
}
}
}
// get the new base docset for this facet
DocSet base = searcher.getDocSet(qlist);
if (rb.grouping() && rb.getGroupingSpec().isTruncateGroups()) {
Grouping grouping = new Grouping(searcher, null, rb.getQueryCommand(), false, 0, false);
grouping.setWithinGroupSort(rb.getGroupingSpec().getSortWithinGroup());
if (rb.getGroupingSpec().getFields().length > 0) {
grouping.addFieldCommand(rb.getGroupingSpec().getFields()[0], req);
} else if (rb.getGroupingSpec().getFunctions().length > 0) {
grouping.addFunctionCommand(rb.getGroupingSpec().getFunctions()[0], req);
} else {
return base;
}
AbstractAllGroupHeadsCollector allGroupHeadsCollector =
grouping.getCommands().get(0).createAllGroupCollector();
searcher.search(base.getTopFilter(), allGroupHeadsCollector);
return new BitDocSet(allGroupHeadsCollector.retrieveGroupHeads(searcher.maxDoc()));
} else {
return base;
}
}
protected ParsedParams parseParams(String type, String param) throws SyntaxError, IOException {
SolrParams localParams = QueryParsing.getLocalParams(param, req.getParams());
DocSet docs = docsOrig;
String facetValue = param;
String key = param;
List<String> tags = Collections.emptyList();
int threads = -1;
if (localParams == null) {
SolrParams params = global;
SolrParams required = new RequiredSolrParams(params);
return new ParsedParams(localParams, params, required, facetValue, docs, key, tags, threads);
}
SolrParams params = SolrParams.wrapDefaults(localParams, global);
SolrParams required = new RequiredSolrParams(params);
// remove local params unless it's a query
if (type != FacetParams.FACET_QUERY) { // TODO Cut over to an Enum here
facetValue = localParams.get(CommonParams.VALUE);
}
// reset set the default key now that localParams have been removed
key = facetValue;
// allow explicit set of the key
key = localParams.get(CommonParams.OUTPUT_KEY, key);
String tagStr = localParams.get(CommonParams.TAG);
tags = tagStr == null ? Collections.<String>emptyList() : StrUtils.splitSmart(tagStr, ',');
String threadStr = localParams.get(CommonParams.THREADS);
if (threadStr != null) {
threads = Integer.parseInt(threadStr);
}
// figure out if we need a new base DocSet
String excludeStr = localParams.get(CommonParams.EXCLUDE);
if (excludeStr == null)
return new ParsedParams(localParams, params, required, facetValue, docs, key, tags, threads);
List<String> excludeTagList = StrUtils.splitSmart(excludeStr, ',');
docs = computeDocSet(docs, excludeTagList);
return new ParsedParams(localParams, params, required, facetValue, docs, key, tags, threads);
}
/**
* Returns a list of value constraints and the associated facet counts for each facet field
* specified in the params.
*
* @see FacetParams#FACET_FIELD
* @see #getFieldMissingCount
* @see #getFacetTermEnumCounts
*/
@SuppressWarnings("unchecked")
public NamedList<Object> getFacetFieldCounts() throws IOException, SyntaxError {
NamedList<Object> res = new SimpleOrderedMap<>();
String[] facetFs = global.getParams(FacetParams.FACET_FIELD);
if (null == facetFs) {
return res;
}
// Passing a negative number for FACET_THREADS implies an unlimited number of threads is
// acceptable.
// Also, a subtlety of directExecutor is that no matter how many times you "submit" a job, it's
// really
// just a method call in that it's run by the calling thread.
int maxThreads = req.getParams().getInt(FacetParams.FACET_THREADS, 0);
Executor executor = maxThreads == 0 ? directExecutor : facetExecutor;
final Semaphore semaphore = new Semaphore((maxThreads <= 0) ? Integer.MAX_VALUE : maxThreads);
List<Future<NamedList>> futures = new ArrayList<>(facetFs.length);
try {
// Loop over fields; submit to executor, keeping the future
for (String f : facetFs) {
final ParsedParams parsed = parseParams(FacetParams.FACET_FIELD, f);
final SolrParams localParams = parsed.localParams;
final String termList = localParams == null ? null : localParams.get(CommonParams.TERMS);
final String key = parsed.key;
final String facetValue = parsed.facetValue;
Callable<NamedList> callable =
new Callable<NamedList>() {
@Override
public NamedList call() throws Exception {
try {
NamedList<Object> result = new SimpleOrderedMap<>();
if (termList != null) {
List<String> terms = StrUtils.splitSmart(termList, ",", true);
result.add(key, getListedTermCounts(facetValue, parsed, terms));
} else {
result.add(key, getTermCounts(facetValue, parsed));
}
return result;
} catch (SolrException se) {
throw se;
} catch (Exception e) {
throw new SolrException(
ErrorCode.SERVER_ERROR, "Exception during facet.field: " + facetValue, e);
} finally {
semaphore.release();
}
}
};
RunnableFuture<NamedList> runnableFuture = new FutureTask<>(callable);
semaphore.acquire(); // may block and/or interrupt
executor.execute(runnableFuture); // releases semaphore when done
futures.add(runnableFuture);
} // facetFs loop
// Loop over futures to get the values. The order is the same as facetFs but shouldn't matter.
for (Future<NamedList> future : futures) {
res.addAll(future.get());
}
assert semaphore.availablePermits() >= maxThreads;
} catch (InterruptedException e) {
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"Error while processing facet fields: InterruptedException",
e);
} catch (ExecutionException ee) {
Throwable e = ee.getCause(); // unwrap
if (e instanceof RuntimeException) {
throw (RuntimeException) e;
}
throw new SolrException(
SolrException.ErrorCode.SERVER_ERROR,
"Error while processing facet fields: " + e.toString(),
e);
}
return res;
}