@Test
  public void testNaNInf() throws IOException {
    SolrQueryRequest req = req("dummy");
    SolrQueryResponse rsp = new SolrQueryResponse();
    QueryResponseWriter w = new PythonResponseWriter();

    StringWriter buf = new StringWriter();
    rsp.add("data1", Float.NaN);
    rsp.add("data2", Double.NEGATIVE_INFINITY);
    rsp.add("data3", Float.POSITIVE_INFINITY);
    w.write(buf, req, rsp);
    assertEquals(
        buf.toString(), "{'data1':float('NaN'),'data2':-float('Inf'),'data3':float('Inf')}");

    w = new RubyResponseWriter();
    buf = new StringWriter();
    w.write(buf, req, rsp);
    assertEquals(buf.toString(), "{'data1'=>(0.0/0.0),'data2'=>-(1.0/0.0),'data3'=>(1.0/0.0)}");

    w = new JSONResponseWriter();
    buf = new StringWriter();
    w.write(buf, req, rsp);
    assertEquals(
        buf.toString(), "{\"data1\":\"NaN\",\"data2\":\"-Infinity\",\"data3\":\"Infinity\"}");
    req.close();
  }
  public void testJSON() throws IOException {
    SolrQueryRequest req = req("wt", "json", "json.nl", "arrarr");
    SolrQueryResponse rsp = new SolrQueryResponse();
    JSONResponseWriter w = new JSONResponseWriter();

    StringWriter buf = new StringWriter();
    NamedList nl = new NamedList();
    nl.add("data1", "hello");
    nl.add(null, 42);
    rsp.add("nl", nl);

    w.write(buf, req, rsp);
    assertEquals(buf.toString(), "{\"nl\":[[\"data1\",\"hello\"],[null,42]]}");
    req.close();
  }
Esempio n. 3
0
 public SimpleFacets(SolrQueryRequest req, DocSet docs, SolrParams params, ResponseBuilder rb) {
   this.req = req;
   this.searcher = req.getSearcher();
   this.docsOrig = docs;
   this.global = params;
   this.rb = rb;
 }
  void doPerf(String writerName, SolrQueryRequest req, int encIter, int decIter) throws Exception {
    SolrQueryResponse rsp = getResponse(req);
    QueryResponseWriter w = h.getCore().getQueryResponseWriter(writerName);

    ByteArrayOutputStream out = null;

    System.gc();
    long start = System.currentTimeMillis();
    for (int i = 0; i < encIter; i++) {
      if (w instanceof BinaryQueryResponseWriter) {
        BinaryQueryResponseWriter binWriter = (BinaryQueryResponseWriter) w;
        out = new ByteArrayOutputStream();
        binWriter.write(out, req, rsp);
        out.close();
      } else {
        out = new ByteArrayOutputStream();
        // to be fair, from my previous tests, much of the performance will be sucked up
        // by java's UTF-8 encoding/decoding, not the actual writing
        Writer writer = new OutputStreamWriter(out, "UTF-8");
        w.write(writer, req, rsp);
        writer.close();
      }
    }

    long encodeTime = Math.max(System.currentTimeMillis() - start, 1);

    byte[] arr = out.toByteArray();

    start = System.currentTimeMillis();
    writerName = writerName.intern();
    for (int i = 0; i < decIter; i++) {
      ResponseParser rp = null;
      if (writerName == "xml") {
        rp = new XMLResponseParser();
      } else if (writerName == "javabin") {
        rp = new BinaryResponseParser();
      } else {
        break;
      }
      ByteArrayInputStream in = new ByteArrayInputStream(arr);
      rp.processResponse(in, "UTF-8");
    }

    long decodeTime = Math.max(System.currentTimeMillis() - start, 1);

    log.info(
        "writer "
            + writerName
            + ", size="
            + out.size()
            + ", encodeRate="
            + (encodeTime == 1 ? "N/A" : "" + (encIter * 1000L / encodeTime))
            + ", decodeRate="
            + (decodeTime == 1 ? "N/A" : "" + (decIter * 1000L / decodeTime)));

    req.close();
    SolrRequestInfo.clearRequestInfo();
  }
Esempio n. 5
0
  protected DocSet computeDocSet(DocSet baseDocSet, List<String> excludeTagList)
      throws SyntaxError, IOException {
    Map<?, ?> tagMap = (Map<?, ?>) req.getContext().get("tags");
    // rb can be null if facets are being calculated from a RequestHandler e.g. MoreLikeThisHandler
    if (tagMap == null || rb == null) {
      return baseDocSet;
    }

    IdentityHashMap<Query, Boolean> excludeSet = new IdentityHashMap<>();
    for (String excludeTag : excludeTagList) {
      Object olst = tagMap.get(excludeTag);
      // tagMap has entries of List<String,List<QParser>>, but subject to change in the future
      if (!(olst instanceof Collection)) continue;
      for (Object o : (Collection<?>) olst) {
        if (!(o instanceof QParser)) continue;
        QParser qp = (QParser) o;
        excludeSet.put(qp.getQuery(), Boolean.TRUE);
      }
    }
    if (excludeSet.size() == 0) return baseDocSet;

    List<Query> qlist = new ArrayList<>();

    // add the base query
    if (!excludeSet.containsKey(rb.getQuery())) {
      qlist.add(rb.getQuery());
    }

    // add the filters
    if (rb.getFilters() != null) {
      for (Query q : rb.getFilters()) {
        if (!excludeSet.containsKey(q)) {
          qlist.add(q);
        }
      }
    }

    // get the new base docset for this facet
    DocSet base = searcher.getDocSet(qlist);
    if (rb.grouping() && rb.getGroupingSpec().isTruncateGroups()) {
      Grouping grouping = new Grouping(searcher, null, rb.getQueryCommand(), false, 0, false);
      grouping.setWithinGroupSort(rb.getGroupingSpec().getSortWithinGroup());
      if (rb.getGroupingSpec().getFields().length > 0) {
        grouping.addFieldCommand(rb.getGroupingSpec().getFields()[0], req);
      } else if (rb.getGroupingSpec().getFunctions().length > 0) {
        grouping.addFunctionCommand(rb.getGroupingSpec().getFunctions()[0], req);
      } else {
        return base;
      }
      AbstractAllGroupHeadsCollector allGroupHeadsCollector =
          grouping.getCommands().get(0).createAllGroupCollector();
      searcher.search(base.getTopFilter(), allGroupHeadsCollector);
      return new BitDocSet(allGroupHeadsCollector.retrieveGroupHeads(searcher.maxDoc()));
    } else {
      return base;
    }
  }
Esempio n. 6
0
  protected ParsedParams parseParams(String type, String param) throws SyntaxError, IOException {
    SolrParams localParams = QueryParsing.getLocalParams(param, req.getParams());
    DocSet docs = docsOrig;
    String facetValue = param;
    String key = param;
    List<String> tags = Collections.emptyList();
    int threads = -1;

    if (localParams == null) {
      SolrParams params = global;
      SolrParams required = new RequiredSolrParams(params);
      return new ParsedParams(localParams, params, required, facetValue, docs, key, tags, threads);
    }

    SolrParams params = SolrParams.wrapDefaults(localParams, global);
    SolrParams required = new RequiredSolrParams(params);

    // remove local params unless it's a query
    if (type != FacetParams.FACET_QUERY) { // TODO Cut over to an Enum here
      facetValue = localParams.get(CommonParams.VALUE);
    }

    // reset set the default key now that localParams have been removed
    key = facetValue;

    // allow explicit set of the key
    key = localParams.get(CommonParams.OUTPUT_KEY, key);

    String tagStr = localParams.get(CommonParams.TAG);
    tags = tagStr == null ? Collections.<String>emptyList() : StrUtils.splitSmart(tagStr, ',');

    String threadStr = localParams.get(CommonParams.THREADS);
    if (threadStr != null) {
      threads = Integer.parseInt(threadStr);
    }

    // figure out if we need a new base DocSet
    String excludeStr = localParams.get(CommonParams.EXCLUDE);
    if (excludeStr == null)
      return new ParsedParams(localParams, params, required, facetValue, docs, key, tags, threads);

    List<String> excludeTagList = StrUtils.splitSmart(excludeStr, ',');
    docs = computeDocSet(docs, excludeTagList);
    return new ParsedParams(localParams, params, required, facetValue, docs, key, tags, threads);
  }
Esempio n. 7
0
  /**
   * Returns a list of value constraints and the associated facet counts for each facet field
   * specified in the params.
   *
   * @see FacetParams#FACET_FIELD
   * @see #getFieldMissingCount
   * @see #getFacetTermEnumCounts
   */
  @SuppressWarnings("unchecked")
  public NamedList<Object> getFacetFieldCounts() throws IOException, SyntaxError {

    NamedList<Object> res = new SimpleOrderedMap<>();
    String[] facetFs = global.getParams(FacetParams.FACET_FIELD);
    if (null == facetFs) {
      return res;
    }

    // Passing a negative number for FACET_THREADS implies an unlimited number of threads is
    // acceptable.
    // Also, a subtlety of directExecutor is that no matter how many times you "submit" a job, it's
    // really
    // just a method call in that it's run by the calling thread.
    int maxThreads = req.getParams().getInt(FacetParams.FACET_THREADS, 0);
    Executor executor = maxThreads == 0 ? directExecutor : facetExecutor;
    final Semaphore semaphore = new Semaphore((maxThreads <= 0) ? Integer.MAX_VALUE : maxThreads);
    List<Future<NamedList>> futures = new ArrayList<>(facetFs.length);

    try {
      // Loop over fields; submit to executor, keeping the future
      for (String f : facetFs) {
        final ParsedParams parsed = parseParams(FacetParams.FACET_FIELD, f);
        final SolrParams localParams = parsed.localParams;
        final String termList = localParams == null ? null : localParams.get(CommonParams.TERMS);
        final String key = parsed.key;
        final String facetValue = parsed.facetValue;
        Callable<NamedList> callable =
            new Callable<NamedList>() {
              @Override
              public NamedList call() throws Exception {
                try {
                  NamedList<Object> result = new SimpleOrderedMap<>();
                  if (termList != null) {
                    List<String> terms = StrUtils.splitSmart(termList, ",", true);
                    result.add(key, getListedTermCounts(facetValue, parsed, terms));
                  } else {
                    result.add(key, getTermCounts(facetValue, parsed));
                  }
                  return result;
                } catch (SolrException se) {
                  throw se;
                } catch (Exception e) {
                  throw new SolrException(
                      ErrorCode.SERVER_ERROR, "Exception during facet.field: " + facetValue, e);
                } finally {
                  semaphore.release();
                }
              }
            };

        RunnableFuture<NamedList> runnableFuture = new FutureTask<>(callable);
        semaphore.acquire(); // may block and/or interrupt
        executor.execute(runnableFuture); // releases semaphore when done
        futures.add(runnableFuture);
      } // facetFs loop

      // Loop over futures to get the values. The order is the same as facetFs but shouldn't matter.
      for (Future<NamedList> future : futures) {
        res.addAll(future.get());
      }
      assert semaphore.availablePermits() >= maxThreads;
    } catch (InterruptedException e) {
      throw new SolrException(
          SolrException.ErrorCode.SERVER_ERROR,
          "Error while processing facet fields: InterruptedException",
          e);
    } catch (ExecutionException ee) {
      Throwable e = ee.getCause(); // unwrap
      if (e instanceof RuntimeException) {
        throw (RuntimeException) e;
      }
      throw new SolrException(
          SolrException.ErrorCode.SERVER_ERROR,
          "Error while processing facet fields: " + e.toString(),
          e);
    }

    return res;
  }