Exemplo n.º 1
0
  /** test range query performance */
  public void XtestFilteringPerformance() throws Exception {
    int indexSize = 19999;
    float fractionCovered = .1f;

    String l = t(0);
    String u = t((int) (indexSize * 10 * fractionCovered));

    SolrQueryRequest req = lrf.makeRequest();

    QParser parser = QParser.getParser("foomany_s:[" + l + " TO " + u + "]", null, req);
    Query rangeQ = parser.parse();
    List<Query> filters = new ArrayList<Query>();
    filters.add(rangeQ);
    req.close();

    parser =
        QParser.getParser(
            "{!dismax qf=t10_100_ws pf=t10_100_ws ps=20}" + t(0) + ' ' + t(1) + ' ' + t(2),
            null,
            req);
    Query q = parser.parse();

    // SolrIndexSearcher searcher = req.getSearcher();
    // DocSet range = searcher.getDocSet(rangeQ, null);

    createIndex2(indexSize, "foomany_s", "t10_100_ws");

    // doListGen(100, q, filters, false, true);
    doListGen(500, q, filters, false, true);

    req.close();
  }
Exemplo n.º 2
0
  /** test range query performance */
  public void XtestRangePerformance() throws Exception {
    int indexSize = 1999;
    float fractionCovered = 1.0f;

    String l = t(0);
    String u = t((int) (indexSize * 10 * fractionCovered));

    SolrQueryRequest req = lrf.makeRequest();
    QParser parser = QParser.getParser("foomany_s:[" + l + " TO " + u + "]", null, req);
    Query range = parser.parse();

    QParser parser2 = QParser.getParser("{!frange l=" + l + " u=" + u + "}foomany_s", null, req);
    Query frange = parser2.parse();
    req.close();

    createIndex2(indexSize, "foomany_s");

    doSetGen(1, range);
    doSetGen(1, frange); // load field cache

    doSetGen(100, range);
    doSetGen(10000, frange);
  }
    public Query parse() throws SyntaxError {
      String reRankQueryString = localParams.get("reRankQuery");
      boolean scale = localParams.getBool("scale", false);
      QParser reRankParser = QParser.getParser(reRankQueryString, null, req);
      Query reRankQuery = reRankParser.parse();

      int reRankDocs = localParams.getInt("reRankDocs", 200);
      reRankDocs = Math.max(1, reRankDocs); //

      double reRankWeight = localParams.getDouble("reRankWeight", 2.0d);

      int start = params.getInt(CommonParams.START, 0);
      int rows = params.getInt(CommonParams.ROWS, 10);
      int length = start + rows;
      return new ReRankQuery(reRankQuery, reRankDocs, reRankWeight, length, scale);
    }
Exemplo n.º 4
0
  /**
   * The form of the sort specification string currently parsed is:
   *
   * <pre>
   * SortSpec ::= SingleSort [, SingleSort]*
   * SingleSort ::= &lt;fieldname|function&gt; SortDirection
   * SortDirection ::= top | desc | bottom | asc
   * </pre>
   *
   * Examples:
   *
   * <pre>
   *   score desc               #normal sort by score (will return null)
   *   weight bottom            #sort by weight ascending
   *   weight desc              #sort by weight descending
   *   height desc,weight desc  #sort by height descending, and use weight descending to break any ties
   *   height desc,weight asc   #sort by height descending, using weight ascending as a tiebreaker
   * </pre>
   *
   * @return a SortSpec object populated with the appropriate Sort (which may be null if default
   *     score sort is used) and SchemaFields (where applicable) using hardcoded default count &amp;
   *     offset values.
   */
  public static SortSpec parseSortSpec(String sortSpec, SolrQueryRequest req) {
    if (sortSpec == null || sortSpec.length() == 0) return newEmptySortSpec();

    List<SortField> sorts = new ArrayList<SortField>(4);
    List<SchemaField> fields = new ArrayList<SchemaField>(4);

    try {

      StrParser sp = new StrParser(sortSpec);
      while (sp.pos < sp.end) {
        sp.eatws();

        final int start = sp.pos;

        // short circuit test for a really simple field name
        String field = sp.getId(null);
        Exception qParserException = null;

        if (field == null || !Character.isWhitespace(sp.peekChar())) {
          // let's try it as a function instead
          field = null;
          String funcStr = sp.val.substring(start);

          QParser parser = QParser.getParser(funcStr, FunctionQParserPlugin.NAME, req);
          Query q = null;
          try {
            if (parser instanceof FunctionQParser) {
              FunctionQParser fparser = (FunctionQParser) parser;
              fparser.setParseMultipleSources(false);
              fparser.setParseToEnd(false);

              q = fparser.getQuery();

              if (fparser.localParams != null) {
                if (fparser.valFollowedParams) {
                  // need to find the end of the function query via the string parser
                  int leftOver = fparser.sp.end - fparser.sp.pos;
                  sp.pos = sp.end - leftOver; // reset our parser to the same amount of leftover
                } else {
                  // the value was via the "v" param in localParams, so we need to find
                  // the end of the local params themselves to pick up where we left off
                  sp.pos = start + fparser.localParamsEnd;
                }
              } else {
                // need to find the end of the function query via the string parser
                int leftOver = fparser.sp.end - fparser.sp.pos;
                sp.pos = sp.end - leftOver; // reset our parser to the same amount of leftover
              }
            } else {
              // A QParser that's not for function queries.
              // It must have been specified via local params.
              q = parser.getQuery();

              assert parser.getLocalParams() != null;
              sp.pos = start + parser.localParamsEnd;
            }

            Boolean top = sp.getSortDirection();
            if (null != top) {
              // we have a Query and a valid direction
              if (q instanceof FunctionQuery) {
                sorts.add(((FunctionQuery) q).getValueSource().getSortField(top));
              } else {
                sorts.add((new QueryValueSource(q, 0.0f)).getSortField(top));
              }
              fields.add(null);
              continue;
            }
          } catch (Exception e) {
            // hang onto this in case the string isn't a full field name either
            qParserException = e;
          }
        }

        // if we made it here, we either have a "simple" field name,
        // or there was a problem parsing the string as a complex func/quer

        if (field == null) {
          // try again, simple rules for a field name with no whitespace
          sp.pos = start;
          field = sp.getSimpleString();
        }
        Boolean top = sp.getSortDirection();
        if (null == top) {
          throw new SolrException(
              SolrException.ErrorCode.BAD_REQUEST,
              "Can't determine a Sort Order (asc or desc) in sort spec " + sp);
        }

        if (SCORE.equals(field)) {
          if (top) {
            sorts.add(SortField.FIELD_SCORE);
          } else {
            sorts.add(new SortField(null, SortField.Type.SCORE, true));
          }
          fields.add(null);
        } else if (DOCID.equals(field)) {
          sorts.add(new SortField(null, SortField.Type.DOC, top));
          fields.add(null);
        } else {
          // try to find the field
          SchemaField sf = req.getSchema().getFieldOrNull(field);
          if (null == sf) {
            if (null != qParserException) {
              throw new SolrException(
                  SolrException.ErrorCode.BAD_REQUEST,
                  "sort param could not be parsed as a query, and is not a "
                      + "field that exists in the index: "
                      + field,
                  qParserException);
            }
            throw new SolrException(
                SolrException.ErrorCode.BAD_REQUEST, "sort param field can't be found: " + field);
          }
          sorts.add(sf.getSortField(top));
          fields.add(sf);
        }
      }

    } catch (SyntaxError e) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "error in sort: " + sortSpec, e);
    }

    // normalize a sort on score desc to null
    if (sorts.size() == 1 && sorts.get(0) == SortField.FIELD_SCORE) {
      return newEmptySortSpec();
    }

    Sort s = new Sort(sorts.toArray(new SortField[sorts.size()]));
    return new SortSpec(s, fields);
  }
Exemplo n.º 5
0
  @Override
  public void prepare(ResponseBuilder rb) throws IOException {

    SolrQueryRequest req = rb.req;
    SolrParams params = req.getParams();
    if (!params.getBool(COMPONENT_NAME, true)) {
      return;
    }
    SolrQueryResponse rsp = rb.rsp;

    // Set field flags
    String fl = params.get(CommonParams.FL);
    int fieldFlags = 0;
    if (fl != null) {
      fieldFlags |= SolrPluginUtils.setReturnFields(fl, rsp);
    }
    rb.setFieldFlags(fieldFlags);

    String defType = params.get(QueryParsing.DEFTYPE, QParserPlugin.DEFAULT_QTYPE);

    // get it from the response builder to give a different component a chance
    // to set it.
    String queryString = rb.getQueryString();
    if (queryString == null) {
      // this is the normal way it's set.
      queryString = params.get(CommonParams.Q);
      rb.setQueryString(queryString);
    }

    try {
      QParser parser = QParser.getParser(rb.getQueryString(), defType, req);
      Query q = parser.getQuery();
      if (q == null) {
        // normalize a null query to a query that matches nothing
        q = new BooleanQuery();
      }
      rb.setQuery(q);
      rb.setSortSpec(parser.getSort(true));
      rb.setQparser(parser);

      String[] fqs = req.getParams().getParams(CommonParams.FQ);
      if (fqs != null && fqs.length != 0) {
        List<Query> filters = rb.getFilters();
        if (filters == null) {
          filters = new ArrayList<Query>();
          rb.setFilters(filters);
        }
        for (String fq : fqs) {
          if (fq != null && fq.trim().length() != 0) {
            QParser fqp = QParser.getParser(fq, null, req);
            filters.add(fqp.getQuery());
          }
        }
      }
    } catch (ParseException e) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
    }

    // TODO: temporary... this should go in a different component.
    String shards = params.get(ShardParams.SHARDS);
    if (shards != null) {
      List<String> lst = StrUtils.splitSmart(shards, ",", true);
      rb.shards = lst.toArray(new String[lst.size()]);
    }
    String shards_rows = params.get(ShardParams.SHARDS_ROWS);
    if (shards_rows != null) {
      rb.shards_rows = Integer.parseInt(shards_rows);
    }
    String shards_start = params.get(ShardParams.SHARDS_START);
    if (shards_start != null) {
      rb.shards_start = Integer.parseInt(shards_start);
    }
  }
Exemplo n.º 6
0
  /** Actually run the query */
  @Override
  public void process(ResponseBuilder rb) throws IOException {
    SolrQueryRequest req = rb.req;
    SolrQueryResponse rsp = rb.rsp;
    SolrParams params = req.getParams();
    if (!params.getBool(COMPONENT_NAME, true)) {
      return;
    }
    SolrIndexSearcher searcher = req.getSearcher();

    if (rb.getQueryCommand().getOffset() < 0) {
      throw new SolrException(
          SolrException.ErrorCode.BAD_REQUEST, "'start' parameter cannot be negative");
    }

    // -1 as flag if not set.
    long timeAllowed = (long) params.getInt(CommonParams.TIME_ALLOWED, -1);

    // Optional: This could also be implemented by the top-level searcher sending
    // a filter that lists the ids... that would be transparent to
    // the request handler, but would be more expensive (and would preserve score
    // too if desired).
    String ids = params.get(ShardParams.IDS);
    if (ids != null) {
      SchemaField idField = req.getSchema().getUniqueKeyField();
      List<String> idArr = StrUtils.splitSmart(ids, ",", true);
      int[] luceneIds = new int[idArr.size()];
      int docs = 0;
      for (int i = 0; i < idArr.size(); i++) {
        int id =
            req.getSearcher()
                .getFirstMatch(
                    new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
        if (id >= 0) luceneIds[docs++] = id;
      }

      DocListAndSet res = new DocListAndSet();
      res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
      if (rb.isNeedDocSet()) {
        List<Query> queries = new ArrayList<Query>();
        queries.add(rb.getQuery());
        List<Query> filters = rb.getFilters();
        if (filters != null) queries.addAll(filters);
        res.docSet = searcher.getDocSet(queries);
      }
      rb.setResults(res);
      rsp.add("response", rb.getResults().docList);
      return;
    }

    SolrIndexSearcher.QueryCommand cmd = rb.getQueryCommand();
    cmd.setTimeAllowed(timeAllowed);
    SolrIndexSearcher.QueryResult result = new SolrIndexSearcher.QueryResult();

    //
    // grouping / field collapsing
    //
    boolean doGroup = params.getBool(GroupParams.GROUP, false);
    if (doGroup) {
      try {
        cmd.groupCommands = new ArrayList<Grouping.Command>();

        String[] fields = params.getParams(GroupParams.GROUP_FIELD);
        String[] funcs = params.getParams(GroupParams.GROUP_FUNC);
        String[] queries = params.getParams(GroupParams.GROUP_QUERY);
        String groupSortStr = params.get(GroupParams.GROUP_SORT);
        Sort groupSort =
            groupSortStr != null ? QueryParsing.parseSort(groupSortStr, req.getSchema()) : null;

        int limitDefault = cmd.getLen(); // this is normally from "rows"
        int docsPerGroupDefault = params.getInt(GroupParams.GROUP_LIMIT, 1);

        // temporary: implement all group-by-field as group-by-func
        if (funcs == null) {
          funcs = fields;
        } else if (fields != null) {
          // catenate functions and fields
          String[] both = new String[fields.length + funcs.length];
          System.arraycopy(fields, 0, both, 0, fields.length);
          System.arraycopy(funcs, 0, both, fields.length, funcs.length);
          funcs = both;
        }

        if (funcs != null) {
          for (String groupByStr : funcs) {
            QParser parser = QParser.getParser(groupByStr, "func", rb.req);
            Query q = parser.getQuery();
            Grouping.CommandFunc gc = new Grouping.CommandFunc();
            gc.groupSort = groupSort;

            if (q instanceof FunctionQuery) {
              gc.groupBy = ((FunctionQuery) q).getValueSource();
            } else {
              gc.groupBy = new QueryValueSource(q, 0.0f);
            }
            gc.key = groupByStr;
            gc.groupLimit = limitDefault;
            gc.docsPerGroup = docsPerGroupDefault;

            cmd.groupCommands.add(gc);
          }
        }

        if (cmd.groupCommands.size() == 0) cmd.groupCommands = null;

        if (cmd.groupCommands != null) {
          if (rb.doHighlights || rb.isDebug()) {
            // we need a single list of the returned docs
            cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
          }

          searcher.search(result, cmd);
          rb.setResult(result);
          rsp.add("grouped", result.groupedResults);
          // TODO: get "hits" a different way to log
          return;
        }
      } catch (ParseException e) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
      }
    }

    // normal search result
    searcher.search(result, cmd);
    rb.setResult(result);

    rsp.add("response", rb.getResults().docList);
    rsp.getToLog().add("hits", rb.getResults().docList.matches());

    doFieldSortValues(rb, searcher);
    doPrefetch(rb);
  }