Ejemplo n.º 1
0
  /** test range query performance */
  public void XtestFilteringPerformance() throws Exception {
    int indexSize = 19999;
    float fractionCovered = .1f;

    String l = t(0);
    String u = t((int) (indexSize * 10 * fractionCovered));

    SolrQueryRequest req = lrf.makeRequest();

    QParser parser = QParser.getParser("foomany_s:[" + l + " TO " + u + "]", null, req);
    Query rangeQ = parser.parse();
    List<Query> filters = new ArrayList<Query>();
    filters.add(rangeQ);
    req.close();

    parser =
        QParser.getParser(
            "{!dismax qf=t10_100_ws pf=t10_100_ws ps=20}" + t(0) + ' ' + t(1) + ' ' + t(2),
            null,
            req);
    Query q = parser.parse();

    // SolrIndexSearcher searcher = req.getSearcher();
    // DocSet range = searcher.getDocSet(rangeQ, null);

    createIndex2(indexSize, "foomany_s", "t10_100_ws");

    // doListGen(100, q, filters, false, true);
    doListGen(500, q, filters, false, true);

    req.close();
  }
Ejemplo n.º 2
0
 public Sort toSort(List<? extends OrderSpecifier<?>> orderBys) {
   List<SortField> sorts = new ArrayList<SortField>(orderBys.size());
   for (OrderSpecifier<?> order : orderBys) {
     if (!(order.getTarget() instanceof Path<?>)) {
       throw new IllegalArgumentException("argument was not of type Path.");
     }
     Class<?> type = order.getTarget().getType();
     boolean reverse = !order.isAscending();
     Path<?> path = getPath(order.getTarget());
     if (Number.class.isAssignableFrom(type)) {
       sorts.add(new SortField(toField(path), sortFields.get(type), reverse));
     } else {
       sorts.add(new SortField(toField(path), sortLocale, reverse));
     }
   }
   Sort sort = new Sort();
   sort.setSort(sorts.toArray(new SortField[sorts.size()]));
   return sort;
 }
 private static Object getReferencedId(DataRecord next, ReferenceFieldMetadata field) {
   DataRecord record = (DataRecord) next.get(field);
   if (record != null) {
     Collection<FieldMetadata> keyFields = record.getType().getKeyFields();
     if (keyFields.size() == 1) {
       return record.get(keyFields.iterator().next());
     } else {
       List<Object> compositeKeyValues = new ArrayList<Object>(keyFields.size());
       for (FieldMetadata keyField : keyFields) {
         compositeKeyValues.add(record.get(keyField));
       }
       return compositeKeyValues;
     }
   } else {
     return StringUtils.EMPTY;
   }
 }
Ejemplo n.º 4
0
  @Override
  public void prepare(ResponseBuilder rb) throws IOException {

    SolrQueryRequest req = rb.req;
    SolrParams params = req.getParams();
    if (!params.getBool(COMPONENT_NAME, true)) {
      return;
    }
    SolrQueryResponse rsp = rb.rsp;

    // Set field flags
    String fl = params.get(CommonParams.FL);
    int fieldFlags = 0;
    if (fl != null) {
      fieldFlags |= SolrPluginUtils.setReturnFields(fl, rsp);
    }
    rb.setFieldFlags(fieldFlags);

    String defType = params.get(QueryParsing.DEFTYPE, QParserPlugin.DEFAULT_QTYPE);

    // get it from the response builder to give a different component a chance
    // to set it.
    String queryString = rb.getQueryString();
    if (queryString == null) {
      // this is the normal way it's set.
      queryString = params.get(CommonParams.Q);
      rb.setQueryString(queryString);
    }

    try {
      QParser parser = QParser.getParser(rb.getQueryString(), defType, req);
      Query q = parser.getQuery();
      if (q == null) {
        // normalize a null query to a query that matches nothing
        q = new BooleanQuery();
      }
      rb.setQuery(q);
      rb.setSortSpec(parser.getSort(true));
      rb.setQparser(parser);

      String[] fqs = req.getParams().getParams(CommonParams.FQ);
      if (fqs != null && fqs.length != 0) {
        List<Query> filters = rb.getFilters();
        if (filters == null) {
          filters = new ArrayList<Query>();
          rb.setFilters(filters);
        }
        for (String fq : fqs) {
          if (fq != null && fq.trim().length() != 0) {
            QParser fqp = QParser.getParser(fq, null, req);
            filters.add(fqp.getQuery());
          }
        }
      }
    } catch (ParseException e) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
    }

    // TODO: temporary... this should go in a different component.
    String shards = params.get(ShardParams.SHARDS);
    if (shards != null) {
      List<String> lst = StrUtils.splitSmart(shards, ",", true);
      rb.shards = lst.toArray(new String[lst.size()]);
    }
    String shards_rows = params.get(ShardParams.SHARDS_ROWS);
    if (shards_rows != null) {
      rb.shards_rows = Integer.parseInt(shards_rows);
    }
    String shards_start = params.get(ShardParams.SHARDS_START);
    if (shards_start != null) {
      rb.shards_start = Integer.parseInt(shards_start);
    }
  }
Ejemplo n.º 5
0
  /** Actually run the query */
  @Override
  public void process(ResponseBuilder rb) throws IOException {
    SolrQueryRequest req = rb.req;
    SolrQueryResponse rsp = rb.rsp;
    SolrParams params = req.getParams();
    if (!params.getBool(COMPONENT_NAME, true)) {
      return;
    }
    SolrIndexSearcher searcher = req.getSearcher();

    if (rb.getQueryCommand().getOffset() < 0) {
      throw new SolrException(
          SolrException.ErrorCode.BAD_REQUEST, "'start' parameter cannot be negative");
    }

    // -1 as flag if not set.
    long timeAllowed = (long) params.getInt(CommonParams.TIME_ALLOWED, -1);

    // Optional: This could also be implemented by the top-level searcher sending
    // a filter that lists the ids... that would be transparent to
    // the request handler, but would be more expensive (and would preserve score
    // too if desired).
    String ids = params.get(ShardParams.IDS);
    if (ids != null) {
      SchemaField idField = req.getSchema().getUniqueKeyField();
      List<String> idArr = StrUtils.splitSmart(ids, ",", true);
      int[] luceneIds = new int[idArr.size()];
      int docs = 0;
      for (int i = 0; i < idArr.size(); i++) {
        int id =
            req.getSearcher()
                .getFirstMatch(
                    new Term(idField.getName(), idField.getType().toInternal(idArr.get(i))));
        if (id >= 0) luceneIds[docs++] = id;
      }

      DocListAndSet res = new DocListAndSet();
      res.docList = new DocSlice(0, docs, luceneIds, null, docs, 0);
      if (rb.isNeedDocSet()) {
        List<Query> queries = new ArrayList<Query>();
        queries.add(rb.getQuery());
        List<Query> filters = rb.getFilters();
        if (filters != null) queries.addAll(filters);
        res.docSet = searcher.getDocSet(queries);
      }
      rb.setResults(res);
      rsp.add("response", rb.getResults().docList);
      return;
    }

    SolrIndexSearcher.QueryCommand cmd = rb.getQueryCommand();
    cmd.setTimeAllowed(timeAllowed);
    SolrIndexSearcher.QueryResult result = new SolrIndexSearcher.QueryResult();

    //
    // grouping / field collapsing
    //
    boolean doGroup = params.getBool(GroupParams.GROUP, false);
    if (doGroup) {
      try {
        cmd.groupCommands = new ArrayList<Grouping.Command>();

        String[] fields = params.getParams(GroupParams.GROUP_FIELD);
        String[] funcs = params.getParams(GroupParams.GROUP_FUNC);
        String[] queries = params.getParams(GroupParams.GROUP_QUERY);
        String groupSortStr = params.get(GroupParams.GROUP_SORT);
        Sort groupSort =
            groupSortStr != null ? QueryParsing.parseSort(groupSortStr, req.getSchema()) : null;

        int limitDefault = cmd.getLen(); // this is normally from "rows"
        int docsPerGroupDefault = params.getInt(GroupParams.GROUP_LIMIT, 1);

        // temporary: implement all group-by-field as group-by-func
        if (funcs == null) {
          funcs = fields;
        } else if (fields != null) {
          // catenate functions and fields
          String[] both = new String[fields.length + funcs.length];
          System.arraycopy(fields, 0, both, 0, fields.length);
          System.arraycopy(funcs, 0, both, fields.length, funcs.length);
          funcs = both;
        }

        if (funcs != null) {
          for (String groupByStr : funcs) {
            QParser parser = QParser.getParser(groupByStr, "func", rb.req);
            Query q = parser.getQuery();
            Grouping.CommandFunc gc = new Grouping.CommandFunc();
            gc.groupSort = groupSort;

            if (q instanceof FunctionQuery) {
              gc.groupBy = ((FunctionQuery) q).getValueSource();
            } else {
              gc.groupBy = new QueryValueSource(q, 0.0f);
            }
            gc.key = groupByStr;
            gc.groupLimit = limitDefault;
            gc.docsPerGroup = docsPerGroupDefault;

            cmd.groupCommands.add(gc);
          }
        }

        if (cmd.groupCommands.size() == 0) cmd.groupCommands = null;

        if (cmd.groupCommands != null) {
          if (rb.doHighlights || rb.isDebug()) {
            // we need a single list of the returned docs
            cmd.setFlags(SolrIndexSearcher.GET_DOCLIST);
          }

          searcher.search(result, cmd);
          rb.setResult(result);
          rsp.add("grouped", result.groupedResults);
          // TODO: get "hits" a different way to log
          return;
        }
      } catch (ParseException e) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
      }
    }

    // normal search result
    searcher.search(result, cmd);
    rb.setResult(result);

    rsp.add("response", rb.getResults().docList);
    rsp.getToLog().add("hits", rb.getResults().docList.matches());

    doFieldSortValues(rb, searcher);
    doPrefetch(rb);
  }
 @Override
 public StorageResults visit(Select select) {
   // TMDM-4654: Checks if entity has a composite PK.
   Set<ComplexTypeMetadata> compositeKeyTypes = new HashSet<ComplexTypeMetadata>();
   // TMDM-7496: Search should include references to reused types
   Collection<ComplexTypeMetadata> types =
       new HashSet<ComplexTypeMetadata>(select.accept(new SearchTransitiveClosure()));
   for (ComplexTypeMetadata type : types) {
     if (type.getKeyFields().size() > 1) {
       compositeKeyTypes.add(type);
     }
   }
   if (!compositeKeyTypes.isEmpty()) {
     StringBuilder message = new StringBuilder();
     Iterator it = compositeKeyTypes.iterator();
     while (it.hasNext()) {
       ComplexTypeMetadata compositeKeyType = (ComplexTypeMetadata) it.next();
       message.append(compositeKeyType.getName());
       if (it.hasNext()) {
         message.append(',');
       }
     }
     throw new FullTextQueryCompositeKeyException(message.toString());
   }
   // Removes Joins and joined fields.
   List<Join> joins = select.getJoins();
   if (!joins.isEmpty()) {
     Set<ComplexTypeMetadata> joinedTypes = new HashSet<ComplexTypeMetadata>();
     for (Join join : joins) {
       joinedTypes.add(join.getRightField().getFieldMetadata().getContainingType());
     }
     for (ComplexTypeMetadata joinedType : joinedTypes) {
       types.remove(joinedType);
     }
     List<TypedExpression> filteredFields = new LinkedList<TypedExpression>();
     for (TypedExpression expression : select.getSelectedFields()) {
       if (expression instanceof Field) {
         FieldMetadata fieldMetadata = ((Field) expression).getFieldMetadata();
         if (joinedTypes.contains(fieldMetadata.getContainingType())) {
           TypeMapping mapping =
               mappings.getMappingFromDatabase(fieldMetadata.getContainingType());
           filteredFields.add(
               new Alias(
                   new StringConstant(StringUtils.EMPTY),
                   mapping.getUser(fieldMetadata).getName()));
         } else {
           filteredFields.add(expression);
         }
       } else {
         filteredFields.add(expression);
       }
     }
     selectedFields.clear();
     selectedFields.addAll(filteredFields);
   }
   // Handle condition
   Condition condition = select.getCondition();
   if (condition == null) {
     throw new IllegalArgumentException("Expected a condition in select clause but got 0.");
   }
   // Create Lucene query (concatenates all sub queries together).
   FullTextSession fullTextSession = Search.getFullTextSession(session);
   Query parsedQuery = select.getCondition().accept(new LuceneQueryGenerator(types));
   // Create Hibernate Search query
   Set<Class> classes = new HashSet<Class>();
   for (ComplexTypeMetadata type : types) {
     String className = ClassCreator.getClassName(type.getName());
     try {
       ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
       classes.add(contextClassLoader.loadClass(className));
     } catch (ClassNotFoundException e) {
       throw new RuntimeException("Could not find class '" + className + "'.", e);
     }
   }
   FullTextQuery fullTextQuery =
       fullTextSession.createFullTextQuery(
           parsedQuery, classes.toArray(new Class<?>[classes.size()]));
   // Very important to leave this null (would disable ability to search across different types)
   fullTextQuery.setCriteriaQuery(null);
   fullTextQuery.setSort(Sort.RELEVANCE); // Default sort (if no order by specified).
   query =
       EntityFinder.wrap(
           fullTextQuery,
           (HibernateStorage) storage,
           session); // ensures only MDM entity objects are returned.
   // Order by
   for (OrderBy current : select.getOrderBy()) {
     current.accept(this);
   }
   // Paging
   Paging paging = select.getPaging();
   paging.accept(this);
   pageSize = paging.getLimit();
   boolean hasPaging = pageSize < Integer.MAX_VALUE;
   if (!hasPaging) {
     return createResults(query.scroll(ScrollMode.FORWARD_ONLY));
   } else {
     return createResults(query.list());
   }
 }
  public InternalSearchResponse merge(
      ScoreDoc[] sortedDocs,
      AtomicArray<? extends QuerySearchResultProvider> queryResultsArr,
      AtomicArray<? extends FetchSearchResultProvider> fetchResultsArr) {

    List<? extends AtomicArray.Entry<? extends QuerySearchResultProvider>> queryResults =
        queryResultsArr.asList();
    List<? extends AtomicArray.Entry<? extends FetchSearchResultProvider>> fetchResults =
        fetchResultsArr.asList();

    if (queryResults.isEmpty()) {
      return InternalSearchResponse.empty();
    }

    QuerySearchResult firstResult = queryResults.get(0).value.queryResult();

    boolean sorted = false;
    int sortScoreIndex = -1;
    if (firstResult.topDocs() instanceof TopFieldDocs) {
      sorted = true;
      TopFieldDocs fieldDocs = (TopFieldDocs) firstResult.queryResult().topDocs();
      for (int i = 0; i < fieldDocs.fields.length; i++) {
        if (fieldDocs.fields[i].getType() == SortField.Type.SCORE) {
          sortScoreIndex = i;
        }
      }
    }

    // merge facets
    InternalFacets facets = null;
    if (!queryResults.isEmpty()) {
      // we rely on the fact that the order of facets is the same on all query results
      if (firstResult.facets() != null
          && firstResult.facets().facets() != null
          && !firstResult.facets().facets().isEmpty()) {
        List<Facet> aggregatedFacets = Lists.newArrayList();
        List<Facet> namedFacets = Lists.newArrayList();
        for (Facet facet : firstResult.facets()) {
          // aggregate each facet name into a single list, and aggregate it
          namedFacets.clear();
          for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults) {
            for (Facet facet1 : entry.value.queryResult().facets()) {
              if (facet.getName().equals(facet1.getName())) {
                namedFacets.add(facet1);
              }
            }
          }
          if (!namedFacets.isEmpty()) {
            Facet aggregatedFacet =
                ((InternalFacet) namedFacets.get(0))
                    .reduce(new InternalFacet.ReduceContext(cacheRecycler, namedFacets));
            aggregatedFacets.add(aggregatedFacet);
          }
        }
        facets = new InternalFacets(aggregatedFacets);
      }
    }

    // count the total (we use the query result provider here, since we might not get any hits (we
    // scrolled past them))
    long totalHits = 0;
    float maxScore = Float.NEGATIVE_INFINITY;
    boolean timedOut = false;
    Boolean terminatedEarly = null;
    for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults) {
      QuerySearchResult result = entry.value.queryResult();
      if (result.searchTimedOut()) {
        timedOut = true;
      }
      if (result.terminatedEarly() != null) {
        if (terminatedEarly == null) {
          terminatedEarly = result.terminatedEarly();
        } else if (result.terminatedEarly()) {
          terminatedEarly = true;
        }
      }
      totalHits += result.topDocs().totalHits;
      if (!Float.isNaN(result.topDocs().getMaxScore())) {
        maxScore = Math.max(maxScore, result.topDocs().getMaxScore());
      }
    }
    if (Float.isInfinite(maxScore)) {
      maxScore = Float.NaN;
    }

    // clean the fetch counter
    for (AtomicArray.Entry<? extends FetchSearchResultProvider> entry : fetchResults) {
      entry.value.fetchResult().initCounter();
    }

    // merge hits
    List<InternalSearchHit> hits = new ArrayList<>();
    if (!fetchResults.isEmpty()) {
      for (ScoreDoc shardDoc : sortedDocs) {
        FetchSearchResultProvider fetchResultProvider = fetchResultsArr.get(shardDoc.shardIndex);
        if (fetchResultProvider == null) {
          continue;
        }
        FetchSearchResult fetchResult = fetchResultProvider.fetchResult();
        int index = fetchResult.counterGetAndIncrement();
        if (index < fetchResult.hits().internalHits().length) {
          InternalSearchHit searchHit = fetchResult.hits().internalHits()[index];
          searchHit.score(shardDoc.score);
          searchHit.shard(fetchResult.shardTarget());

          if (sorted) {
            FieldDoc fieldDoc = (FieldDoc) shardDoc;
            searchHit.sortValues(fieldDoc.fields);
            if (sortScoreIndex != -1) {
              searchHit.score(((Number) fieldDoc.fields[sortScoreIndex]).floatValue());
            }
          }

          hits.add(searchHit);
        }
      }
    }

    // merge suggest results
    Suggest suggest = null;
    if (!queryResults.isEmpty()) {
      final Map<String, List<Suggest.Suggestion>> groupedSuggestions = new HashMap<>();
      boolean hasSuggestions = false;
      for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults) {
        Suggest shardResult = entry.value.queryResult().queryResult().suggest();

        if (shardResult == null) {
          continue;
        }
        hasSuggestions = true;
        Suggest.group(groupedSuggestions, shardResult);
      }

      suggest =
          hasSuggestions
              ? new Suggest(Suggest.Fields.SUGGEST, Suggest.reduce(groupedSuggestions))
              : null;
    }

    // merge addAggregation
    InternalAggregations aggregations = null;
    if (!queryResults.isEmpty()) {
      if (firstResult.aggregations() != null && firstResult.aggregations().asList() != null) {
        List<InternalAggregations> aggregationsList = new ArrayList<>(queryResults.size());
        for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults) {
          aggregationsList.add((InternalAggregations) entry.value.queryResult().aggregations());
        }
        aggregations =
            InternalAggregations.reduce(
                aggregationsList, new ReduceContext(null, bigArrays, scriptService));
      }
    }

    InternalSearchHits searchHits =
        new InternalSearchHits(
            hits.toArray(new InternalSearchHit[hits.size()]), totalHits, maxScore);

    return new InternalSearchResponse(
        searchHits, facets, aggregations, suggest, timedOut, terminatedEarly);
  }