示例#1
0
  @Override
  public SearchRequestBuilder initRequestBuilder(SearchRequestBuilder srb) {
    // the dateFilter should not apply to the date facets!
    dateFilter = null;
    srb = super.initRequestBuilder(srb);

    if (dateFilter != null) srb.setFilter(dateFilter);

    if (isDateFacets()) {
      // too much work to convert the generic case with all the date math
      // so cheat for our case:
      String name = ElasticTweetSearch.DATE_FACET;
      RangeFacetBuilder rfb = FacetBuilders.rangeFacet(name).field(ElasticTweetSearch.DATE);
      MyDate date = new MyDate();

      // latest
      rfb.addUnboundedTo(Helper.toLocalDateTime(date.clone().minusHours(8).castToHour().toDate()));
      // first day
      rfb.addUnboundedTo(Helper.toLocalDateTime(date.castToDay().toDate()));

      for (int i = 0; i < 7; i++) {
        // 'from' must be smaller than 'to'!
        Date oldDate = date.toDate();
        rfb.addRange(
            Helper.toLocalDateTime(date.minusDays(1).toDate()), Helper.toLocalDateTime(oldDate));
      }

      // oldest
      rfb.addUnboundedFrom(Helper.toLocalDateTime(date.toDate()));
      srb.addFacet(rfb);
    }

    return srb;
  }
  public String createQuery(
      boolean includeFacets, int simpleSelections, int rangeSelections, int pathSelections) {
    SenseiClientRequest clientRequest =
        senseiQueryProducer.createQuery(
            includeFacets, simpleSelections, rangeSelections, pathSelections);

    FilterBuilder[] filters = new FilterBuilder[clientRequest.getSelections().size()];
    for (int i = 0; i < filters.length; i++) {
      Selection selection = clientRequest.getSelections().get(i);
      if (selection instanceof Terms) {
        filters[i] = createFilterFromTerm((Terms) selection);
      }
      if (selection instanceof Range) {
        Range range = (Range) selection;
        RangeFilterBuilder rangeFilter = FilterBuilders.rangeFilter(range.getField());
        filters[i] = rangeFilter;
        if (!"*".equals(range.getFrom())) rangeFilter.from(range.getFrom());
        if (!"*".equals(range.getTo())) rangeFilter.to(range.getTo());
      }
      if (selection instanceof Path) {
        Path path = (Path) selection;
        filters[i] = FilterBuilders.prefixFilter(path.getField(), path.getValue());
      }
    }
    FilterBuilder selections = FilterBuilders.orFilter(filters);
    SearchRequestBuilder requestBuilder = new SearchRequestBuilder(new MockClient());
    requestBuilder.setFilter(selections);
    try {
      JSONObject request = new JSONObject(requestBuilder.toString());
      JSONObject facets = new JSONObject();
      for (Selection selection : clientRequest.getSelections()) {
        if (selection instanceof Terms) {
          String field = selection.getField();
          facets.put(
              field, new JSONObject("{\"terms\": {\"field\":\"" + field + "\", \"size\" : 300}}"));
        }
      }
      request.put("facets", facets);
      return request.toString();
    } catch (Exception ex) {
      throw new RuntimeException(ex);
    }
  }
  /** All indices has to be created before! */
  public void mergeIndices(
      Collection<String> indexList,
      String intoIndex,
      int hitsPerPage,
      boolean forceRefresh,
      CreateObjectsInterface<T> createObj,
      FilterBuilder additionalFilter) {
    if (forceRefresh) {
      refresh(indexList);
      refresh(intoIndex);
    }

    int keepTime = 100;
    for (String fromIndex : indexList) {
      SearchRequestBuilder srb =
          client
              .prepareSearch(fromIndex)
              .setVersion(true)
              .setQuery(QueryBuilders.matchAllQuery())
              .setSize(hitsPerPage)
              .setSearchType(SearchType.SCAN)
              .setScroll(TimeValue.timeValueMinutes(keepTime));
      if (additionalFilter != null) srb.setFilter(additionalFilter);
      SearchResponse rsp = srb.execute().actionGet();

      try {
        long total = rsp.hits().totalHits();
        int collectedResults = 0;
        while (true) {
          StopWatch queryWatch = new StopWatch().start();
          rsp =
              client
                  .prepareSearchScroll(rsp.scrollId())
                  .setScroll(TimeValue.timeValueMinutes(keepTime))
                  .execute()
                  .actionGet();
          long currentResults = rsp.hits().hits().length;
          if (currentResults == 0) break;

          queryWatch.stop();
          Collection<T> objs = createObj.collectObjects(rsp);
          StopWatch updateWatch = new StopWatch().start();
          int failed = bulkUpdate(objs, intoIndex, false, false).size();
          // trying to enable flushing to avoid memory issues on the server side?
          flush(intoIndex);
          updateWatch.stop();
          collectedResults += currentResults;
          logger.info(
              "Progress "
                  + collectedResults
                  + "/"
                  + total
                  + " fromIndex="
                  + fromIndex
                  + " update:"
                  + updateWatch.totalTime().getSeconds()
                  + " query:"
                  + queryWatch.totalTime().getSeconds()
                  + " failed:"
                  + failed);
        }
        logger.info(
            "Finished copying of index:"
                + fromIndex
                + ". Total:"
                + total
                + " collected:"
                + collectedResults);
      } catch (Exception ex) {
        //                throw new RuntimeException(ex);
        logger.error(
            "Failed to copy data from index " + fromIndex + " into " + intoIndex + ".", ex);
      }
    }

    if (forceRefresh) refresh(intoIndex);
  }