Example #1
0
 @Override
 public ImmutableConciseSet goConcise(final BitmapIndexSelector selector) {
   final Context cx = Context.enter();
   try {
     final Indexed<String> dimValues = selector.getDimensionValues(dimension);
     ImmutableConciseSet conciseSet;
     if (dimValues == null) {
       conciseSet = new ImmutableConciseSet();
     } else {
       conciseSet =
           ImmutableConciseSet.union(
               FunctionalIterable.create(dimValues)
                   .filter(
                       new Predicate<String>() {
                         @Override
                         public boolean apply(@Nullable String input) {
                           return predicate.applyInContext(cx, input);
                         }
                       })
                   .transform(
                       new com.google.common.base.Function<String, ImmutableConciseSet>() {
                         @Override
                         public ImmutableConciseSet apply(@Nullable String input) {
                           return selector.getConciseInvertedIndex(dimension, input);
                         }
                       }));
     }
     return conciseSet;
   } finally {
     Context.exit();
   }
 }
  @Override
  public Iterable<Cursor> makeCursors(Filter filter, Interval interval, QueryGranularity gran) {
    Interval actualInterval = interval;
    if (!actualInterval.overlaps(index.dataInterval)) {
      return ImmutableList.of();
    }

    if (actualInterval.getStart().isBefore(index.dataInterval.getStart())) {
      actualInterval = actualInterval.withStart(index.dataInterval.getStart());
    }
    if (actualInterval.getEnd().isAfter(index.dataInterval.getEnd())) {
      actualInterval = actualInterval.withEnd(index.dataInterval.getEnd());
    }

    final Iterable<Cursor> iterable;
    if (filter == null) {
      iterable = new NoFilterCursorIterable(index, actualInterval, gran);
    } else {
      Offset offset = new ConciseOffset(filter.goConcise(new MMappedInvertedIndexSelector(index)));

      iterable = new CursorIterable(index, actualInterval, gran, offset);
    }

    return FunctionalIterable.create(iterable).keep(Functions.<Cursor>identity());
  }
Example #3
0
  public static File merge(
      List<IndexableAdapter> indexes,
      final AggregatorFactory[] metricAggs,
      File outDir,
      ProgressIndicator progress)
      throws IOException {
    FileUtils.deleteDirectory(outDir);
    if (!outDir.mkdirs()) {
      throw new ISE("Couldn't make outdir[%s].", outDir);
    }

    final AggregatorFactory[] lowerCaseMetricAggs = new AggregatorFactory[metricAggs.length];
    for (int i = 0; i < metricAggs.length; i++) {
      lowerCaseMetricAggs[i] = new ToLowerCaseAggregatorFactory(metricAggs[i]);
    }

    final List<String> mergedDimensions =
        mergeIndexed(
            Lists.transform(
                indexes,
                new Function<IndexableAdapter, Iterable<String>>() {
                  @Override
                  public Iterable<String> apply(@Nullable IndexableAdapter input) {
                    return Iterables.transform(
                        input.getAvailableDimensions(),
                        new Function<String, String>() {
                          @Override
                          public String apply(@Nullable String input) {
                            return input.toLowerCase();
                          }
                        });
                  }
                }));
    final List<String> mergedMetrics =
        Lists.transform(
            mergeIndexed(
                Lists.<Iterable<String>>newArrayList(
                    FunctionalIterable.create(indexes)
                        .transform(
                            new Function<IndexableAdapter, Iterable<String>>() {
                              @Override
                              public Iterable<String> apply(@Nullable IndexableAdapter input) {
                                return Iterables.transform(
                                    input.getAvailableMetrics(),
                                    new Function<String, String>() {
                                      @Override
                                      public String apply(@Nullable String input) {
                                        return input.toLowerCase();
                                      }
                                    });
                              }
                            })
                        .concat(
                            Arrays.<Iterable<String>>asList(
                                new AggFactoryStringIndexed(lowerCaseMetricAggs))))),
            new Function<String, String>() {
              @Override
              public String apply(@Nullable String input) {
                return input.toLowerCase();
              }
            });
    if (mergedMetrics.size() != lowerCaseMetricAggs.length) {
      throw new IAE(
          "Bad number of metrics[%d], expected [%d]",
          mergedMetrics.size(), lowerCaseMetricAggs.length);
    }

    final AggregatorFactory[] sortedMetricAggs = new AggregatorFactory[mergedMetrics.size()];
    for (int i = 0; i < lowerCaseMetricAggs.length; i++) {
      AggregatorFactory metricAgg = lowerCaseMetricAggs[i];
      sortedMetricAggs[mergedMetrics.indexOf(metricAgg.getName())] = metricAgg;
    }

    for (int i = 0; i < mergedMetrics.size(); i++) {
      if (!sortedMetricAggs[i].getName().equals(mergedMetrics.get(i))) {
        throw new IAE(
            "Metric mismatch, index[%d] [%s] != [%s]",
            i, lowerCaseMetricAggs[i].getName(), mergedMetrics.get(i));
      }
    }

    Function<ArrayList<Iterable<Rowboat>>, Iterable<Rowboat>> rowMergerFn =
        new Function<ArrayList<Iterable<Rowboat>>, Iterable<Rowboat>>() {
          @Override
          public Iterable<Rowboat> apply(@Nullable ArrayList<Iterable<Rowboat>> boats) {
            return CombiningIterable.create(
                new MergeIterable<Rowboat>(Ordering.<Rowboat>natural().nullsFirst(), boats),
                Ordering.<Rowboat>natural().nullsFirst(),
                new RowboatMergeFunction(sortedMetricAggs));
          }
        };

    return makeIndexFiles(indexes, outDir, progress, mergedDimensions, mergedMetrics, rowMergerFn);
  }