public IncrementalIndexAdapter(
      Interval dataInterval, IncrementalIndex<?> index, BitmapFactory bitmapFactory) {
    this.dataInterval = dataInterval;
    this.index = index;

    /* Sometimes it's hard to tell whether one dimension contains a null value or not.
     * If one dimension had show a null or empty value explicitly, then yes, it contains
     * null value. But if one dimension's values are all non-null, it still early to say
     * this dimension does not contain null value. Consider a two row case, first row had
     * "dimA=1" and "dimB=2", the second row only had "dimA=3". To dimB, its value are "2" and
     * never showed a null or empty value. But when we combines these two rows, dimB is null
     * in row 2. So we should iterate all rows to determine whether one dimension contains
     * a null value.
     */
    this.hasNullValueDimensions = Sets.newHashSet();

    final List<IncrementalIndex.DimensionDesc> dimensions = index.getDimensions();

    indexers = Maps.newHashMapWithExpectedSize(dimensions.size());
    for (IncrementalIndex.DimensionDesc dimension : dimensions) {
      indexers.put(dimension.getName(), new DimensionIndexer(dimension));
    }

    int rowNum = 0;
    for (IncrementalIndex.TimeAndDims timeAndDims : index.getFacts().keySet()) {
      final int[][] dims = timeAndDims.getDims();

      for (IncrementalIndex.DimensionDesc dimension : dimensions) {
        final int dimIndex = dimension.getIndex();
        DimensionIndexer indexer = indexers.get(dimension.getName());
        if (dimIndex >= dims.length || dims[dimIndex] == null) {
          hasNullValueDimensions.add(dimension.getName());
          continue;
        }
        final IncrementalIndex.DimDim values = dimension.getValues();
        if (hasNullValue(values, dims[dimIndex])) {
          hasNullValueDimensions.add(dimension.getName());
        }

        final MutableBitmap[] bitmapIndexes = indexer.invertedIndexes;

        for (Comparable dimIdxComparable : dims[dimIndex]) {
          Integer dimIdx = (Integer) dimIdxComparable;
          if (bitmapIndexes[dimIdx] == null) {
            bitmapIndexes[dimIdx] = bitmapFactory.makeEmptyMutableBitmap();
          }
          try {
            bitmapIndexes[dimIdx].add(rowNum);
          } catch (Exception e) {
            log.info(e.toString());
          }
        }
      }

      ++rowNum;
    }
  }
Beispiel #2
0
  @Override
  public Sequence<Result<SearchResultValue>> run(
      final Query<Result<SearchResultValue>> input, Map<String, Object> responseContext) {
    if (!(input instanceof SearchQuery)) {
      throw new ISE("Got a [%s] which isn't a %s", input.getClass(), SearchQuery.class);
    }

    final SearchQuery query = (SearchQuery) input;
    final Filter filter = Filters.convertDimensionFilters(query.getDimensionsFilter());
    final List<DimensionSpec> dimensions = query.getDimensions();
    final SearchQuerySpec searchQuerySpec = query.getQuery();
    final int limit = query.getLimit();
    final boolean descending = query.isDescending();

    // Closing this will cause segfaults in unit tests.
    final QueryableIndex index = segment.asQueryableIndex();

    if (index != null) {
      final TreeMap<SearchHit, MutableInt> retVal =
          Maps.newTreeMap(query.getSort().getComparator());

      Iterable<DimensionSpec> dimsToSearch;
      if (dimensions == null || dimensions.isEmpty()) {
        dimsToSearch =
            Iterables.transform(index.getAvailableDimensions(), Druids.DIMENSION_IDENTITY);
      } else {
        dimsToSearch = dimensions;
      }

      final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();

      final ImmutableBitmap baseFilter =
          filter == null
              ? null
              : filter.getBitmapIndex(new ColumnSelectorBitmapIndexSelector(bitmapFactory, index));

      for (DimensionSpec dimension : dimsToSearch) {
        final Column column = index.getColumn(dimension.getDimension());
        if (column == null) {
          continue;
        }

        final BitmapIndex bitmapIndex = column.getBitmapIndex();
        ExtractionFn extractionFn = dimension.getExtractionFn();
        if (extractionFn == null) {
          extractionFn = IdentityExtractionFn.getInstance();
        }
        if (bitmapIndex != null) {
          for (int i = 0; i < bitmapIndex.getCardinality(); ++i) {
            String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i)));
            if (!searchQuerySpec.accept(dimVal)) {
              continue;
            }
            ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
            if (baseFilter != null) {
              bitmap = bitmapFactory.intersection(Arrays.asList(baseFilter, bitmap));
            }
            if (bitmap.size() > 0) {
              MutableInt counter = new MutableInt(bitmap.size());
              MutableInt prev =
                  retVal.put(new SearchHit(dimension.getOutputName(), dimVal), counter);
              if (prev != null) {
                counter.add(prev.intValue());
              }
              if (retVal.size() >= limit) {
                return makeReturnResult(limit, retVal);
              }
            }
          }
        }
      }

      return makeReturnResult(limit, retVal);
    }

    final StorageAdapter adapter = segment.asStorageAdapter();

    if (adapter == null) {
      log.makeAlert("WTF!? Unable to process search query on segment.")
          .addData("segment", segment.getIdentifier())
          .addData("query", query)
          .emit();
      throw new ISE(
          "Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped.");
    }

    final Iterable<DimensionSpec> dimsToSearch;
    if (dimensions == null || dimensions.isEmpty()) {
      dimsToSearch =
          Iterables.transform(adapter.getAvailableDimensions(), Druids.DIMENSION_IDENTITY);
    } else {
      dimsToSearch = dimensions;
    }

    final Sequence<Cursor> cursors =
        adapter.makeCursors(filter, segment.getDataInterval(), QueryGranularity.ALL, descending);

    final TreeMap<SearchHit, MutableInt> retVal =
        cursors.accumulate(
            Maps.<SearchHit, SearchHit, MutableInt>newTreeMap(query.getSort().getComparator()),
            new Accumulator<TreeMap<SearchHit, MutableInt>, Cursor>() {
              @Override
              public TreeMap<SearchHit, MutableInt> accumulate(
                  TreeMap<SearchHit, MutableInt> set, Cursor cursor) {
                if (set.size() >= limit) {
                  return set;
                }

                Map<String, DimensionSelector> dimSelectors = Maps.newHashMap();
                for (DimensionSpec dim : dimsToSearch) {
                  dimSelectors.put(dim.getOutputName(), cursor.makeDimensionSelector(dim));
                }

                while (!cursor.isDone()) {
                  for (Map.Entry<String, DimensionSelector> entry : dimSelectors.entrySet()) {
                    final DimensionSelector selector = entry.getValue();

                    if (selector != null) {
                      final IndexedInts vals = selector.getRow();
                      for (int i = 0; i < vals.size(); ++i) {
                        final String dimVal = selector.lookupName(vals.get(i));
                        if (searchQuerySpec.accept(dimVal)) {
                          MutableInt counter = new MutableInt(1);
                          MutableInt prev = set.put(new SearchHit(entry.getKey(), dimVal), counter);
                          if (prev != null) {
                            counter.add(prev.intValue());
                          }
                          if (set.size() >= limit) {
                            return set;
                          }
                        }
                      }
                    }
                  }

                  cursor.advance();
                }

                return set;
              }
            });

    return makeReturnResult(limit, retVal);
  }