public IncrementalIndexAdapter( Interval dataInterval, IncrementalIndex<?> index, BitmapFactory bitmapFactory) { this.dataInterval = dataInterval; this.index = index; /* Sometimes it's hard to tell whether one dimension contains a null value or not. * If one dimension had show a null or empty value explicitly, then yes, it contains * null value. But if one dimension's values are all non-null, it still early to say * this dimension does not contain null value. Consider a two row case, first row had * "dimA=1" and "dimB=2", the second row only had "dimA=3". To dimB, its value are "2" and * never showed a null or empty value. But when we combines these two rows, dimB is null * in row 2. So we should iterate all rows to determine whether one dimension contains * a null value. */ this.hasNullValueDimensions = Sets.newHashSet(); final List<IncrementalIndex.DimensionDesc> dimensions = index.getDimensions(); indexers = Maps.newHashMapWithExpectedSize(dimensions.size()); for (IncrementalIndex.DimensionDesc dimension : dimensions) { indexers.put(dimension.getName(), new DimensionIndexer(dimension)); } int rowNum = 0; for (IncrementalIndex.TimeAndDims timeAndDims : index.getFacts().keySet()) { final int[][] dims = timeAndDims.getDims(); for (IncrementalIndex.DimensionDesc dimension : dimensions) { final int dimIndex = dimension.getIndex(); DimensionIndexer indexer = indexers.get(dimension.getName()); if (dimIndex >= dims.length || dims[dimIndex] == null) { hasNullValueDimensions.add(dimension.getName()); continue; } final IncrementalIndex.DimDim values = dimension.getValues(); if (hasNullValue(values, dims[dimIndex])) { hasNullValueDimensions.add(dimension.getName()); } final MutableBitmap[] bitmapIndexes = indexer.invertedIndexes; for (Comparable dimIdxComparable : dims[dimIndex]) { Integer dimIdx = (Integer) dimIdxComparable; if (bitmapIndexes[dimIdx] == null) { bitmapIndexes[dimIdx] = bitmapFactory.makeEmptyMutableBitmap(); } try { bitmapIndexes[dimIdx].add(rowNum); } catch (Exception e) { log.info(e.toString()); } } } ++rowNum; } }
@Override public Sequence<Result<SearchResultValue>> run( final Query<Result<SearchResultValue>> input, Map<String, Object> responseContext) { if (!(input instanceof SearchQuery)) { throw new ISE("Got a [%s] which isn't a %s", input.getClass(), SearchQuery.class); } final SearchQuery query = (SearchQuery) input; final Filter filter = Filters.convertDimensionFilters(query.getDimensionsFilter()); final List<DimensionSpec> dimensions = query.getDimensions(); final SearchQuerySpec searchQuerySpec = query.getQuery(); final int limit = query.getLimit(); final boolean descending = query.isDescending(); // Closing this will cause segfaults in unit tests. final QueryableIndex index = segment.asQueryableIndex(); if (index != null) { final TreeMap<SearchHit, MutableInt> retVal = Maps.newTreeMap(query.getSort().getComparator()); Iterable<DimensionSpec> dimsToSearch; if (dimensions == null || dimensions.isEmpty()) { dimsToSearch = Iterables.transform(index.getAvailableDimensions(), Druids.DIMENSION_IDENTITY); } else { dimsToSearch = dimensions; } final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); final ImmutableBitmap baseFilter = filter == null ? null : filter.getBitmapIndex(new ColumnSelectorBitmapIndexSelector(bitmapFactory, index)); for (DimensionSpec dimension : dimsToSearch) { final Column column = index.getColumn(dimension.getDimension()); if (column == null) { continue; } final BitmapIndex bitmapIndex = column.getBitmapIndex(); ExtractionFn extractionFn = dimension.getExtractionFn(); if (extractionFn == null) { extractionFn = IdentityExtractionFn.getInstance(); } if (bitmapIndex != null) { for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i))); if (!searchQuerySpec.accept(dimVal)) { continue; } ImmutableBitmap bitmap = bitmapIndex.getBitmap(i); if (baseFilter != null) { bitmap = bitmapFactory.intersection(Arrays.asList(baseFilter, bitmap)); } if (bitmap.size() > 0) { MutableInt counter = new MutableInt(bitmap.size()); MutableInt prev = retVal.put(new SearchHit(dimension.getOutputName(), dimVal), counter); if (prev != null) { counter.add(prev.intValue()); } if (retVal.size() >= limit) { return makeReturnResult(limit, retVal); } } } } } return makeReturnResult(limit, retVal); } final StorageAdapter adapter = segment.asStorageAdapter(); if (adapter == null) { log.makeAlert("WTF!? Unable to process search query on segment.") .addData("segment", segment.getIdentifier()) .addData("query", query) .emit(); throw new ISE( "Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped."); } final Iterable<DimensionSpec> dimsToSearch; if (dimensions == null || dimensions.isEmpty()) { dimsToSearch = Iterables.transform(adapter.getAvailableDimensions(), Druids.DIMENSION_IDENTITY); } else { dimsToSearch = dimensions; } final Sequence<Cursor> cursors = adapter.makeCursors(filter, segment.getDataInterval(), QueryGranularity.ALL, descending); final TreeMap<SearchHit, MutableInt> retVal = cursors.accumulate( Maps.<SearchHit, SearchHit, MutableInt>newTreeMap(query.getSort().getComparator()), new Accumulator<TreeMap<SearchHit, MutableInt>, Cursor>() { @Override public TreeMap<SearchHit, MutableInt> accumulate( TreeMap<SearchHit, MutableInt> set, Cursor cursor) { if (set.size() >= limit) { return set; } Map<String, DimensionSelector> dimSelectors = Maps.newHashMap(); for (DimensionSpec dim : dimsToSearch) { dimSelectors.put(dim.getOutputName(), cursor.makeDimensionSelector(dim)); } while (!cursor.isDone()) { for (Map.Entry<String, DimensionSelector> entry : dimSelectors.entrySet()) { final DimensionSelector selector = entry.getValue(); if (selector != null) { final IndexedInts vals = selector.getRow(); for (int i = 0; i < vals.size(); ++i) { final String dimVal = selector.lookupName(vals.get(i)); if (searchQuerySpec.accept(dimVal)) { MutableInt counter = new MutableInt(1); MutableInt prev = set.put(new SearchHit(entry.getKey(), dimVal), counter); if (prev != null) { counter.add(prev.intValue()); } if (set.size() >= limit) { return set; } } } } } cursor.advance(); } return set; } }); return makeReturnResult(limit, retVal); }