@Test public void testSanity() throws Exception { File segmentDir = tempFolder.newFolder(); createTestIndex(segmentDir); QueryableIndex qi = null; try { qi = IndexIO.loadIndex(segmentDir); StorageAdapter sa = new QueryableIndexStorageAdapter(qi); WindowedStorageAdapter wsa = new WindowedStorageAdapter(sa, sa.getInterval()); IngestSegmentFirehose firehose = new IngestSegmentFirehose( ImmutableList.of(wsa, wsa), ImmutableList.of("host"), ImmutableList.of("visited_sum", "unique_hosts"), null, QueryGranularity.NONE); int count = 0; while (firehose.hasMore()) { firehose.nextRow(); count++; } Assert.assertEquals(18, count); } finally { if (qi != null) { qi.close(); } } }
@Override public Sequence<Result<SearchResultValue>> run( final Query<Result<SearchResultValue>> input, Map<String, Object> responseContext) { if (!(input instanceof SearchQuery)) { throw new ISE("Got a [%s] which isn't a %s", input.getClass(), SearchQuery.class); } final SearchQuery query = (SearchQuery) input; final Filter filter = Filters.convertDimensionFilters(query.getDimensionsFilter()); final List<DimensionSpec> dimensions = query.getDimensions(); final SearchQuerySpec searchQuerySpec = query.getQuery(); final int limit = query.getLimit(); final boolean descending = query.isDescending(); // Closing this will cause segfaults in unit tests. final QueryableIndex index = segment.asQueryableIndex(); if (index != null) { final TreeMap<SearchHit, MutableInt> retVal = Maps.newTreeMap(query.getSort().getComparator()); Iterable<DimensionSpec> dimsToSearch; if (dimensions == null || dimensions.isEmpty()) { dimsToSearch = Iterables.transform(index.getAvailableDimensions(), Druids.DIMENSION_IDENTITY); } else { dimsToSearch = dimensions; } final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions(); final ImmutableBitmap baseFilter = filter == null ? null : filter.getBitmapIndex(new ColumnSelectorBitmapIndexSelector(bitmapFactory, index)); for (DimensionSpec dimension : dimsToSearch) { final Column column = index.getColumn(dimension.getDimension()); if (column == null) { continue; } final BitmapIndex bitmapIndex = column.getBitmapIndex(); ExtractionFn extractionFn = dimension.getExtractionFn(); if (extractionFn == null) { extractionFn = IdentityExtractionFn.getInstance(); } if (bitmapIndex != null) { for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i))); if (!searchQuerySpec.accept(dimVal)) { continue; } ImmutableBitmap bitmap = bitmapIndex.getBitmap(i); if (baseFilter != null) { bitmap = bitmapFactory.intersection(Arrays.asList(baseFilter, bitmap)); } if (bitmap.size() > 0) { MutableInt counter = new MutableInt(bitmap.size()); MutableInt prev = retVal.put(new SearchHit(dimension.getOutputName(), dimVal), counter); if (prev != null) { counter.add(prev.intValue()); } if (retVal.size() >= limit) { return makeReturnResult(limit, retVal); } } } } } return makeReturnResult(limit, retVal); } final StorageAdapter adapter = segment.asStorageAdapter(); if (adapter == null) { log.makeAlert("WTF!? Unable to process search query on segment.") .addData("segment", segment.getIdentifier()) .addData("query", query) .emit(); throw new ISE( "Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped."); } final Iterable<DimensionSpec> dimsToSearch; if (dimensions == null || dimensions.isEmpty()) { dimsToSearch = Iterables.transform(adapter.getAvailableDimensions(), Druids.DIMENSION_IDENTITY); } else { dimsToSearch = dimensions; } final Sequence<Cursor> cursors = adapter.makeCursors(filter, segment.getDataInterval(), QueryGranularity.ALL, descending); final TreeMap<SearchHit, MutableInt> retVal = cursors.accumulate( Maps.<SearchHit, SearchHit, MutableInt>newTreeMap(query.getSort().getComparator()), new Accumulator<TreeMap<SearchHit, MutableInt>, Cursor>() { @Override public TreeMap<SearchHit, MutableInt> accumulate( TreeMap<SearchHit, MutableInt> set, Cursor cursor) { if (set.size() >= limit) { return set; } Map<String, DimensionSelector> dimSelectors = Maps.newHashMap(); for (DimensionSpec dim : dimsToSearch) { dimSelectors.put(dim.getOutputName(), cursor.makeDimensionSelector(dim)); } while (!cursor.isDone()) { for (Map.Entry<String, DimensionSelector> entry : dimSelectors.entrySet()) { final DimensionSelector selector = entry.getValue(); if (selector != null) { final IndexedInts vals = selector.getRow(); for (int i = 0; i < vals.size(); ++i) { final String dimVal = selector.lookupName(vals.get(i)); if (searchQuerySpec.accept(dimVal)) { MutableInt counter = new MutableInt(1); MutableInt prev = set.put(new SearchHit(entry.getKey(), dimVal), counter); if (prev != null) { counter.add(prev.intValue()); } if (set.size() >= limit) { return set; } } } } } cursor.advance(); } return set; } }); return makeReturnResult(limit, retVal); }