public IncrementalIndexAdapter(
Interval dataInterval, IncrementalIndex<?> index, BitmapFactory bitmapFactory) {
this.dataInterval = dataInterval;
this.index = index;
/* Sometimes it's hard to tell whether one dimension contains a null value or not.
* If one dimension had show a null or empty value explicitly, then yes, it contains
* null value. But if one dimension's values are all non-null, it still early to say
* this dimension does not contain null value. Consider a two row case, first row had
* "dimA=1" and "dimB=2", the second row only had "dimA=3". To dimB, its value are "2" and
* never showed a null or empty value. But when we combines these two rows, dimB is null
* in row 2. So we should iterate all rows to determine whether one dimension contains
* a null value.
*/
this.hasNullValueDimensions = Sets.newHashSet();
final List<IncrementalIndex.DimensionDesc> dimensions = index.getDimensions();
indexers = Maps.newHashMapWithExpectedSize(dimensions.size());
for (IncrementalIndex.DimensionDesc dimension : dimensions) {
indexers.put(dimension.getName(), new DimensionIndexer(dimension));
}
int rowNum = 0;
for (IncrementalIndex.TimeAndDims timeAndDims : index.getFacts().keySet()) {
final int[][] dims = timeAndDims.getDims();
for (IncrementalIndex.DimensionDesc dimension : dimensions) {
final int dimIndex = dimension.getIndex();
DimensionIndexer indexer = indexers.get(dimension.getName());
if (dimIndex >= dims.length || dims[dimIndex] == null) {
hasNullValueDimensions.add(dimension.getName());
continue;
}
final IncrementalIndex.DimDim values = dimension.getValues();
if (hasNullValue(values, dims[dimIndex])) {
hasNullValueDimensions.add(dimension.getName());
}
final MutableBitmap[] bitmapIndexes = indexer.invertedIndexes;
for (Comparable dimIdxComparable : dims[dimIndex]) {
Integer dimIdx = (Integer) dimIdxComparable;
if (bitmapIndexes[dimIdx] == null) {
bitmapIndexes[dimIdx] = bitmapFactory.makeEmptyMutableBitmap();
}
try {
bitmapIndexes[dimIdx].add(rowNum);
} catch (Exception e) {
log.info(e.toString());
}
}
}
++rowNum;
}
}
@Override
public Sequence<Result<SearchResultValue>> run(
final Query<Result<SearchResultValue>> input, Map<String, Object> responseContext) {
if (!(input instanceof SearchQuery)) {
throw new ISE("Got a [%s] which isn't a %s", input.getClass(), SearchQuery.class);
}
final SearchQuery query = (SearchQuery) input;
final Filter filter = Filters.convertDimensionFilters(query.getDimensionsFilter());
final List<DimensionSpec> dimensions = query.getDimensions();
final SearchQuerySpec searchQuerySpec = query.getQuery();
final int limit = query.getLimit();
final boolean descending = query.isDescending();
// Closing this will cause segfaults in unit tests.
final QueryableIndex index = segment.asQueryableIndex();
if (index != null) {
final TreeMap<SearchHit, MutableInt> retVal =
Maps.newTreeMap(query.getSort().getComparator());
Iterable<DimensionSpec> dimsToSearch;
if (dimensions == null || dimensions.isEmpty()) {
dimsToSearch =
Iterables.transform(index.getAvailableDimensions(), Druids.DIMENSION_IDENTITY);
} else {
dimsToSearch = dimensions;
}
final BitmapFactory bitmapFactory = index.getBitmapFactoryForDimensions();
final ImmutableBitmap baseFilter =
filter == null
? null
: filter.getBitmapIndex(new ColumnSelectorBitmapIndexSelector(bitmapFactory, index));
for (DimensionSpec dimension : dimsToSearch) {
final Column column = index.getColumn(dimension.getDimension());
if (column == null) {
continue;
}
final BitmapIndex bitmapIndex = column.getBitmapIndex();
ExtractionFn extractionFn = dimension.getExtractionFn();
if (extractionFn == null) {
extractionFn = IdentityExtractionFn.getInstance();
}
if (bitmapIndex != null) {
for (int i = 0; i < bitmapIndex.getCardinality(); ++i) {
String dimVal = Strings.nullToEmpty(extractionFn.apply(bitmapIndex.getValue(i)));
if (!searchQuerySpec.accept(dimVal)) {
continue;
}
ImmutableBitmap bitmap = bitmapIndex.getBitmap(i);
if (baseFilter != null) {
bitmap = bitmapFactory.intersection(Arrays.asList(baseFilter, bitmap));
}
if (bitmap.size() > 0) {
MutableInt counter = new MutableInt(bitmap.size());
MutableInt prev =
retVal.put(new SearchHit(dimension.getOutputName(), dimVal), counter);
if (prev != null) {
counter.add(prev.intValue());
}
if (retVal.size() >= limit) {
return makeReturnResult(limit, retVal);
}
}
}
}
}
return makeReturnResult(limit, retVal);
}
final StorageAdapter adapter = segment.asStorageAdapter();
if (adapter == null) {
log.makeAlert("WTF!? Unable to process search query on segment.")
.addData("segment", segment.getIdentifier())
.addData("query", query)
.emit();
throw new ISE(
"Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped.");
}
final Iterable<DimensionSpec> dimsToSearch;
if (dimensions == null || dimensions.isEmpty()) {
dimsToSearch =
Iterables.transform(adapter.getAvailableDimensions(), Druids.DIMENSION_IDENTITY);
} else {
dimsToSearch = dimensions;
}
final Sequence<Cursor> cursors =
adapter.makeCursors(filter, segment.getDataInterval(), QueryGranularity.ALL, descending);
final TreeMap<SearchHit, MutableInt> retVal =
cursors.accumulate(
Maps.<SearchHit, SearchHit, MutableInt>newTreeMap(query.getSort().getComparator()),
new Accumulator<TreeMap<SearchHit, MutableInt>, Cursor>() {
@Override
public TreeMap<SearchHit, MutableInt> accumulate(
TreeMap<SearchHit, MutableInt> set, Cursor cursor) {
if (set.size() >= limit) {
return set;
}
Map<String, DimensionSelector> dimSelectors = Maps.newHashMap();
for (DimensionSpec dim : dimsToSearch) {
dimSelectors.put(dim.getOutputName(), cursor.makeDimensionSelector(dim));
}
while (!cursor.isDone()) {
for (Map.Entry<String, DimensionSelector> entry : dimSelectors.entrySet()) {
final DimensionSelector selector = entry.getValue();
if (selector != null) {
final IndexedInts vals = selector.getRow();
for (int i = 0; i < vals.size(); ++i) {
final String dimVal = selector.lookupName(vals.get(i));
if (searchQuerySpec.accept(dimVal)) {
MutableInt counter = new MutableInt(1);
MutableInt prev = set.put(new SearchHit(entry.getKey(), dimVal), counter);
if (prev != null) {
counter.add(prev.intValue());
}
if (set.size() >= limit) {
return set;
}
}
}
}
}
cursor.advance();
}
return set;
}
});
return makeReturnResult(limit, retVal);
}