public static Object getAllValue(FieldSpec spec) {
Object allValue;
switch (spec.getDataType()) {
case INT:
allValue = V1Constants.STARTREE_ALL_NUMBER.intValue();
break;
case LONG:
allValue = V1Constants.STARTREE_ALL_NUMBER.longValue();
break;
case FLOAT:
allValue = V1Constants.STARTREE_ALL_NUMBER.floatValue();
break;
case DOUBLE:
allValue = V1Constants.STARTREE_ALL_NUMBER.doubleValue();
break;
case STRING:
case BOOLEAN:
allValue = V1Constants.STARTREE_ALL;
break;
default:
throw new UnsupportedOperationException(
"unsupported data type : "
+ spec.getDataType()
+ " : "
+ " for column : "
+ spec.getName());
}
return allValue;
}
/** Initialize dictionaries for dimension values */
private void initializeAndBuildDictionaries(
Schema schema, Map<String, ColumnIndexCreationInfo> columnInfo, File file) throws Exception {
for (final FieldSpec spec : schema.getAllFieldSpecs()) {
final ColumnIndexCreationInfo info = columnInfo.get(spec.getName());
if (info.isCreateDictionary()) {
dictionaryCreatorMap.put(
spec.getName(),
new SegmentDictionaryCreator(
info.hasNulls(), info.getSortedUniqueElementsArray(), spec, file));
} else {
throw new RuntimeException("Creation of indices without dictionaries is not implemented!");
}
dictionaryCreatorMap.get(spec.getName()).build();
}
// Add __ALL__ to dimension dictionaries
for (DimensionFieldSpec spec : schema.getDimensionFieldSpecs()) {
Object allValue = StarTreeIndexNode.getAllValue(spec);
if (schema.getFieldSpecFor(spec.getName()).isSingleValueField()) {
Object allIndex = dictionaryCreatorMap.get(spec.getName()).indexOfSV(allValue);
} else {
Object allIndex = dictionaryCreatorMap.get(spec.getName()).indexOfMV(allValue);
}
}
}
private Block getBlock() {
if (!blockReturned) {
blockReturned = true;
isPredicateEvaluated = true;
if (fieldSpec.isSingleValueField()) {
Block SvBlock =
new RealtimeSingleValueBlock(
filteredDocIdBitmap,
fieldSpec,
dictionary,
offset,
(FixedByteSingleColumnSingleValueReaderWriter) indexReader);
return SvBlock;
} else {
Block mvBlock =
new RealtimeMultiValueBlock(
fieldSpec,
dictionary,
filteredDocIdBitmap,
offset,
maxNumberOfMultiValues,
(FixedByteSingleColumnMultiValueReaderWriter) indexReader);
return mvBlock;
}
}
return null;
}
@Override
public void init(
SegmentGeneratorConfig segmentCreationSpec,
Map<String, ColumnIndexCreationInfo> indexCreationInfoMap,
Schema schema,
int totalDocs,
File outDir)
throws Exception {
docIdCounter = 0;
config = segmentCreationSpec;
this.indexCreationInfoMap = indexCreationInfoMap;
dictionaryCreatorMap = new HashMap<String, SegmentDictionaryCreator>();
forwardIndexCreatorMap = new HashMap<String, ForwardIndexCreator>();
this.indexCreationInfoMap = indexCreationInfoMap;
invertedIndexCreatorMap = new HashMap<String, InvertedIndexCreator>();
file = outDir;
// Check that the output directory does not exist
if (file.exists()) {
throw new RuntimeException(
"Segment output directory " + file.getAbsolutePath() + " already exists.");
}
file.mkdir();
this.schema = schema;
this.totalDocs = totalDocs;
// Initialize and build dictionaries
for (final FieldSpec spec : schema.getAllFieldSpecs()) {
final ColumnIndexCreationInfo info = indexCreationInfoMap.get(spec.getName());
if (info.isCreateDictionary()) {
dictionaryCreatorMap.put(
spec.getName(),
new SegmentDictionaryCreator(
info.hasNulls(), info.getSortedUniqueElementsArray(), spec, file));
} else {
throw new RuntimeException("Creation of indices without dictionaries is not implemented!");
}
}
// For each column, build its dictionary and initialize a forwards and an inverted index
for (final String column : dictionaryCreatorMap.keySet()) {
dictionaryCreatorMap.get(column).build();
dictionaryCache.put(column, new HashMap<Object, Object>());
ColumnIndexCreationInfo indexCreationInfo = indexCreationInfoMap.get(column);
if (schema.getFieldSpecFor(column).isSingleValueField()) {
if (indexCreationInfo.isSorted()) {
forwardIndexCreatorMap.put(
column,
new SingleValueSortedForwardIndexCreator(
file,
indexCreationInfo.getSortedUniqueElementsArray().length,
schema.getFieldSpecFor(column)));
} else {
forwardIndexCreatorMap.put(
column,
new SingleValueUnsortedForwardIndexCreator(
schema.getFieldSpecFor(column),
file,
indexCreationInfo.getSortedUniqueElementsArray().length,
totalDocs,
indexCreationInfo.getTotalNumberOfEntries(),
indexCreationInfo.hasNulls()));
}
} else {
forwardIndexCreatorMap.put(
column,
new MultiValueUnsortedForwardIndexCreator(
schema.getFieldSpecFor(column),
file,
indexCreationInfo.getSortedUniqueElementsArray().length,
totalDocs,
indexCreationInfo.getTotalNumberOfEntries(),
indexCreationInfo.hasNulls()));
}
if (config.createInvertedIndexEnabled()) {
invertedIndexCreatorMap.put(
column,
new BitmapInvertedIndexCreator(
file,
indexCreationInfo.getSortedUniqueElementsArray().length,
schema.getFieldSpecFor(column)));
}
}
}