/** Converts a raw row into its (possibly partial) dimension and complete metric values */
private StarTreeTableRow extractValues(GenericRow row) {
List<Integer> dimensions = new ArrayList<Integer>();
for (String dimensionName : schema.getDimensionNames()) {
Integer valueId;
if (schema.getFieldSpecFor(dimensionName).isSingleValueField()
&& !starTreeIndexSpec.getExcludedDimensions().contains(dimensionName)) {
Object value = row.getValue(dimensionName);
valueId = dictionaryCreatorMap.get(dimensionName).indexOfSV(value);
} else {
// Multi-value fields are not supported - always ALL
valueId = V1Constants.STARTREE_ALL_NUMBER.intValue();
}
dimensions.add(valueId);
}
List<Number> metrics = new ArrayList<Number>(schema.getMetricNames().size());
for (MetricFieldSpec metricFieldSpec : schema.getMetricFieldSpecs()) {
Object value = row.getValue(metricFieldSpec.getName());
switch (metricFieldSpec.getDataType()) {
case INT:
metrics.add((Integer) value);
break;
case LONG:
metrics.add((Long) value);
break;
case DOUBLE:
metrics.add((Double) value);
break;
case FLOAT:
metrics.add((Float) value);
break;
default:
throw new IllegalStateException("Unsupported data type " + metricFieldSpec.getDataType());
}
}
return new StarTreeTableRow(dimensions, metrics);
}
@Override
public void indexRow(GenericRow row) {
// Find matching leaves in StarTree for row
currentMatchingNodes.clear();
StarTreeTableRow tableRow = extractValues(row);
findMatchingLeaves(starTreeBuilder.getTree(), tableRow.getDimensions(), currentMatchingNodes);
// Only write the raw value, maintaining sort order (we will write aggregates when sealing)
for (StarTreeIndexNode node : currentMatchingNodes) {
Map<Integer, Integer> pathValues = node.getPathValues();
if (!pathValues.containsValue(StarTreeIndexNode.all())) {
StarTreeTableRange range = starTreeBuilder.getDocumentIdRange(node.getNodeId());
StarTreeTable subTable =
starTreeBuilder.getTable().view(range.getStartDocumentId(), range.getDocumentCount());
Integer nextMatchingDocumentId =
starTreeBuilder.getNextDocumentId(tableRow.getDimensions());
if (nextMatchingDocumentId == null) {
throw new IllegalStateException("Could not assign document ID for row " + tableRow);
}
// Write using that document ID to all columns
for (final String column : dictionaryCreatorMap.keySet()) {
Object columnValueToIndex = row.getValue(column);
if (schema.getFieldSpecFor(column).isSingleValueField()) {
System.out.println(column + ": " + columnValueToIndex);
int dictionaryIndex = dictionaryCreatorMap.get(column).indexOfSV(columnValueToIndex);
((SingleValueForwardIndexCreator) forwardIndexCreatorMap.get(column))
.index(nextMatchingDocumentId, dictionaryIndex);
if (config.createInvertedIndexEnabled()) {
invertedIndexCreatorMap
.get(column)
.add(nextMatchingDocumentId, (Object) dictionaryIndex);
}
} else {
int[] dictionaryIndex = dictionaryCreatorMap.get(column).indexOfMV(columnValueToIndex);
((MultiValueForwardIndexCreator) forwardIndexCreatorMap.get(column))
.index(nextMatchingDocumentId, dictionaryIndex);
if (config.createInvertedIndexEnabled()) {
invertedIndexCreatorMap.get(column).add(nextMatchingDocumentId, dictionaryIndex);
}
}
}
}
}
}
@Override
public void indexRow(GenericRow row) {
for (final String column : dictionaryCreatorMap.keySet()) {
Object columnValueToIndex = row.getValue(column);
Object dictionaryIndex;
if (dictionaryCache.get(column).containsKey(columnValueToIndex)) {
dictionaryIndex = dictionaryCache.get(column).get(columnValueToIndex);
} else {
dictionaryIndex = dictionaryCreatorMap.get(column).indexOf(columnValueToIndex);
dictionaryCache.get(column).put(columnValueToIndex, dictionaryIndex);
}
forwardIndexCreatorMap.get(column).index(docIdCounter, dictionaryIndex);
if (config.createInvertedIndexEnabled()) {
invertedIndexCreatorMap.get(column).add(docIdCounter, dictionaryIndex);
}
}
docIdCounter++;
}