/** Collects all the StarTree leaves that match the provided dimension values */
private void findMatchingLeaves(
StarTreeIndexNode node, List<Integer> values, Set<StarTreeIndexNode> leaves) {
if (node.isLeaf()) {
leaves.add(node);
} else {
Integer value = values.get(node.getChildDimensionName());
findMatchingLeaves(node.getChildren().get(value), values, leaves);
findMatchingLeaves(node.getChildren().get(StarTreeIndexNode.all()), values, leaves);
}
}
/**
* Returns the unique metric values for each column.
*
* <p>The original unique values cannot be used because after aggregation, we almost certainly
* have new values to encode that were not present in the original data set.
*/
private Map<String, Set<Object>> computeUniqueMetricValues() {
Map<String, Set<Object>> uniqueMetricValues = new HashMap<String, Set<Object>>();
Iterator<StarTreeTableRow> tableIterator = starTreeBuilder.getTable().getAllCombinations();
while (tableIterator.hasNext()) {
StarTreeTableRow row = tableIterator.next();
for (int i = 0; i < schema.getMetricNames().size(); i++) {
String metricName = schema.getMetricNames().get(i);
Object metricValue = row.getMetrics().get(i);
Set<Object> uniqueValues = uniqueMetricValues.get(metricName);
if (uniqueValues == null) {
uniqueValues = new HashSet<Object>();
uniqueMetricValues.put(metricName, uniqueValues);
}
uniqueValues.add(metricValue);
}
}
return uniqueMetricValues;
}
@Override
public void indexRow(GenericRow row) {
// Find matching leaves in StarTree for row
currentMatchingNodes.clear();
StarTreeTableRow tableRow = extractValues(row);
findMatchingLeaves(starTreeBuilder.getTree(), tableRow.getDimensions(), currentMatchingNodes);
// Only write the raw value, maintaining sort order (we will write aggregates when sealing)
for (StarTreeIndexNode node : currentMatchingNodes) {
Map<Integer, Integer> pathValues = node.getPathValues();
if (!pathValues.containsValue(StarTreeIndexNode.all())) {
StarTreeTableRange range = starTreeBuilder.getDocumentIdRange(node.getNodeId());
StarTreeTable subTable =
starTreeBuilder.getTable().view(range.getStartDocumentId(), range.getDocumentCount());
Integer nextMatchingDocumentId =
starTreeBuilder.getNextDocumentId(tableRow.getDimensions());
if (nextMatchingDocumentId == null) {
throw new IllegalStateException("Could not assign document ID for row " + tableRow);
}
// Write using that document ID to all columns
for (final String column : dictionaryCreatorMap.keySet()) {
Object columnValueToIndex = row.getValue(column);
if (schema.getFieldSpecFor(column).isSingleValueField()) {
System.out.println(column + ": " + columnValueToIndex);
int dictionaryIndex = dictionaryCreatorMap.get(column).indexOfSV(columnValueToIndex);
((SingleValueForwardIndexCreator) forwardIndexCreatorMap.get(column))
.index(nextMatchingDocumentId, dictionaryIndex);
if (config.createInvertedIndexEnabled()) {
invertedIndexCreatorMap
.get(column)
.add(nextMatchingDocumentId, (Object) dictionaryIndex);
}
} else {
int[] dictionaryIndex = dictionaryCreatorMap.get(column).indexOfMV(columnValueToIndex);
((MultiValueForwardIndexCreator) forwardIndexCreatorMap.get(column))
.index(nextMatchingDocumentId, dictionaryIndex);
if (config.createInvertedIndexEnabled()) {
invertedIndexCreatorMap.get(column).add(nextMatchingDocumentId, dictionaryIndex);
}
}
}
}
}
}