/** * Returns the unique metric values for each column. * * <p>The original unique values cannot be used because after aggregation, we almost certainly * have new values to encode that were not present in the original data set. */ private Map<String, Set<Object>> computeUniqueMetricValues() { Map<String, Set<Object>> uniqueMetricValues = new HashMap<String, Set<Object>>(); Iterator<StarTreeTableRow> tableIterator = starTreeBuilder.getTable().getAllCombinations(); while (tableIterator.hasNext()) { StarTreeTableRow row = tableIterator.next(); for (int i = 0; i < schema.getMetricNames().size(); i++) { String metricName = schema.getMetricNames().get(i); Object metricValue = row.getMetrics().get(i); Set<Object> uniqueValues = uniqueMetricValues.get(metricName); if (uniqueValues == null) { uniqueValues = new HashSet<Object>(); uniqueMetricValues.put(metricName, uniqueValues); } uniqueValues.add(metricValue); } } return uniqueMetricValues; }
@Override public void seal() throws ConfigurationException, IOException { // Write all the aggregate rows to the aggregate segment LOG.info("Writing aggregate segment..."); long startMillis = System.currentTimeMillis(); int currentAggregateDocumentId = 0; Iterator<StarTreeTableRow> itr = starTreeBuilder.getTable().getAllCombinations(); while (itr.hasNext()) { StarTreeTableRow next = itr.next(); if (next.getDimensions().contains(StarTreeIndexNode.all())) { // Write using that document ID to all columns for (final String column : dictionaryCreatorMap.keySet()) { Object dictionaryIndex = null; // TODO: Is this okay? if (starTreeDimensionDictionary.containsKey(column)) { // Index the dimension value Integer dimensionId = starTreeDimensionDictionary.get(column); Integer dimensionValue = next.getDimensions().get(dimensionId); if (dimensionValue == StarTreeIndexNode.all()) { // Use all value Object allValue = StarTreeIndexNode.getAllValue(schema.getFieldSpecFor(column)); if (schema.getFieldSpecFor(column).isSingleValueField()) { dictionaryIndex = dictionaryCreatorMap.get(column).indexOfSV(allValue); } else { dictionaryIndex = dictionaryCreatorMap.get(column).indexOfMV(allValue); } } else { dictionaryIndex = dimensionValue; } } else if (starTreeMetricDictionary.containsKey(column)) { // Index the aggregate metric Integer metricId = starTreeMetricDictionary.get(column); Object columnValueToIndex = next.getMetrics().get(metricId); if (schema.getFieldSpecFor(column).isSingleValueField()) { dictionaryIndex = dictionaryCreatorMap.get(column).indexOfSV(columnValueToIndex); } else { dictionaryIndex = dictionaryCreatorMap.get(column).indexOfMV(columnValueToIndex); } } else { // Just index the raw value Object columnValueToIndex = StarTreeIndexNode.getAllValue(schema.getFieldSpecFor(column)); if (schema.getFieldSpecFor(column).isSingleValueField()) { dictionaryIndex = dictionaryCreatorMap.get(column).indexOfSV(columnValueToIndex); } else { dictionaryIndex = dictionaryCreatorMap.get(column).indexOfMV(columnValueToIndex); } } if (schema.getFieldSpecFor(column).isSingleValueField()) { ((SingleValueForwardIndexCreator) aggregateForwardIndexCreatorMap.get(column)) .index(currentAggregateDocumentId, (Integer) dictionaryIndex); } else { ((MultiValueForwardIndexCreator) aggregateForwardIndexCreatorMap.get(column)) .index(currentAggregateDocumentId, (int[]) dictionaryIndex); } if (config.createInvertedIndexEnabled()) { aggregateInvertedIndexCreatorMap .get(column) .add(currentAggregateDocumentId, dictionaryIndex); } } currentAggregateDocumentId++; } } long endMillis = System.currentTimeMillis(); LOG.info("Done writing aggregate segment (took {} ms)", endMillis - startMillis); for (final String column : forwardIndexCreatorMap.keySet()) { forwardIndexCreatorMap.get(column).close(); if (config.createInvertedIndexEnabled()) { invertedIndexCreatorMap.get(column).seal(); } dictionaryCreatorMap.get(column).close(); } for (final String column : aggregateForwardIndexCreatorMap.keySet()) { aggregateForwardIndexCreatorMap.get(column).close(); if (config.createInvertedIndexEnabled()) { aggregateInvertedIndexCreatorMap.get(column).seal(); } // n.b. The dictionary from raw data is used } writeMetadata(outDir, starTreeBuilder.getTotalRawDocumentCount()); // Write star tree LOG.info("Writing " + V1Constants.STARTREE_FILE); startMillis = System.currentTimeMillis(); File starTreeFile = new File(starTreeDir, V1Constants.STARTREE_FILE); OutputStream starTreeOutputStream = new FileOutputStream(starTreeFile); starTreeBuilder.getTree().writeTree(starTreeOutputStream); starTreeOutputStream.close(); endMillis = System.currentTimeMillis(); LOG.info("Wrote StarTree file (took {} ms)", endMillis - startMillis); // Copy the dictionary files into startree directory // n.b. this is done so the segment is as stand-alone as possible, though could be removed as an // optimization File[] dictionaryFiles = outDir.listFiles( new FilenameFilter() { @Override public boolean accept(File dir, String name) { return name.endsWith(V1Constants.Dict.FILE_EXTENTION); } }); for (File dictionaryFile : dictionaryFiles) { FileUtils.copyFile(dictionaryFile, new File(starTreeDir, dictionaryFile.getName())); } // Write star tree metadata writeMetadata(starTreeDir, starTreeBuilder.getTotalAggregateDocumentCount()); }