/** Returns the dimension values, in order of tree level, to this node. */
 public Map<Integer, Integer> getPathValues() {
   StarTreeIndexNode current = this;
   Map<Integer, Integer> values = new HashMap<Integer, Integer>();
   while (current != null && current.getParent() != null) {
     values.put(current.getDimensionName(), current.getDimensionValue());
     current = current.getParent();
   }
   return values;
 }
 public StarTreeIndexNode getMatchingNode(StarTreeIndexNode node, List<Integer> dimensions) {
   if (node == null || node.isLeaf()) {
     return node;
   }
   Integer childDimensionName = node.getChildDimensionName();
   Integer childDimensionValue = dimensions.get(childDimensionName);
   StarTreeIndexNode child = node.getChildren().get(childDimensionValue);
   return getMatchingNode(child, dimensions);
 }
 /** Returns the dimension IDs, in order of tree level, to this node. */
 public List<Integer> getPathDimensions() {
   StarTreeIndexNode current = this;
   List<Integer> dimensions = new LinkedList<Integer>();
   while (current != null && current.getParent() != null) {
     dimensions.add(0, current.getDimensionName());
     current = current.getParent();
   }
   return dimensions;
 }
 /** Collects all the StarTree leaves that match the provided dimension values */
 private void findMatchingLeaves(
     StarTreeIndexNode node, List<Integer> values, Set<StarTreeIndexNode> leaves) {
   if (node.isLeaf()) {
     leaves.add(node);
   } else {
     Integer value = values.get(node.getChildDimensionName());
     findMatchingLeaves(node.getChildren().get(value), values, leaves);
     findMatchingLeaves(node.getChildren().get(StarTreeIndexNode.all()), values, leaves);
   }
 }
  public static void printTree(StarTreeIndexNode node, int level) {
    for (int i = 0; i < level; i++) {
      System.out.print("  ");
    }
    System.out.println(node);

    if (!node.isLeaf()) {
      for (StarTreeIndexNode child : node.getChildren().values()) {
        printTree(child, level + 1);
      }
    }
  }
  /** Initialize dictionaries for dimension values */
  private void initializeAndBuildDictionaries(
      Schema schema, Map<String, ColumnIndexCreationInfo> columnInfo, File file) throws Exception {
    for (final FieldSpec spec : schema.getAllFieldSpecs()) {
      final ColumnIndexCreationInfo info = columnInfo.get(spec.getName());
      if (info.isCreateDictionary()) {
        dictionaryCreatorMap.put(
            spec.getName(),
            new SegmentDictionaryCreator(
                info.hasNulls(), info.getSortedUniqueElementsArray(), spec, file));
      } else {
        throw new RuntimeException("Creation of indices without dictionaries is not implemented!");
      }
      dictionaryCreatorMap.get(spec.getName()).build();
    }

    // Add __ALL__ to dimension dictionaries
    for (DimensionFieldSpec spec : schema.getDimensionFieldSpecs()) {
      Object allValue = StarTreeIndexNode.getAllValue(spec);
      if (schema.getFieldSpecFor(spec.getName()).isSingleValueField()) {
        Object allIndex = dictionaryCreatorMap.get(spec.getName()).indexOfSV(allValue);
      } else {
        Object allIndex = dictionaryCreatorMap.get(spec.getName()).indexOfMV(allValue);
      }
    }
  }
  @Override
  public void indexRow(GenericRow row) {
    // Find matching leaves in StarTree for row
    currentMatchingNodes.clear();
    StarTreeTableRow tableRow = extractValues(row);
    findMatchingLeaves(starTreeBuilder.getTree(), tableRow.getDimensions(), currentMatchingNodes);

    // Only write the raw value, maintaining sort order (we will write aggregates when sealing)
    for (StarTreeIndexNode node : currentMatchingNodes) {
      Map<Integer, Integer> pathValues = node.getPathValues();
      if (!pathValues.containsValue(StarTreeIndexNode.all())) {
        StarTreeTableRange range = starTreeBuilder.getDocumentIdRange(node.getNodeId());
        StarTreeTable subTable =
            starTreeBuilder.getTable().view(range.getStartDocumentId(), range.getDocumentCount());

        Integer nextMatchingDocumentId =
            starTreeBuilder.getNextDocumentId(tableRow.getDimensions());
        if (nextMatchingDocumentId == null) {
          throw new IllegalStateException("Could not assign document ID for row " + tableRow);
        }

        // Write using that document ID to all columns
        for (final String column : dictionaryCreatorMap.keySet()) {
          Object columnValueToIndex = row.getValue(column);
          if (schema.getFieldSpecFor(column).isSingleValueField()) {
            System.out.println(column + ": " + columnValueToIndex);
            int dictionaryIndex = dictionaryCreatorMap.get(column).indexOfSV(columnValueToIndex);
            ((SingleValueForwardIndexCreator) forwardIndexCreatorMap.get(column))
                .index(nextMatchingDocumentId, dictionaryIndex);
            if (config.createInvertedIndexEnabled()) {
              invertedIndexCreatorMap
                  .get(column)
                  .add(nextMatchingDocumentId, (Object) dictionaryIndex);
            }
          } else {
            int[] dictionaryIndex = dictionaryCreatorMap.get(column).indexOfMV(columnValueToIndex);
            ((MultiValueForwardIndexCreator) forwardIndexCreatorMap.get(column))
                .index(nextMatchingDocumentId, dictionaryIndex);
            if (config.createInvertedIndexEnabled()) {
              invertedIndexCreatorMap.get(column).add(nextMatchingDocumentId, dictionaryIndex);
            }
          }
        }
      }
    }
  }
 @Override
 public boolean equals(Object o) {
   if (!(o instanceof StarTreeIndexNode)) {
     return false;
   }
   StarTreeIndexNode n = (StarTreeIndexNode) o;
   return Objects.equal(nodeId, n.getNodeId())
       && Objects.equal(level, n.getLevel())
       && Objects.equal(dimensionName, n.getDimensionName())
       && Objects.equal(dimensionValue, n.getDimensionValue())
       && Objects.equal(childDimensionName, n.getChildDimensionName())
       && Objects.equal(children, n.getChildren())
       && Objects.equal(startDocumentId, n.getStartDocumentId())
       && Objects.equal(documentCount, n.getDocumentCount());
 }
Exemple #9
0
    public static com.linkedin.pinot.core.indexsegment.IndexSegment load(
        File indexDir, ReadMode readMode, IndexLoadingConfigMetadata indexLoadingConfigMetadata)
        throws Exception {
      SegmentMetadataImpl metadata = new SegmentMetadataImpl(indexDir);
      if (!metadata
          .getVersion()
          .equalsIgnoreCase(IndexSegmentImpl.EXPECTED_SEGMENT_VERSION.toString())) {

        SegmentVersion from = SegmentVersion.valueOf(metadata.getVersion());
        SegmentVersion to =
            SegmentVersion.valueOf(IndexSegmentImpl.EXPECTED_SEGMENT_VERSION.toString());
        LOGGER.info(
            "segment:{} needs to be converted from :{} to {} version.",
            indexDir.getName(),
            from,
            to);
        SegmentFormatConverter converter = SegmentFormatConverterFactory.getConverter(from, to);
        LOGGER.info("Using converter:{} to up-convert the format", converter.getClass().getName());
        converter.convert(indexDir);
        LOGGER.info(
            "Successfully up-converted segment:{} from :{} to {} version.",
            indexDir.getName(),
            from,
            to);
      }

      Map<String, ColumnIndexContainer> indexContainerMap =
          new HashMap<String, ColumnIndexContainer>();

      for (String column : metadata.getColumnMetadataMap().keySet()) {
        indexContainerMap.put(
            column,
            ColumnIndexContainer.init(
                column,
                indexDir,
                metadata.getColumnMetadataFor(column),
                indexLoadingConfigMetadata,
                readMode));
      }

      // The star tree index (if available)
      StarTreeIndexNode starTreeRoot = null;
      if (metadata.hasStarTree()) {
        File starTreeFile = new File(indexDir, V1Constants.STARTREE_FILE);
        LOGGER.debug("Loading star tree index file {}", starTreeFile);
        starTreeRoot = StarTreeIndexNode.fromBytes(new FileInputStream(starTreeFile));
      }
      return new IndexSegmentImpl(indexDir, metadata, indexContainerMap, starTreeRoot);
    }
  @Override
  public void seal() throws ConfigurationException, IOException {
    // Write all the aggregate rows to the aggregate segment
    LOG.info("Writing aggregate segment...");
    long startMillis = System.currentTimeMillis();
    int currentAggregateDocumentId = 0;
    Iterator<StarTreeTableRow> itr = starTreeBuilder.getTable().getAllCombinations();
    while (itr.hasNext()) {
      StarTreeTableRow next = itr.next();
      if (next.getDimensions().contains(StarTreeIndexNode.all())) {
        // Write using that document ID to all columns
        for (final String column : dictionaryCreatorMap.keySet()) {
          Object dictionaryIndex = null; // TODO: Is this okay?

          if (starTreeDimensionDictionary.containsKey(column)) {
            // Index the dimension value
            Integer dimensionId = starTreeDimensionDictionary.get(column);
            Integer dimensionValue = next.getDimensions().get(dimensionId);
            if (dimensionValue == StarTreeIndexNode.all()) {
              // Use all value
              Object allValue = StarTreeIndexNode.getAllValue(schema.getFieldSpecFor(column));
              if (schema.getFieldSpecFor(column).isSingleValueField()) {
                dictionaryIndex = dictionaryCreatorMap.get(column).indexOfSV(allValue);
              } else {
                dictionaryIndex = dictionaryCreatorMap.get(column).indexOfMV(allValue);
              }
            } else {
              dictionaryIndex = dimensionValue;
            }
          } else if (starTreeMetricDictionary.containsKey(column)) {
            // Index the aggregate metric
            Integer metricId = starTreeMetricDictionary.get(column);
            Object columnValueToIndex = next.getMetrics().get(metricId);
            if (schema.getFieldSpecFor(column).isSingleValueField()) {
              dictionaryIndex = dictionaryCreatorMap.get(column).indexOfSV(columnValueToIndex);
            } else {
              dictionaryIndex = dictionaryCreatorMap.get(column).indexOfMV(columnValueToIndex);
            }
          } else {
            // Just index the raw value
            Object columnValueToIndex =
                StarTreeIndexNode.getAllValue(schema.getFieldSpecFor(column));
            if (schema.getFieldSpecFor(column).isSingleValueField()) {
              dictionaryIndex = dictionaryCreatorMap.get(column).indexOfSV(columnValueToIndex);
            } else {
              dictionaryIndex = dictionaryCreatorMap.get(column).indexOfMV(columnValueToIndex);
            }
          }

          if (schema.getFieldSpecFor(column).isSingleValueField()) {
            ((SingleValueForwardIndexCreator) aggregateForwardIndexCreatorMap.get(column))
                .index(currentAggregateDocumentId, (Integer) dictionaryIndex);
          } else {
            ((MultiValueForwardIndexCreator) aggregateForwardIndexCreatorMap.get(column))
                .index(currentAggregateDocumentId, (int[]) dictionaryIndex);
          }

          if (config.createInvertedIndexEnabled()) {
            aggregateInvertedIndexCreatorMap
                .get(column)
                .add(currentAggregateDocumentId, dictionaryIndex);
          }
        }
        currentAggregateDocumentId++;
      }
    }
    long endMillis = System.currentTimeMillis();
    LOG.info("Done writing aggregate segment (took {} ms)", endMillis - startMillis);

    for (final String column : forwardIndexCreatorMap.keySet()) {
      forwardIndexCreatorMap.get(column).close();
      if (config.createInvertedIndexEnabled()) {
        invertedIndexCreatorMap.get(column).seal();
      }
      dictionaryCreatorMap.get(column).close();
    }

    for (final String column : aggregateForwardIndexCreatorMap.keySet()) {
      aggregateForwardIndexCreatorMap.get(column).close();
      if (config.createInvertedIndexEnabled()) {
        aggregateInvertedIndexCreatorMap.get(column).seal();
      }
      // n.b. The dictionary from raw data is used
    }

    writeMetadata(outDir, starTreeBuilder.getTotalRawDocumentCount());

    // Write star tree
    LOG.info("Writing " + V1Constants.STARTREE_FILE);
    startMillis = System.currentTimeMillis();
    File starTreeFile = new File(starTreeDir, V1Constants.STARTREE_FILE);
    OutputStream starTreeOutputStream = new FileOutputStream(starTreeFile);
    starTreeBuilder.getTree().writeTree(starTreeOutputStream);
    starTreeOutputStream.close();
    endMillis = System.currentTimeMillis();
    LOG.info("Wrote StarTree file (took {} ms)", endMillis - startMillis);

    // Copy the dictionary files into startree directory
    // n.b. this is done so the segment is as stand-alone as possible, though could be removed as an
    // optimization
    File[] dictionaryFiles =
        outDir.listFiles(
            new FilenameFilter() {
              @Override
              public boolean accept(File dir, String name) {
                return name.endsWith(V1Constants.Dict.FILE_EXTENTION);
              }
            });
    for (File dictionaryFile : dictionaryFiles) {
      FileUtils.copyFile(dictionaryFile, new File(starTreeDir, dictionaryFile.getName()));
    }

    // Write star tree metadata
    writeMetadata(starTreeDir, starTreeBuilder.getTotalAggregateDocumentCount());
  }