Example #1
0
    public static com.linkedin.pinot.core.indexsegment.IndexSegment load(
        File indexDir, ReadMode readMode, IndexLoadingConfigMetadata indexLoadingConfigMetadata)
        throws Exception {
      SegmentMetadataImpl metadata = new SegmentMetadataImpl(indexDir);
      if (!metadata
          .getVersion()
          .equalsIgnoreCase(IndexSegmentImpl.EXPECTED_SEGMENT_VERSION.toString())) {

        SegmentVersion from = SegmentVersion.valueOf(metadata.getVersion());
        SegmentVersion to =
            SegmentVersion.valueOf(IndexSegmentImpl.EXPECTED_SEGMENT_VERSION.toString());
        LOGGER.info(
            "segment:{} needs to be converted from :{} to {} version.",
            indexDir.getName(),
            from,
            to);
        SegmentFormatConverter converter = SegmentFormatConverterFactory.getConverter(from, to);
        LOGGER.info("Using converter:{} to up-convert the format", converter.getClass().getName());
        converter.convert(indexDir);
        LOGGER.info(
            "Successfully up-converted segment:{} from :{} to {} version.",
            indexDir.getName(),
            from,
            to);
      }

      Map<String, ColumnIndexContainer> indexContainerMap =
          new HashMap<String, ColumnIndexContainer>();

      for (String column : metadata.getColumnMetadataMap().keySet()) {
        indexContainerMap.put(
            column,
            ColumnIndexContainer.init(
                column,
                indexDir,
                metadata.getColumnMetadataFor(column),
                indexLoadingConfigMetadata,
                readMode));
      }

      // The star tree index (if available)
      StarTreeIndexNode starTreeRoot = null;
      if (metadata.hasStarTree()) {
        File starTreeFile = new File(indexDir, V1Constants.STARTREE_FILE);
        LOGGER.debug("Loading star tree index file {}", starTreeFile);
        starTreeRoot = StarTreeIndexNode.fromBytes(new FileInputStream(starTreeFile));
      }
      return new IndexSegmentImpl(indexDir, metadata, indexContainerMap, starTreeRoot);
    }
Example #2
0
    public static com.linkedin.pinot.core.indexsegment.IndexSegment load(
        File indexDir, ReadMode readMode, IndexLoadingConfigMetadata indexLoadingConfigMetadata)
        throws Exception {

      Preconditions.checkNotNull(indexDir);
      Preconditions.checkArgument(
          indexDir.exists(), "Index directory: {} does not exist", indexDir);
      Preconditions.checkArgument(
          indexDir.isDirectory(), "Index directory: {} is not a directory", indexDir);
      // NOTE: indexLoadingConfigMetadata can be null

      SegmentMetadataImpl metadata = new SegmentMetadataImpl(indexDir);
      SegmentVersion configuredVersionToLoad = getSegmentVersionToLoad(indexLoadingConfigMetadata);
      SegmentVersion metadataVersion = metadata.getSegmentVersion();
      if (shouldConvertFormat(metadataVersion, configuredVersionToLoad)
          && !targetFormatAlreadyExists(indexDir, configuredVersionToLoad)) {
        LOGGER.info(
            "segment:{} needs to be converted from :{} to {} version.",
            indexDir.getName(),
            metadataVersion,
            configuredVersionToLoad);
        SegmentFormatConverter converter =
            SegmentFormatConverterFactory.getConverter(metadataVersion, configuredVersionToLoad);
        LOGGER.info("Using converter:{} to up-convert the format", converter.getClass().getName());
        converter.convert(indexDir);
        LOGGER.info(
            "Successfully up-converted segment:{} from :{} to {} version.",
            indexDir.getName(),
            metadataVersion,
            configuredVersionToLoad);
      }

      File segmentDirectoryPath =
          SegmentDirectoryPaths.segmentDirectoryFor(indexDir, configuredVersionToLoad);
      // load the metadata again since converter may have changed it
      metadata = new SegmentMetadataImpl(segmentDirectoryPath);

      // add or removes indexes based on indexLoadingConfigurationMetadata
      try (SegmentPreProcessor preProcessor =
          new SegmentPreProcessor(segmentDirectoryPath, metadata, indexLoadingConfigMetadata)) {
        preProcessor.process();
      }

      SegmentDirectory segmentDirectory =
          SegmentDirectory.createFromLocalFS(segmentDirectoryPath, metadata, readMode);

      Map<String, ColumnIndexContainer> indexContainerMap =
          new HashMap<String, ColumnIndexContainer>();
      SegmentDirectory.Reader segmentReader = segmentDirectory.createReader();
      for (String column : metadata.getColumnMetadataMap().keySet()) {
        indexContainerMap.put(
            column,
            ColumnIndexContainer.init(
                segmentReader, metadata.getColumnMetadataFor(column), indexLoadingConfigMetadata));
      }

      // load star tree index if it exists
      StarTree starTree = null;
      if (segmentReader.hasStarTree()) {
        LOGGER.debug("Loading star tree for segment: {}", segmentDirectory);
        starTree = StarTree.fromBytes(segmentReader.getStarTreeStream());
      }
      return new IndexSegmentImpl(segmentDirectory, metadata, indexContainerMap, starTree);
    }