Exemplo n.º 1
0
    public static com.linkedin.pinot.core.indexsegment.IndexSegment load(
        File indexDir, ReadMode readMode, IndexLoadingConfigMetadata indexLoadingConfigMetadata)
        throws Exception {

      Preconditions.checkNotNull(indexDir);
      Preconditions.checkArgument(
          indexDir.exists(), "Index directory: {} does not exist", indexDir);
      Preconditions.checkArgument(
          indexDir.isDirectory(), "Index directory: {} is not a directory", indexDir);
      // NOTE: indexLoadingConfigMetadata can be null

      SegmentMetadataImpl metadata = new SegmentMetadataImpl(indexDir);
      SegmentVersion configuredVersionToLoad = getSegmentVersionToLoad(indexLoadingConfigMetadata);
      SegmentVersion metadataVersion = metadata.getSegmentVersion();
      if (shouldConvertFormat(metadataVersion, configuredVersionToLoad)
          && !targetFormatAlreadyExists(indexDir, configuredVersionToLoad)) {
        LOGGER.info(
            "segment:{} needs to be converted from :{} to {} version.",
            indexDir.getName(),
            metadataVersion,
            configuredVersionToLoad);
        SegmentFormatConverter converter =
            SegmentFormatConverterFactory.getConverter(metadataVersion, configuredVersionToLoad);
        LOGGER.info("Using converter:{} to up-convert the format", converter.getClass().getName());
        converter.convert(indexDir);
        LOGGER.info(
            "Successfully up-converted segment:{} from :{} to {} version.",
            indexDir.getName(),
            metadataVersion,
            configuredVersionToLoad);
      }

      File segmentDirectoryPath =
          SegmentDirectoryPaths.segmentDirectoryFor(indexDir, configuredVersionToLoad);
      // load the metadata again since converter may have changed it
      metadata = new SegmentMetadataImpl(segmentDirectoryPath);

      // add or removes indexes based on indexLoadingConfigurationMetadata
      try (SegmentPreProcessor preProcessor =
          new SegmentPreProcessor(segmentDirectoryPath, metadata, indexLoadingConfigMetadata)) {
        preProcessor.process();
      }

      SegmentDirectory segmentDirectory =
          SegmentDirectory.createFromLocalFS(segmentDirectoryPath, metadata, readMode);

      Map<String, ColumnIndexContainer> indexContainerMap =
          new HashMap<String, ColumnIndexContainer>();
      SegmentDirectory.Reader segmentReader = segmentDirectory.createReader();
      for (String column : metadata.getColumnMetadataMap().keySet()) {
        indexContainerMap.put(
            column,
            ColumnIndexContainer.init(
                segmentReader, metadata.getColumnMetadataFor(column), indexLoadingConfigMetadata));
      }

      // load star tree index if it exists
      StarTree starTree = null;
      if (segmentReader.hasStarTree()) {
        LOGGER.debug("Loading star tree for segment: {}", segmentDirectory);
        starTree = StarTree.fromBytes(segmentReader.getStarTreeStream());
      }
      return new IndexSegmentImpl(segmentDirectory, metadata, indexContainerMap, starTree);
    }
Exemplo n.º 2
0
 static boolean targetFormatAlreadyExists(File indexDir, SegmentVersion expectedSegmentVersion) {
   return SegmentDirectoryPaths.segmentDirectoryFor(indexDir, expectedSegmentVersion).exists();
 }