public static com.linkedin.pinot.core.indexsegment.IndexSegment load(
File indexDir, ReadMode readMode, IndexLoadingConfigMetadata indexLoadingConfigMetadata)
throws Exception {
Preconditions.checkNotNull(indexDir);
Preconditions.checkArgument(
indexDir.exists(), "Index directory: {} does not exist", indexDir);
Preconditions.checkArgument(
indexDir.isDirectory(), "Index directory: {} is not a directory", indexDir);
// NOTE: indexLoadingConfigMetadata can be null
SegmentMetadataImpl metadata = new SegmentMetadataImpl(indexDir);
SegmentVersion configuredVersionToLoad = getSegmentVersionToLoad(indexLoadingConfigMetadata);
SegmentVersion metadataVersion = metadata.getSegmentVersion();
if (shouldConvertFormat(metadataVersion, configuredVersionToLoad)
&& !targetFormatAlreadyExists(indexDir, configuredVersionToLoad)) {
LOGGER.info(
"segment:{} needs to be converted from :{} to {} version.",
indexDir.getName(),
metadataVersion,
configuredVersionToLoad);
SegmentFormatConverter converter =
SegmentFormatConverterFactory.getConverter(metadataVersion, configuredVersionToLoad);
LOGGER.info("Using converter:{} to up-convert the format", converter.getClass().getName());
converter.convert(indexDir);
LOGGER.info(
"Successfully up-converted segment:{} from :{} to {} version.",
indexDir.getName(),
metadataVersion,
configuredVersionToLoad);
}
File segmentDirectoryPath =
SegmentDirectoryPaths.segmentDirectoryFor(indexDir, configuredVersionToLoad);
// load the metadata again since converter may have changed it
metadata = new SegmentMetadataImpl(segmentDirectoryPath);
// add or removes indexes based on indexLoadingConfigurationMetadata
try (SegmentPreProcessor preProcessor =
new SegmentPreProcessor(segmentDirectoryPath, metadata, indexLoadingConfigMetadata)) {
preProcessor.process();
}
SegmentDirectory segmentDirectory =
SegmentDirectory.createFromLocalFS(segmentDirectoryPath, metadata, readMode);
Map<String, ColumnIndexContainer> indexContainerMap =
new HashMap<String, ColumnIndexContainer>();
SegmentDirectory.Reader segmentReader = segmentDirectory.createReader();
for (String column : metadata.getColumnMetadataMap().keySet()) {
indexContainerMap.put(
column,
ColumnIndexContainer.init(
segmentReader, metadata.getColumnMetadataFor(column), indexLoadingConfigMetadata));
}
// load star tree index if it exists
StarTree starTree = null;
if (segmentReader.hasStarTree()) {
LOGGER.debug("Loading star tree for segment: {}", segmentDirectory);
starTree = StarTree.fromBytes(segmentReader.getStarTreeStream());
}
return new IndexSegmentImpl(segmentDirectory, metadata, indexContainerMap, starTree);
}
static boolean targetFormatAlreadyExists(File indexDir, SegmentVersion expectedSegmentVersion) {
return SegmentDirectoryPaths.segmentDirectoryFor(indexDir, expectedSegmentVersion).exists();
}