public static com.linkedin.pinot.core.indexsegment.IndexSegment load(
File indexDir, ReadMode readMode, IndexLoadingConfigMetadata indexLoadingConfigMetadata)
throws Exception {
SegmentMetadataImpl metadata = new SegmentMetadataImpl(indexDir);
if (!metadata
.getVersion()
.equalsIgnoreCase(IndexSegmentImpl.EXPECTED_SEGMENT_VERSION.toString())) {
SegmentVersion from = SegmentVersion.valueOf(metadata.getVersion());
SegmentVersion to =
SegmentVersion.valueOf(IndexSegmentImpl.EXPECTED_SEGMENT_VERSION.toString());
LOGGER.info(
"segment:{} needs to be converted from :{} to {} version.",
indexDir.getName(),
from,
to);
SegmentFormatConverter converter = SegmentFormatConverterFactory.getConverter(from, to);
LOGGER.info("Using converter:{} to up-convert the format", converter.getClass().getName());
converter.convert(indexDir);
LOGGER.info(
"Successfully up-converted segment:{} from :{} to {} version.",
indexDir.getName(),
from,
to);
}
Map<String, ColumnIndexContainer> indexContainerMap =
new HashMap<String, ColumnIndexContainer>();
for (String column : metadata.getColumnMetadataMap().keySet()) {
indexContainerMap.put(
column,
ColumnIndexContainer.init(
column,
indexDir,
metadata.getColumnMetadataFor(column),
indexLoadingConfigMetadata,
readMode));
}
// The star tree index (if available)
StarTreeIndexNode starTreeRoot = null;
if (metadata.hasStarTree()) {
File starTreeFile = new File(indexDir, V1Constants.STARTREE_FILE);
LOGGER.debug("Loading star tree index file {}", starTreeFile);
starTreeRoot = StarTreeIndexNode.fromBytes(new FileInputStream(starTreeFile));
}
return new IndexSegmentImpl(indexDir, metadata, indexContainerMap, starTreeRoot);
}
public static com.linkedin.pinot.core.indexsegment.IndexSegment load(
File indexDir, ReadMode readMode, IndexLoadingConfigMetadata indexLoadingConfigMetadata)
throws Exception {
Preconditions.checkNotNull(indexDir);
Preconditions.checkArgument(
indexDir.exists(), "Index directory: {} does not exist", indexDir);
Preconditions.checkArgument(
indexDir.isDirectory(), "Index directory: {} is not a directory", indexDir);
// NOTE: indexLoadingConfigMetadata can be null
SegmentMetadataImpl metadata = new SegmentMetadataImpl(indexDir);
SegmentVersion configuredVersionToLoad = getSegmentVersionToLoad(indexLoadingConfigMetadata);
SegmentVersion metadataVersion = metadata.getSegmentVersion();
if (shouldConvertFormat(metadataVersion, configuredVersionToLoad)
&& !targetFormatAlreadyExists(indexDir, configuredVersionToLoad)) {
LOGGER.info(
"segment:{} needs to be converted from :{} to {} version.",
indexDir.getName(),
metadataVersion,
configuredVersionToLoad);
SegmentFormatConverter converter =
SegmentFormatConverterFactory.getConverter(metadataVersion, configuredVersionToLoad);
LOGGER.info("Using converter:{} to up-convert the format", converter.getClass().getName());
converter.convert(indexDir);
LOGGER.info(
"Successfully up-converted segment:{} from :{} to {} version.",
indexDir.getName(),
metadataVersion,
configuredVersionToLoad);
}
File segmentDirectoryPath =
SegmentDirectoryPaths.segmentDirectoryFor(indexDir, configuredVersionToLoad);
// load the metadata again since converter may have changed it
metadata = new SegmentMetadataImpl(segmentDirectoryPath);
// add or removes indexes based on indexLoadingConfigurationMetadata
try (SegmentPreProcessor preProcessor =
new SegmentPreProcessor(segmentDirectoryPath, metadata, indexLoadingConfigMetadata)) {
preProcessor.process();
}
SegmentDirectory segmentDirectory =
SegmentDirectory.createFromLocalFS(segmentDirectoryPath, metadata, readMode);
Map<String, ColumnIndexContainer> indexContainerMap =
new HashMap<String, ColumnIndexContainer>();
SegmentDirectory.Reader segmentReader = segmentDirectory.createReader();
for (String column : metadata.getColumnMetadataMap().keySet()) {
indexContainerMap.put(
column,
ColumnIndexContainer.init(
segmentReader, metadata.getColumnMetadataFor(column), indexLoadingConfigMetadata));
}
// load star tree index if it exists
StarTree starTree = null;
if (segmentReader.hasStarTree()) {
LOGGER.debug("Loading star tree for segment: {}", segmentDirectory);
starTree = StarTree.fromBytes(segmentReader.getStarTreeStream());
}
return new IndexSegmentImpl(segmentDirectory, metadata, indexContainerMap, starTree);
}