@Override
public void writeValueMetadataToFile(FileOutputSupplier valueEncodingFile) throws IOException {
final SerializerUtils serializerUtils = new SerializerUtils();
dictionaryWriter.close();
serializerUtils.writeString(valueEncodingFile, dimensionName);
ByteStreams.copy(dictionaryWriter.combineStreams(), valueEncodingFile);
// save this File reference, we will read from it later when building bitmap/spatial indexes
dictionaryFile = valueEncodingFile.getFile();
}
@Override
public void writeIndexesToFiles(
ByteSink invertedIndexFile, OutputSupplier<FileOutputStream> spatialIndexFile)
throws IOException {
final SerializerUtils serializerUtils = new SerializerUtils();
bitmapWriter.close();
serializerUtils.writeString(invertedIndexFile, dimensionName);
ByteStreams.copy(bitmapWriter.combineStreams(), invertedIndexFile);
if (capabilities.hasSpatialIndexes()) {
spatialWriter.close();
serializerUtils.writeString(spatialIndexFile, dimensionName);
ByteStreams.copy(spatialWriter.combineStreams(), spatialIndexFile);
spatialIoPeon.cleanup();
}
}
@Override
public void writeIndexes(List<IntBuffer> segmentRowNumConversions, Closer closer)
throws IOException {
final SerializerUtils serializerUtils = new SerializerUtils();
long dimStartTime = System.currentTimeMillis();
String bmpFilename = String.format("%s.inverted", dimensionName);
bitmapWriter =
new GenericIndexedWriter<>(
ioPeon, bmpFilename, indexSpec.getBitmapSerdeFactory().getObjectStrategy());
bitmapWriter.open();
final MappedByteBuffer dimValsMapped = Files.map(dictionaryFile);
closer.register(
new Closeable() {
@Override
public void close() throws IOException {
ByteBufferUtils.unmap(dimValsMapped);
}
});
if (!dimensionName.equals(serializerUtils.readString(dimValsMapped))) {
throw new ISE("dimensions[%s] didn't equate!? This is a major WTF moment.", dimensionName);
}
Indexed<String> dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY);
log.info("Starting dimension[%s] with cardinality[%,d]", dimensionName, dimVals.size());
final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory();
final BitmapFactory bitmapFactory = bitmapSerdeFactory.getBitmapFactory();
RTree tree = null;
spatialWriter = null;
boolean hasSpatial = capabilities.hasSpatialIndexes();
spatialIoPeon = new TmpFileIOPeon();
if (hasSpatial) {
BitmapFactory bmpFactory = bitmapSerdeFactory.getBitmapFactory();
String spatialFilename = String.format("%s.spatial", dimensionName);
spatialWriter =
new ByteBufferWriter<ImmutableRTree>(
spatialIoPeon,
spatialFilename,
new IndexedRTree.ImmutableRTreeObjectStrategy(bmpFactory));
spatialWriter.open();
tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bitmapFactory), bitmapFactory);
}
IndexSeeker[] dictIdSeeker = toIndexSeekers(adapters, dimConversions, dimensionName);
// Iterate all dim values's dictionary id in ascending order which in line with dim values's
// compare result.
for (int dictId = 0; dictId < dimVals.size(); dictId++) {
progress.progress();
List<Iterable<Integer>> convertedInverteds = Lists.newArrayListWithCapacity(adapters.size());
for (int j = 0; j < adapters.size(); ++j) {
int seekedDictId = dictIdSeeker[j].seek(dictId);
if (seekedDictId != IndexSeeker.NOT_EXIST) {
convertedInverteds.add(
new ConvertingIndexedInts(
adapters.get(j).getBitmapIndex(dimensionName, seekedDictId),
segmentRowNumConversions.get(j)));
}
}
MutableBitmap bitset = bitmapSerdeFactory.getBitmapFactory().makeEmptyMutableBitmap();
for (Integer row :
CombiningIterable.createSplatted(
convertedInverteds, Ordering.<Integer>natural().nullsFirst())) {
if (row != IndexMerger.INVALID_ROW) {
bitset.add(row);
}
}
if ((dictId == 0) && (Iterables.getFirst(dimVals, "") == null)) {
bitset.or(nullRowsBitmap);
}
bitmapWriter.write(bitmapSerdeFactory.getBitmapFactory().makeImmutableBitmap(bitset));
if (hasSpatial) {
String dimVal = dimVals.get(dictId);
if (dimVal != null) {
List<String> stringCoords = Lists.newArrayList(SPLITTER.split(dimVal));
float[] coords = new float[stringCoords.size()];
for (int j = 0; j < coords.length; j++) {
coords[j] = Float.valueOf(stringCoords.get(j));
}
tree.insert(coords, bitset);
}
}
}
log.info(
"Completed dimension[%s] in %,d millis.",
dimensionName, System.currentTimeMillis() - dimStartTime);
if (hasSpatial) {
spatialWriter.write(ImmutableRTree.newImmutableFromMutable(tree));
}
}