@Override
public long getGlobalOrd(long segmentOrd) {
return segmentOrd + segmentOrdToGlobalOrdLookup.get((int) segmentOrd);
}
public OrdinalMappingSource[] build(long maxOrd) {
// If we find out that there are less then predefined number of ordinals, it is better to put
// the the
// segment ordinal to global ordinal mapping in a packed ints, since the amount values are
// small and
// will most likely fit in the CPU caches and MonotonicAppendingLongBuffer's compression will
// just be
// unnecessary.
if (maxOrd <= threshold) {
// Rebuilding from MonotonicAppendingLongBuffer to PackedInts.Mutable is fast
PackedInts.Mutable[] newSegmentOrdToGlobalOrdDeltas = new PackedInts.Mutable[numSegments];
for (int i = 0; i < segmentOrdToGlobalOrdDeltas.length; i++) {
newSegmentOrdToGlobalOrdDeltas[i] =
PackedInts.getMutable(
(int) segmentOrdToGlobalOrdDeltas[i].size(),
PackedInts.bitsRequired(maxOrd),
acceptableOverheadRatio);
}
for (int readerIndex = 0; readerIndex < segmentOrdToGlobalOrdDeltas.length; readerIndex++) {
MonotonicAppendingLongBuffer segmentOrdToGlobalOrdDelta =
segmentOrdToGlobalOrdDeltas[readerIndex];
for (long ordIndex = 0; ordIndex < segmentOrdToGlobalOrdDelta.size(); ordIndex++) {
long ordDelta = segmentOrdToGlobalOrdDelta.get(ordIndex);
newSegmentOrdToGlobalOrdDeltas[readerIndex].set((int) ordIndex, ordDelta);
}
}
PackedIntOrdinalMappingSource[] sources = new PackedIntOrdinalMappingSource[numSegments];
for (int i = 0; i < newSegmentOrdToGlobalOrdDeltas.length; i++) {
PackedInts.Reader segmentOrdToGlobalOrdDelta = newSegmentOrdToGlobalOrdDeltas[i];
if (segmentOrdToGlobalOrdDelta.size() == maxOrd) {
// This means that a segment contains all the value and in that case segment ordinals
// can be used as global ordinals. This will save an extra lookup per hit.
sources[i] = null;
} else {
long ramUsed = segmentOrdToGlobalOrdDelta.ramBytesUsed();
sources[i] =
new PackedIntOrdinalMappingSource(segmentOrdToGlobalOrdDelta, ramUsed, maxOrd);
memorySizeInBytesCounter += ramUsed;
}
}
return sources;
} else {
OrdinalMappingSource[] sources =
new OrdinalMappingSource[segmentOrdToGlobalOrdDeltas.length];
for (int i = 0; i < segmentOrdToGlobalOrdDeltas.length; i++) {
MonotonicAppendingLongBuffer segmentOrdToGlobalOrdLookup = segmentOrdToGlobalOrdDeltas[i];
if (segmentOrdToGlobalOrdLookup.size() == maxOrd) {
// idem as above
sources[i] = null;
} else {
segmentOrdToGlobalOrdLookup.freeze();
long ramUsed = segmentOrdToGlobalOrdLookup.ramBytesUsed();
sources[i] =
new CompressedOrdinalMappingSource(segmentOrdToGlobalOrdLookup, ramUsed, maxOrd);
memorySizeInBytesCounter += ramUsed;
}
}
return sources;
}
}
@Override
protected int position(int docID) throws IOException {
data.seek(baseOffset + index.get(docID) * size);
return size;
}
private NumericDocValues loadNumeric(FieldInfo field) throws IOException {
NumericEntry entry = numerics.get(field.name);
data.seek(entry.offset);
switch (entry.format) {
case TABLE_COMPRESSED:
int size = data.readVInt();
if (size > 256) {
throw new CorruptIndexException(
"TABLE_COMPRESSED cannot have more than 256 distinct values, got=" + size, data);
}
final long decode[] = new long[size];
for (int i = 0; i < decode.length; i++) {
decode[i] = data.readLong();
}
final int formatID = data.readVInt();
final int bitsPerValue = data.readVInt();
final PackedInts.Reader ordsReader =
PackedInts.getReaderNoHeader(
data,
PackedInts.Format.byId(formatID),
entry.packedIntsVersion,
maxDoc,
bitsPerValue);
if (!merging) {
ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed());
numericInfo.put(field.name, ordsReader);
}
return new NumericDocValues() {
@Override
public long get(int docID) {
return decode[(int) ordsReader.get(docID)];
}
};
case DELTA_COMPRESSED:
final int blockSize = data.readVInt();
final BlockPackedReader reader =
new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false);
if (!merging) {
ramBytesUsed.addAndGet(reader.ramBytesUsed());
numericInfo.put(field.name, reader);
}
return reader;
case UNCOMPRESSED:
final byte bytes[] = new byte[maxDoc];
data.readBytes(bytes, 0, bytes.length);
if (!merging) {
ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes));
numericInfo.put(field.name, Accountables.namedAccountable("byte array", maxDoc));
}
return new NumericDocValues() {
@Override
public long get(int docID) {
return bytes[docID];
}
};
case GCD_COMPRESSED:
final long min = data.readLong();
final long mult = data.readLong();
final int quotientBlockSize = data.readVInt();
final BlockPackedReader quotientReader =
new BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false);
if (!merging) {
ramBytesUsed.addAndGet(quotientReader.ramBytesUsed());
numericInfo.put(field.name, quotientReader);
}
return new NumericDocValues() {
@Override
public long get(int docID) {
return min + mult * quotientReader.get(docID);
}
};
default:
throw new AssertionError();
}
}
@Override
public BytesRef getBytes(int docID, BytesRef bytesRef) {
return data.fillSlice(bytesRef, addresses.get(docID) * size, size);
}
