public static BitIndex wrap(byte[] data, int position, int limit) {
// create a BitSet underneath
BitSet bitSet = new BitSet();
BitSet scalarFields = new BitSet();
BitSet floatFields = new BitSet();
BitSet doubleFields = new BitSet();
BitReader bitReader = new BitReader(data, position, limit, 5, true);
int bitPos = 0;
while (bitReader.hasNext()) {
int zeros = bitReader.nextSetBit();
if (zeros < 0) break;
bitPos += zeros;
bitSet.set(bitPos);
if (bitReader.next() != 0) {
// either float or scalar
if (bitReader.next() != 0) {
scalarFields.set(bitPos);
} else floatFields.set(bitPos);
} else {
// either a double or untyped
if (bitReader.next() != 0) doubleFields.set(bitPos);
}
bitPos++;
}
return new UncompressedBitIndex(bitSet, scalarFields, floatFields, doubleFields);
}
public static void main(final String[] args) {
final com.carrotsearch.hppc.BitSet hppcBitSet =
new com.carrotsearch.hppc.BitSet(Long.MAX_VALUE);
hppcBitSet.set(Integer.MAX_VALUE);
final java.util.BitSet javaBitSet = new java.util.BitSet(Integer.MAX_VALUE);
javaBitSet.set(Integer.MAX_VALUE - 1);
System.out.println(ObjectSizeCalculator.getObjectSize(hppcBitSet));
System.out.println(ObjectSizeCalculator.getObjectSize(javaBitSet));
}
private ByteEntryAccumulator getKeyAccumulator() {
if (indexKeyAccumulator == null) {
BitSet keyFields = new BitSet();
for (int keyColumn : mainColToIndexPosMap) {
if (keyColumn >= 0) keyFields.set(keyColumn);
}
indexKeyAccumulator =
new ByteEntryAccumulator(EntryPredicateFilter.emptyPredicate(), keyFields);
}
return indexKeyAccumulator;
}
/** Adds stem index to the set with a check on the stem's document frequency. */
private void addStemIndex(
final int[] wordsStemIndex,
int documentCount,
int[][] stemsTfByDocument,
final BitSet requiredStemIndices,
final int featureIndex) {
final int stemIndex = wordsStemIndex[featureIndex];
final int df = stemsTfByDocument[stemIndex].length / 2;
if (((double) df / documentCount) <= maxWordDf) {
requiredStemIndices.set(stemIndex);
}
}
@Test
public void testAlwaysAcceptEntryWorks() throws Exception {
BitSet fields = new BitSet();
fields.set(0);
fields.set(2);
EntryPredicateFilter predicateFilter = new EntryPredicateFilter(fields);
EntryAccumulator accumulator = new ByteEntryAccumulator(predicateFilter, false, null);
byte[] encodedOne = Encoding.encode(1);
accumulator.add(2, encodedOne, 0, encodedOne.length);
byte[] encodedTwo = Encoding.encode(2);
accumulator.add(0, encodedTwo, 0, encodedTwo.length);
byte[] bytes = accumulator.finish();
MultiFieldDecoder decoder = MultiFieldDecoder.wrap(bytes);
Assert.assertEquals(2, decoder.decodeNextInt());
Assert.assertEquals(1, decoder.decodeNextInt());
}
private EntryEncoder getRowEncoder() {
if (indexValueEncoder == null) {
BitSet nonNullFields = new BitSet();
int highestSetPosition = 0;
for (int keyColumn : mainColToIndexPosMap) {
if (keyColumn > highestSetPosition) highestSetPosition = keyColumn;
}
nonNullFields.set(highestSetPosition + 1);
indexValueEncoder =
EntryEncoder.create(
SpliceKryoRegistry.getInstance(),
1,
nonNullFields,
new BitSet(),
new BitSet(),
new BitSet());
}
return indexValueEncoder;
}