public static BitIndex wrap(byte[] data, int position, int limit) {
    // create a BitSet underneath
    BitSet bitSet = new BitSet();
    BitSet scalarFields = new BitSet();
    BitSet floatFields = new BitSet();
    BitSet doubleFields = new BitSet();
    BitReader bitReader = new BitReader(data, position, limit, 5, true);

    int bitPos = 0;
    while (bitReader.hasNext()) {
      int zeros = bitReader.nextSetBit();
      if (zeros < 0) break;
      bitPos += zeros;
      bitSet.set(bitPos);
      if (bitReader.next() != 0) {
        // either float or scalar
        if (bitReader.next() != 0) {
          scalarFields.set(bitPos);
        } else floatFields.set(bitPos);
      } else {
        // either a double or untyped
        if (bitReader.next() != 0) doubleFields.set(bitPos);
      }
      bitPos++;
    }
    return new UncompressedBitIndex(bitSet, scalarFields, floatFields, doubleFields);
  }
  public static void main(final String[] args) {
    final com.carrotsearch.hppc.BitSet hppcBitSet =
        new com.carrotsearch.hppc.BitSet(Long.MAX_VALUE);
    hppcBitSet.set(Integer.MAX_VALUE);
    final java.util.BitSet javaBitSet = new java.util.BitSet(Integer.MAX_VALUE);
    javaBitSet.set(Integer.MAX_VALUE - 1);

    System.out.println(ObjectSizeCalculator.getObjectSize(hppcBitSet));
    System.out.println(ObjectSizeCalculator.getObjectSize(javaBitSet));
  }
  private ByteEntryAccumulator getKeyAccumulator() {
    if (indexKeyAccumulator == null) {
      BitSet keyFields = new BitSet();
      for (int keyColumn : mainColToIndexPosMap) {
        if (keyColumn >= 0) keyFields.set(keyColumn);
      }
      indexKeyAccumulator =
          new ByteEntryAccumulator(EntryPredicateFilter.emptyPredicate(), keyFields);
    }

    return indexKeyAccumulator;
  }
Ejemplo n.º 4
0
 /** Adds stem index to the set with a check on the stem's document frequency. */
 private void addStemIndex(
     final int[] wordsStemIndex,
     int documentCount,
     int[][] stemsTfByDocument,
     final BitSet requiredStemIndices,
     final int featureIndex) {
   final int stemIndex = wordsStemIndex[featureIndex];
   final int df = stemsTfByDocument[stemIndex].length / 2;
   if (((double) df / documentCount) <= maxWordDf) {
     requiredStemIndices.set(stemIndex);
   }
 }
  @Test
  public void testAlwaysAcceptEntryWorks() throws Exception {
    BitSet fields = new BitSet();
    fields.set(0);
    fields.set(2);
    EntryPredicateFilter predicateFilter = new EntryPredicateFilter(fields);
    EntryAccumulator accumulator = new ByteEntryAccumulator(predicateFilter, false, null);
    byte[] encodedOne = Encoding.encode(1);
    accumulator.add(2, encodedOne, 0, encodedOne.length);
    byte[] encodedTwo = Encoding.encode(2);
    accumulator.add(0, encodedTwo, 0, encodedTwo.length);

    byte[] bytes = accumulator.finish();
    MultiFieldDecoder decoder = MultiFieldDecoder.wrap(bytes);
    Assert.assertEquals(2, decoder.decodeNextInt());
    Assert.assertEquals(1, decoder.decodeNextInt());
  }
 private EntryEncoder getRowEncoder() {
   if (indexValueEncoder == null) {
     BitSet nonNullFields = new BitSet();
     int highestSetPosition = 0;
     for (int keyColumn : mainColToIndexPosMap) {
       if (keyColumn > highestSetPosition) highestSetPosition = keyColumn;
     }
     nonNullFields.set(highestSetPosition + 1);
     indexValueEncoder =
         EntryEncoder.create(
             SpliceKryoRegistry.getInstance(),
             1,
             nonNullFields,
             new BitSet(),
             new BitSet(),
             new BitSet());
   }
   return indexValueEncoder;
 }