Пример #1
0
  public static LinkedByteBuffer toByteBuffer(ResultHeap rheap, byte compression) {
    int size = rheap.size();
    int[] docids = new int[size];
    double[] scores = new double[size];
    rheap.decrSortResults(docids, scores);

    LinkedByteBuffer buffer = new LinkedByteBuffer();

    if (compression == NOCOMPRESSION) {
      for (int i = 0; i < size; i++) {
        buffer.putInt(docids[i]); // store docids
        buffer.putFloat((float) scores[i]); // store scores
      }
    } else if (compression == DOCIDSONLY) {
      NewPForEncoding encoder = new NewPForEncoding(padLimit);
      int spos;

      for (spos = 0; spos + 128 < size; spos += 128) {
        for (int i = 0; i < 128; i++) {
          buffer.putFloat((float) scores[spos + i]); // store scores
        }
        encoder.encodeSingle(buffer, docids, spos, 128); // encode docids
      }

      int rest = size - spos;
      for (int i = 0; i < rest; i++) {
        buffer.putFloat((float) scores[spos + i]); // store scores
      }
      encoder.encodeSingle(buffer, docids, spos, rest);
    } else if (compression == CROPFLOATS) {
      int fin[] = new int[128], curbits, bitshift = 23 - mantissaBits, eof = mantissaBits << 2;
      int fout[] = new int[eof];
      double tmp, prev = size > 0 ? scores[0] : 0;

      NewPForEncoding encoder = new NewPForEncoding(padLimit);
      int spos;

      for (spos = 0; spos + 128 < size; spos += 128) {
        for (int i = 0; i < 128; i++) {
          tmp = scores[spos + i];
          curbits = Float.floatToIntBits((float) (prev - tmp)); // separate scores
          prev = tmp;
          buffer.put((byte) (curbits >> 23)); // store exponens
          fin[i] = (curbits >> bitshift) & PackUtils.masks[mantissaBits];
        }

        PackUtils.cleanArray(fout); // pack mantissas
        PackUtils.pack(mantissaBits, 128, 0, 0, fin, fout);

        for (int i = 0; i < eof; i++) {
          buffer.putInt(fout[i]); // store mantissas
        }

        encoder.encodeSingle(buffer, docids, spos, 128); // encode docs
      }

      int rest = size - spos;

      for (int i = 0; i < rest; i++) {
        tmp = scores[spos + i];
        curbits =
            Float.floatToIntBits((float) (prev - tmp)); // calculate differences, separate scores
        prev = tmp;
        buffer.put((byte) (curbits >> 23));
        fin[i] = (curbits >> bitshift) & PackUtils.masks[mantissaBits];
      }

      for (int i = rest; i < 128; i++) fin[i] = 0; // for the last group we pad zeros.
      PackUtils.cleanArray(fout);
      PackUtils.pack(mantissaBits, 128, 0, 0, fin, fout);

      for (int i = 0; i < eof; i++) {
        buffer.putInt(fout[i]);
      }
      encoder.encodeSingle(buffer, docids, spos, rest);

    } else if (compression == MAXCOMPRESSION) {
      // PForCompress dgaps
      // PForCompress exponents
      // PForCompress mantissas

      NewPForEncoding encoder = new NewPForEncoding(padLimit);

      int spos;
      int e[] = new int[128], m[] = new int[128], bits, bitshift = 23 - mantissaBits;
      double tmp, prev = size > 0 ? scores[0] : 0;

      for (spos = 0; spos + 128 < size; spos += 128) {
        for (int i = 0; i < 128; i++) {
          tmp = scores[spos + i];
          bits =
              Float.floatToIntBits((float) (prev - tmp)); // calculate differences, separate scores
          prev = tmp;
          e[i] = (bits >> 23) & 0xff;
          m[i] = (bits >> bitshift) & PackUtils.masks[mantissaBits];
        }

        encoder.encodeSingle(buffer, e, 0, 128); // encode exponents
        encoder.encodeSingle(buffer, m, 0, 128); // encode mantissas
        encoder.encodeSingle(buffer, docids, spos, 128);
      }

      int rest = size - spos;
      for (int i = 0; i < rest; i++) {
        tmp = scores[spos + i];
        bits = Float.floatToIntBits((float) (prev - tmp)); // calculate differences, separate scores
        prev = tmp;
        e[i] = (bits >> 23) & 0xff;
        m[i] = ((bits >> bitshift) + 1) & PackUtils.masks[mantissaBits];
      }

      if (rest < padLimit) for (int i = 0; i < rest; i++) m[i]++;
      encoder.encodeSingle(buffer, e, 0, rest); // encode exponents
      encoder.encodeSingle(buffer, m, 0, rest); // encode mantissas
      encoder.encodeSingle(buffer, docids, spos, rest);
    }

    return buffer;
  }
Пример #2
0
  public static QueryResults fromByteBuffer(
      ByteBuffer buffer, int cnt, int realCnt, byte compression) {
    int[] docids = new int[cnt];
    double[] scores = new double[cnt];

    // decompress docids
    if (compression == NOCOMPRESSION) {
      for (int i = 0; i < cnt; i++) {
        docids[i] = buffer.getInt();
        scores[i] = buffer.getFloat();
      }
    } else if (compression == DOCIDSONLY) {
      NewPForDecoding decoder = new NewPForDecoding(padLimit);

      double drbuffer[] = new double[128];
      int spos;
      for (spos = 0; spos + 128 < cnt; spos += 128) {
        for (int i = 0; i < 128; i++) scores[spos + i] = buffer.getFloat();
        decoder.decodeSingle(buffer, 128, docids, spos);
      }

      int rest = cnt - spos;
      for (int i = 0; i < rest; i++) drbuffer[i] = buffer.getFloat();
      decoder.decodeSingle(buffer, rest, docids, spos);

    } else if (compression == CROPFLOATS) {
      NewPForDecoding decoder = new NewPForDecoding(padLimit);

      int spos;

      int fout[] = new int[128], curbits, bitshift = 23 - mantissaBits, eof = mantissaBits << 2;
      int fin[] = new int[eof];
      byte exponents[] = new byte[128];
      double last = 0d, diff;

      for (spos = 0; spos + 128 < cnt; spos += 128) {
        // read exponents
        for (int i = 0; i < 128; i++) {
          exponents[i] = buffer.get();
        }

        // read mantissas
        for (int i = 0; i < eof; i++) {
          fin[i] = buffer.getInt();
        }

        // unpack mantissas
        PackUtils.unpack(mantissaBits, 128, 0, 0, fin, fout);

        // decode dgaps
        decoder.decodeSingle(buffer, 128, docids, spos);

        // restore docids and scores
        for (int i = 0; i < 128; i++) {
          curbits = (exponents[i] << 23) & 0x7FFFFFFF;
          curbits |= fout[i] << bitshift;
          diff = Float.intBitsToFloat(curbits);
          last = scores[spos + i] = (spos > 0 || i > 0) ? last - diff : diff;
        }
      }

      int rest = cnt - spos;

      // read exponents
      for (int i = 0; i < rest; i++) {
        exponents[i] = buffer.get();
        // out[i] = buffer.getInt();
      }

      // read mantissas
      for (int i = 0; i < eof; i++) {
        fin[i] = buffer.getInt();
      }

      // unpack mantissas
      PackUtils.unpack(mantissaBits, 128, 0, 0, fin, fout);

      // decode dgaps
      decoder.decodeSingle(buffer, rest, docids, spos);

      // restore floats!
      for (int i = 0; i < rest; i++) {

        curbits = (exponents[i] << 23) & 0x7FFFFFFF;
        curbits |= fout[i] << bitshift;

        diff = Float.intBitsToFloat(curbits);
        last = scores[spos + i] = (spos > 0 || i > 0) ? last - diff : diff;
      }
    } else if (compression == MAXCOMPRESSION) {
      // PForDecompress exponents
      // PForDecompress mantissas
      NewPForDecoding decoder = new NewPForDecoding(padLimit);

      int spos, e[] = new int[128], m[] = new int[128], bitshift = 23 - mantissaBits;
      double diff, last = 0d;

      for (spos = 0; spos + 128 < cnt; spos += 128) {
        decoder.decodeSingle(buffer, 128, e, 0); // decode exponents
        decoder.decodeSingle(buffer, 128, m, 0); // decode mantissas
        decoder.decodeSingle(buffer, 128, docids, spos); // decode docids

        for (int i = 0; i < 128; i++) {
          diff = Float.intBitsToFloat((e[i] << 23) | (m[i] << bitshift));
          last = scores[spos + i] = (spos > 0 || i > 0) ? last - diff : diff;
        }
      }

      int rest = cnt - spos;
      decoder.decodeSingle(buffer, rest, e, 0); // decode exponents
      decoder.decodeSingle(buffer, rest, m, 0); // decode mantissas
      decoder.decodeSingle(buffer, rest, docids, spos); // decode docids

      if (rest < padLimit) for (int i = 0; i < rest; i++) m[i]--;

      for (int i = 0; i < rest; i++) {
        diff = Float.intBitsToFloat((e[i] << 23) | (m[i] << bitshift));
        last = scores[spos + i] = (spos > 0 || i > 0) ? last - diff : diff;
      }
    }

    return new QueryResults(docids, scores, cnt, realCnt);
  }