Exemple #1
0
 public void next() throws IOException {
   streamOffset = getRecordOffset();
   inputStream.close();
   if (encodedInputStream != null && encodedInputStream.available() > 0) {
     Preconditions.checkState(
         blockCursor.getBlockType() == Constants.FIRST_LZMA_BLOCK
             || blockCursor.getBlockType() == Constants.NEXT_LZMA_BLOCK);
     resumeLzmaDecoder();
     return;
   }
   encodedInputStream = null;
   blockCursor.next();
   if (blockCursor.getBlockType() == Constants.NEXT_LZMA_BLOCK) {
     startLzmaDecoder(false);
     return;
   }
   initChunk();
 }
  public static void writeStringBTree(String directory, String stringField, File btreeDir)
      throws IOException {
    final String termsFilename =
        Files.buildPath(directory, SimpleStringFieldWriter.getTermsFilename(stringField));
    if (!new File(termsFilename).exists() || new File(termsFilename).length() == 0L) return;
    final CountingInputStream termsList =
        new CountingInputStream(new BufferedInputStream(new FileInputStream(termsFilename), 65536));
    try {
      ImmutableBTreeIndex.Writer.write(
          btreeDir,
          new AbstractIterator<Generation.Entry<String, LongPair>>() {
            private String key;
            private LongPair value;

            private byte[] lastTerm = new byte[10];
            private int lastTermLen = 0;
            private long lastTermDocOffset = 0L;
            private long lastTermFileOffset = 0L;

            @Override
            public Generation.Entry<String, LongPair> computeNext() {
              try {
                if (!nextTerm()) return endOfData();

                key = new String(lastTerm, 0, lastTermLen, Charsets.UTF_8);
                value = new LongPair(lastTermFileOffset, lastTermDocOffset);

                for (int i = 0; i < BLOCK_SIZE - 1; ++i) {
                  if (!nextTerm()) {
                    break;
                  }
                }

                return Generation.Entry.create(key, value);
              } catch (IOException e) {
                throw new RuntimeException(e);
              }
            }

            private boolean nextTerm() throws IOException {
              final int removeLen;
              // sorry
              try {
                removeLen = (int) FlamdexUtils.readVLong(termsList);
              } catch (EOFException e) {
                return false;
              }

              final int newLen = (int) FlamdexUtils.readVLong(termsList);

              lastTerm = ensureCapacity(lastTerm, lastTermLen - removeLen + newLen);
              ByteStreams.readFully(termsList, lastTerm, lastTermLen - removeLen, newLen);
              lastTermLen = lastTermLen - removeLen + newLen;

              final long offsetDelta = FlamdexUtils.readVLong(termsList);
              lastTermDocOffset += offsetDelta;

              lastTermFileOffset = termsList.getCount();

              FlamdexUtils.readVLong(termsList); // termDocFreq

              return true;
            }
          },
          new StringSerializer(),
          new LongPairSerializer(),
          65536,
          false);
    } finally {
      termsList.close();
    }
  }
  public static void writeIntBTree(String directory, String intField, File btreeDir)
      throws IOException {
    final String termsFilename =
        Files.buildPath(directory, SimpleIntFieldWriter.getTermsFilename(intField));
    if (!new File(termsFilename).exists() || new File(termsFilename).length() == 0L) return;
    final CountingInputStream termsList =
        new CountingInputStream(new BufferedInputStream(new FileInputStream(termsFilename), 65536));
    try {
      ImmutableBTreeIndex.Writer.write(
          btreeDir,
          new AbstractIterator<Generation.Entry<Long, LongPair>>() {
            private long lastTerm = 0;
            private long lastTermDocOffset = 0L;
            private long lastTermFileOffset = 0L;

            private long key;
            private LongPair value;

            @Override
            protected Generation.Entry<Long, LongPair> computeNext() {
              try {
                if (!nextTerm()) return endOfData();

                key = lastTerm;
                value = new LongPair(lastTermFileOffset, lastTermDocOffset);

                for (int i = 0; i < BLOCK_SIZE - 1; ++i) {
                  if (!nextTerm()) {
                    break;
                  }
                }

                return Generation.Entry.create(key, value);
              } catch (IOException e) {
                throw new RuntimeException(e);
              }
            }

            private boolean nextTerm() throws IOException {
              final long termDelta;
              // sorry
              try {
                termDelta = FlamdexUtils.readVLong(termsList);
              } catch (EOFException e) {
                return false;
              }

              lastTerm += termDelta;

              final long offsetDelta = FlamdexUtils.readVLong(termsList);
              lastTermDocOffset += offsetDelta;

              lastTermFileOffset = termsList.getCount();

              FlamdexUtils.readVLong(termsList); // termDocFreq

              return true;
            }
          },
          new LongSerializer(),
          new LongPairSerializer(),
          65536,
          false);
    } finally {
      termsList.close();
    }
  }
Exemple #4
0
 @Override
 public void close() throws IOException {
   inputStream.close();
 }
Exemple #5
0
 public void seek(BlockPointer pointer) throws IOException {
   inputStream.close();
   blockCursor.seek(pointer);
   initChunk();
 }