예제 #1
0
 /**
  * writes a single page
  *
  * @param valueCount count of values
  * @param uncompressedPageSize the size of the data once uncompressed
  * @param bytes the compressed data for the page without header
  * @param rlEncoding encoding of the repetition level
  * @param dlEncoding encoding of the definition level
  * @param valuesEncoding encoding of values
  */
 public void writeDataPage(
     int valueCount,
     int uncompressedPageSize,
     BytesInput bytes,
     Statistics statistics,
     parquet.column.Encoding rlEncoding,
     parquet.column.Encoding dlEncoding,
     parquet.column.Encoding valuesEncoding)
     throws IOException {
   state = state.write();
   long beforeHeader = out.getPos();
   if (DEBUG) LOG.debug(beforeHeader + ": write data page: " + valueCount + " values");
   int compressedPageSize = (int) bytes.size();
   metadataConverter.writeDataPageHeader(
       uncompressedPageSize,
       compressedPageSize,
       valueCount,
       statistics,
       rlEncoding,
       dlEncoding,
       valuesEncoding,
       out);
   long headerSize = out.getPos() - beforeHeader;
   this.uncompressedLength += uncompressedPageSize + headerSize;
   this.compressedLength += compressedPageSize + headerSize;
   if (DEBUG) LOG.debug(out.getPos() + ": write data page content " + compressedPageSize);
   bytes.writeAllTo(out);
   currentStatistics.mergeStatistics(statistics);
   currentEncodings.add(rlEncoding);
   currentEncodings.add(dlEncoding);
   currentEncodings.add(valuesEncoding);
 }
예제 #2
0
 /**
  * writes a dictionary page page
  *
  * @param dictionaryPage the dictionary page
  */
 public void writeDictionaryPage(DictionaryPage dictionaryPage) throws IOException {
   state = state.write();
   if (DEBUG)
     LOG.debug(
         out.getPos()
             + ": write dictionary page: "
             + dictionaryPage.getDictionarySize()
             + " values");
   currentChunkDictionaryPageOffset = out.getPos();
   int uncompressedSize = dictionaryPage.getUncompressedSize();
   int compressedPageSize = (int) dictionaryPage.getBytes().size(); // TODO: fix casts
   metadataConverter.writeDictionaryPageHeader(
       uncompressedSize,
       compressedPageSize,
       dictionaryPage.getDictionarySize(),
       dictionaryPage.getEncoding(),
       out);
   long headerSize = out.getPos() - currentChunkDictionaryPageOffset;
   this.uncompressedLength += uncompressedSize + headerSize;
   this.compressedLength += compressedPageSize + headerSize;
   if (DEBUG) LOG.debug(out.getPos() + ": write dictionary page content " + compressedPageSize);
   dictionaryPage.getBytes().writeAllTo(out);
   currentEncodings.add(dictionaryPage.getEncoding());
 }