private void verifyCompressedFile(Path f, int expectedNumLines) throws IOException {
    Configuration conf = new Configuration();
    conf.set("fs.default.name", "file:///");
    FileSystem fs = FileSystem.get(conf);
    InputStream is = fs.open(f);
    CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
    CompressionCodec codec = ccf.getCodec(f);
    LOG.info("gzip check codec is " + codec);
    Decompressor decompressor = CodecPool.getDecompressor(codec);
    if (null == decompressor) {
      LOG.info("Verifying gzip sanity with null decompressor");
    } else {
      LOG.info("Verifying gzip sanity with decompressor: " + decompressor.toString());
    }
    is = codec.createInputStream(is, decompressor);
    BufferedReader r = new BufferedReader(new InputStreamReader(is));
    int numLines = 0;
    while (true) {
      String ln = r.readLine();
      if (ln == null) {
        break;
      }
      numLines++;
    }

    r.close();
    assertEquals("Did not read back correct number of lines", expectedNumLines, numLines);
    LOG.info("gzip sanity check returned " + numLines + " lines; ok.");
  }
Exemple #2
0
 public void returnDecompressor(Decompressor decompressor) {
   if (decompressor != null) {
     CodecPool.returnDecompressor(decompressor);
     if (decompressor.getClass().isAnnotationPresent(DoNotPool.class)) {
       decompressor.end();
     }
   }
 }
Exemple #3
0
 public void returnDecompressor(Decompressor decompressor) {
   if (decompressor != null) {
     if (LOG.isTraceEnabled()) LOG.trace("Returning decompressor " + decompressor + " to pool.");
     CodecPool.returnDecompressor(decompressor);
     if (decompressor.getClass().isAnnotationPresent(DoNotPool.class)) {
       if (LOG.isTraceEnabled()) LOG.trace("Ending decompressor " + decompressor);
       decompressor.end();
     }
   }
 }
Exemple #4
0
    public Decompressor getDecompressor() {
      CompressionCodec codec = getCodec(conf);
      if (codec != null) {
        Decompressor decompressor = CodecPool.getDecompressor(codec);
        if (LOG.isTraceEnabled())
          LOG.trace("Retrieved decompressor " + decompressor + " from pool.");
        if (decompressor != null) {
          if (decompressor.finished()) {
            // Somebody returns the decompressor to CodecPool but is still using it.
            LOG.warn("Deompressor obtained from CodecPool is already finished()");
          }
          decompressor.reset();
        }
        return decompressor;
      }

      return null;
    }
 private boolean checkSetDictionaryArrayIndexOutOfBoundsException(Decompressor decompressor) {
   try {
     decompressor.setDictionary(new byte[] {(byte) 0}, 0, -1);
   } catch (ArrayIndexOutOfBoundsException e) {
     return true;
   } catch (Exception e) {
   }
   return false;
 }
 private boolean checkSetDictionaryNullPointerException(Decompressor decompressor) {
   try {
     decompressor.setDictionary(null, 0, 1);
   } catch (NullPointerException ex) {
     return true;
   } catch (Exception ex) {
   }
   return false;
 }
Exemple #7
0
    public Decompressor getDecompressor() {
      CompressionCodec codec = getCodec(conf);
      if (codec != null) {
        Decompressor decompressor = CodecPool.getDecompressor(codec);
        if (decompressor != null) {
          if (decompressor.finished()) {
            // Somebody returns the decompressor to CodecPool but is still using
            // it.
            LOG.warn("Deompressor obtained from CodecPool is already finished()");
            // throw new AssertionError(
            // "Decompressor obtained from CodecPool is already finished()");
          }
          decompressor.reset();
        }
        return decompressor;
      }

      return null;
    }
  private void shuffleToMemory(
      MapHost host,
      MapOutput mapOutput,
      InputStream input,
      int decompressedLength,
      int compressedLength)
      throws IOException {
    IFileInputStream checksumIn = new IFileInputStream(input, compressedLength, job);

    input = checksumIn;

    // Are map-outputs compressed?
    if (codec != null) {
      decompressor.reset();
      input = codec.createInputStream(input, decompressor);
    }

    // Copy map-output into an in-memory buffer
    byte[] shuffleData = mapOutput.getMemory();

    try {
      IOUtils.readFully(input, shuffleData, 0, shuffleData.length);
      metrics.inputBytes(shuffleData.length);
      LOG.info(
          "Read "
              + shuffleData.length
              + " bytes from map-output for "
              + mapOutput.getAttemptIdentifier());
    } catch (IOException ioe) {
      // Close the streams
      IOUtils.cleanup(LOG, input);

      // Re-throw
      throw ioe;
    }
  }