/** Test <code>long skip(long len)</code>. */
  @Test
  public void skipTest() throws IOException {
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      WriteType op = WriteType.THROUGH;
      int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

      TachyonFile file = mTfs.getFile(fileId);
      InStream is = file.getInStream(ReadType.CACHE);
      Assert.assertTrue(is instanceof RemoteBlockInStream);
      Assert.assertEquals(k / 2, is.skip(k / 2));
      Assert.assertEquals(k / 2, is.read());
      is.close();
      Assert.assertFalse(file.isInMemory());

      if (k >= 3) {
        is = file.getInStream(ReadType.CACHE);
        Assert.assertTrue(is instanceof RemoteBlockInStream);
        int t = k / 3;
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals(t, is.read());
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals(2 * t + 1, is.read());
        is.close();
        Assert.assertFalse(file.isInMemory());
      }
    }
  }
Esempio n. 2
0
 @Test
 public void testUncompressed() throws Exception {
   OutputCollector collect = new OutputCollector();
   OutStream out = new OutStream("test", 100, null, collect);
   PositionCollector[] positions = new PositionCollector[1024];
   for (int i = 0; i < 1024; ++i) {
     positions[i] = new PositionCollector();
     out.getPosition(positions[i]);
     out.write(i);
   }
   out.flush();
   assertEquals(1024, collect.buffer.size());
   for (int i = 0; i < 1024; ++i) {
     assertEquals((byte) i, collect.buffer.get(i));
   }
   ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
   collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
   inBuf.flip();
   InStream in =
       InStream.create(
           "test", new ByteBuffer[] {inBuf}, new long[] {0}, inBuf.remaining(), null, 100);
   assertEquals(
       "uncompressed stream test position: 0 length: 1024" + " range: 0 offset: 0 limit: 0",
       in.toString());
   for (int i = 0; i < 1024; ++i) {
     int x = in.read();
     assertEquals(i & 0xff, x);
   }
   for (int i = 1023; i >= 0; --i) {
     in.seek(positions[i]);
     assertEquals(i & 0xff, in.read());
   }
 }
Esempio n. 3
0
 @Test
 public void testCompressed() throws Exception {
   OutputCollector collect = new OutputCollector();
   CompressionCodec codec = new ZlibCodec();
   OutStream out = new OutStream("test", 300, codec, collect);
   PositionCollector[] positions = new PositionCollector[1024];
   for (int i = 0; i < 1024; ++i) {
     positions[i] = new PositionCollector();
     out.getPosition(positions[i]);
     out.write(i);
   }
   out.flush();
   assertEquals("test", out.toString());
   assertEquals(961, collect.buffer.size());
   ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
   collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
   inBuf.flip();
   InStream in =
       InStream.create(
           "test", new ByteBuffer[] {inBuf}, new long[] {0}, inBuf.remaining(), codec, 300);
   assertEquals(
       "compressed stream test position: 0 length: 961 range: 0"
           + " offset: 0 limit: 0 range 0 = 0 to 961",
       in.toString());
   for (int i = 0; i < 1024; ++i) {
     int x = in.read();
     assertEquals(i & 0xff, x);
   }
   assertEquals(0, in.available());
   for (int i = 1023; i >= 0; --i) {
     in.seek(positions[i]);
     assertEquals(i & 0xff, in.read());
   }
 }
Esempio n. 4
0
 private void readValues(boolean ignoreEof) throws IOException {
   int control = input.read();
   used = 0;
   if (control == -1) {
     if (!ignoreEof) {
       throw new EOFException("Read past end of buffer RLE byte from " + input);
     }
     used = numLiterals = 0;
     return;
   } else if (control < 0x80) {
     repeat = true;
     numLiterals = control + RunLengthByteWriter.MIN_REPEAT_SIZE;
     int val = input.read();
     if (val == -1) {
       throw new EOFException("Reading RLE byte got EOF");
     }
     literals[0] = (byte) val;
   } else {
     repeat = false;
     numLiterals = 0x100 - control;
     int bytes = 0;
     while (bytes < numLiterals) {
       int result = input.read(literals, bytes, numLiterals - bytes);
       if (result == -1) {
         throw new EOFException("Reading RLE byte literal got EOF in " + this);
       }
       bytes += result;
     }
   }
 }
Esempio n. 5
0
  /** Test <code>void read()</code>. */
  @Test
  public void readTest1() throws IOException {
    for (int k = 100; k <= 200; k += 33) {
      for (WriteType op : WriteType.values()) {
        int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

        TachyonFile file = mTfs.getFile(fileId);
        InStream is =
            (k < 150 ? file.getInStream(ReadType.CACHE) : file.getInStream(ReadType.NO_CACHE));
        Assert.assertTrue(is instanceof BlockInStream);
        byte[] ret = new byte[k];
        int value = is.read();
        int cnt = 0;
        while (value != -1) {
          ret[cnt++] = (byte) value;
          value = is.read();
        }
        Assert.assertTrue(TestUtils.equalIncreasingByteArray(k, ret));
        is.close();

        is = (k < 150 ? file.getInStream(ReadType.CACHE) : file.getInStream(ReadType.NO_CACHE));
        Assert.assertTrue(is instanceof BlockInStream);
        ret = new byte[k];
        value = is.read();
        cnt = 0;
        while (value != -1) {
          ret[cnt++] = (byte) value;
          value = is.read();
        }
        Assert.assertTrue(TestUtils.equalIncreasingByteArray(k, ret));
        is.close();
      }
    }
  }
Esempio n. 6
0
  /** Test <code>long skip(long len)</code>. */
  @Test
  public void skipTest() throws IOException {
    for (int k = 10; k <= 200; k += 33) {
      for (WriteType op : WriteType.values()) {
        int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

        TachyonFile file = mTfs.getFile(fileId);
        InStream is =
            (k < 100 ? file.getInStream(ReadType.CACHE) : file.getInStream(ReadType.NO_CACHE));
        Assert.assertTrue(is instanceof BlockInStream);
        Assert.assertEquals(k / 2, is.skip(k / 2));
        Assert.assertEquals(k / 2, is.read());
        is.close();

        is = (k < 100 ? file.getInStream(ReadType.CACHE) : file.getInStream(ReadType.NO_CACHE));
        Assert.assertTrue(is instanceof BlockInStream);
        int t = k / 3;
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals(t, is.read());
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals((byte) (2 * t + 1), is.read());
        is.close();
      }
    }
  }
  /** Test <code>void read(byte[] b, int off, int len)</code>. Read from underfs. */
  @Test
  public void readTest3() throws IOException {
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      WriteType op = WriteType.THROUGH;
      int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

      TachyonFile file = mTfs.getFile(fileId);
      InStream is = file.getInStream(ReadType.NO_CACHE);
      if (k == 0) {
        Assert.assertTrue(is instanceof EmptyBlockInStream);
      } else {
        Assert.assertTrue(is instanceof RemoteBlockInStream);
      }
      byte[] ret = new byte[k / 2];
      Assert.assertEquals(k / 2, is.read(ret, 0, k / 2));
      Assert.assertTrue(TestUtils.equalIncreasingByteArray(k / 2, ret));
      is.close();
      if (k == 0) {
        Assert.assertTrue(file.isInMemory());
      } else {
        Assert.assertFalse(file.isInMemory());
      }

      is = file.getInStream(ReadType.CACHE);
      if (k == 0) {
        Assert.assertTrue(is instanceof EmptyBlockInStream);
      } else {
        Assert.assertTrue(is instanceof RemoteBlockInStream);
      }
      ret = new byte[k];
      Assert.assertEquals(k, is.read(ret, 0, k));
      Assert.assertTrue(TestUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(file.isInMemory());

      is = file.getInStream(ReadType.CACHE);
      if (k == 0) {
        Assert.assertTrue(is instanceof EmptyBlockInStream);
      } else {
        Assert.assertTrue(is instanceof LocalBlockInStream);
      }
      ret = new byte[k];
      Assert.assertEquals(k, is.read(ret));
      Assert.assertTrue(TestUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(file.isInMemory());
    }
  }
Esempio n. 8
0
  @Test
  public void testCorruptStream() throws Exception {
    OutputCollector collect = new OutputCollector();
    CompressionCodec codec = new ZlibCodec();
    OutStream out = new OutStream("test", 500, codec, collect);
    PositionCollector[] positions = new PositionCollector[1024];
    for (int i = 0; i < 1024; ++i) {
      positions[i] = new PositionCollector();
      out.getPosition(positions[i]);
      out.write(i);
    }
    out.flush();

    // now try to read the stream with a buffer that is too small
    ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
    collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
    inBuf.flip();
    InStream in =
        InStream.create(
            "test", new ByteBuffer[] {inBuf}, new long[] {0}, inBuf.remaining(), codec, 100);
    byte[] contents = new byte[1024];
    try {
      in.read(contents);
      fail();
    } catch (IllegalArgumentException iae) {
      // EXPECTED
    }

    // make a corrupted header
    inBuf.clear();
    inBuf.put((byte) 32);
    inBuf.put((byte) 0);
    inBuf.flip();
    in =
        InStream.create(
            "test2", new ByteBuffer[] {inBuf}, new long[] {0}, inBuf.remaining(), codec, 300);
    try {
      in.read();
      fail();
    } catch (IllegalStateException ise) {
      // EXPECTED
    }
  }
  /**
   * Test <code>void seek(long pos)</code>.
   *
   * @throws IOException
   */
  @Test
  public void seekTest() throws IOException {
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      WriteType op = WriteType.THROUGH;
      int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

      TachyonFile file = mTfs.getFile(fileId);
      InStream is = file.getInStream(ReadType.NO_CACHE);
      if (k == 0) {
        Assert.assertTrue(is instanceof EmptyBlockInStream);
      } else {
        Assert.assertTrue(is instanceof RemoteBlockInStream);
      }

      is.seek(k / 3);
      Assert.assertEquals(k / 3, is.read());
      is.seek(k / 2);
      Assert.assertEquals(k / 2, is.read());
      is.seek(k / 4);
      Assert.assertEquals(k / 4, is.read());
      is.close();
    }
  }
Esempio n. 10
0
 private void readValues(boolean ignoreEof) throws IOException {
   int control = input.read();
   if (control == -1) {
     if (!ignoreEof) {
       throw new EOFException("Read past end of RLE integer from " + input);
     }
     used = numLiterals = 0;
     return;
   } else if (control < 0x80) {
     numLiterals = control + RunLengthIntegerWriter.MIN_REPEAT_SIZE;
     used = 0;
     repeat = true;
     delta = input.read();
     if (delta == -1) {
       throw new EOFException("End of stream in RLE Integer from " + input);
     }
     // convert from 0 to 255 to -128 to 127 by converting to a signed byte
     delta = (byte) (0 + delta);
     if (signed) {
       literals[0] = utils.readVslong(input);
     } else {
       literals[0] = utils.readVulong(input);
     }
   } else {
     repeat = false;
     numLiterals = 0x100 - control;
     used = 0;
     for (int i = 0; i < numLiterals; ++i) {
       if (signed) {
         literals[i] = utils.readVslong(input);
       } else {
         literals[i] = utils.readVulong(input);
       }
     }
   }
 }
Esempio n. 11
0
  /** Test <code>void read(byte[] b, int off, int len)</code>. */
  @Test
  public void readTest3() throws IOException {
    for (int k = 100; k <= 300; k += 33) {
      for (WriteType op : WriteType.values()) {
        int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

        TachyonFile file = mTfs.getFile(fileId);
        InStream is =
            (k < 200 ? file.getInStream(ReadType.CACHE) : file.getInStream(ReadType.NO_CACHE));
        Assert.assertTrue(is instanceof BlockInStream);
        byte[] ret = new byte[k / 2];
        Assert.assertEquals(k / 2, is.read(ret, 0, k / 2));
        Assert.assertTrue(TestUtils.equalIncreasingByteArray(k / 2, ret));
        is.close();

        is = (k < 200 ? file.getInStream(ReadType.CACHE) : file.getInStream(ReadType.NO_CACHE));
        Assert.assertTrue(is instanceof BlockInStream);
        ret = new byte[k];
        Assert.assertEquals(k, is.read(ret, 0, k));
        Assert.assertTrue(TestUtils.equalIncreasingByteArray(k, ret));
        is.close();
      }
    }
  }
Esempio n. 12
0
  public ReaderImpl(FileSystem fs, Path path, Configuration conf) throws IOException {
    this.fileSystem = fs;
    this.path = path;
    this.conf = conf;
    FSDataInputStream file = fs.open(path);
    long size = fs.getFileStatus(path).getLen();
    int readSize = (int) Math.min(size, DIRECTORY_SIZE_GUESS);
    ByteBuffer buffer = ByteBuffer.allocate(readSize);
    InStream.read(
        file,
        size - readSize,
        buffer.array(),
        buffer.arrayOffset() + buffer.position(),
        buffer.remaining());
    int psLen = buffer.get(readSize - 1);
    int psOffset = readSize - 1 - psLen;
    CodedInputStream in =
        CodedInputStream.newInstance(buffer.array(), buffer.arrayOffset() + psOffset, psLen);
    OrcProto.PostScript ps = OrcProto.PostScript.parseFrom(in);
    int footerSize = (int) ps.getFooterLength();
    bufferSize = (int) ps.getCompressionBlockSize();
    switch (ps.getCompression()) {
      case NONE:
        compressionKind = CompressionKind.NONE;
        break;
      case ZLIB:
        compressionKind = CompressionKind.ZLIB;
        break;
      case SNAPPY:
        compressionKind = CompressionKind.SNAPPY;
        break;
      case LZO:
        compressionKind = CompressionKind.LZO;
        break;
      default:
        throw new IllegalArgumentException("Unknown compression");
    }
    codec = WriterImpl.createCodec(compressionKind);

    InputStream instream =
        InStream.create(
            "footer", file, size - 1 - psLen - footerSize, footerSize, codec, bufferSize);
    footer = OrcProto.Footer.parseFrom(instream);
    inspector = new OrcLazyRowObjectInspector(0, footer.getTypesList());
    file.close();
  }
Esempio n. 13
0
  /** Test <code>void read(byte[] b, int off, int len)</code>. Read from remote data server. */
  @Test
  public void readTest6() throws IOException {
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      WriteType op = WriteType.MUST_CACHE;
      int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

      TachyonFile file = mTfs.getFile(fileId);
      InStream is = new RemoteBlockInStream(file, ReadType.NO_CACHE, 0);
      Assert.assertTrue(is instanceof RemoteBlockInStream);
      byte[] ret = new byte[k / 2];
      int start = 0;
      while (start < k / 2) {
        int read = is.read(ret, 0, (k / 2) - start);
        Assert.assertTrue(TestUtils.equalIncreasingByteArray(start, read, ret));
        start += read;
      }
      is.close();
      Assert.assertTrue(file.isInMemory());
    }
  }