public HdfsFileInputStream(
      TachyonFS tfs, int fileId, Path hdfsPath, Configuration conf, int bufferSize) {
    LOG.debug(
        "PartitionInputStreamHdfs("
            + tfs
            + ", "
            + fileId
            + ", "
            + hdfsPath
            + ", "
            + conf
            + ", "
            + bufferSize
            + ")");
    mCurrentPosition = 0;
    mTFS = tfs;
    mFileId = fileId;
    mHdfsPath = hdfsPath;
    mHadoopConf = conf;
    mHadoopBufferSize = bufferSize;

    TachyonFile tachyonFile = mTFS.getFile(mFileId);
    try {
      mTachyonFileInputStream = tachyonFile.getInStream(ReadType.CACHE);
    } catch (IOException e) {
      LOG.error(e.getMessage());
      return;
    }
  }
Example #2
0
 @Test
 public void copyFromLocalTest() throws IOException {
   File testDir = new File(mLocalTachyonCluster.getTachyonHome() + "/testDir");
   testDir.mkdir();
   File testDirInner = new File(mLocalTachyonCluster.getTachyonHome() + "/testDir/testDirInner");
   testDirInner.mkdir();
   File testFile =
       generateFileContent("/testDir/testFile", BufferUtils.getIncreasingByteArray(10));
   generateFileContent(
       "/testDir/testDirInner/testFile2", BufferUtils.getIncreasingByteArray(10, 20));
   mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getParent(), "/testDir"});
   Assert.assertEquals(
       getCommandOutput(new String[] {"copyFromLocal", testFile.getParent(), "/testDir"}),
       mOutput.toString());
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/testDir/testFile"));
   TachyonFile tFile2 = mTfs.getFile(new TachyonURI("/testDir/testDirInner/testFile2"));
   Assert.assertNotNull(tFile);
   Assert.assertNotNull(tFile2);
   Assert.assertEquals(10, tFile.length());
   Assert.assertEquals(20, tFile2.length());
   byte[] read = readContent(tFile, 10);
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, read));
   read = readContent(tFile2, 20);
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(10, 20, read));
 }
Example #3
0
  /** Test <code>void read()</code>. */
  @Test
  public void readTest1() throws IOException {
    for (int k = 100; k <= 200; k += 33) {
      for (WriteType op : WriteType.values()) {
        int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

        TachyonFile file = mTfs.getFile(fileId);
        InStream is =
            (k < 150 ? file.getInStream(ReadType.CACHE) : file.getInStream(ReadType.NO_CACHE));
        Assert.assertTrue(is instanceof BlockInStream);
        byte[] ret = new byte[k];
        int value = is.read();
        int cnt = 0;
        while (value != -1) {
          ret[cnt++] = (byte) value;
          value = is.read();
        }
        Assert.assertTrue(TestUtils.equalIncreasingByteArray(k, ret));
        is.close();

        is = (k < 150 ? file.getInStream(ReadType.CACHE) : file.getInStream(ReadType.NO_CACHE));
        Assert.assertTrue(is instanceof BlockInStream);
        ret = new byte[k];
        value = is.read();
        cnt = 0;
        while (value != -1) {
          ret[cnt++] = (byte) value;
          value = is.read();
        }
        Assert.assertTrue(TestUtils.equalIncreasingByteArray(k, ret));
        is.close();
      }
    }
  }
  public static void writeParition()
      throws IOException, TableDoesNotExistException, InvalidPathException,
          FileAlreadyExistException, TException {
    RawTable rawTable = sTachyonClient.getRawTable(sTablePath);

    LOG.info("Writing data...");
    for (int column = 0; column < COLS; column++) {
      RawColumn rawColumn = rawTable.getRawColumn(column);
      if (!rawColumn.createPartition(0)) {
        CommonUtils.runtimeException(
            "Failed to create partition in table " + sTablePath + " under column " + column);
      }

      ByteBuffer buf = ByteBuffer.allocate(80);
      buf.order(ByteOrder.nativeOrder());
      for (int k = 0; k < 20; k++) {
        buf.putInt(k);
      }
      buf.flip();
      CommonUtils.printByteBuffer(LOG, buf);
      buf.flip();

      TachyonFile tFile = rawColumn.getPartition(0);
      OutStream os = tFile.createOutStream(sWriteType);
      os.write(buf);
      os.close();
    }
  }
  /** Test <code>long skip(long len)</code>. */
  @Test
  public void skipTest() throws IOException {
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      WriteType op = WriteType.THROUGH;
      int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

      TachyonFile file = mTfs.getFile(fileId);
      InStream is = file.getInStream(ReadType.CACHE);
      Assert.assertTrue(is instanceof RemoteBlockInStream);
      Assert.assertEquals(k / 2, is.skip(k / 2));
      Assert.assertEquals(k / 2, is.read());
      is.close();
      Assert.assertFalse(file.isInMemory());

      if (k >= 3) {
        is = file.getInStream(ReadType.CACHE);
        Assert.assertTrue(is instanceof RemoteBlockInStream);
        int t = k / 3;
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals(t, is.read());
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals(2 * t + 1, is.read());
        is.close();
        Assert.assertFalse(file.isInMemory());
      }
    }
  }
  /**
   * This function displays the first 5KB of a file if it is in ASCII format.
   *
   * @param path The path of the file to display
   * @param request The HttpServletRequest object
   * @throws FileDoesNotExistException
   * @throws IOException
   * @throws InvalidPathException
   * @throws TException
   */
  private void displayFile(String path, HttpServletRequest request)
      throws FileDoesNotExistException, InvalidPathException, IOException {
    TachyonClient tachyonClient = TachyonClient.getClient(mMasterInfo.getMasterAddress());
    TachyonFile tFile = tachyonClient.getFile(path);
    if (tFile == null) {
      throw new FileDoesNotExistException(path);
    }

    InStream is = tFile.getInStream(OpType.READ_NO_CACHE);
    int len = Math.min(5 * Constants.KB, (int) tFile.getSize());
    byte[] data = new byte[len];
    is.read(data, 0, len);
    String fileData = CommonUtils.convertByteArrayToString(data);
    if (fileData == null) {
      fileData = "The requested file is not completely encoded in ascii";
    }
    is.close();

    try {
      tachyonClient.close();
    } catch (TException e) {
      LOG.error(e.getMessage());
    }
    request.setAttribute("fileData", fileData);
    return;
  }
Example #7
0
  /** Test <code>long skip(long len)</code>. */
  @Test
  public void skipTest() throws IOException {
    for (int k = 10; k <= 200; k += 33) {
      for (WriteType op : WriteType.values()) {
        int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

        TachyonFile file = mTfs.getFile(fileId);
        InStream is =
            (k < 100 ? file.getInStream(ReadType.CACHE) : file.getInStream(ReadType.NO_CACHE));
        Assert.assertTrue(is instanceof BlockInStream);
        Assert.assertEquals(k / 2, is.skip(k / 2));
        Assert.assertEquals(k / 2, is.read());
        is.close();

        is = (k < 100 ? file.getInStream(ReadType.CACHE) : file.getInStream(ReadType.NO_CACHE));
        Assert.assertTrue(is instanceof BlockInStream);
        int t = k / 3;
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals(t, is.read());
        Assert.assertEquals(t, is.skip(t));
        Assert.assertEquals((byte) (2 * t + 1), is.read());
        is.close();
      }
    }
  }
Example #8
0
 @Test
 public void touchTest() throws IOException {
   String[] argv = new String[] {"touch", "/testFile"};
   mFsShell.touch(argv);
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/testFile"));
   Assert.assertNotNull(tFile);
   Assert.assertEquals(getCommandOutput(argv), mOutput.toString());
   Assert.assertTrue(tFile.isFile());
 }
Example #9
0
 @Test
 public void mkdirShortPathTest() throws IOException {
   mFsShell.mkdir(new String[] {"mkdir", "/root/testFile1"});
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/root/testFile1"));
   Assert.assertNotNull(tFile);
   Assert.assertEquals(
       getCommandOutput(new String[] {"mkdir", "/root/testFile1"}), mOutput.toString());
   Assert.assertTrue(tFile.isDirectory());
 }
Example #10
0
 @Test
 public void mkdirComplexPathTest() throws IOException {
   mFsShell.mkdir(new String[] {"mkdir", "/Complex!@#$%^&*()-_=+[]{};\"'<>,.?/File"});
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/Complex!@#$%^&*()-_=+[]{};\"'<>,.?/File"));
   Assert.assertNotNull(tFile);
   Assert.assertEquals(
       getCommandOutput(new String[] {"mkdir", "/Complex!@#$%^&*()-_=+[]{};\"'<>,.?/File"}),
       mOutput.toString());
   Assert.assertTrue(tFile.isDirectory());
 }
Example #11
0
  public FileInStream(TachyonFile file, ReadType opType) throws IOException {
    super(file, opType);

    FILE_LENGTH = file.length();
    BLOCK_CAPACITY = file.getBlockSizeByte();

    mCurrentPosition = 0;
    mCurrentBlockIndex = -1;
    mCurrentBlockInStream = null;
    mCurrentBlockLeft = 0;
  }
 private int createAndWriteFile(TachyonURI filePath, WriteType writeType, int len)
     throws IOException {
   ByteBuffer buf = ByteBuffer.allocate(len);
   buf.order(ByteOrder.nativeOrder());
   for (int k = 0; k < len; k++) {
     buf.put((byte) k);
   }
   int fileId = mTFS.createFile(filePath);
   TachyonFile file = mTFS.getFile(fileId);
   OutStream os = file.getOutStream(writeType);
   os.write(buf.array());
   os.close();
   return fileId;
 }
Example #13
0
 @Test
 public void mkdirTest() throws IOException {
   String qualifiedPath =
       "tachyon://"
           + NetworkAddressUtils.getLocalHostName(Constants.DEFAULT_HOST_RESOLUTION_TIMEOUT_MS)
           + ":"
           + mLocalTachyonCluster.getMasterPort()
           + "/root/testFile1";
   mFsShell.mkdir(new String[] {"mkdir", qualifiedPath});
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/root/testFile1"));
   Assert.assertNotNull(tFile);
   Assert.assertEquals(
       getCommandOutput(new String[] {"mkdir", qualifiedPath}), mOutput.toString());
   Assert.assertTrue(tFile.isDirectory());
 }
 private void deleteDuringEviction(int i) throws IOException {
   final String fileName1 = "/file" + i + "_1";
   final String fileName2 = "/file" + i + "_2";
   int fileId =
       createAndWriteFile(new TachyonURI(fileName1), WriteType.CACHE_THROUGH, MEM_CAPACITY_BYTES);
   TachyonFile file = mTFS.getFile(fileId);
   Assert.assertTrue(file.isInMemory());
   // Deleting file1, command will be sent by master to worker asynchronously
   mTFS.delete(new TachyonURI(fileName1), false);
   // Meanwhile creating file2. If creation arrives earlier than deletion, it will evict file1
   fileId =
       createAndWriteFile(
           new TachyonURI(fileName2), WriteType.CACHE_THROUGH, MEM_CAPACITY_BYTES / 4);
   file = mTFS.getFile(fileId);
   Assert.assertTrue(file.isInMemory());
   mTFS.delete(new TachyonURI(fileName2), false);
 }
Example #15
0
 @Test
 public void touchTestWithFullURI() throws IOException {
   String tachyonURI =
       "tachyon://"
           + mLocalTachyonCluster.getMasterHostname()
           + ":"
           + mLocalTachyonCluster.getMasterPort()
           + "/destFileURI";
   // when
   String[] argv = new String[] {"touch", tachyonURI};
   mFsShell.touch(argv);
   // then
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/destFileURI"));
   Assert.assertThat(tFile, CoreMatchers.notNullValue());
   Assert.assertThat(getCommandOutput(argv), CoreMatchers.equalTo(mOutput.toString()));
   Assert.assertThat(tFile.isFile(), CoreMatchers.equalTo(true));
 }
Example #16
0
 public static void readFile() throws IOException {
   for (int i = 0; i < sFiles; i++) {
     String filePath = sFileFolder + "/part-" + i;
     LOG.debug("Reading data from " + filePath);
     TachyonFile file = sTachyonClient.getFile(filePath);
     TachyonByteBuffer buf = file.readByteBuffer();
     if (buf == null) {
       file.recache();
       buf = file.readByteBuffer();
     }
     buf.DATA.order(ByteOrder.nativeOrder());
     for (int k = 0; k < sNumbers; k++) {
       sPass = sPass && (buf.DATA.getInt() == k);
     }
     buf.close();
   }
 }
Example #17
0
  public static void writeFile() throws IOException {
    for (int i = 0; i < sFiles; i++) {
      String filePath = sFileFolder + "/part-" + i;
      TachyonFile file = sTachyonClient.getFile(filePath);
      OutputStream os = file.getOutStream(WriteType.ASYNC_THROUGH);

      ByteBuffer buf = ByteBuffer.allocate(80);
      buf.order(ByteOrder.nativeOrder());
      for (int k = 0; k < sNumbers; k++) {
        buf.putInt(k);
      }
      buf.flip();
      LOG.debug("Writing data to " + filePath);
      os.write(buf.array());
      os.close();
    }
  }
Example #18
0
 @Test
 public void locationTest() throws IOException {
   int fileId = TachyonFSTestUtils.createByteFile(mTfs, "/testFile", WriteType.MUST_CACHE, 10);
   mFsShell.location(new String[] {"location", "/testFile"});
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/testFile"));
   Assert.assertNotNull(tFile);
   List<String> locationsList = tFile.getLocationHosts();
   String[] commandParameters = new String[3 + locationsList.size()];
   commandParameters[0] = "location";
   commandParameters[1] = "/testFile";
   commandParameters[2] = String.valueOf(fileId);
   Iterator<String> iter = locationsList.iterator();
   int i = 3;
   while (iter.hasNext()) {
     commandParameters[i++] = iter.next();
   }
   Assert.assertEquals(getCommandOutput(commandParameters), mOutput.toString());
 }
Example #19
0
    public void readPartition()
        throws IOException, SuspectedFileSizeException, InvalidPathException, TException {
      ByteBuffer buf;
      if (DEBUG_MODE) {
        LOG.info("Verifying the reading data...");

        for (int pId = mLeft; pId < mRight; pId++) {
          TachyonFile file = mTC.getFile(FILE_NAME + mWorkerId);
          buf = file.readByteBuffer();
          IntBuffer intBuf;
          intBuf = buf.asIntBuffer();
          int tmp;
          for (int i = 0; i < BLOCKS_PER_FILE; i++) {
            for (int k = 0; k < BLOCK_SIZE_BYTES / 4; k++) {
              tmp = intBuf.get();
              if ((k == 0 && tmp == (i + mWorkerId)) || (k != 0 && tmp == k)) {
              } else {
                CommonUtils.runtimeException("WHAT? " + tmp + " " + k);
              }
            }
          }
          file.readByteBuffer();
        }
      }

      long sum = 0;
      for (int pId = mLeft; pId < mRight; pId++) {
        long startTimeMs = System.currentTimeMillis();
        TachyonFile file = mTC.getFile(FILE_NAME + (mWorkerId + BASE_FILE_NUMBER));
        buf = file.readByteBuffer();
        for (int i = 0; i < BLOCKS_PER_FILE; i++) {
          buf.get(mBuf.array());
        }
        sum += mBuf.get(pId % 16);

        if (DEBUG_MODE) {
          buf.flip();
          CommonUtils.printByteBuffer(LOG, buf);
        }
        buf.clear();
        logPerIteration(startTimeMs, pId, "th ReadTachyonFile @ Worker ", pId);
      }
      Results[mWorkerId] = sum;
    }
Example #20
0
    public void writeParition()
        throws IOException, SuspectedFileSizeException, InvalidPathException, TException {
      if (DEBUG_MODE) {
        mBuf.flip();
        CommonUtils.printByteBuffer(LOG, mBuf);
      }

      mBuf.flip();
      for (int pId = mLeft; pId < mRight; pId++) {
        long startTimeMs = System.currentTimeMillis();
        TachyonFile file = mTC.getFile(FILE_NAME + (mWorkerId + BASE_FILE_NUMBER));
        OutStream os = file.getOutStream(WriteType.CACHE);
        for (int k = 0; k < BLOCKS_PER_FILE; k++) {
          mBuf.array()[0] = (byte) (k + mWorkerId);
          os.write(mBuf.array());
        }
        os.close();
        logPerIteration(startTimeMs, pId, "th WriteTachyonFile @ Worker ", pId);
      }
    }
Example #21
0
 @Test
 public void copyFromLocalLargeTest() throws IOException {
   File testFile = new File(mLocalTachyonCluster.getTachyonHome() + "/testFile");
   testFile.createNewFile();
   FileOutputStream fos = new FileOutputStream(testFile);
   byte[] toWrite = BufferUtils.getIncreasingByteArray(SIZE_BYTES);
   fos.write(toWrite);
   fos.close();
   mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"});
   Assert.assertEquals(
       getCommandOutput(new String[] {"copyFromLocal", testFile.getAbsolutePath(), "/testFile"}),
       mOutput.toString());
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/testFile"));
   Assert.assertNotNull(tFile);
   Assert.assertEquals(SIZE_BYTES, tFile.length());
   InStream tfis = tFile.getInStream(ReadType.NO_CACHE);
   byte[] read = new byte[SIZE_BYTES];
   tfis.read(read);
   Assert.assertTrue(BufferUtils.equalIncreasingByteArray(SIZE_BYTES, read));
 }
Example #22
0
 @Test
 public void copyFromLocalTestWithFullURI() throws IOException {
   File testFile = generateFileContent("/srcFileURI", BufferUtils.getIncreasingByteArray(10));
   String tachyonURI =
       "tachyon://"
           + mLocalTachyonCluster.getMasterHostname()
           + ":"
           + mLocalTachyonCluster.getMasterPort()
           + "/destFileURI";
   // when
   mFsShell.copyFromLocal(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
   String cmdOut =
       getCommandOutput(new String[] {"copyFromLocal", testFile.getPath(), tachyonURI});
   // then
   Assert.assertThat(cmdOut, CoreMatchers.equalTo(mOutput.toString()));
   TachyonFile tFile = mTfs.getFile(new TachyonURI("/destFileURI"));
   Assert.assertThat(tFile.length(), CoreMatchers.equalTo(10L));
   byte[] read = readContent(tFile, 10);
   Assert.assertThat(BufferUtils.equalIncreasingByteArray(10, read), CoreMatchers.equalTo(true));
 }
  /** Test <code>void read(byte[] b, int off, int len)</code>. Read from remote data server. */
  @Test
  public void readTest6() throws IOException {
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      WriteType op = WriteType.MUST_CACHE;
      int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

      TachyonFile file = mTfs.getFile(fileId);
      InStream is = new RemoteBlockInStream(file, ReadType.NO_CACHE, 0);
      Assert.assertTrue(is instanceof RemoteBlockInStream);
      byte[] ret = new byte[k / 2];
      int start = 0;
      while (start < k / 2) {
        int read = is.read(ret, 0, (k / 2) - start);
        Assert.assertTrue(TestUtils.equalIncreasingByteArray(start, read, ret));
        start += read;
      }
      is.close();
      Assert.assertTrue(file.isInMemory());
    }
  }
  /** Test <code>void read()</code>. Read from remote data server. */
  @Test
  public void readTest4() throws IOException {
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      WriteType op = WriteType.MUST_CACHE;
      int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

      TachyonFile file = mTfs.getFile(fileId);
      RemoteBlockInStream is = new RemoteBlockInStream(file, ReadType.NO_CACHE, 0);
      Assert.assertTrue(is instanceof RemoteBlockInStream);
      byte[] ret = new byte[k];
      int value = is.read();
      int cnt = 0;
      while (value != -1) {
        ret[cnt++] = (byte) value;
        value = is.read();
      }
      Assert.assertTrue(TestUtils.equalIncreasingByteArray(k, ret));
      is.close();
      Assert.assertTrue(file.isInMemory());
    }
  }
  /** Test <code>void read(byte b[])</code>. Read from underfs. */
  @Test
  public void readTest7() throws IOException {
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      WriteType op = WriteType.THROUGH;
      int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

      TachyonFile file = mTfs.getFile(fileId);
      InStream is = file.getInStream(ReadType.NO_CACHE);
      if (k == 0) {
        Assert.assertTrue(is instanceof EmptyBlockInStream);
      } else {
        Assert.assertTrue(is instanceof RemoteBlockInStream);
      }
      byte[] ret = new byte[k];
      Assert.assertEquals(k, is.read(ret));
      Assert.assertTrue(TestUtils.equalIncreasingByteArray(k, ret));
      Assert.assertEquals(-1, is.read(ret));
      is.close();
      Assert.assertFalse(file.isInMemory());
    }
  }
 /**
  * This function displays 5KB of a file from a specific offset if it is in ASCII format.
  *
  * @param path The path of the file to display
  * @param request The HttpServletRequest object
  * @param offset Where the file starts to display.
  * @throws FileDoesNotExistException
  * @throws IOException
  * @throws InvalidPathException
  */
 private void displayFile(String path, HttpServletRequest request, long offset)
     throws FileDoesNotExistException, InvalidPathException, IOException {
   String masterAddress =
       Constants.HEADER
           + mMasterInfo.getMasterAddress().getHostName()
           + ":"
           + mMasterInfo.getMasterAddress().getPort();
   TachyonFS tachyonClient = TachyonFS.get(masterAddress);
   TachyonFile tFile = tachyonClient.getFile(path);
   String fileData = null;
   if (tFile == null) {
     throw new FileDoesNotExistException(path);
   }
   if (tFile.isComplete()) {
     InStream is = tFile.getInStream(ReadType.NO_CACHE);
     int len = (int) Math.min(5 * Constants.KB, tFile.length() - offset);
     byte[] data = new byte[len];
     is.skip(offset);
     is.read(data, 0, len);
     fileData = CommonUtils.convertByteArrayToStringWithoutEscape(data);
     if (fileData == null) {
       fileData = "The requested file is not completely encoded in ascii";
     }
     is.close();
   } else {
     fileData = "The requested file is not complete yet.";
   }
   try {
     tachyonClient.close();
   } catch (IOException e) {
     LOG.error(e.getMessage());
   }
   List<BlockInfo> rawBlockList = mMasterInfo.getBlockList(path);
   List<UiBlockInfo> uiBlockInfo = new ArrayList<UiBlockInfo>();
   for (BlockInfo blockInfo : rawBlockList) {
     uiBlockInfo.add(new UiBlockInfo(blockInfo));
   }
   request.setAttribute("fileBlocks", uiBlockInfo);
   request.setAttribute("fileData", fileData);
 }
  public static void readPartition()
      throws IOException, TableDoesNotExistException, InvalidPathException, TException {
    LOG.info("Reading data...");
    RawTable rawTable = sTachyonClient.getRawTable(mId);
    ByteBuffer metadata = rawTable.getMetadata();
    LOG.info("Metadata: ");
    LOG.info(metadata.getInt() + " ");
    LOG.info(metadata.getInt() + " ");
    LOG.info(metadata.getInt() + " ");

    for (int column = 0; column < COLS; column++) {
      RawColumn rawColumn = rawTable.getRawColumn(column);
      TachyonFile tFile = rawColumn.getPartition(0);

      ByteBuffer buf = tFile.readByteBuffer();
      if (buf == null) {
        tFile.recacheData();
      }
      CommonUtils.printByteBuffer(LOG, tFile.readByteBuffer());
      tFile.releaseFileLock();
    }
  }
  /**
   * Test <code>void seek(long pos)</code>.
   *
   * @throws IOException
   */
  @Test
  public void seekTest() throws IOException {
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
      WriteType op = WriteType.THROUGH;
      int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

      TachyonFile file = mTfs.getFile(fileId);
      InStream is = file.getInStream(ReadType.NO_CACHE);
      if (k == 0) {
        Assert.assertTrue(is instanceof EmptyBlockInStream);
      } else {
        Assert.assertTrue(is instanceof RemoteBlockInStream);
      }

      is.seek(k / 3);
      Assert.assertEquals(k / 3, is.read());
      is.seek(k / 2);
      Assert.assertEquals(k / 2, is.read());
      is.seek(k / 4);
      Assert.assertEquals(k / 4, is.read());
      is.close();
    }
  }
  /**
   * Test <code>void seek(long pos)</code>.
   *
   * @throws IOException
   */
  @Test
  public void seekExceptionTest() throws IOException {
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
      WriteType op = WriteType.THROUGH;
      int fileId = TestUtils.createByteFile(mTfs, "/root/testFile_" + k + "_" + op, op, k);

      TachyonFile file = mTfs.getFile(fileId);
      InStream is = file.getInStream(ReadType.NO_CACHE);
      if (k == 0) {
        Assert.assertTrue(is instanceof EmptyBlockInStream);
      } else {
        Assert.assertTrue(is instanceof RemoteBlockInStream);
      }

      try {
        is.seek(-1);
      } catch (IOException e) {
        // This is expected
        continue;
      }
      is.close();
      throw new IOException("Except seek IOException");
    }
  }
Example #30
0
  @Test
  public void rawtablePerfTest()
      throws TableDoesNotExistException, InvalidPathException, FileAlreadyExistException,
          TableColumnException, IOException, TException {
    int col = 200;

    long sMs = System.currentTimeMillis();
    int fileId = mClient.createRawTable("/table", col);
    //    System.out.println("A " + (System.currentTimeMillis() - sMs));

    sMs = System.currentTimeMillis();
    RawTable table = mClient.getRawTable(fileId);
    Assert.assertEquals(col, table.getColumns());
    table = mClient.getRawTable("/table");
    Assert.assertEquals(col, table.getColumns());
    //    System.out.println("B " + (System.currentTimeMillis() - sMs));

    sMs = System.currentTimeMillis();
    for (int k = 0; k < col; k++) {
      RawColumn rawCol = table.getRawColumn(k);
      rawCol.createPartition(0);
      TachyonFile file = rawCol.getPartition(0);
      OutStream outStream = file.getOutStream(OpType.WRITE_CACHE);
      outStream.write(TestUtils.getIncreasingByteArray(10));
      outStream.close();
    }
    //    System.out.println("C " + (System.currentTimeMillis() - sMs));

    sMs = System.currentTimeMillis();
    for (int k = 0; k < col; k++) {
      RawColumn rawCol = table.getRawColumn(k);
      TachyonFile file = rawCol.getPartition(0, true);
      Assert.assertEquals(TestUtils.getIncreasingByteBuffer(10), file.readByteBuffer());
      file.releaseFileLock();
    }
    //    System.out.println("D " + (System.currentTimeMillis() - sMs));

    sMs = System.currentTimeMillis();
    for (int k = 0; k < col; k++) {
      RawColumn rawCol = table.getRawColumn(k);
      TachyonFile file = rawCol.getPartition(0, true);
      Assert.assertEquals(TestUtils.getIncreasingByteBuffer(10), file.readByteBuffer());
      file.releaseFileLock();
    }
    //    System.out.println("E " + (System.currentTimeMillis() - sMs));
  }