Exemplo n.º 1
0
  public static void writeParition()
      throws IOException, TableDoesNotExistException, InvalidPathException,
          FileAlreadyExistException, TException {
    RawTable rawTable = sTachyonClient.getRawTable(sTablePath);

    LOG.info("Writing data...");
    for (int column = 0; column < COLS; column++) {
      RawColumn rawColumn = rawTable.getRawColumn(column);
      if (!rawColumn.createPartition(0)) {
        CommonUtils.runtimeException(
            "Failed to create partition in table " + sTablePath + " under column " + column);
      }

      ByteBuffer buf = ByteBuffer.allocate(80);
      buf.order(ByteOrder.nativeOrder());
      for (int k = 0; k < 20; k++) {
        buf.putInt(k);
      }
      buf.flip();
      CommonUtils.printByteBuffer(LOG, buf);
      buf.flip();

      TachyonFile tFile = rawColumn.getPartition(0);
      OutStream os = tFile.createOutStream(sWriteType);
      os.write(buf);
      os.close();
    }
  }
Exemplo n.º 2
0
 public static void createFiles() throws IOException {
   long startTimeMs = CommonUtils.getCurrentMs();
   for (int k = 0; k < THREADS; k++) {
     int fileId = MTC.createFile(FILE_NAME + (k + BASE_FILE_NUMBER));
     CommonUtils.printTimeTakenMs(startTimeMs, LOG, "user_createFiles with fileId " + fileId);
   }
 }
Exemplo n.º 3
0
 public static void createRawTable() throws InvalidPathException {
   long startTimeMs = CommonUtils.getCurrentMs();
   ByteBuffer data = ByteBuffer.allocate(12);
   data.putInt(-1);
   data.putInt(-2);
   data.putInt(-3);
   data.flip();
   mId = sTachyonClient.createRawTable(sTablePath, 3, data);
   CommonUtils.printTimeTakenMs(startTimeMs, LOG, "createRawTable with id " + mId);
 }
Exemplo n.º 4
0
 public HdfsClient() {
   try {
     mFs = FileSystem.get(new Configuration());
   } catch (IOException e) {
     CommonUtils.runtimeException(e);
   }
 }
Exemplo n.º 5
0
    public void memoryCopyParition() throws IOException {
      if (DEBUG_MODE) {
        mBuf.flip();
        CommonUtils.printByteBuffer(LOG, mBuf);
      }
      mBuf.flip();
      long sum = 0;
      String str = "th " + mMsg + " @ Worker ";

      if (mOneToMany) {
        ByteBuffer dst = null;
        RandomAccessFile file = null;
        if (mMemoryOnly) {
          dst = ByteBuffer.allocateDirect(FILE_BYTES);
        }
        for (int times = mLeft; times < mRight; times++) {
          long startTimeMs = System.currentTimeMillis();
          if (!mMemoryOnly) {
            file = new RandomAccessFile(FOLDER + (mWorkerId + BASE_FILE_NUMBER), "rw");
            dst = file.getChannel().map(MapMode.READ_WRITE, 0, FILE_BYTES);
          }
          dst.order(ByteOrder.nativeOrder());
          for (int k = 0; k < BLOCKS_PER_FILE; k++) {
            mBuf.array()[0] = (byte) (k + mWorkerId);
            dst.put(mBuf.array());
          }
          dst.clear();
          sum += dst.get(times);
          dst.clear();
          if (!mMemoryOnly) {
            file.close();
          }
          logPerIteration(startTimeMs, times, str, mWorkerId);
        }
      } else {
        ByteBuffer dst = null;
        RandomAccessFile file = null;
        if (mMemoryOnly) {
          dst = ByteBuffer.allocateDirect(FILE_BYTES);
        }
        for (int times = mLeft; times < mRight; times++) {
          long startTimeMs = System.currentTimeMillis();
          if (!mMemoryOnly) {
            file = new RandomAccessFile(FOLDER + (mWorkerId + BASE_FILE_NUMBER), "rw");
            dst = file.getChannel().map(MapMode.READ_WRITE, 0, FILE_BYTES);
          }
          dst.order(ByteOrder.nativeOrder());
          for (int k = 0; k < BLOCKS_PER_FILE; k++) {
            dst.get(mBuf.array());
          }
          sum += mBuf.get(times % 16);
          dst.clear();
          if (!mMemoryOnly) {
            file.close();
          }
          logPerIteration(startTimeMs, times, str, mWorkerId);
        }
      }
      Results[mWorkerId] = sum;
    }
Exemplo n.º 6
0
 public String getSize() {
   if (IS_DIRECTORY) {
     return " ";
   } else {
     return CommonUtils.getSizeFromBytes(SIZE);
   }
 }
Exemplo n.º 7
0
  /**
   * This function displays the first 5KB of a file if it is in ASCII format.
   *
   * @param path The path of the file to display
   * @param request The HttpServletRequest object
   * @throws FileDoesNotExistException
   * @throws IOException
   * @throws InvalidPathException
   * @throws TException
   */
  private void displayFile(String path, HttpServletRequest request)
      throws FileDoesNotExistException, InvalidPathException, IOException {
    TachyonClient tachyonClient = TachyonClient.getClient(mMasterInfo.getMasterAddress());
    TachyonFile tFile = tachyonClient.getFile(path);
    if (tFile == null) {
      throw new FileDoesNotExistException(path);
    }

    InStream is = tFile.getInStream(OpType.READ_NO_CACHE);
    int len = Math.min(5 * Constants.KB, (int) tFile.getSize());
    byte[] data = new byte[len];
    is.read(data, 0, len);
    String fileData = CommonUtils.convertByteArrayToString(data);
    if (fileData == null) {
      fileData = "The requested file is not completely encoded in ascii";
    }
    is.close();

    try {
      tachyonClient.close();
    } catch (TException e) {
      LOG.error(e.getMessage());
    }
    request.setAttribute("fileData", fileData);
    return;
  }
Exemplo n.º 8
0
 @Override
 public void run() {
   try {
     readPartition();
   } catch (Exception e) {
     CommonUtils.runtimeException(e);
   }
   LOG.info("ReadWorker " + mWorkerId + " just finished.");
 }
Exemplo n.º 9
0
 @Override
 public void run() {
   try {
     memoryCopyParition();
   } catch (IOException e) {
     CommonUtils.runtimeException(e);
   }
   LOG.info(mMsg + mWorkerId + " just finished.");
 }
Exemplo n.º 10
0
    public void readPartition()
        throws IOException, SuspectedFileSizeException, InvalidPathException, TException {
      ByteBuffer buf;
      if (DEBUG_MODE) {
        LOG.info("Verifying the reading data...");

        for (int pId = mLeft; pId < mRight; pId++) {
          TachyonFile file = mTC.getFile(FILE_NAME + mWorkerId);
          buf = file.readByteBuffer();
          IntBuffer intBuf;
          intBuf = buf.asIntBuffer();
          int tmp;
          for (int i = 0; i < BLOCKS_PER_FILE; i++) {
            for (int k = 0; k < BLOCK_SIZE_BYTES / 4; k++) {
              tmp = intBuf.get();
              if ((k == 0 && tmp == (i + mWorkerId)) || (k != 0 && tmp == k)) {
              } else {
                CommonUtils.runtimeException("WHAT? " + tmp + " " + k);
              }
            }
          }
          file.readByteBuffer();
        }
      }

      long sum = 0;
      for (int pId = mLeft; pId < mRight; pId++) {
        long startTimeMs = System.currentTimeMillis();
        TachyonFile file = mTC.getFile(FILE_NAME + (mWorkerId + BASE_FILE_NUMBER));
        buf = file.readByteBuffer();
        for (int i = 0; i < BLOCKS_PER_FILE; i++) {
          buf.get(mBuf.array());
        }
        sum += mBuf.get(pId % 16);

        if (DEBUG_MODE) {
          buf.flip();
          CommonUtils.printByteBuffer(LOG, buf);
        }
        buf.clear();
        logPerIteration(startTimeMs, pId, "th ReadTachyonFile @ Worker ", pId);
      }
      Results[mWorkerId] = sum;
    }
Exemplo n.º 11
0
  /**
   * Populates attribute fields with data from the MasterInfo associated with this servlet. Errors
   * will be displayed in an error field. Debugging can be enabled to display additional data. Will
   * eventually redirect the request to a jsp.
   *
   * @param request The HttpServletRequest object
   * @param response The HttpServletResponse object
   * @throws ServletException
   * @throws IOException
   * @throws UnknownHostException
   */
  @Override
  protected void doGet(HttpServletRequest request, HttpServletResponse response)
      throws ServletException, IOException, UnknownHostException {
    request.setAttribute("debug", Constants.DEBUG);

    request.setAttribute("masterNodeAddress", mMasterInfo.getMasterAddress().toString());
    request.setAttribute("invalidPathError", "");
    List<ClientFileInfo> filesInfo = null;
    String currentPath = request.getParameter("path");

    if (currentPath.isEmpty()) {
      currentPath = "/";
    }
    request.setAttribute("currentPath", currentPath);
    try {
      UiFileInfo currentFileInfo = new UiFileInfo(mMasterInfo.getFileInfo(currentPath));
      request.setAttribute("currentDirectory", currentFileInfo);
      if (!currentFileInfo.getIsDirectory()) {
        displayFile(currentFileInfo.getAbsolutePath(), request);
        getServletContext().getRequestDispatcher("/viewFile.jsp").forward(request, response);
        return;
      }
      CommonUtils.validatePath(currentPath);
      setPathDirectories(currentPath, request);
      filesInfo = mMasterInfo.getFilesInfo(currentPath);
    } catch (FileDoesNotExistException fdne) {
      request.setAttribute("invalidPathError", "Error: Invalid Path " + fdne.getMessage());
      getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
      return;
    } catch (InvalidPathException ipe) {
      request.setAttribute("invalidPathError", "Error: Invalid Path " + ipe.getLocalizedMessage());
      getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
      return;
    }

    List<UiFileInfo> fileInfos = new ArrayList<UiFileInfo>(filesInfo.size());
    for (ClientFileInfo fileInfo : filesInfo) {
      UiFileInfo toAdd = new UiFileInfo(fileInfo);
      try {
        if (!toAdd.getIsDirectory()) {
          toAdd.setFileLocations(mMasterInfo.getFileLocations(toAdd.getId()));
        }
      } catch (FileDoesNotExistException fdne) {
        request.setAttribute("invalidPathError", "Error: Invalid Path " + fdne.getMessage());
        getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
        return;
      }
      fileInfos.add(toAdd);
    }
    Collections.sort(fileInfos);
    request.setAttribute("fileInfos", fileInfos);

    getServletContext().getRequestDispatcher("/browse.jsp").forward(request, response);
  }
Exemplo n.º 12
0
  public RawColumn getRawColumn(int columnIndex) {
    if (columnIndex < 0 || columnIndex >= CLIENT_RAW_TABLE_INFO.getColumns()) {
      CommonUtils.runtimeException(
          CLIENT_RAW_TABLE_INFO.getPath()
              + " does not have column "
              + columnIndex
              + ". It has "
              + CLIENT_RAW_TABLE_INFO.getColumns()
              + " columns.");
    }

    return new RawColumn(TACHYON_CLIENT, this, columnIndex);
  }
Exemplo n.º 13
0
  private static void memoryCopyTest(boolean write, boolean memoryOnly) {
    ByteBuffer[] bufs = new ByteBuffer[THREADS];

    for (int thread = 0; thread < THREADS; thread++) {
      ByteBuffer sRawData = ByteBuffer.allocate(BLOCK_SIZE_BYTES);
      sRawData.order(ByteOrder.nativeOrder());
      for (int k = 0; k < BLOCK_SIZE_BYTES / 4; k++) {
        sRawData.putInt(k);
      }
      bufs[thread] = sRawData;
    }

    String msg = (write ? "Write" : "Read") + (memoryOnly ? "_Memory " : "_RamFile ");

    GeneralWorker[] WWs = new GeneralWorker[THREADS];
    int t = FILES / THREADS;
    for (int thread = 0; thread < THREADS; thread++) {
      WWs[thread] =
          new GeneralWorker(
              thread, t * thread, t * (thread + 1), bufs[thread], write, memoryOnly, msg);
    }

    long startTimeMs = System.currentTimeMillis();
    for (int thread = 0; thread < THREADS; thread++) {
      WWs[thread].start();
    }
    for (int thread = 0; thread < THREADS; thread++) {
      try {
        WWs[thread].join();
      } catch (InterruptedException e) {
        CommonUtils.runtimeException(e);
      }
    }
    long takenTimeMs = System.currentTimeMillis() - startTimeMs;
    double result = 1000L * FILES_BYTES / takenTimeMs / 1024 / 1024;

    LOG.info(
        result
            + " Mb/sec. "
            + RESULT_PREFIX
            + "Entire "
            + msg
            + " Test : "
            + " Took "
            + takenTimeMs
            + " ms. Current System Time: "
            + System.currentTimeMillis());
  }
Exemplo n.º 14
0
 public void delete(Path f, boolean recursive) {
   IOException te = null;
   int cnt = 0;
   while (cnt < MAX_TRY) {
     try {
       mFs.delete(f, recursive);
     } catch (IOException e) {
       cnt++;
       LOG.error(cnt + " : " + e.getMessage(), e);
       te = e;
       continue;
     }
     return;
   }
   CommonUtils.runtimeException(te);
 }
Exemplo n.º 15
0
 public void copyToLocalFile(boolean delSrc, Path src, Path dst) {
   IOException te = null;
   int cnt = 0;
   while (cnt < MAX_TRY) {
     try {
       mFs.copyToLocalFile(delSrc, src, dst);
     } catch (IOException e) {
       cnt++;
       LOG.error(cnt + " : " + e.getMessage(), e);
       te = e;
       continue;
     }
     return;
   }
   CommonUtils.runtimeException(te);
 }
Exemplo n.º 16
0
  private static void TachyonTest(boolean write) {
    ByteBuffer[] bufs = new ByteBuffer[THREADS];

    for (int thread = 0; thread < THREADS; thread++) {
      ByteBuffer sRawData = ByteBuffer.allocate(BLOCK_SIZE_BYTES);
      sRawData.order(ByteOrder.nativeOrder());
      for (int k = 0; k < BLOCK_SIZE_BYTES / 4; k++) {
        sRawData.putInt(k);
      }
      bufs[thread] = sRawData;
    }

    Worker[] WWs = new Worker[THREADS];
    int t = FILES / THREADS;
    for (int thread = 0; thread < THREADS; thread++) {
      if (write) {
        WWs[thread] = new TachyonWriterWorker(thread, t * thread, t * (thread + 1), bufs[thread]);
      } else {
        WWs[thread] = new TachyonReadWorker(thread, t * thread, t * (thread + 1), bufs[thread]);
      }
    }

    long startTimeMs = System.currentTimeMillis();
    for (int thread = 0; thread < THREADS; thread++) {
      WWs[thread].start();
    }
    for (int thread = 0; thread < THREADS; thread++) {
      try {
        WWs[thread].join();
      } catch (InterruptedException e) {
        CommonUtils.runtimeException(e);
      }
    }
    long takenTimeMs = System.currentTimeMillis() - startTimeMs;
    double result = FILES_BYTES * 1000L / takenTimeMs / 1024 / 1024;
    LOG.info(
        result
            + " Mb/sec. "
            + RESULT_PREFIX
            + "Entire "
            + (write ? "Write " : "Read ")
            + " Took "
            + takenTimeMs
            + " ms. Current System Time: "
            + System.currentTimeMillis());
  }
Exemplo n.º 17
0
 public void copyFromLocalFile(boolean delSrc, boolean overwrite, String src, String dst) {
   IOException te = null;
   LOG.info("Trying to copy from " + src + " to " + dst);
   int cnt = 0;
   while (cnt < MAX_TRY) {
     try {
       mFs.copyFromLocalFile(delSrc, overwrite, new Path(src), new Path(dst));
     } catch (IOException e) {
       cnt++;
       LOG.error(cnt + " : " + e.getMessage(), e);
       te = e;
       continue;
     }
     LOG.info("Finished the copy from " + src + " to " + dst);
     return;
   }
   CommonUtils.runtimeException(te);
 }
Exemplo n.º 18
0
    public void writeParition()
        throws IOException, SuspectedFileSizeException, InvalidPathException, TException {
      if (DEBUG_MODE) {
        mBuf.flip();
        CommonUtils.printByteBuffer(LOG, mBuf);
      }

      mBuf.flip();
      for (int pId = mLeft; pId < mRight; pId++) {
        long startTimeMs = System.currentTimeMillis();
        TachyonFile file = mTC.getFile(FILE_NAME + (mWorkerId + BASE_FILE_NUMBER));
        OutStream os = file.getOutStream(WriteType.CACHE);
        for (int k = 0; k < BLOCKS_PER_FILE; k++) {
          mBuf.array()[0] = (byte) (k + mWorkerId);
          os.write(mBuf.array());
        }
        os.close();
        logPerIteration(startTimeMs, pId, "th WriteTachyonFile @ Worker ", pId);
      }
    }
Exemplo n.º 19
0
  public static void readPartition()
      throws IOException, TableDoesNotExistException, InvalidPathException, TException {
    LOG.info("Reading data...");
    RawTable rawTable = sTachyonClient.getRawTable(mId);
    ByteBuffer metadata = rawTable.getMetadata();
    LOG.info("Metadata: ");
    LOG.info(metadata.getInt() + " ");
    LOG.info(metadata.getInt() + " ");
    LOG.info(metadata.getInt() + " ");

    for (int column = 0; column < COLS; column++) {
      RawColumn rawColumn = rawTable.getRawColumn(column);
      TachyonFile tFile = rawColumn.getPartition(0);

      ByteBuffer buf = tFile.readByteBuffer();
      if (buf == null) {
        tFile.recacheData();
      }
      CommonUtils.printByteBuffer(LOG, tFile.readByteBuffer());
      tFile.releaseFileLock();
    }
  }
Exemplo n.º 20
0
 public ByteBuffer getMetadata() {
   return CommonUtils.cloneByteBuffer(CLIENT_RAW_TABLE_INFO.metadata);
 }
Exemplo n.º 21
0
 public void updateMetadata(ByteBuffer metadata) throws IOException {
   TACHYON_CLIENT.updateRawTableMetadata(CLIENT_RAW_TABLE_INFO.getId(), metadata);
   CLIENT_RAW_TABLE_INFO.setMetadata(CommonUtils.cloneByteBuffer(metadata));
 }
Exemplo n.º 22
0
  public static void main(String[] args)
      throws IOException, InvalidPathException, FileAlreadyExistException {
    if (args.length != 9) {
      System.out.println(
          "java -cp target/tachyon-"
              + Version.VERSION
              + "-jar-with-dependencies.jar tachyon.examples.Performance "
              + "<MasterIp> <FileName> <WriteBlockSizeInBytes> <BlocksPerFile> "
              + "<DebugMode:true/false> <Threads> <FilesPerThread> <TestCaseNumber> <BaseFileNumber>\n"
              + "1: Files Write Test\n"
              + "2: Files Read Test\n"
              + "3: RamFile Write Test \n"
              + "4: RamFile Read Test \n"
              + "5: ByteBuffer Write Test \n"
              + "6: ByteBuffer Read Test \n");
      System.exit(-1);
    }

    MASTER_ADDRESS = args[0];
    FILE_NAME = args[1];
    BLOCK_SIZE_BYTES = Integer.parseInt(args[2]);
    BLOCKS_PER_FILE = Integer.parseInt(args[3]);
    DEBUG_MODE = ("true".equals(args[4]));
    THREADS = Integer.parseInt(args[5]);
    FILES = Integer.parseInt(args[6]) * THREADS;
    int testCase = Integer.parseInt(args[7]);
    BASE_FILE_NUMBER = Integer.parseInt(args[8]);

    FILE_BYTES = BLOCKS_PER_FILE * BLOCK_SIZE_BYTES;
    FILES_BYTES = 1L * FILE_BYTES * FILES;

    RESULT_PREFIX =
        String.format(
            "Threads %d FilesPerThread %d TotalFiles %d "
                + "BLOCK_SIZE_KB %d BLOCKS_PER_FILE %d FILE_SIZE_MB %d "
                + "Tachyon_WRITE_BUFFER_SIZE_KB %d BaseFileNumber %d : ",
            THREADS,
            FILES / THREADS,
            FILES,
            BLOCK_SIZE_BYTES / 1024,
            BLOCKS_PER_FILE,
            CommonUtils.getMB(FILE_BYTES),
            UserConf.get().FILE_BUFFER_BYTES / 1024,
            BASE_FILE_NUMBER);

    if (testCase == 1) {
      RESULT_PREFIX = "TachyonFilesWriteTest " + RESULT_PREFIX;
      LOG.info(RESULT_PREFIX);
      MTC = TachyonFS.get(MASTER_ADDRESS);
      createFiles();
      TachyonTest(true);
    } else if (testCase == 2) {
      RESULT_PREFIX = "TachyonFilesReadTest " + RESULT_PREFIX;
      LOG.info(RESULT_PREFIX);
      MTC = TachyonFS.get(MASTER_ADDRESS);
      TachyonTest(false);
    } else if (testCase == 3) {
      RESULT_PREFIX = "RamFile Write " + RESULT_PREFIX;
      LOG.info(RESULT_PREFIX);
      memoryCopyTest(true, false);
    } else if (testCase == 4) {
      RESULT_PREFIX = "RamFile Read " + RESULT_PREFIX;
      LOG.info(RESULT_PREFIX);
      memoryCopyTest(false, false);
    } else if (testCase == 5) {
      RESULT_PREFIX = "ByteBuffer Write Test " + RESULT_PREFIX;
      LOG.info(RESULT_PREFIX);
      memoryCopyTest(true, true);
    } else if (testCase == 6) {
      RESULT_PREFIX = "ByteBuffer Read Test " + RESULT_PREFIX;
      LOG.info(RESULT_PREFIX);
      memoryCopyTest(false, true);
    } else {
      CommonUtils.runtimeException("No Test Case " + testCase);
    }

    for (int k = 0; k < RESULT_ARRAY_SIZE; k++) {
      System.out.print(Results[k] + " ");
    }
    System.out.println();
    System.exit(0);
  }
Exemplo n.º 23
0
 public String getCreationTime() {
   return CommonUtils.convertMsToDate(CREATION_TIME_MS);
 }