Example #1
0
    public void memoryCopyParition() throws IOException {
      if (DEBUG_MODE) {
        mBuf.flip();
        CommonUtils.printByteBuffer(LOG, mBuf);
      }
      mBuf.flip();
      long sum = 0;
      String str = "th " + mMsg + " @ Worker ";

      if (mOneToMany) {
        ByteBuffer dst = null;
        RandomAccessFile file = null;
        if (mMemoryOnly) {
          dst = ByteBuffer.allocateDirect(FILE_BYTES);
        }
        for (int times = mLeft; times < mRight; times++) {
          long startTimeMs = System.currentTimeMillis();
          if (!mMemoryOnly) {
            file = new RandomAccessFile(FOLDER + (mWorkerId + BASE_FILE_NUMBER), "rw");
            dst = file.getChannel().map(MapMode.READ_WRITE, 0, FILE_BYTES);
          }
          dst.order(ByteOrder.nativeOrder());
          for (int k = 0; k < BLOCKS_PER_FILE; k++) {
            mBuf.array()[0] = (byte) (k + mWorkerId);
            dst.put(mBuf.array());
          }
          dst.clear();
          sum += dst.get(times);
          dst.clear();
          if (!mMemoryOnly) {
            file.close();
          }
          logPerIteration(startTimeMs, times, str, mWorkerId);
        }
      } else {
        ByteBuffer dst = null;
        RandomAccessFile file = null;
        if (mMemoryOnly) {
          dst = ByteBuffer.allocateDirect(FILE_BYTES);
        }
        for (int times = mLeft; times < mRight; times++) {
          long startTimeMs = System.currentTimeMillis();
          if (!mMemoryOnly) {
            file = new RandomAccessFile(FOLDER + (mWorkerId + BASE_FILE_NUMBER), "rw");
            dst = file.getChannel().map(MapMode.READ_WRITE, 0, FILE_BYTES);
          }
          dst.order(ByteOrder.nativeOrder());
          for (int k = 0; k < BLOCKS_PER_FILE; k++) {
            dst.get(mBuf.array());
          }
          sum += mBuf.get(times % 16);
          dst.clear();
          if (!mMemoryOnly) {
            file.close();
          }
          logPerIteration(startTimeMs, times, str, mWorkerId);
        }
      }
      Results[mWorkerId] = sum;
    }
  public static void writeParition()
      throws IOException, TableDoesNotExistException, InvalidPathException,
          FileAlreadyExistException, TException {
    RawTable rawTable = sTachyonClient.getRawTable(sTablePath);

    LOG.info("Writing data...");
    for (int column = 0; column < COLS; column++) {
      RawColumn rawColumn = rawTable.getRawColumn(column);
      if (!rawColumn.createPartition(0)) {
        CommonUtils.runtimeException(
            "Failed to create partition in table " + sTablePath + " under column " + column);
      }

      ByteBuffer buf = ByteBuffer.allocate(80);
      buf.order(ByteOrder.nativeOrder());
      for (int k = 0; k < 20; k++) {
        buf.putInt(k);
      }
      buf.flip();
      CommonUtils.printByteBuffer(LOG, buf);
      buf.flip();

      TachyonFile tFile = rawColumn.getPartition(0);
      OutStream os = tFile.createOutStream(sWriteType);
      os.write(buf);
      os.close();
    }
  }
Example #3
0
    public void readPartition()
        throws IOException, SuspectedFileSizeException, InvalidPathException, TException {
      ByteBuffer buf;
      if (DEBUG_MODE) {
        LOG.info("Verifying the reading data...");

        for (int pId = mLeft; pId < mRight; pId++) {
          TachyonFile file = mTC.getFile(FILE_NAME + mWorkerId);
          buf = file.readByteBuffer();
          IntBuffer intBuf;
          intBuf = buf.asIntBuffer();
          int tmp;
          for (int i = 0; i < BLOCKS_PER_FILE; i++) {
            for (int k = 0; k < BLOCK_SIZE_BYTES / 4; k++) {
              tmp = intBuf.get();
              if ((k == 0 && tmp == (i + mWorkerId)) || (k != 0 && tmp == k)) {
              } else {
                CommonUtils.runtimeException("WHAT? " + tmp + " " + k);
              }
            }
          }
          file.readByteBuffer();
        }
      }

      long sum = 0;
      for (int pId = mLeft; pId < mRight; pId++) {
        long startTimeMs = System.currentTimeMillis();
        TachyonFile file = mTC.getFile(FILE_NAME + (mWorkerId + BASE_FILE_NUMBER));
        buf = file.readByteBuffer();
        for (int i = 0; i < BLOCKS_PER_FILE; i++) {
          buf.get(mBuf.array());
        }
        sum += mBuf.get(pId % 16);

        if (DEBUG_MODE) {
          buf.flip();
          CommonUtils.printByteBuffer(LOG, buf);
        }
        buf.clear();
        logPerIteration(startTimeMs, pId, "th ReadTachyonFile @ Worker ", pId);
      }
      Results[mWorkerId] = sum;
    }
Example #4
0
    public void writeParition()
        throws IOException, SuspectedFileSizeException, InvalidPathException, TException {
      if (DEBUG_MODE) {
        mBuf.flip();
        CommonUtils.printByteBuffer(LOG, mBuf);
      }

      mBuf.flip();
      for (int pId = mLeft; pId < mRight; pId++) {
        long startTimeMs = System.currentTimeMillis();
        TachyonFile file = mTC.getFile(FILE_NAME + (mWorkerId + BASE_FILE_NUMBER));
        OutStream os = file.getOutStream(WriteType.CACHE);
        for (int k = 0; k < BLOCKS_PER_FILE; k++) {
          mBuf.array()[0] = (byte) (k + mWorkerId);
          os.write(mBuf.array());
        }
        os.close();
        logPerIteration(startTimeMs, pId, "th WriteTachyonFile @ Worker ", pId);
      }
    }
  public static void readPartition()
      throws IOException, TableDoesNotExistException, InvalidPathException, TException {
    LOG.info("Reading data...");
    RawTable rawTable = sTachyonClient.getRawTable(mId);
    ByteBuffer metadata = rawTable.getMetadata();
    LOG.info("Metadata: ");
    LOG.info(metadata.getInt() + " ");
    LOG.info(metadata.getInt() + " ");
    LOG.info(metadata.getInt() + " ");

    for (int column = 0; column < COLS; column++) {
      RawColumn rawColumn = rawTable.getRawColumn(column);
      TachyonFile tFile = rawColumn.getPartition(0);

      ByteBuffer buf = tFile.readByteBuffer();
      if (buf == null) {
        tFile.recacheData();
      }
      CommonUtils.printByteBuffer(LOG, tFile.readByteBuffer());
      tFile.releaseFileLock();
    }
  }