Ejemplo n.º 1
0
  private ServerBootstrap createBootstrap() {
    final ServerBootstrap boot =
        createBootstrapOfType(
            mTachyonConf.getEnum(Constants.WORKER_NETWORK_NETTY_CHANNEL, ChannelType.class));

    // use pooled buffers
    boot.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
    boot.childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);

    // set write buffer
    // this is the default, but its recommended to set it in case of change in future netty.
    boot.childOption(
        ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK,
        (int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_WATERMARK_HIGH));
    boot.childOption(
        ChannelOption.WRITE_BUFFER_LOW_WATER_MARK,
        (int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_WATERMARK_LOW));

    // more buffer settings on Netty socket option, one can tune them by specifying
    // properties, e.g.:
    // tachyon.worker.network.netty.backlog=50
    // tachyon.worker.network.netty.buffer.send=64KB
    // tachyon.worker.network.netty.buffer.receive=64KB
    if (mTachyonConf.containsKey(Constants.WORKER_NETWORK_NETTY_BACKLOG)) {
      boot.option(
          ChannelOption.SO_BACKLOG, mTachyonConf.getInt(Constants.WORKER_NETWORK_NETTY_BACKLOG));
    }
    if (mTachyonConf.containsKey(Constants.WORKER_NETWORK_NETTY_BUFFER_SEND)) {
      boot.option(
          ChannelOption.SO_SNDBUF,
          (int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_BUFFER_SEND));
    }
    if (mTachyonConf.containsKey(Constants.WORKER_NETWORK_NETTY_BUFFER_RECEIVE)) {
      boot.option(
          ChannelOption.SO_RCVBUF,
          (int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_BUFFER_RECEIVE));
    }
    return boot;
  }
Ejemplo n.º 2
0
  /**
   * Re-cache the block into memory
   *
   * @param blockIndex The block index of the current file.
   * @return true if succeed, false otherwise
   * @throws IOException
   */
  boolean recache(int blockIndex) throws IOException {
    String path = getUfsPath();
    UnderFileSystem underFsClient = UnderFileSystem.get(path, mTachyonConf);

    InputStream inputStream = null;
    BlockOutStream bos = null;
    try {
      inputStream = underFsClient.open(path);

      long length = getBlockSizeByte();
      long offset = blockIndex * length;
      inputStream.skip(offset);

      int bufferBytes = (int) mTachyonConf.getBytes(Constants.USER_FILE_BUFFER_BYTES, Constants.MB);
      byte[] buffer = new byte[bufferBytes];
      bos = BlockOutStream.get(this, WriteType.TRY_CACHE, blockIndex, mTachyonConf);
      int limit;
      while (length > 0 && ((limit = inputStream.read(buffer)) >= 0)) {
        if (limit != 0) {
          if (length >= limit) {
            bos.write(buffer, 0, limit);
            length -= limit;
          } else {
            bos.write(buffer, 0, (int) length);
            length = 0;
          }
        }
      }
      bos.close();
    } catch (IOException e) {
      LOG.warn(e.getMessage(), e);
      if (bos != null) {
        bos.cancel();
      }
      return false;
    } finally {
      if (inputStream != null) {
        inputStream.close();
      }
    }

    return true;
  }
Ejemplo n.º 3
0
 /** @return a newly allocated byte buffer of the user defined default size */
 private ByteBuffer allocateBuffer() {
   TachyonConf conf = ClientContext.getConf();
   return ByteBuffer.allocate((int) conf.getBytes(Constants.USER_FILE_BUFFER_BYTES));
 }