Ejemplo n.º 1
0
 /**
  * Returns the local filename for the block if that file exists on the local file system. This is
  * an alpha power-api feature for applications that want short-circuit-read files directly. There
  * is no guarantee that the file still exists after this call returns, as Tachyon may evict blocks
  * from memory at any time.
  *
  * @param blockIndex The index of the block in the file.
  * @return filename on local file system or null if file not present on local file system.
  * @throws IOException
  */
 public String getLocalFilename(int blockIndex) throws IOException {
   ClientBlockInfo blockInfo = getClientBlockInfo(blockIndex);
   long blockId = blockInfo.getBlockId();
   int blockLockId = mTachyonFS.getBlockLockId();
   String filename = mTachyonFS.lockBlock(blockId, blockLockId);
   if (filename != null) {
     mTachyonFS.unlockBlock(blockId, blockLockId);
   }
   return filename;
 }
Ejemplo n.º 2
0
  private ByteBuffer readRemoteByteBuffer(ClientBlockInfo blockInfo, long offset, long len) {
    ByteBuffer buf = null;

    try {
      List<NetAddress> blockLocations = blockInfo.getLocations();
      LOG.info("Block locations:" + blockLocations);

      for (int k = 0; k < blockLocations.size(); k++) {
        String host = blockLocations.get(k).mHost;
        int port = blockLocations.get(k).mSecondaryPort;

        // The data is not in remote machine's memory if port == -1.
        if (port == -1) {
          continue;
        }
        if (host.equals(InetAddress.getLocalHost().getHostName())
            || host.equals(InetAddress.getLocalHost().getHostAddress())) {
          String localFileName = CommonUtils.concat(TFS.getRootFolder(), blockInfo.blockId);
          LOG.warn("Master thinks the local machine has data " + localFileName + "! But not!");
        }
        LOG.info(
            host
                + ":"
                + port
                + " current host is "
                + InetAddress.getLocalHost().getHostName()
                + " "
                + InetAddress.getLocalHost().getHostAddress());

        try {
          buf =
              retrieveByteBufferFromRemoteMachine(
                  new InetSocketAddress(host, port), blockInfo.blockId, offset, len);
          if (buf != null) {
            break;
          }
        } catch (IOException e) {
          LOG.error(e.getMessage());
          buf = null;
        }
      }
    } catch (IOException e) {
      LOG.error("Failed to get read data from remote " + e.getMessage());
      buf = null;
    }

    return buf;
  }
Ejemplo n.º 3
0
  /**
   * Reads from a remote byte buffer.
   *
   * @param tachyonFS a TachyonFS
   * @param blockInfo the block information
   * @param offset offset to start the read at
   * @param len number of bytes to read
   * @param conf Tachyon configuration
   * @return <code>ByteBuffer</code> containing the bytes read
   */
  public ByteBuffer readRemoteByteBuffer(
      TachyonFS tachyonFS, ClientBlockInfo blockInfo, long offset, long len, TachyonConf conf) {
    ByteBuffer buf = null;

    try {
      List<NetAddress> blockLocations = blockInfo.getLocations();
      LOG.info("Block locations:" + blockLocations);
      String localhost = NetworkAddressUtils.getConnectHost(ServiceType.WORKER_RPC, conf);

      for (NetAddress blockLocation : blockLocations) {
        String host = blockLocation.mHost;
        int port = blockLocation.mSecondaryPort;

        // The data is not in remote machine's memory if primary port == -1. We check primary port
        // because if the data is in the under storage, the secondary port (data transfer port)
        // will be set.
        if (blockLocation.mPort == -1) {
          continue;
        }

        if (host.equals(InetAddress.getLocalHost().getHostName())
            || host.equals(InetAddress.getLocalHost().getHostAddress())
            || host.equals(localhost)) {
          LOG.warn(
              "Master thinks the local machine has data, but not!"
                  + "(or local read is disabled) blockId:{}",
              blockInfo.blockId);
        }
        LOG.info(
            host
                + ":"
                + port
                + " current host is "
                + localhost
                + " "
                + NetworkAddressUtils.getLocalIpAddress(conf));

        try {
          buf =
              retrieveByteBufferFromRemoteMachine(
                  new InetSocketAddress(host, port), blockInfo.blockId, offset, len, conf);
          if (buf != null) {
            break;
          }
        } catch (IOException e) {
          LOG.error(
              "Fail to retrieve byte buffer for block "
                  + blockInfo.blockId
                  + " from remote "
                  + host
                  + ":"
                  + port
                  + " with offset "
                  + offset
                  + " and length "
                  + len,
              e);
          buf = null;
        }
      }
    } catch (IOException e) {
      LOG.error("Failed to get read data from remote ", e);
      buf = null;
    }

    return buf;
  }
Ejemplo n.º 4
0
  @Override
  public int read(byte[] b, int off, int len) throws IOException {
    if (b == null) {
      throw new NullPointerException();
    } else if (off < 0 || len < 0 || len > b.length - off) {
      throw new IndexOutOfBoundsException();
    } else if (len == 0) {
      return 0;
    } else if (mBlockPos == mBlockInfo.length) {
      return -1;
    }

    // We read at most len bytes, but if mBlockPos + len exceeds the length of the file, we only
    // read up to the end of the file
    len = (int) Math.min(len, mBlockInfo.length - mBlockPos);
    int bytesLeft = len;
    // Lazy initialization of the out stream for caching to avoid collisions with other caching
    // attempts that are invalidated later due to seek/skips
    if (bytesLeft > 0 && mBlockOutStream == null && mRecache) {
      try {
        mBlockOutStream = BlockOutStream.get(mFile, WriteType.TRY_CACHE, mBlockIndex, mTachyonConf);
        // We should only cache when we are writing to a local worker
        if (mBlockOutStream instanceof RemoteBlockOutStream) {
          LOG.info("Cannot find a local worker to write to, recache attempt cancelled.");
          cancelRecache();
        }
      } catch (IOException ioe) {
        LOG.warn("Recache attempt failed.", ioe);
        cancelRecache();
      }
    }

    // While we still have bytes to read, make sure the buffer is set to read the byte at mBlockPos.
    // If we fail to set mCurrentBuffer, we stream the rest from the underfs
    while (bytesLeft > 0 && mAttemptReadFromWorkers && updateCurrentBuffer()) {
      int bytesToRead = Math.min(bytesLeft, mCurrentBuffer.remaining());
      mCurrentBuffer.get(b, off, bytesToRead);
      if (mRecache) {
        mBlockOutStream.write(b, off, bytesToRead);
      }
      off += bytesToRead;
      bytesLeft -= bytesToRead;
      mBlockPos += bytesToRead;
    }
    mBytesReadRemote += len - bytesLeft;
    mTachyonFS.getClientMetrics().incBytesReadRemote(len - bytesLeft);

    if (bytesLeft > 0) {
      // Unable to read from worker memory, reading this block from underfs in the future.
      mAttemptReadFromWorkers = false;
      // We failed to read everything from mCurrentBuffer, so we need to stream the rest from the
      // underfs
      if (!setupStreamFromUnderFs()) {
        LOG.error(
            "Failed to read at position "
                + mBlockPos
                + " in block "
                + mBlockInfo.getBlockId()
                + " from workers or underfs");
        // Return the number of bytes we managed to read
        return len - bytesLeft;
      }
      while (bytesLeft > 0) {
        int readBytes = mCheckpointInputStream.read(b, off, bytesLeft);
        if (readBytes <= 0) {
          LOG.error("Checkpoint stream read 0 bytes, which shouldn't ever happen");
          return len - bytesLeft;
        }
        if (mRecache) {
          mBlockOutStream.write(b, off, readBytes);
        }
        off += readBytes;
        bytesLeft -= readBytes;
        mBlockPos += readBytes;
        mCheckpointPos += readBytes;
        mTachyonFS.getClientMetrics().incBytesReadUfs(readBytes);
      }
    }
    return len;
  }
Ejemplo n.º 5
0
 /**
  * Promote block back to top layer after access
  *
  * @param blockIndex the index of the block
  * @return true if success, false otherwise
  * @throws IOException
  */
 public boolean promoteBlock(int blockIndex) throws IOException {
   ClientBlockInfo blockInfo = getClientBlockInfo(blockIndex);
   return mTachyonFS.promoteBlock(blockInfo.getBlockId());
 }