Exemplo n.º 1
0
 /**
  * Gets a stream to write data to a block. The stream can only be backed by Alluxio storage.
  *
  * @param blockId the block to write
  * @param blockSize the standard block size to write, or -1 if the block already exists (and this
  *     stream is just storing the block in Alluxio again)
  * @param address the address of the worker to write the block to, fails if the worker cannot
  *     serve the request
  * @return a {@link BufferedBlockOutStream} which can be used to write data to the block in a
  *     streaming fashion
  * @throws IOException if the block cannot be written
  */
 public BufferedBlockOutStream getOutStream(long blockId, long blockSize, WorkerNetAddress address)
     throws IOException {
   if (blockSize == -1) {
     try (CloseableResource<BlockMasterClient> blockMasterClientResource =
         mContext.acquireMasterClientResource()) {
       blockSize = blockMasterClientResource.get().getBlockInfo(blockId).getLength();
     } catch (AlluxioException e) {
       throw new IOException(e);
     }
   }
   // No specified location to write to.
   if (address == null) {
     throw new RuntimeException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage());
   }
   // Location is local.
   if (NetworkAddressUtils.getLocalHostName(ClientContext.getConf()).equals(address.getHost())) {
     if (mContext.hasLocalWorker()) {
       return new LocalBlockOutStream(blockId, blockSize);
     } else {
       throw new IOException(ExceptionMessage.NO_LOCAL_WORKER.getMessage("write"));
     }
   }
   // Location is specified and it is remote.
   return new RemoteBlockOutStream(blockId, blockSize, address);
 }
Exemplo n.º 2
0
 /**
  * Gets the block info of a block, if it exists.
  *
  * @param blockId the blockId to obtain information about
  * @return a {@link BlockInfo} containing the metadata of the block
  * @throws IOException if the block does not exist
  */
 public BlockInfo getInfo(long blockId) throws IOException {
   try (CloseableResource<BlockMasterClient> masterClientResource =
       mContext.acquireMasterClientResource()) {
     return masterClientResource.get().getBlockInfo(blockId);
   } catch (AlluxioException e) {
     throw new IOException(e);
   }
 }
Exemplo n.º 3
0
 /**
  * Gets the used bytes of Alluxio's BlockStore.
  *
  * @return the used bytes of Alluxio's BlockStore
  * @throws IOException when the connection to the client fails
  */
 public long getUsedBytes() throws IOException {
   try (CloseableResource<BlockMasterClient> blockMasterClientResource =
       mContext.acquireMasterClientResource()) {
     return blockMasterClientResource.get().getUsedBytes();
   } catch (ConnectionFailedException e) {
     throw new IOException(e);
   }
 }
Exemplo n.º 4
0
 /**
  * @return the info of all active block workers
  * @throws IOException when work info list cannot be obtained from master
  * @throws AlluxioException if network connection failed
  */
 public List<BlockWorkerInfo> getWorkerInfoList() throws IOException, AlluxioException {
   List<BlockWorkerInfo> infoList = Lists.newArrayList();
   try (CloseableResource<BlockMasterClient> masterClientResource =
       mContext.acquireMasterClientResource()) {
     for (WorkerInfo workerInfo : masterClientResource.get().getWorkerInfoList()) {
       infoList.add(
           new BlockWorkerInfo(
               workerInfo.getAddress(), workerInfo.getCapacityBytes(), workerInfo.getUsedBytes()));
     }
     return infoList;
   }
 }
Exemplo n.º 5
0
 /**
  * Gets the worker addresses with the given hostname by querying the master. Returns all the
  * addresses, if the hostname is an empty string.
  *
  * @param hostname hostname of the worker to query, empty string denotes any worker
  * @return a list of {@link WorkerNetAddress} with the given hostname
  */
 private List<WorkerNetAddress> getWorkerAddresses(String hostname) {
   List<WorkerNetAddress> addresses = new ArrayList<>();
   try (CloseableResource<BlockMasterClient> masterClient = acquireMasterClientResource()) {
     List<WorkerInfo> workers = masterClient.get().getWorkerInfoList();
     for (WorkerInfo worker : workers) {
       if (hostname.isEmpty() || worker.getAddress().getHost().equals(hostname)) {
         addresses.add(worker.getAddress());
       }
     }
   } catch (Exception e) {
     Throwables.propagate(e);
   }
   return addresses;
 }
Exemplo n.º 6
0
  /**
   * Gets a stream to read the data of a block. The stream is backed by Alluxio storage.
   *
   * @param blockId the block to read from
   * @return a {@link BlockInStream} which can be used to read the data in a streaming fashion
   * @throws IOException if the block does not exist
   */
  public BufferedBlockInStream getInStream(long blockId) throws IOException {
    BlockInfo blockInfo;
    try (CloseableResource<BlockMasterClient> masterClientResource =
        mContext.acquireMasterClientResource()) {
      blockInfo = masterClientResource.get().getBlockInfo(blockId);
    } catch (AlluxioException e) {
      throw new IOException(e);
    }

    if (blockInfo.getLocations().isEmpty()) {
      throw new IOException("Block " + blockId + " is not available in Alluxio");
    }
    // TODO(calvin): Get location via a policy.
    // Although blockInfo.locations are sorted by tier, we prefer reading from the local worker.
    // But when there is no local worker or there are no local blocks, we prefer the first
    // location in blockInfo.locations that is nearest to memory tier.
    // Assuming if there is no local worker, there are no local blocks in blockInfo.locations.
    // TODO(cc): Check mContext.hasLocalWorker before finding for a local block when the TODO
    // for hasLocalWorker is fixed.
    String localHostName = NetworkAddressUtils.getLocalHostName(ClientContext.getConf());
    for (BlockLocation location : blockInfo.getLocations()) {
      WorkerNetAddress workerNetAddress = location.getWorkerAddress();
      if (workerNetAddress.getHost().equals(localHostName)) {
        // There is a local worker and the block is local.
        try {
          return new LocalBlockInStream(blockId, blockInfo.getLength());
        } catch (IOException e) {
          LOG.warn("Failed to open local stream for block " + blockId + ". " + e.getMessage());
          // Getting a local stream failed, do not try again
          break;
        }
      }
    }
    // No local worker/block, get the first location since it's nearest to memory tier.
    WorkerNetAddress workerNetAddress = blockInfo.getLocations().get(0).getWorkerAddress();
    return new RemoteBlockInStream(blockId, blockInfo.getLength(), workerNetAddress);
  }
Exemplo n.º 7
0
 /**
  * Attempts to promote a block in Alluxio space. If the block is not present, this method will
  * return without an error. If the block is present in multiple workers, only one worker will
  * receive the promotion request.
  *
  * @param blockId the id of the block to promote
  * @throws IOException if the block does not exist
  */
 public void promote(long blockId) throws IOException {
   BlockInfo info;
   try (CloseableResource<BlockMasterClient> blockMasterClientResource =
       mContext.acquireMasterClientResource()) {
     info = blockMasterClientResource.get().getBlockInfo(blockId);
   } catch (AlluxioException e) {
     throw new IOException(e);
   }
   if (info.getLocations().isEmpty()) {
     // Nothing to promote
     return;
   }
   // Get the first worker address for now, as this will likely be the location being read from
   // TODO(calvin): Get this location via a policy (possibly location is a parameter to promote)
   WorkerNetAddress workerAddr = info.getLocations().get(0).getWorkerAddress();
   BlockWorkerClient blockWorkerClient = mContext.acquireWorkerClient(workerAddr.getHost());
   try {
     blockWorkerClient.promoteBlock(blockId);
   } catch (AlluxioException e) {
     throw new IOException(e);
   } finally {
     mContext.releaseWorkerClient(blockWorkerClient);
   }
 }