/** * Gets a stream to write data to a block. The stream can only be backed by Alluxio storage. * * @param blockId the block to write * @param blockSize the standard block size to write, or -1 if the block already exists (and this * stream is just storing the block in Alluxio again) * @param address the address of the worker to write the block to, fails if the worker cannot * serve the request * @return a {@link BufferedBlockOutStream} which can be used to write data to the block in a * streaming fashion * @throws IOException if the block cannot be written */ public BufferedBlockOutStream getOutStream(long blockId, long blockSize, WorkerNetAddress address) throws IOException { if (blockSize == -1) { try (CloseableResource<BlockMasterClient> blockMasterClientResource = mContext.acquireMasterClientResource()) { blockSize = blockMasterClientResource.get().getBlockInfo(blockId).getLength(); } catch (AlluxioException e) { throw new IOException(e); } } // No specified location to write to. if (address == null) { throw new RuntimeException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage()); } // Location is local. if (NetworkAddressUtils.getLocalHostName(ClientContext.getConf()).equals(address.getHost())) { if (mContext.hasLocalWorker()) { return new LocalBlockOutStream(blockId, blockSize); } else { throw new IOException(ExceptionMessage.NO_LOCAL_WORKER.getMessage("write")); } } // Location is specified and it is remote. return new RemoteBlockOutStream(blockId, blockSize, address); }
/** * Gets the block info of a block, if it exists. * * @param blockId the blockId to obtain information about * @return a {@link BlockInfo} containing the metadata of the block * @throws IOException if the block does not exist */ public BlockInfo getInfo(long blockId) throws IOException { try (CloseableResource<BlockMasterClient> masterClientResource = mContext.acquireMasterClientResource()) { return masterClientResource.get().getBlockInfo(blockId); } catch (AlluxioException e) { throw new IOException(e); } }
/** * Gets the used bytes of Alluxio's BlockStore. * * @return the used bytes of Alluxio's BlockStore * @throws IOException when the connection to the client fails */ public long getUsedBytes() throws IOException { try (CloseableResource<BlockMasterClient> blockMasterClientResource = mContext.acquireMasterClientResource()) { return blockMasterClientResource.get().getUsedBytes(); } catch (ConnectionFailedException e) { throw new IOException(e); } }
/** * @return the info of all active block workers * @throws IOException when work info list cannot be obtained from master * @throws AlluxioException if network connection failed */ public List<BlockWorkerInfo> getWorkerInfoList() throws IOException, AlluxioException { List<BlockWorkerInfo> infoList = Lists.newArrayList(); try (CloseableResource<BlockMasterClient> masterClientResource = mContext.acquireMasterClientResource()) { for (WorkerInfo workerInfo : masterClientResource.get().getWorkerInfoList()) { infoList.add( new BlockWorkerInfo( workerInfo.getAddress(), workerInfo.getCapacityBytes(), workerInfo.getUsedBytes())); } return infoList; } }
/** * Gets the worker addresses with the given hostname by querying the master. Returns all the * addresses, if the hostname is an empty string. * * @param hostname hostname of the worker to query, empty string denotes any worker * @return a list of {@link WorkerNetAddress} with the given hostname */ private List<WorkerNetAddress> getWorkerAddresses(String hostname) { List<WorkerNetAddress> addresses = new ArrayList<>(); try (CloseableResource<BlockMasterClient> masterClient = acquireMasterClientResource()) { List<WorkerInfo> workers = masterClient.get().getWorkerInfoList(); for (WorkerInfo worker : workers) { if (hostname.isEmpty() || worker.getAddress().getHost().equals(hostname)) { addresses.add(worker.getAddress()); } } } catch (Exception e) { Throwables.propagate(e); } return addresses; }
/** * Gets a stream to read the data of a block. The stream is backed by Alluxio storage. * * @param blockId the block to read from * @return a {@link BlockInStream} which can be used to read the data in a streaming fashion * @throws IOException if the block does not exist */ public BufferedBlockInStream getInStream(long blockId) throws IOException { BlockInfo blockInfo; try (CloseableResource<BlockMasterClient> masterClientResource = mContext.acquireMasterClientResource()) { blockInfo = masterClientResource.get().getBlockInfo(blockId); } catch (AlluxioException e) { throw new IOException(e); } if (blockInfo.getLocations().isEmpty()) { throw new IOException("Block " + blockId + " is not available in Alluxio"); } // TODO(calvin): Get location via a policy. // Although blockInfo.locations are sorted by tier, we prefer reading from the local worker. // But when there is no local worker or there are no local blocks, we prefer the first // location in blockInfo.locations that is nearest to memory tier. // Assuming if there is no local worker, there are no local blocks in blockInfo.locations. // TODO(cc): Check mContext.hasLocalWorker before finding for a local block when the TODO // for hasLocalWorker is fixed. String localHostName = NetworkAddressUtils.getLocalHostName(ClientContext.getConf()); for (BlockLocation location : blockInfo.getLocations()) { WorkerNetAddress workerNetAddress = location.getWorkerAddress(); if (workerNetAddress.getHost().equals(localHostName)) { // There is a local worker and the block is local. try { return new LocalBlockInStream(blockId, blockInfo.getLength()); } catch (IOException e) { LOG.warn("Failed to open local stream for block " + blockId + ". " + e.getMessage()); // Getting a local stream failed, do not try again break; } } } // No local worker/block, get the first location since it's nearest to memory tier. WorkerNetAddress workerNetAddress = blockInfo.getLocations().get(0).getWorkerAddress(); return new RemoteBlockInStream(blockId, blockInfo.getLength(), workerNetAddress); }
/** * Attempts to promote a block in Alluxio space. If the block is not present, this method will * return without an error. If the block is present in multiple workers, only one worker will * receive the promotion request. * * @param blockId the id of the block to promote * @throws IOException if the block does not exist */ public void promote(long blockId) throws IOException { BlockInfo info; try (CloseableResource<BlockMasterClient> blockMasterClientResource = mContext.acquireMasterClientResource()) { info = blockMasterClientResource.get().getBlockInfo(blockId); } catch (AlluxioException e) { throw new IOException(e); } if (info.getLocations().isEmpty()) { // Nothing to promote return; } // Get the first worker address for now, as this will likely be the location being read from // TODO(calvin): Get this location via a policy (possibly location is a parameter to promote) WorkerNetAddress workerAddr = info.getLocations().get(0).getWorkerAddress(); BlockWorkerClient blockWorkerClient = mContext.acquireWorkerClient(workerAddr.getHost()); try { blockWorkerClient.promoteBlock(blockId); } catch (AlluxioException e) { throw new IOException(e); } finally { mContext.releaseWorkerClient(blockWorkerClient); } }