Пример #1
0
 /**
  * Creates a new file output stream.
  *
  * @param path the file path
  * @param options the client options
  * @throws IOException if an I/O error occurs
  */
 public FileOutStream(AlluxioURI path, OutStreamOptions options) throws IOException {
   mUri = Preconditions.checkNotNull(path);
   mNonce = IdUtils.getRandomNonNegativeLong();
   mBlockSize = options.getBlockSizeBytes();
   mAlluxioStorageType = options.getAlluxioStorageType();
   mUnderStorageType = options.getUnderStorageType();
   mContext = FileSystemContext.INSTANCE;
   mPreviousBlockOutStreams = new LinkedList<BufferedBlockOutStream>();
   if (mUnderStorageType.isSyncPersist()) {
     updateUfsPath();
     String tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
     UnderFileSystem ufs = UnderFileSystem.get(tmpPath, ClientContext.getConf());
     // TODO(jiri): Implement collection of temporary files left behind by dead clients.
     mUnderStorageOutputStream = ufs.create(tmpPath, (int) mBlockSize);
   } else {
     mUfsPath = null;
     mUnderStorageOutputStream = null;
   }
   mClosed = false;
   mCanceled = false;
   mShouldCacheCurrentBlock = mAlluxioStorageType.isStore();
   mBytesWritten = 0;
   mLocationPolicy =
       Preconditions.checkNotNull(
           options.getLocationPolicy(),
           PreconditionMessage.FILE_WRITE_LOCATION_POLICY_UNSPECIFIED);
 }
Пример #2
0
  private void getNextBlock() throws IOException {
    if (mCurrentBlockOutStream != null) {
      Preconditions.checkState(
          mCurrentBlockOutStream.remaining() <= 0, PreconditionMessage.ERR_BLOCK_REMAINING);
      mPreviousBlockOutStreams.add(mCurrentBlockOutStream);
    }

    if (mAlluxioStorageType.isStore()) {
      try {
        WorkerNetAddress address =
            mLocationPolicy.getWorkerForNextBlock(
                mContext.getAlluxioBlockStore().getWorkerInfoList(), mBlockSize);
        mCurrentBlockOutStream =
            mContext.getAlluxioBlockStore().getOutStream(getNextBlockId(), mBlockSize, address);
        mShouldCacheCurrentBlock = true;
      } catch (AlluxioException e) {
        throw new IOException(e);
      }
    }
  }
Пример #3
0
  @Override
  public void close() throws IOException {
    if (mClosed) {
      return;
    }
    if (mCurrentBlockOutStream != null) {
      mPreviousBlockOutStreams.add(mCurrentBlockOutStream);
    }

    CompleteFileOptions options = CompleteFileOptions.defaults();
    if (mUnderStorageType.isSyncPersist()) {
      String tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
      UnderFileSystem ufs = UnderFileSystem.get(tmpPath, ClientContext.getConf());
      if (mCanceled) {
        // TODO(yupeng): Handle this special case in under storage integrations.
        mUnderStorageOutputStream.close();
        if (!ufs.exists(tmpPath)) {
          // Location of the temporary file has changed, recompute it.
          updateUfsPath();
          tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
        }
        ufs.delete(tmpPath, false);
      } else {
        mUnderStorageOutputStream.flush();
        mUnderStorageOutputStream.close();
        if (!ufs.exists(tmpPath)) {
          // Location of the temporary file has changed, recompute it.
          updateUfsPath();
          tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
        }
        if (!ufs.rename(tmpPath, mUfsPath)) {
          throw new IOException("Failed to rename " + tmpPath + " to " + mUfsPath);
        }
        options.setUfsLength(ufs.getFileSize(mUfsPath));
      }
    }

    if (mAlluxioStorageType.isStore()) {
      try {
        if (mCanceled) {
          for (BufferedBlockOutStream bos : mPreviousBlockOutStreams) {
            bos.cancel();
          }
        } else {
          for (BufferedBlockOutStream bos : mPreviousBlockOutStreams) {
            bos.close();
          }
        }
      } catch (IOException e) {
        handleCacheWriteException(e);
      }
    }

    // Complete the file if it's ready to be completed.
    if (!mCanceled && (mUnderStorageType.isSyncPersist() || mAlluxioStorageType.isStore())) {
      FileSystemMasterClient masterClient = mContext.acquireMasterClient();
      try {
        masterClient.completeFile(mUri, options);
      } catch (AlluxioException e) {
        throw new IOException(e);
      } finally {
        mContext.releaseMasterClient(masterClient);
      }
    }

    if (mUnderStorageType.isAsyncPersist()) {
      scheduleAsyncPersist();
    }
    mClosed = true;
  }