예제 #1
0
    @Override
    public FileSystemWriter<E> load(StorageKey key) throws Exception {
      Preconditions.checkState(
          view.getDataset() instanceof FileSystemDataset,
          "FileSystemWriters cannot create writer for " + view.getDataset());

      FileSystemDataset dataset = (FileSystemDataset) view.getDataset();
      Path partition = convert.fromKey(key);
      FileSystemWriter<E> writer =
          FileSystemWriter.newWriter(
              dataset.getFileSystem(),
              new Path(dataset.getDirectory(), partition),
              dataset.getDescriptor());

      PartitionListener listener = dataset.getPartitionListener();
      if (listener != null) {
        listener.partitionAdded(dataset.getNamespace(), dataset.getName(), partition.toString());
      }

      // initialize the writer after calling the listener
      // this lets the listener decide if and how to create the
      // partition directory
      writer.initialize();

      return writer;
    }
예제 #2
0
    @Override
    @edu.umd.cs.findbugs.annotations.SuppressWarnings(
        value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE",
        justification = "Writer is known to be IncrementalWriter")
    public FileSystemWriter.IncrementalWriter<E> load(StorageKey key) throws Exception {
      Preconditions.checkState(
          view.getDataset() instanceof FileSystemDataset,
          "FileSystemWriters cannot create writer for " + view.getDataset());

      FileSystemDataset dataset = (FileSystemDataset) view.getDataset();
      Path partition = convert.fromKey(key);
      FileSystemWriter<E> writer =
          FileSystemWriter.newWriter(
              dataset.getFileSystem(),
              new Path(dataset.getDirectory(), partition),
              dataset.getDescriptor());

      PartitionListener listener = dataset.getPartitionListener();
      if (listener != null) {
        listener.partitionAdded(dataset.getNamespace(), dataset.getName(), partition.toString());
      }

      // initialize the writer after calling the listener
      // this lets the listener decide if and how to create the
      // partition directory
      writer.initialize();

      return (FileSystemWriter.IncrementalWriter<E>) writer;
    }
예제 #3
0
  @Override
  public void initialize() {
    Preconditions.checkState(
        state.equals(ReaderWriterState.NEW), "Unable to open a writer from state:%s", state);

    DatasetDescriptor descriptor = view.getDataset().getDescriptor();
    ValidationException.check(
        FileSystemWriter.isSupportedFormat(descriptor),
        "Not a supported format: %s",
        descriptor.getFormat());

    LOG.debug("Opening partitioned dataset writer w/strategy:{}", partitionStrategy);

    cachedWriters =
        CacheBuilder.newBuilder()
            .maximumSize(maxWriters)
            .removalListener(new DatasetWriterCloser<E>())
            .build(createCacheLoader());

    state = ReaderWriterState.OPEN;
  }