Esempio n. 1
0
    @Override
    @edu.umd.cs.findbugs.annotations.SuppressWarnings(
        value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE",
        justification = "Writer is known to be IncrementalWriter")
    public FileSystemWriter.IncrementalWriter<E> load(StorageKey key) throws Exception {
      Preconditions.checkState(
          view.getDataset() instanceof FileSystemDataset,
          "FileSystemWriters cannot create writer for " + view.getDataset());

      FileSystemDataset dataset = (FileSystemDataset) view.getDataset();
      Path partition = convert.fromKey(key);
      FileSystemWriter<E> writer =
          FileSystemWriter.newWriter(
              dataset.getFileSystem(),
              new Path(dataset.getDirectory(), partition),
              dataset.getDescriptor());

      PartitionListener listener = dataset.getPartitionListener();
      if (listener != null) {
        listener.partitionAdded(dataset.getNamespace(), dataset.getName(), partition.toString());
      }

      // initialize the writer after calling the listener
      // this lets the listener decide if and how to create the
      // partition directory
      writer.initialize();

      return (FileSystemWriter.IncrementalWriter<E>) writer;
    }
Esempio n. 2
0
    @Override
    public FileSystemWriter<E> load(StorageKey key) throws Exception {
      Preconditions.checkState(
          view.getDataset() instanceof FileSystemDataset,
          "FileSystemWriters cannot create writer for " + view.getDataset());

      FileSystemDataset dataset = (FileSystemDataset) view.getDataset();
      Path partition = convert.fromKey(key);
      FileSystemWriter<E> writer =
          FileSystemWriter.newWriter(
              dataset.getFileSystem(),
              new Path(dataset.getDirectory(), partition),
              dataset.getDescriptor());

      PartitionListener listener = dataset.getPartitionListener();
      if (listener != null) {
        listener.partitionAdded(dataset.getNamespace(), dataset.getName(), partition.toString());
      }

      // initialize the writer after calling the listener
      // this lets the listener decide if and how to create the
      // partition directory
      writer.initialize();

      return writer;
    }
Esempio n. 3
0
  @Test
  @SuppressWarnings("unchecked")
  public void toDirNameIdentityWithSlashes() {
    PartitionStrategy strategy =
        new PartitionStrategy.Builder().identity("name").identity("address").build();

    StorageKey key = new StorageKey(strategy);
    key.replaceValues((List) Lists.newArrayList("John Doe", "NY/USA"));

    Assert.assertEquals(new Path("name_copy=John+Doe/address_copy=NY%2FUSA"), convert.fromKey(key));
  }
Esempio n. 4
0
  @Test
  @SuppressWarnings("unchecked")
  public void testFromKey() {
    PartitionStrategy strategy =
        new PartitionStrategy.Builder()
            .year("timestamp")
            .month("timestamp")
            .day("timestamp")
            .build();

    StorageKey key = new StorageKey(strategy);
    key.replaceValues((List) Lists.newArrayList(2013, 11, 5));

    Assert.assertEquals(new Path("year=2013/month=11/day=05"), convert.fromKey(key));
  }
    @Override
    public DatasetWriter<E> load(StorageKey key) throws Exception {
      Preconditions.checkState(
          view.getDataset() instanceof FileSystemDataset,
          "FileSystemWriters cannot create writer for " + view.getDataset());

      FileSystemDataset dataset = (FileSystemDataset) view.getDataset();
      Path partition = convert.fromKey(key);
      DatasetWriter<E> writer =
          new FileSystemWriter<E>(
              dataset.getFileSystem(),
              new Path(dataset.getDirectory(), partition),
              dataset.getDescriptor());

      PartitionListener listener = dataset.getPartitionListener();
      if (listener != null) {
        listener.partitionAdded(dataset.getName(), partition.toString());
      }

      writer.open();

      return writer;
    }