예제 #1
0
 @SuppressWarnings("unchecked")
 private Path toDirectoryName(Path dir, PartitionKey key) {
   Path result = dir;
   for (int i = 0; i < key.getLength(); i++) {
     final FieldPartitioner fp = partitionStrategy.getFieldPartitioners().get(i);
     if (result != null) {
       result = new Path(result, convert.dirnameForValue(fp, key.get(i)));
     } else {
       result = new Path(convert.dirnameForValue(fp, key.get(i)));
     }
   }
   return result;
 }
예제 #2
0
    @Override
    @edu.umd.cs.findbugs.annotations.SuppressWarnings(
        value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE",
        justification = "Writer is known to be IncrementalWriter")
    public FileSystemWriter.IncrementalWriter<E> load(StorageKey key) throws Exception {
      Preconditions.checkState(
          view.getDataset() instanceof FileSystemDataset,
          "FileSystemWriters cannot create writer for " + view.getDataset());

      FileSystemDataset dataset = (FileSystemDataset) view.getDataset();
      Path partition = convert.fromKey(key);
      FileSystemWriter<E> writer =
          FileSystemWriter.newWriter(
              dataset.getFileSystem(),
              new Path(dataset.getDirectory(), partition),
              dataset.getDescriptor());

      PartitionListener listener = dataset.getPartitionListener();
      if (listener != null) {
        listener.partitionAdded(dataset.getNamespace(), dataset.getName(), partition.toString());
      }

      // initialize the writer after calling the listener
      // this lets the listener decide if and how to create the
      // partition directory
      writer.initialize();

      return (FileSystemWriter.IncrementalWriter<E>) writer;
    }
예제 #3
0
    @Override
    public FileSystemWriter<E> load(StorageKey key) throws Exception {
      Preconditions.checkState(
          view.getDataset() instanceof FileSystemDataset,
          "FileSystemWriters cannot create writer for " + view.getDataset());

      FileSystemDataset dataset = (FileSystemDataset) view.getDataset();
      Path partition = convert.fromKey(key);
      FileSystemWriter<E> writer =
          FileSystemWriter.newWriter(
              dataset.getFileSystem(),
              new Path(dataset.getDirectory(), partition),
              dataset.getDescriptor());

      PartitionListener listener = dataset.getPartitionListener();
      if (listener != null) {
        listener.partitionAdded(dataset.getNamespace(), dataset.getName(), partition.toString());
      }

      // initialize the writer after calling the listener
      // this lets the listener decide if and how to create the
      // partition directory
      writer.initialize();

      return writer;
    }
예제 #4
0
 @Test
 public void testDirnameMinWidth() {
   Assert.assertEquals(
       "min=01",
       PathConversion.dirnameForValue(new MinuteFieldPartitioner("timestamp", "min"), 1));
   Assert.assertEquals(
       "hour=01",
       PathConversion.dirnameForValue(new HourFieldPartitioner("timestamp", "hour"), 1));
   Assert.assertEquals(
       "day=01",
       PathConversion.dirnameForValue(new DayOfMonthFieldPartitioner("timestamp", "day"), 1));
   Assert.assertEquals(
       "month=01",
       PathConversion.dirnameForValue(new MonthFieldPartitioner("timestamp", "month"), 1));
   Assert.assertEquals(
       "year=2013",
       PathConversion.dirnameForValue(new YearFieldPartitioner("timestamp", "year"), 2013));
 }
예제 #5
0
 @Test
 public void testUsesFieldName() {
   Assert.assertEquals(
       "day_of_month_field",
       Iterables.getFirst(
           EQ.split(
               PathConversion.dirnameForValue(
                   new DayOfMonthFieldPartitioner("day", "day_of_month_field"), 10)),
           null));
 }
예제 #6
0
  @Test
  @SuppressWarnings("unchecked")
  public void toDirNameIdentityWithSlashes() {
    PartitionStrategy strategy =
        new PartitionStrategy.Builder().identity("name").identity("address").build();

    StorageKey key = new StorageKey(strategy);
    key.replaceValues((List) Lists.newArrayList("John Doe", "NY/USA"));

    Assert.assertEquals(new Path("name_copy=John+Doe/address_copy=NY%2FUSA"), convert.fromKey(key));
  }
예제 #7
0
  @SuppressWarnings("unchecked")
  PartitionKey fromDirectoryName(Path dir) {
    final FieldPartitioner fp = partitionStrategy.getFieldPartitioners().get(0);
    final List<Object> values = Lists.newArrayList();

    if (partitionKey != null) {
      values.addAll(partitionKey.getValues());
    }

    values.add(convert.valueForDirname(fp, dir.getName()));

    return Accessor.getDefault().newPartitionKey(values.toArray());
  }
예제 #8
0
  @Test
  @SuppressWarnings("unchecked")
  public void toDirNameIdentityWithNonString() {
    PartitionStrategy strategy = new PartitionStrategy.Builder().identity("id").build();

    StorageKey expected = new StorageKey(strategy);
    expected.replace(0, 0L);

    Assert.assertEquals(
        "Should convert to schema type",
        expected,
        convert.toKey(new Path("id=0"), new StorageKey(strategy)));
  }
예제 #9
0
  @Test
  @SuppressWarnings("unchecked")
  public void testFromKey() {
    PartitionStrategy strategy =
        new PartitionStrategy.Builder()
            .year("timestamp")
            .month("timestamp")
            .day("timestamp")
            .build();

    StorageKey key = new StorageKey(strategy);
    key.replaceValues((List) Lists.newArrayList(2013, 11, 5));

    Assert.assertEquals(new Path("year=2013/month=11/day=05"), convert.fromKey(key));
  }
예제 #10
0
 @Test
 public void testIgnoresPartitionName() {
   Assert.assertEquals("10", PathConversion.dirnameToValueString("10"));
   Assert.assertEquals("10", PathConversion.dirnameToValueString("=10"));
   Assert.assertEquals("10", PathConversion.dirnameToValueString("anything=10"));
   Assert.assertEquals(
       10, (int) convert.valueForDirname(new MonthFieldPartitioner("timestamp", "month"), "10"));
   Assert.assertEquals(
       10, (int) convert.valueForDirname(new MonthFieldPartitioner("timestamp", "month"), "=10"));
   Assert.assertEquals(
       10,
       (int)
           convert.valueForDirname(
               new MonthFieldPartitioner("timestamp", "month"), "anything=10"));
   Assert.assertEquals(
       10,
       (int)
           convert.valueForDirname(
               new MonthFieldPartitioner("timestamp", "month"), "even=strange=10"));
 }
예제 #11
0
    @Override
    public DatasetWriter<E> load(StorageKey key) throws Exception {
      Preconditions.checkState(
          view.getDataset() instanceof FileSystemDataset,
          "FileSystemWriters cannot create writer for " + view.getDataset());

      FileSystemDataset dataset = (FileSystemDataset) view.getDataset();
      Path partition = convert.fromKey(key);
      DatasetWriter<E> writer =
          new FileSystemWriter<E>(
              dataset.getFileSystem(),
              new Path(dataset.getDirectory(), partition),
              dataset.getDescriptor());

      PartitionListener listener = dataset.getPartitionListener();
      if (listener != null) {
        listener.partitionAdded(dataset.getName(), partition.toString());
      }

      writer.open();

      return writer;
    }
예제 #12
0
 @Test
 public void testNoValidation() {
   Assert.assertEquals(
       13,
       (int) convert.valueForDirname(new MonthFieldPartitioner("timestamp", "month"), "month=13"));
 }