@SuppressWarnings("unchecked")
private Path toDirectoryName(Path dir, PartitionKey key) {
Path result = dir;
for (int i = 0; i < key.getLength(); i++) {
final FieldPartitioner fp = partitionStrategy.getFieldPartitioners().get(i);
if (result != null) {
result = new Path(result, convert.dirnameForValue(fp, key.get(i)));
} else {
result = new Path(convert.dirnameForValue(fp, key.get(i)));
}
}
return result;
}
@Override
@edu.umd.cs.findbugs.annotations.SuppressWarnings(
value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE",
justification = "Writer is known to be IncrementalWriter")
public FileSystemWriter.IncrementalWriter<E> load(StorageKey key) throws Exception {
Preconditions.checkState(
view.getDataset() instanceof FileSystemDataset,
"FileSystemWriters cannot create writer for " + view.getDataset());
FileSystemDataset dataset = (FileSystemDataset) view.getDataset();
Path partition = convert.fromKey(key);
FileSystemWriter<E> writer =
FileSystemWriter.newWriter(
dataset.getFileSystem(),
new Path(dataset.getDirectory(), partition),
dataset.getDescriptor());
PartitionListener listener = dataset.getPartitionListener();
if (listener != null) {
listener.partitionAdded(dataset.getNamespace(), dataset.getName(), partition.toString());
}
// initialize the writer after calling the listener
// this lets the listener decide if and how to create the
// partition directory
writer.initialize();
return (FileSystemWriter.IncrementalWriter<E>) writer;
}
@Override
public FileSystemWriter<E> load(StorageKey key) throws Exception {
Preconditions.checkState(
view.getDataset() instanceof FileSystemDataset,
"FileSystemWriters cannot create writer for " + view.getDataset());
FileSystemDataset dataset = (FileSystemDataset) view.getDataset();
Path partition = convert.fromKey(key);
FileSystemWriter<E> writer =
FileSystemWriter.newWriter(
dataset.getFileSystem(),
new Path(dataset.getDirectory(), partition),
dataset.getDescriptor());
PartitionListener listener = dataset.getPartitionListener();
if (listener != null) {
listener.partitionAdded(dataset.getNamespace(), dataset.getName(), partition.toString());
}
// initialize the writer after calling the listener
// this lets the listener decide if and how to create the
// partition directory
writer.initialize();
return writer;
}
@Test
public void testDirnameMinWidth() {
Assert.assertEquals(
"min=01",
PathConversion.dirnameForValue(new MinuteFieldPartitioner("timestamp", "min"), 1));
Assert.assertEquals(
"hour=01",
PathConversion.dirnameForValue(new HourFieldPartitioner("timestamp", "hour"), 1));
Assert.assertEquals(
"day=01",
PathConversion.dirnameForValue(new DayOfMonthFieldPartitioner("timestamp", "day"), 1));
Assert.assertEquals(
"month=01",
PathConversion.dirnameForValue(new MonthFieldPartitioner("timestamp", "month"), 1));
Assert.assertEquals(
"year=2013",
PathConversion.dirnameForValue(new YearFieldPartitioner("timestamp", "year"), 2013));
}
@Test
public void testUsesFieldName() {
Assert.assertEquals(
"day_of_month_field",
Iterables.getFirst(
EQ.split(
PathConversion.dirnameForValue(
new DayOfMonthFieldPartitioner("day", "day_of_month_field"), 10)),
null));
}
@Test
@SuppressWarnings("unchecked")
public void toDirNameIdentityWithSlashes() {
PartitionStrategy strategy =
new PartitionStrategy.Builder().identity("name").identity("address").build();
StorageKey key = new StorageKey(strategy);
key.replaceValues((List) Lists.newArrayList("John Doe", "NY/USA"));
Assert.assertEquals(new Path("name_copy=John+Doe/address_copy=NY%2FUSA"), convert.fromKey(key));
}
@SuppressWarnings("unchecked")
PartitionKey fromDirectoryName(Path dir) {
final FieldPartitioner fp = partitionStrategy.getFieldPartitioners().get(0);
final List<Object> values = Lists.newArrayList();
if (partitionKey != null) {
values.addAll(partitionKey.getValues());
}
values.add(convert.valueForDirname(fp, dir.getName()));
return Accessor.getDefault().newPartitionKey(values.toArray());
}
@Test
@SuppressWarnings("unchecked")
public void toDirNameIdentityWithNonString() {
PartitionStrategy strategy = new PartitionStrategy.Builder().identity("id").build();
StorageKey expected = new StorageKey(strategy);
expected.replace(0, 0L);
Assert.assertEquals(
"Should convert to schema type",
expected,
convert.toKey(new Path("id=0"), new StorageKey(strategy)));
}
@Test
@SuppressWarnings("unchecked")
public void testFromKey() {
PartitionStrategy strategy =
new PartitionStrategy.Builder()
.year("timestamp")
.month("timestamp")
.day("timestamp")
.build();
StorageKey key = new StorageKey(strategy);
key.replaceValues((List) Lists.newArrayList(2013, 11, 5));
Assert.assertEquals(new Path("year=2013/month=11/day=05"), convert.fromKey(key));
}
@Test
public void testIgnoresPartitionName() {
Assert.assertEquals("10", PathConversion.dirnameToValueString("10"));
Assert.assertEquals("10", PathConversion.dirnameToValueString("=10"));
Assert.assertEquals("10", PathConversion.dirnameToValueString("anything=10"));
Assert.assertEquals(
10, (int) convert.valueForDirname(new MonthFieldPartitioner("timestamp", "month"), "10"));
Assert.assertEquals(
10, (int) convert.valueForDirname(new MonthFieldPartitioner("timestamp", "month"), "=10"));
Assert.assertEquals(
10,
(int)
convert.valueForDirname(
new MonthFieldPartitioner("timestamp", "month"), "anything=10"));
Assert.assertEquals(
10,
(int)
convert.valueForDirname(
new MonthFieldPartitioner("timestamp", "month"), "even=strange=10"));
}
@Override
public DatasetWriter<E> load(StorageKey key) throws Exception {
Preconditions.checkState(
view.getDataset() instanceof FileSystemDataset,
"FileSystemWriters cannot create writer for " + view.getDataset());
FileSystemDataset dataset = (FileSystemDataset) view.getDataset();
Path partition = convert.fromKey(key);
DatasetWriter<E> writer =
new FileSystemWriter<E>(
dataset.getFileSystem(),
new Path(dataset.getDirectory(), partition),
dataset.getDescriptor());
PartitionListener listener = dataset.getPartitionListener();
if (listener != null) {
listener.partitionAdded(dataset.getName(), partition.toString());
}
writer.open();
return writer;
}
@Test
public void testNoValidation() {
Assert.assertEquals(
13,
(int) convert.valueForDirname(new MonthFieldPartitioner("timestamp", "month"), "month=13"));
}