private PartitionedDatasetWriter(FileSystemView<E> view) {
final DatasetDescriptor descriptor = view.getDataset().getDescriptor();
Preconditions.checkArgument(
descriptor.isPartitioned(), "Dataset " + view.getDataset() + " is not partitioned");
this.view = view;
this.partitionStrategy = descriptor.getPartitionStrategy();
int maxWriters = DEFAULT_WRITER_CACHE_SIZE;
if (descriptor.hasProperty(FileSystemProperties.WRITER_CACHE_SIZE_PROP)) {
try {
maxWriters =
Integer.parseInt(descriptor.getProperty(FileSystemProperties.WRITER_CACHE_SIZE_PROP));
} catch (NumberFormatException e) {
LOG.warn(
"Not an integer: "
+ FileSystemProperties.WRITER_CACHE_SIZE_PROP
+ "="
+ descriptor.getProperty(FileSystemProperties.WRITER_CACHE_SIZE_PROP));
}
} else if (partitionStrategy.getCardinality() != FieldPartitioner.UNKNOWN_CARDINALITY) {
maxWriters = Math.min(maxWriters, partitionStrategy.getCardinality());
}
this.maxWriters = maxWriters;
this.state = ReaderWriterState.NEW;
this.reusedKey = new StorageKey(partitionStrategy);
this.accessor = view.getAccessor();
this.provided = view.getProvidedValues();
}
private FileSystemWriter(FileSystem fs, Path path, DatasetDescriptor descriptor) {
Preconditions.checkNotNull(fs, "File system is not defined");
Preconditions.checkNotNull(path, "Destination directory is not defined");
Preconditions.checkNotNull(descriptor, "Descriptor is not defined");
this.fs = fs;
this.directory = path;
this.descriptor = descriptor;
this.conf = new Configuration(fs.getConf());
this.state = ReaderWriterState.NEW;
// copy file format settings from custom properties to the Configuration
for (String prop : descriptor.listProperties()) {
conf.set(prop, descriptor.getProperty(prop));
}
}