@VisibleForTesting
@SuppressWarnings("unchecked")
<E> FileAppender<E> newAppender(Path temp) {
Format format = descriptor.getFormat();
if (Formats.PARQUET.equals(format)) {
// by default, Parquet is not durable
if (DescriptorUtil.isDisabled(FileSystemProperties.NON_DURABLE_PARQUET_PROP, descriptor)) {
return (FileAppender<E>)
new DurableParquetAppender(
fs, temp, descriptor.getSchema(), conf, descriptor.getCompressionType());
} else {
return (FileAppender<E>)
new ParquetAppender(
fs, temp, descriptor.getSchema(), conf, descriptor.getCompressionType());
}
} else if (Formats.AVRO.equals(format)) {
return new AvroAppender<E>(fs, temp, descriptor.getSchema(), descriptor.getCompressionType());
} else if (Formats.CSV.equals(format)
&& DescriptorUtil.isEnabled(FileSystemProperties.ALLOW_CSV_PROP, descriptor)) {
return new CSVAppender<E>(fs, temp, descriptor);
} else {
this.state = ReaderWriterState.ERROR;
throw new UnknownFormatException("Unknown format " + descriptor);
}
}
@SuppressWarnings("unchecked")
private <E> FileAppender<E> newAppender(Path temp) {
Format format = descriptor.getFormat();
if (Formats.PARQUET.equals(format)) {
// by default, guarantee durability with the more costly writer
if (DescriptorUtil.isEnabled(FileSystemProperties.NON_DURABLE_PARQUET_PROP, descriptor)) {
return (FileAppender<E>) new ParquetAppender(fs, temp, descriptor.getSchema(), conf, true);
} else {
return (FileAppender<E>)
new DurableParquetAppender(fs, temp, descriptor.getSchema(), conf, true);
}
} else if (Formats.AVRO.equals(format)) {
return new AvroAppender<E>(fs, temp, descriptor.getSchema(), true);
} else if (Formats.CSV.equals(format)
&& DescriptorUtil.isEnabled(FileSystemProperties.ALLOW_CSV_PROP, descriptor)) {
return new CSVAppender<E>(fs, temp, descriptor);
} else {
this.state = ReaderWriterState.ERROR;
throw new DatasetWriterException("Unknown format " + descriptor);
}
}
static <E> FileSystemWriter<E> newWriter(FileSystem fs, Path path, DatasetDescriptor descriptor) {
Format format = descriptor.getFormat();
if (Formats.PARQUET.equals(format)) {
// by default, Parquet is not durable
if (DescriptorUtil.isDisabled(FileSystemProperties.NON_DURABLE_PARQUET_PROP, descriptor)) {
return new IncrementalWriter<E>(fs, path, descriptor);
} else {
return new FileSystemWriter<E>(fs, path, descriptor);
}
} else if (Formats.AVRO.equals(format) || Formats.CSV.equals(format)) {
return new IncrementalWriter<E>(fs, path, descriptor);
} else {
return new FileSystemWriter<E>(fs, path, descriptor);
}
}
static <E> PartitionedDatasetWriter<E, ?> newWriter(FileSystemView<E> view) {
DatasetDescriptor descriptor = view.getDataset().getDescriptor();
Format format = descriptor.getFormat();
if (Formats.PARQUET.equals(format)) {
// by default, Parquet is not durable
if (DescriptorUtil.isDisabled(FileSystemProperties.NON_DURABLE_PARQUET_PROP, descriptor)) {
return new IncrementalPartitionedDatasetWriter<E>(view);
} else {
return new NonDurablePartitionedDatasetWriter<E>(view);
}
} else if (Formats.AVRO.equals(format) || Formats.CSV.equals(format)) {
return new IncrementalPartitionedDatasetWriter<E>(view);
} else {
return new NonDurablePartitionedDatasetWriter<E>(view);
}
}
static boolean isSupportedFormat(DatasetDescriptor descriptor) {
Format format = descriptor.getFormat();
return (SUPPORTED_FORMATS.contains(format)
|| (Formats.CSV.equals(format)
&& DescriptorUtil.isEnabled(FileSystemProperties.ALLOW_CSV_PROP, descriptor)));
}
