public WriteAvroToByteArray(
CommandBuilder builder,
Config config,
Command parent,
Command child,
MorphlineContext context) {
super(builder, config, parent, child, context);
this.format =
new Validator<Format>()
.validateEnum(
config,
getConfigs().getString(config, "format", Format.container.toString()),
Format.class);
String codec = getConfigs().getString(config, "codec", null);
if (codec == null) {
this.codecFactory = null;
} else {
this.codecFactory = CodecFactory.fromString(codec);
}
Config metadataConfig = getConfigs().getConfig(config, "metadata", ConfigFactory.empty());
for (Map.Entry<String, Object> entry : new Configs().getEntrySet(metadataConfig)) {
this.metadata.put(entry.getKey(), entry.getValue().toString());
}
validateArguments();
}
@Test
public void testHadoopCodecFactoryGZip() {
CodecFactory hadoopSnappyCodec =
HadoopCodecFactory.fromHadoopString("org.apache.hadoop.io.compress.GZipCodec");
CodecFactory avroSnappyCodec = CodecFactory.fromString("deflate");
assertTrue(hadoopSnappyCodec.getClass().equals(avroSnappyCodec.getClass()));
}
/**
* Generates an input Avro file containing the given records in the temporary directory and
* returns the full path of the file.
*/
private <T> String generateTestFile(
String filename,
List<T> elems,
SyncBehavior syncBehavior,
int syncInterval,
AvroCoder<T> coder,
String codec)
throws IOException {
Random random = new Random(0);
File tmpFile = tmpFolder.newFile(filename);
String path = tmpFile.toString();
FileOutputStream os = new FileOutputStream(tmpFile);
DatumWriter<T> datumWriter = coder.createDatumWriter();
try (DataFileWriter<T> writer = new DataFileWriter<>(datumWriter)) {
writer.setCodec(CodecFactory.fromString(codec));
writer.create(coder.getSchema(), os);
int recordIndex = 0;
int syncIndex = syncBehavior == SyncBehavior.SYNC_RANDOM ? random.nextInt(syncInterval) : 0;
for (T elem : elems) {
writer.append(elem);
recordIndex++;
switch (syncBehavior) {
case SYNC_REGULAR:
if (recordIndex == syncInterval) {
recordIndex = 0;
writer.sync();
}
break;
case SYNC_RANDOM:
if (recordIndex == syncIndex) {
recordIndex = 0;
writer.sync();
syncIndex = random.nextInt(syncInterval);
}
break;
case SYNC_DEFAULT:
default:
}
}
}
return path;
}