private void writeContainer(Record src, OutputStream dst) {
DataFileWriter dataFileWriter = null;
try {
try {
Schema schema = null;
for (Object attachment : src.get(Fields.ATTACHMENT_BODY)) {
Preconditions.checkNotNull(attachment);
GenericContainer datum = (GenericContainer) attachment;
schema = getSchema(datum, schema);
assert schema != null;
if (dataFileWriter == null) { // init
GenericDatumWriter datumWriter = new GenericDatumWriter(schema);
dataFileWriter = new DataFileWriter(datumWriter);
if (codecFactory != null) {
dataFileWriter.setCodec(codecFactory);
}
for (Map.Entry<String, String> entry : metadata.entrySet()) {
dataFileWriter.setMeta(entry.getKey(), entry.getValue());
}
dataFileWriter.create(schema, dst);
}
dataFileWriter.append(datum);
}
if (dataFileWriter != null) {
dataFileWriter.flush();
}
} catch (IOException e) {
throw new MorphlineRuntimeException(e);
}
} finally {
Closeables.closeQuietly(dataFileWriter);
}
}
@Test
public void test() throws IOException {
File file = new File("target/AvroDocument.avro");
Schema schema = AvroDocument._SCHEMA;
{
System.out.println("Writing to: " + file.getAbsolutePath());
DatumWriter<Object> datumWriter = new SpecificDatumWriter(AvroDocument.class);
FileOutputStream outputStream = new FileOutputStream(file);
DataFileWriter<Object> dfw = new DataFileWriter<Object>(schema, outputStream, datumWriter);
AvroDocument d = createTestDocument();
dfw.append(d);
dfw.flush();
dfw.close();
}
{
System.out.println("Reading from: " + file.getAbsolutePath());
DatumReader<Object> datumReader = new SpecificDatumReader(AvroDocument.class);
SeekableInput seekableInput = new SeekableFileInput(file);
DataFileReader<Object> dfr = new DataFileReader<Object>(seekableInput, datumReader);
AvroDocument d = new AvroDocument();
dfr.next(d);
AvroDocumentReader.dumpAvroDocument(d, System.out);
}
}
public void execute(TridentTuple tuple, TridentCollector collector) {
GenericRecord docEntry = new GenericData.Record(schema);
docEntry.put("docid", tuple.getStringByField("documentId"));
docEntry.put("time", Time.currentTimeMillis());
docEntry.put("line", tuple.getStringByField("document"));
try {
dataFileWriter.append(docEntry);
dataFileWriter.flush();
} catch (IOException e) {
LOG.error("Error writing to document record: " + e);
throw new RuntimeException(e);
}
}
public static void main(String[] args) throws IOException {
DatumWriter<ArchivePlace> datumWriter =
new SpecificDatumWriter<ArchivePlace>(ArchivePlace.class);
DataFileWriter<ArchivePlace> falloutDatafileWriter =
new DataFileWriter<ArchivePlace>(datumWriter);
FileOutputStream falloutOutputStream =
new FileOutputStream("src/test/resources/archive-places/input.avro", true);
falloutDatafileWriter.create(ArchivePlace.SCHEMA$, falloutOutputStream);
List<ArchivePlace> places =
SerializationUtil.loadFromJsons(
ArchivePlace.SCHEMA$, "src/test/resources/archive-places/input.json");
for (ArchivePlace place : places) {
falloutDatafileWriter.append(place);
falloutDatafileWriter.flush();
}
falloutDatafileWriter.close();
falloutOutputStream.close();
}
@Override
public void close() throws IOException {
dataFileWriter.flush();
dataFileWriter.close();
super.close();
}
@Override
protected void writeFooter() throws Exception {
dataFileWriter.flush();
}