// It is assumed that record has following format.
// byte 1 : version, static 0x1
// byte 2-5 : int schemaId
// remaining bytes would have avro data
@Override
public GenericRecord parse(ByteBuffer bytes) {
if (bytes.remaining() < 5) {
throw new ParseException("record must have at least 5 bytes carrying version and schemaId");
}
byte version = bytes.get();
if (version != V1) {
throw new ParseException("found record of arbitrary version [%s]", version);
}
int schemaId = bytes.getInt();
Schema schemaObj = schemaObjs.get(schemaId);
if (schemaObj == null) {
throw new ParseException("Failed to find schema for id [%s]", schemaId);
}
try {
DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(schemaObj);
ByteBufferInputStream inputStream =
new ByteBufferInputStream(Collections.singletonList(bytes));
return reader.read(null, DecoderFactory.get().binaryDecoder(inputStream, null));
} catch (Exception e) {
throw new ParseException(e, "Fail to decode avro message with schemaId [%s].", schemaId);
}
}
@Override
public int run(InputStream stdin, PrintStream out, PrintStream err, List<String> args)
throws Exception {
if (args.size() != 2) {
err.println("Expected 2 arguments: schema binary_data_file");
err.println("Use '-' as binary_data_file for stdin.");
return 1;
}
Schema schema = Schema.parse(args.get(0));
InputStream input;
boolean needsClosing;
if (args.get(1).equals("-")) {
input = stdin;
needsClosing = false;
} else {
input = new FileInputStream(args.get(1));
needsClosing = true;
}
try {
DatumReader<Object> reader = new GenericDatumReader<Object>(schema);
Object datum = reader.read(null, DecoderFactory.get().binaryDecoder(input, null));
DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
JsonGenerator g = new JsonFactory().createJsonGenerator(out, JsonEncoding.UTF8);
g.useDefaultPrettyPrinter();
writer.write(datum, EncoderFactory.get().jsonEncoder(schema, g));
g.flush();
out.println();
out.flush();
} finally {
if (needsClosing) {
input.close();
}
}
return 0;
}
public Record fromStream(
final InputStream stream, @Nullable final Set<URI> propertiesToDeserialize)
throws IOException {
final Decoder decoder = DecoderFactory.get().directBinaryDecoder(stream, null);
final DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(Schemas.NODE);
final GenericRecord generic = reader.read(null, decoder);
return decodeRecord(generic, propertiesToDeserialize);
}
@Test(expected = AvroTypeException.class)
public void testNoDefaultField() throws Exception {
Schema expected =
Schema.parse(
"{\"type\":\"record\", \"name\":\"Foo\", \"fields\":"
+ "[{\"name\":\"f\", \"type\": \"string\"}]}");
DatumReader<Object> in = new GenericDatumReader<Object>(ACTUAL, expected);
in.read(null, DecoderFactory.get().binaryDecoder(new ByteArrayInputStream(new byte[0]), null));
}
public URI expandURI(final byte[] bytes) {
Preconditions.checkNotNull(bytes);
try {
final InputStream stream = new ByteArrayInputStream(bytes);
final Decoder decoder = DecoderFactory.get().directBinaryDecoder(stream, null);
final DatumReader<Object> reader =
new GenericDatumReader<Object>(Schemas.COMPRESSED_IDENTIFIER);
final Object generic = reader.read(null, decoder);
return (URI) decodeNode(generic);
} catch (final IOException ex) {
throw new Error("Unexpected exception (!): " + ex.getMessage(), ex);
}
}
/**
* Decode an entity from the initialized Avro Decoder using the DatumReader.
*
* @param decoder The decoder to decode the entity fields
* @param reader The Avro DatumReader that will read the entity with the decoder.
* @return The entity.
*/
public static <T> T readAvroEntity(Decoder decoder, DatumReader<T> reader) {
try {
return reader.read(null, decoder);
} catch (IOException e) {
throw new SerializationException("Could not deserialize Avro entity", e);
}
}
/**
* Create a deep copy of an Avro value.
*
* @param source The value to be copied
* @return The deep copy of the value
*/
@Override
public T deepCopy(T source) {
if (source == null) {
return null;
}
if (datumReader == null) {
datumReader = createDatumReader(conf);
}
if (datumWriter == null) {
datumWriter = createDatumWriter(conf);
}
ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream();
binaryEncoder = EncoderFactory.get().binaryEncoder(byteOutStream, binaryEncoder);
T target = createCopyTarget();
try {
datumWriter.write(source, binaryEncoder);
binaryEncoder.flush();
binaryDecoder =
DecoderFactory.get().binaryDecoder(byteOutStream.toByteArray(), binaryDecoder);
return datumReader.read(target, binaryDecoder);
} catch (Exception e) {
throw new CrunchRuntimeException("Error while deep copying avro value " + source, e);
}
}
public static void checkBlockingBinary(
Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader)
throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
writer.setSchema(schema);
Encoder encoder = EncoderFactory.get().blockingBinaryEncoder(out, null);
writer.write(datum, encoder);
encoder.flush();
byte[] data = out.toByteArray();
reader.setSchema(schema);
Object decoded = reader.read(null, DecoderFactory.get().binaryDecoder(data, null));
assertEquals("Decoded data does not match.", datum, decoded);
}
private static void checkDefault(String schemaJson, String defaultJson, Object defaultValue)
throws Exception {
String recordJson =
"{\"type\":\"record\", \"name\":\"Foo\", \"fields\":[{\"name\":\"f\", "
+ "\"type\":"
+ schemaJson
+ ", "
+ "\"default\":"
+ defaultJson
+ "}]}";
Schema expected = Schema.parse(recordJson);
DatumReader<Object> in = new GenericDatumReader<Object>(ACTUAL, expected);
GenericData.Record record =
(GenericData.Record) in.read(null, DecoderFactory.get().binaryDecoder(new byte[0], null));
assertEquals("Wrong default.", defaultValue, record.get("f"));
assertEquals("Wrong toString", expected, Schema.parse(expected.toString()));
}
public static void checkBinaryJson(String json) throws Exception {
Object node = Json.parseJson(json);
ByteArrayOutputStream out = new ByteArrayOutputStream();
DatumWriter<Object> writer = new Json.ObjectWriter();
Encoder encoder = EncoderFactory.get().binaryEncoder(out, null);
encoder = EncoderFactory.get().validatingEncoder(Json.SCHEMA, encoder);
writer.write(node, encoder);
encoder.flush();
byte[] bytes = out.toByteArray();
DatumReader<Object> reader = new Json.ObjectReader();
Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null);
decoder = DecoderFactory.get().validatingDecoder(Json.SCHEMA, decoder);
Object decoded = reader.read(null, decoder);
assertEquals("Decoded json does not match.", Json.toString(node), Json.toString(decoded));
}
private static void checkJson(
Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader)
throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out);
writer.setSchema(schema);
writer.write(datum, encoder);
writer.write(datum, encoder);
encoder.flush();
byte[] data = out.toByteArray();
reader.setSchema(schema);
Decoder decoder = DecoderFactory.get().jsonDecoder(schema, new ByteArrayInputStream(data));
Object decoded = reader.read(null, decoder);
assertEquals("Decoded data does not match.", datum, decoded);
decoded = reader.read(decoded, decoder);
assertEquals("Decoded data does not match.", datum, decoded);
}
private static void checkJson(Schema schema, Object datum, String json) throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out);
DatumWriter<Object> writer = new GenericDatumWriter<Object>();
writer.setSchema(schema);
writer.write(datum, encoder);
encoder.flush();
byte[] data = out.toByteArray();
String encoded = new String(data, "UTF-8");
assertEquals("Encoded data does not match.", json, encoded);
DatumReader<Object> reader = new GenericDatumReader<Object>();
reader.setSchema(schema);
Object decoded =
reader.read(null, DecoderFactory.get().jsonDecoder(schema, new ByteArrayInputStream(data)));
assertEquals("Decoded data does not match.", datum, decoded);
}
@Test
public void testEnumMismatch() throws Exception {
Schema actual = Schema.parse("{\"type\":\"enum\",\"name\":\"E\",\"symbols\":[\"X\",\"Y\"]}");
Schema expected = Schema.parse("{\"type\":\"enum\",\"name\":\"E\",\"symbols\":[\"Y\",\"Z\"]}");
ByteArrayOutputStream out = new ByteArrayOutputStream();
DatumWriter<Object> writer = new GenericDatumWriter<Object>(actual);
Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);
writer.write(new GenericData.EnumSymbol(actual, "Y"), encoder);
writer.write(new GenericData.EnumSymbol(actual, "X"), encoder);
encoder.flush();
byte[] data = out.toByteArray();
Decoder decoder = DecoderFactory.get().binaryDecoder(data, null);
DatumReader<String> in = new GenericDatumReader<String>(actual, expected);
assertEquals("Wrong value", new GenericData.EnumSymbol(expected, "Y"), in.read(null, decoder));
try {
in.read(null, decoder);
fail("Should have thrown exception.");
} catch (AvroTypeException e) {
// expected
}
}
public void serializeGeneric() throws IOException {
// Create a datum to serialize.
Schema schema = new Schema.Parser().parse(getClass().getResourceAsStream("/MyPair.avsc"));
GenericRecord datum = new GenericData.Record(schema);
datum.put("left", new Utf8("dog"));
datum.put("right", new Utf8("cat"));
// Serialize it.
ByteArrayOutputStream out = new ByteArrayOutputStream();
DatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(schema);
Encoder encoder = EncoderFactory.get().binaryEncoder(out, null);
writer.write(datum, encoder);
encoder.flush();
out.close();
System.out.println("Serialization: " + out);
// Deserialize it.
DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(schema);
BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(out.toByteArray(), null);
GenericRecord result = reader.read(null, decoder);
System.out.printf("Left: %s, Right: %s\n", result.get("left"), result.get("right"));
}
@Override
public void readExternal(java.io.ObjectInput in) throws java.io.IOException {
READER$.read(this, org.apache.avro.specific.SpecificData.getDecoder(in));
}
/**
* Get the next event from the stream
*
* @return the next event
* @throws IOException
*/
@SuppressWarnings("unchecked")
public HistoryEvent getNextEvent() throws IOException {
Event wrapper;
try {
wrapper = (Event) reader.read(null, decoder);
} catch (EOFException e) { // at EOF
return null;
}
HistoryEvent result;
switch (wrapper.type) {
case JOB_SUBMITTED:
result = new JobSubmittedEvent();
break;
case JOB_INITED:
result = new JobInitedEvent();
break;
case JOB_FINISHED:
result = new JobFinishedEvent();
break;
case JOB_PRIORITY_CHANGED:
result = new JobPriorityChangeEvent();
break;
case JOB_STATUS_CHANGED:
result = new JobStatusChangedEvent();
break;
case JOB_FAILED:
result = new JobUnsuccessfulCompletionEvent();
break;
case JOB_KILLED:
result = new JobUnsuccessfulCompletionEvent();
break;
case JOB_INFO_CHANGED:
result = new JobInfoChangeEvent();
break;
case TASK_STARTED:
result = new TaskStartedEvent();
break;
case TASK_FINISHED:
result = new TaskFinishedEvent();
break;
case TASK_FAILED:
result = new TaskFailedEvent();
break;
case TASK_UPDATED:
result = new TaskUpdatedEvent();
break;
case MAP_ATTEMPT_STARTED:
result = new TaskAttemptStartedEvent();
break;
case MAP_ATTEMPT_FINISHED:
result = new MapAttemptFinishedEvent();
break;
case MAP_ATTEMPT_FAILED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case MAP_ATTEMPT_KILLED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case REDUCE_ATTEMPT_STARTED:
result = new TaskAttemptStartedEvent();
break;
case REDUCE_ATTEMPT_FINISHED:
result = new ReduceAttemptFinishedEvent();
break;
case REDUCE_ATTEMPT_FAILED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case REDUCE_ATTEMPT_KILLED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case SETUP_ATTEMPT_STARTED:
result = new TaskAttemptStartedEvent();
break;
case SETUP_ATTEMPT_FINISHED:
result = new TaskAttemptFinishedEvent();
break;
case SETUP_ATTEMPT_FAILED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case SETUP_ATTEMPT_KILLED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case CLEANUP_ATTEMPT_STARTED:
result = new TaskAttemptStartedEvent();
break;
case CLEANUP_ATTEMPT_FINISHED:
result = new TaskAttemptFinishedEvent();
break;
case CLEANUP_ATTEMPT_FAILED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case CLEANUP_ATTEMPT_KILLED:
result = new TaskAttemptUnsuccessfulCompletionEvent();
break;
case AM_STARTED:
result = new AMStartedEvent();
break;
default:
throw new RuntimeException("unexpected event type: " + wrapper.type);
}
result.setDatum(wrapper.event);
return result;
}
public Object fromStream(final InputStream stream) throws IOException {
final Decoder decoder = DecoderFactory.get().directBinaryDecoder(stream, null);
final DatumReader<Object> reader = new GenericDatumReader<Object>(Schemas.NODE);
final Object generic = reader.read(null, decoder);
return decodeNode(generic);
}
