@Override
public int run(InputStream stdin, PrintStream out, PrintStream err, List<String> args)
throws Exception {
if (args.size() != 1) {
// Unlike other commands, "-" can't be used for stdin, because
// we can only use seekable files.
err.println("Expected 1 argument: input_file.");
return 1;
}
GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
FileReader<Object> fileReader = DataFileReader.openReader(new File(args.get(0)), reader);
try {
Schema schema = fileReader.getSchema();
DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
JsonEncoder encoder = EncoderFactory.get().jsonEncoder(schema, out);
for (Object datum : fileReader) writer.write(datum, encoder);
encoder.flush();
out.println();
out.flush();
} finally {
fileReader.close();
}
return 0;
}
@Override
public int run(InputStream stdin, PrintStream out, PrintStream err, List<String> args)
throws Exception {
if (args.size() != 2) {
err.println("Expected 2 arguments: schema binary_data_file");
err.println("Use '-' as binary_data_file for stdin.");
return 1;
}
Schema schema = Schema.parse(args.get(0));
InputStream input;
boolean needsClosing;
if (args.get(1).equals("-")) {
input = stdin;
needsClosing = false;
} else {
input = new FileInputStream(args.get(1));
needsClosing = true;
}
try {
DatumReader<Object> reader = new GenericDatumReader<Object>(schema);
Object datum = reader.read(null, DecoderFactory.get().binaryDecoder(input, null));
DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
JsonGenerator g = new JsonFactory().createJsonGenerator(out, JsonEncoding.UTF8);
g.useDefaultPrettyPrinter();
writer.write(datum, EncoderFactory.get().jsonEncoder(schema, g));
g.flush();
out.println();
out.flush();
} finally {
if (needsClosing) {
input.close();
}
}
return 0;
}
public void toStream(final OutputStream stream, final Object object) throws IOException {
final Object generic = encodeNode(object);
final Encoder encoder = EncoderFactory.get().directBinaryEncoder(stream, null);
final DatumWriter<Object> writer = new GenericDatumWriter<Object>(Schemas.NODE);
writer.write(generic, encoder);
encoder.flush();
}
public void toStream(
final OutputStream stream,
final Record object,
@Nullable final Set<URI> propertiesToSerialize)
throws IOException {
final Object generic = encodeRecord(object, propertiesToSerialize);
final Encoder encoder = EncoderFactory.get().directBinaryEncoder(stream, null);
final DatumWriter<Object> writer = new GenericDatumWriter<Object>(Schemas.NODE);
writer.write(generic, encoder);
encoder.flush();
}
public byte[] compressURI(final URI uri) {
Preconditions.checkNotNull(uri);
try {
final ByteArrayOutputStream stream = new ByteArrayOutputStream();
final Encoder encoder = EncoderFactory.get().directBinaryEncoder(stream, null);
final DatumWriter<Object> writer =
new GenericDatumWriter<Object>(Schemas.COMPRESSED_IDENTIFIER);
this.dictionary.keyFor(uri); // ensure a compressed version of URI is available
final Object generic = encodeIdentifier(uri);
writer.write(generic, encoder);
return stream.toByteArray();
} catch (final IOException ex) {
throw new Error("Unexpected exception (!): " + ex.getMessage(), ex);
}
}
/**
* Create a deep copy of an Avro value.
*
* @param source The value to be copied
* @return The deep copy of the value
*/
@Override
public T deepCopy(T source) {
if (source == null) {
return null;
}
if (datumReader == null) {
datumReader = createDatumReader(conf);
}
if (datumWriter == null) {
datumWriter = createDatumWriter(conf);
}
ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream();
binaryEncoder = EncoderFactory.get().binaryEncoder(byteOutStream, binaryEncoder);
T target = createCopyTarget();
try {
datumWriter.write(source, binaryEncoder);
binaryEncoder.flush();
binaryDecoder =
DecoderFactory.get().binaryDecoder(byteOutStream.toByteArray(), binaryDecoder);
return datumReader.read(target, binaryDecoder);
} catch (Exception e) {
throw new CrunchRuntimeException("Error while deep copying avro value " + source, e);
}
}
public static void checkBlockingBinary(
Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader)
throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
writer.setSchema(schema);
Encoder encoder = EncoderFactory.get().blockingBinaryEncoder(out, null);
writer.write(datum, encoder);
encoder.flush();
byte[] data = out.toByteArray();
reader.setSchema(schema);
Object decoded = reader.read(null, DecoderFactory.get().binaryDecoder(data, null));
assertEquals("Decoded data does not match.", datum, decoded);
}
/**
* Given an entity, an avro schema, and an encoder, write the entity to the encoder's underlying
* output stream.
*
* @param entity The entity we want to encode.
* @param encoder The Avro Encoder we will write to.
* @param writer The DatumWriter we'll use to encode the entity to the encoder.
*/
public static <T> void writeAvroEntity(T entity, Encoder encoder, DatumWriter<T> writer) {
try {
writer.write(entity, encoder);
encoder.flush();
} catch (IOException e) {
throw new SerializationException("Could not serialize Avro entity", e);
}
}
public static void checkBinaryJson(String json) throws Exception {
Object node = Json.parseJson(json);
ByteArrayOutputStream out = new ByteArrayOutputStream();
DatumWriter<Object> writer = new Json.ObjectWriter();
Encoder encoder = EncoderFactory.get().binaryEncoder(out, null);
encoder = EncoderFactory.get().validatingEncoder(Json.SCHEMA, encoder);
writer.write(node, encoder);
encoder.flush();
byte[] bytes = out.toByteArray();
DatumReader<Object> reader = new Json.ObjectReader();
Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null);
decoder = DecoderFactory.get().validatingDecoder(Json.SCHEMA, decoder);
Object decoded = reader.read(null, decoder);
assertEquals("Decoded json does not match.", Json.toString(node), Json.toString(decoded));
}
@Test
public void testValidate() throws Exception {
getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
manager.storePolicy("user", "info:name", TestProducer.class, new AlwaysFreshen());
assertEquals(
BaseTool.SUCCESS,
runTool(
new FreshTool(),
KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
"--do=validate-all"));
KijiFreshnessPolicyRecord record =
KijiFreshnessPolicyRecord.newBuilder()
.setRecordVersion(ProtocolVersion.parse("policyrecord-0.1").toCanonicalString())
.setProducerClass(TestProducer.class.getName())
.setFreshnessPolicyClass(AlwaysFreshen.class.getName())
.setFreshnessPolicyState("")
.build();
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
final EncoderFactory encoderFactory = EncoderFactory.get();
Encoder encoder = encoderFactory.directBinaryEncoder(outputStream, null);
final DatumWriter<KijiFreshnessPolicyRecord> recordWriter =
new SpecificDatumWriter<KijiFreshnessPolicyRecord>(KijiFreshnessPolicyRecord.SCHEMA$);
recordWriter.write(record, encoder);
getKiji()
.getMetaTable()
.putValue("user", "kiji.scoring.fresh.columnName", outputStream.toByteArray());
assertEquals(
BaseTool.FAILURE,
runTool(
new FreshTool(),
KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
"--do=validate-all"));
assertEquals(
"Freshness policy attached to column: columnName is not valid.", mToolOutputLines[1]);
assertEquals(
"NO_FAMILY_IN_TABLE: java.lang.IllegalArgumentException: Table: user does not "
+ "contain family: columnName",
mToolOutputLines[2]);
}
private static void checkJson(Schema schema, Object datum, String json) throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out);
DatumWriter<Object> writer = new GenericDatumWriter<Object>();
writer.setSchema(schema);
writer.write(datum, encoder);
encoder.flush();
byte[] data = out.toByteArray();
String encoded = new String(data, "UTF-8");
assertEquals("Encoded data does not match.", json, encoded);
DatumReader<Object> reader = new GenericDatumReader<Object>();
reader.setSchema(schema);
Object decoded =
reader.read(null, DecoderFactory.get().jsonDecoder(schema, new ByteArrayInputStream(data)));
assertEquals("Decoded data does not match.", datum, decoded);
}
private static void checkJson(
Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader)
throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out);
writer.setSchema(schema);
writer.write(datum, encoder);
writer.write(datum, encoder);
encoder.flush();
byte[] data = out.toByteArray();
reader.setSchema(schema);
Decoder decoder = DecoderFactory.get().jsonDecoder(schema, new ByteArrayInputStream(data));
Object decoded = reader.read(null, decoder);
assertEquals("Decoded data does not match.", datum, decoded);
decoded = reader.read(decoded, decoder);
assertEquals("Decoded data does not match.", datum, decoded);
}
@Test
public void testEnumMismatch() throws Exception {
Schema actual = Schema.parse("{\"type\":\"enum\",\"name\":\"E\",\"symbols\":[\"X\",\"Y\"]}");
Schema expected = Schema.parse("{\"type\":\"enum\",\"name\":\"E\",\"symbols\":[\"Y\",\"Z\"]}");
ByteArrayOutputStream out = new ByteArrayOutputStream();
DatumWriter<Object> writer = new GenericDatumWriter<Object>(actual);
Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);
writer.write(new GenericData.EnumSymbol(actual, "Y"), encoder);
writer.write(new GenericData.EnumSymbol(actual, "X"), encoder);
encoder.flush();
byte[] data = out.toByteArray();
Decoder decoder = DecoderFactory.get().binaryDecoder(data, null);
DatumReader<String> in = new GenericDatumReader<String>(actual, expected);
assertEquals("Wrong value", new GenericData.EnumSymbol(expected, "Y"), in.read(null, decoder));
try {
in.read(null, decoder);
fail("Should have thrown exception.");
} catch (AvroTypeException e) {
// expected
}
}
public void serializeGeneric() throws IOException {
// Create a datum to serialize.
Schema schema = new Schema.Parser().parse(getClass().getResourceAsStream("/MyPair.avsc"));
GenericRecord datum = new GenericData.Record(schema);
datum.put("left", new Utf8("dog"));
datum.put("right", new Utf8("cat"));
// Serialize it.
ByteArrayOutputStream out = new ByteArrayOutputStream();
DatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(schema);
Encoder encoder = EncoderFactory.get().binaryEncoder(out, null);
writer.write(datum, encoder);
encoder.flush();
out.close();
System.out.println("Serialization: " + out);
// Deserialize it.
DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(schema);
BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(out.toByteArray(), null);
GenericRecord result = reader.read(null, decoder);
System.out.printf("Left: %s, Right: %s\n", result.get("left"), result.get("right"));
}
/**
* Gives the output message
*
* @param outputType output data type
* @param result mapping result
* @return the output as a String
* @throws IOException
*/
public String getOutputMessage(String outputType, GenericRecord result)
throws SynapseException, IOException {
DatumWriter<GenericRecord> writer = null;
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
Encoder encoder = new DummyEncoder(byteArrayOutputStream);
// OMElement outMessage = null;
String outMessage = null;
try {
writer = WriterRegistry.getInstance().get(outputType).newInstance();
writer.setSchema(result.getSchema());
writer.write(result, encoder);
if (log.isDebugEnabled()) {
log.debug("Output received from datum writer.." + byteArrayOutputStream.toString());
}
} catch (Exception e) {
handleException("Data coversion Failed at JSONWriter..", e);
} finally {
encoder.flush();
}
/*try {
// Converts the result into an OMElement
outMessage = getOutputResult(byteArrayOutputStream.toString());
} catch (XMLStreamException e) {
handleException(
"Failed at generating the OMElement for the JSON output received...",
e);
}*/
outMessage = byteArrayOutputStream.toString();
return outMessage;
}
@Override
public void writeExternal(java.io.ObjectOutput out) throws java.io.IOException {
WRITER$.write(this, org.apache.avro.specific.SpecificData.getEncoder(out));
}
