コード例 #1
0
ファイル: DataFileReadTool.java プロジェクト: nagaral/warp
  @Override
  public int run(InputStream stdin, PrintStream out, PrintStream err, List<String> args)
      throws Exception {
    if (args.size() != 1) {
      // Unlike other commands, "-" can't be used for stdin, because
      // we can only use seekable files.
      err.println("Expected 1 argument: input_file.");
      return 1;
    }

    GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
    FileReader<Object> fileReader = DataFileReader.openReader(new File(args.get(0)), reader);
    try {
      Schema schema = fileReader.getSchema();
      DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
      JsonEncoder encoder = EncoderFactory.get().jsonEncoder(schema, out);
      for (Object datum : fileReader) writer.write(datum, encoder);
      encoder.flush();
      out.println();
      out.flush();
    } finally {
      fileReader.close();
    }
    return 0;
  }
コード例 #2
0
 @Override
 public int run(InputStream stdin, PrintStream out, PrintStream err, List<String> args)
     throws Exception {
   if (args.size() != 2) {
     err.println("Expected 2 arguments: schema binary_data_file");
     err.println("Use '-' as binary_data_file for stdin.");
     return 1;
   }
   Schema schema = Schema.parse(args.get(0));
   InputStream input;
   boolean needsClosing;
   if (args.get(1).equals("-")) {
     input = stdin;
     needsClosing = false;
   } else {
     input = new FileInputStream(args.get(1));
     needsClosing = true;
   }
   try {
     DatumReader<Object> reader = new GenericDatumReader<Object>(schema);
     Object datum = reader.read(null, DecoderFactory.get().binaryDecoder(input, null));
     DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
     JsonGenerator g = new JsonFactory().createJsonGenerator(out, JsonEncoding.UTF8);
     g.useDefaultPrettyPrinter();
     writer.write(datum, EncoderFactory.get().jsonEncoder(schema, g));
     g.flush();
     out.println();
     out.flush();
   } finally {
     if (needsClosing) {
       input.close();
     }
   }
   return 0;
 }
コード例 #3
0
 public void toStream(final OutputStream stream, final Object object) throws IOException {
   final Object generic = encodeNode(object);
   final Encoder encoder = EncoderFactory.get().directBinaryEncoder(stream, null);
   final DatumWriter<Object> writer = new GenericDatumWriter<Object>(Schemas.NODE);
   writer.write(generic, encoder);
   encoder.flush();
 }
コード例 #4
0
 public void toStream(
     final OutputStream stream,
     final Record object,
     @Nullable final Set<URI> propertiesToSerialize)
     throws IOException {
   final Object generic = encodeRecord(object, propertiesToSerialize);
   final Encoder encoder = EncoderFactory.get().directBinaryEncoder(stream, null);
   final DatumWriter<Object> writer = new GenericDatumWriter<Object>(Schemas.NODE);
   writer.write(generic, encoder);
   encoder.flush();
 }
コード例 #5
0
  public byte[] compressURI(final URI uri) {
    Preconditions.checkNotNull(uri);
    try {
      final ByteArrayOutputStream stream = new ByteArrayOutputStream();
      final Encoder encoder = EncoderFactory.get().directBinaryEncoder(stream, null);
      final DatumWriter<Object> writer =
          new GenericDatumWriter<Object>(Schemas.COMPRESSED_IDENTIFIER);
      this.dictionary.keyFor(uri); // ensure a compressed version of URI is available
      final Object generic = encodeIdentifier(uri);
      writer.write(generic, encoder);
      return stream.toByteArray();

    } catch (final IOException ex) {
      throw new Error("Unexpected exception (!): " + ex.getMessage(), ex);
    }
  }
コード例 #6
0
ファイル: AvroDeepCopier.java プロジェクト: relateiq/crunch
  /**
   * Create a deep copy of an Avro value.
   *
   * @param source The value to be copied
   * @return The deep copy of the value
   */
  @Override
  public T deepCopy(T source) {

    if (source == null) {
      return null;
    }

    if (datumReader == null) {
      datumReader = createDatumReader(conf);
    }
    if (datumWriter == null) {
      datumWriter = createDatumWriter(conf);
    }
    ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream();
    binaryEncoder = EncoderFactory.get().binaryEncoder(byteOutStream, binaryEncoder);
    T target = createCopyTarget();
    try {
      datumWriter.write(source, binaryEncoder);
      binaryEncoder.flush();
      binaryDecoder =
          DecoderFactory.get().binaryDecoder(byteOutStream.toByteArray(), binaryDecoder);
      return datumReader.read(target, binaryDecoder);
    } catch (Exception e) {
      throw new CrunchRuntimeException("Error while deep copying avro value " + source, e);
    }
  }
コード例 #7
0
ファイル: TestSchema.java プロジェクト: krschultz/avro
  public static void checkBlockingBinary(
      Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader)
      throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    writer.setSchema(schema);
    Encoder encoder = EncoderFactory.get().blockingBinaryEncoder(out, null);
    writer.write(datum, encoder);
    encoder.flush();
    byte[] data = out.toByteArray();

    reader.setSchema(schema);

    Object decoded = reader.read(null, DecoderFactory.get().binaryDecoder(data, null));

    assertEquals("Decoded data does not match.", datum, decoded);
  }
コード例 #8
0
ファイル: AvroUtils.java プロジェクト: jmhsieh/kite
 /**
  * Given an entity, an avro schema, and an encoder, write the entity to the encoder's underlying
  * output stream.
  *
  * @param entity The entity we want to encode.
  * @param encoder The Avro Encoder we will write to.
  * @param writer The DatumWriter we'll use to encode the entity to the encoder.
  */
 public static <T> void writeAvroEntity(T entity, Encoder encoder, DatumWriter<T> writer) {
   try {
     writer.write(entity, encoder);
     encoder.flush();
   } catch (IOException e) {
     throw new SerializationException("Could not serialize Avro entity", e);
   }
 }
コード例 #9
0
ファイル: TestSchema.java プロジェクト: krschultz/avro
  public static void checkBinaryJson(String json) throws Exception {
    Object node = Json.parseJson(json);
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    DatumWriter<Object> writer = new Json.ObjectWriter();
    Encoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    encoder = EncoderFactory.get().validatingEncoder(Json.SCHEMA, encoder);
    writer.write(node, encoder);
    encoder.flush();
    byte[] bytes = out.toByteArray();

    DatumReader<Object> reader = new Json.ObjectReader();
    Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null);
    decoder = DecoderFactory.get().validatingDecoder(Json.SCHEMA, decoder);
    Object decoded = reader.read(null, decoder);

    assertEquals("Decoded json does not match.", Json.toString(node), Json.toString(decoded));
  }
コード例 #10
0
  @Test
  public void testValidate() throws Exception {
    getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
    final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
    manager.storePolicy("user", "info:name", TestProducer.class, new AlwaysFreshen());
    assertEquals(
        BaseTool.SUCCESS,
        runTool(
            new FreshTool(),
            KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
            "--do=validate-all"));

    KijiFreshnessPolicyRecord record =
        KijiFreshnessPolicyRecord.newBuilder()
            .setRecordVersion(ProtocolVersion.parse("policyrecord-0.1").toCanonicalString())
            .setProducerClass(TestProducer.class.getName())
            .setFreshnessPolicyClass(AlwaysFreshen.class.getName())
            .setFreshnessPolicyState("")
            .build();

    final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    final EncoderFactory encoderFactory = EncoderFactory.get();
    Encoder encoder = encoderFactory.directBinaryEncoder(outputStream, null);
    final DatumWriter<KijiFreshnessPolicyRecord> recordWriter =
        new SpecificDatumWriter<KijiFreshnessPolicyRecord>(KijiFreshnessPolicyRecord.SCHEMA$);
    recordWriter.write(record, encoder);
    getKiji()
        .getMetaTable()
        .putValue("user", "kiji.scoring.fresh.columnName", outputStream.toByteArray());

    assertEquals(
        BaseTool.FAILURE,
        runTool(
            new FreshTool(),
            KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
            "--do=validate-all"));
    assertEquals(
        "Freshness policy attached to column: columnName is not valid.", mToolOutputLines[1]);
    assertEquals(
        "NO_FAMILY_IN_TABLE: java.lang.IllegalArgumentException: Table: user does not "
            + "contain family: columnName",
        mToolOutputLines[2]);
  }
コード例 #11
0
ファイル: TestSchema.java プロジェクト: krschultz/avro
  private static void checkJson(Schema schema, Object datum, String json) throws Exception {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out);
    DatumWriter<Object> writer = new GenericDatumWriter<Object>();
    writer.setSchema(schema);
    writer.write(datum, encoder);
    encoder.flush();
    byte[] data = out.toByteArray();

    String encoded = new String(data, "UTF-8");
    assertEquals("Encoded data does not match.", json, encoded);

    DatumReader<Object> reader = new GenericDatumReader<Object>();
    reader.setSchema(schema);
    Object decoded =
        reader.read(null, DecoderFactory.get().jsonDecoder(schema, new ByteArrayInputStream(data)));

    assertEquals("Decoded data does not match.", datum, decoded);
  }
コード例 #12
0
ファイル: TestSchema.java プロジェクト: krschultz/avro
  private static void checkJson(
      Schema schema, Object datum, DatumWriter<Object> writer, DatumReader<Object> reader)
      throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    Encoder encoder = EncoderFactory.get().jsonEncoder(schema, out);
    writer.setSchema(schema);
    writer.write(datum, encoder);
    writer.write(datum, encoder);
    encoder.flush();
    byte[] data = out.toByteArray();

    reader.setSchema(schema);
    Decoder decoder = DecoderFactory.get().jsonDecoder(schema, new ByteArrayInputStream(data));
    Object decoded = reader.read(null, decoder);
    assertEquals("Decoded data does not match.", datum, decoded);

    decoded = reader.read(decoded, decoder);
    assertEquals("Decoded data does not match.", datum, decoded);
  }
コード例 #13
0
ファイル: TestSchema.java プロジェクト: krschultz/avro
 @Test
 public void testEnumMismatch() throws Exception {
   Schema actual = Schema.parse("{\"type\":\"enum\",\"name\":\"E\",\"symbols\":[\"X\",\"Y\"]}");
   Schema expected = Schema.parse("{\"type\":\"enum\",\"name\":\"E\",\"symbols\":[\"Y\",\"Z\"]}");
   ByteArrayOutputStream out = new ByteArrayOutputStream();
   DatumWriter<Object> writer = new GenericDatumWriter<Object>(actual);
   Encoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);
   writer.write(new GenericData.EnumSymbol(actual, "Y"), encoder);
   writer.write(new GenericData.EnumSymbol(actual, "X"), encoder);
   encoder.flush();
   byte[] data = out.toByteArray();
   Decoder decoder = DecoderFactory.get().binaryDecoder(data, null);
   DatumReader<String> in = new GenericDatumReader<String>(actual, expected);
   assertEquals("Wrong value", new GenericData.EnumSymbol(expected, "Y"), in.read(null, decoder));
   try {
     in.read(null, decoder);
     fail("Should have thrown exception.");
   } catch (AvroTypeException e) {
     // expected
   }
 }
コード例 #14
0
ファイル: AvroExample.java プロジェクト: cwilkes/AvroExample
  public void serializeGeneric() throws IOException {
    // Create a datum to serialize.
    Schema schema = new Schema.Parser().parse(getClass().getResourceAsStream("/MyPair.avsc"));
    GenericRecord datum = new GenericData.Record(schema);
    datum.put("left", new Utf8("dog"));
    datum.put("right", new Utf8("cat"));

    // Serialize it.
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(schema);
    Encoder encoder = EncoderFactory.get().binaryEncoder(out, null);
    writer.write(datum, encoder);
    encoder.flush();
    out.close();
    System.out.println("Serialization: " + out);

    // Deserialize it.
    DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(schema);
    BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(out.toByteArray(), null);
    GenericRecord result = reader.read(null, decoder);
    System.out.printf("Left: %s, Right: %s\n", result.get("left"), result.get("right"));
  }
コード例 #15
0
  /**
   * Gives the output message
   *
   * @param outputType output data type
   * @param result mapping result
   * @return the output as a String
   * @throws IOException
   */
  public String getOutputMessage(String outputType, GenericRecord result)
      throws SynapseException, IOException {

    DatumWriter<GenericRecord> writer = null;
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    Encoder encoder = new DummyEncoder(byteArrayOutputStream);
    // OMElement outMessage = null;
    String outMessage = null;

    try {

      writer = WriterRegistry.getInstance().get(outputType).newInstance();
      writer.setSchema(result.getSchema());
      writer.write(result, encoder);

      if (log.isDebugEnabled()) {
        log.debug("Output received from datum writer.." + byteArrayOutputStream.toString());
      }

    } catch (Exception e) {
      handleException("Data coversion Failed at JSONWriter..", e);
    } finally {
      encoder.flush();
    }

    /*try {
    	// Converts the result into an OMElement
    	outMessage = getOutputResult(byteArrayOutputStream.toString());
    } catch (XMLStreamException e) {
    	handleException(
    			"Failed at generating the OMElement for the JSON output received...",
    			e);
    }*/

    outMessage = byteArrayOutputStream.toString();

    return outMessage;
  }
コード例 #16
0
ファイル: DirreqStats.java プロジェクト: tomlurge/converTor
 @Override
 public void writeExternal(java.io.ObjectOutput out) throws java.io.IOException {
   WRITER$.write(this, org.apache.avro.specific.SpecificData.getEncoder(out));
 }