Пример #1
0
 @SuppressWarnings("unchecked")
 public static byte[] toBytes(Object o, Schema schema) throws IOException {
   Type type = schema.getType();
   switch (type) {
     case STRING:
       return Bytes.toBytes(((Utf8) o).toString()); // TODO: maybe ((Utf8)o).getBytes(); ?
     case BYTES:
       return ((ByteBuffer) o).array();
     case INT:
       return Bytes.toBytes((Integer) o);
     case LONG:
       return Bytes.toBytes((Long) o);
     case FLOAT:
       return Bytes.toBytes((Float) o);
     case DOUBLE:
       return Bytes.toBytes((Double) o);
     case BOOLEAN:
       return (Boolean) o ? new byte[] {1} : new byte[] {0};
     case ENUM:
       return new byte[] {(byte) ((Enum<?>) o).ordinal()};
     case RECORD:
       // TODO: This is TOO SLOW... OPTIMIZE
       writer.setSchema(schema);
       ByteArrayOutputStream os = new ByteArrayOutputStream();
       BinaryEncoder encoder = new BinaryEncoder(os);
       writer.write(o, encoder);
       encoder.flush();
       return os.toByteArray();
     default:
       throw new RuntimeException("Unknown type: " + type);
   }
 }
Пример #2
0
  @Override
  public void persist() {
    if (hasUpdate) {
      ByteArrayOutputStream baos = new ByteArrayOutputStream();
      BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(baos, null);
      SpecificDatumWriter<Topic> datumWriter = new SpecificDatumWriter<>(Topic.class);
      try {
        for (Topic topic : topicMap.values()) {
          datumWriter.write(topic, encoder);
          LOG.info("Persisted {}", topic);
        }
        encoder.flush();
        String base64Str =
            new String(base64.encodeBase64(baos.toByteArray()), Charset.forName("UTF-8"));
        state.setProperty(TOPIC_LIST, base64Str);
      } catch (IOException e) {
        LOG.error("Can't persist topic list info", e);
      }

      baos = new ByteArrayOutputStream();
      try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
        oos.writeObject(nfSubscriptions);
        String base64Str =
            new String(base64.encodeBase64(baos.toByteArray()), Charset.forName("UTF-8"));
        state.setProperty(NF_SUBSCRIPTIONS, base64Str);
      } catch (IOException e) {
        LOG.error("Can't persist notification subscription info", e);
      }

      StringBuilder attachedEndpointsString = new StringBuilder();
      for (Map.Entry<EndpointAccessToken, EndpointKeyHash> attached :
          attachedEndpoints.entrySet()) {
        attachedEndpointsString
            .append(attached.getKey().getToken())
            .append(":")
            .append(attached.getValue().getKeyHash())
            .append(',');
      }
      state.setProperty(ATTACHED_ENDPOINTS, attachedEndpointsString.toString());
      state.setProperty(EVENT_SEQ_NUM, "" + eventSequence.get());
      if (topicListHash != null) {
        state.setProperty(TOPIC_LIST_HASH, "" + topicListHash);
      }

      OutputStream os = null;
      try {
        storage.renameTo(stateFileLocation, stateFileLocation + "_bckp");
        os = storage.openForWrite(stateFileLocation);
        state.store(os, null);
        hasUpdate = false;
      } catch (IOException e) {
        LOG.error("Can't persist state file", e);
      } finally {
        IOUtils.closeQuietly(os);
      }
    }
  }
 private byte[] serializeAvro(Object datum, Schema schema) throws IOException {
   ByteArrayOutputStream out = new ByteArrayOutputStream();
   ReflectDatumWriter<Object> writer = new ReflectDatumWriter<Object>(schema);
   BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null);
   out.reset();
   writer.write(datum, encoder);
   encoder.flush();
   return out.toByteArray();
 }
 private int testAvroSer(GenericRecord value, int reps, ByteArrayOutputStream result)
     throws Exception {
   BinaryEncoder avroEncoder = null;
   for (int i = 0; i < reps; ++i) {
     result.reset();
     // reuse?
     // avroEncoder = ENCODER_FACTORY.binaryEncoder(result, null);
     avroEncoder = ENCODER_FACTORY.binaryEncoder(result, avroEncoder);
     WRITER.write(value, avroEncoder);
     avroEncoder.flush();
   }
   return result.size(); // just to get some non-optimizable number
 }
Пример #5
0
  /**
   * Create a deep copy of an Avro value.
   *
   * @param source The value to be copied
   * @return The deep copy of the value
   */
  @Override
  public T deepCopy(T source) {

    if (source == null) {
      return null;
    }

    if (datumReader == null) {
      datumReader = createDatumReader(conf);
    }
    if (datumWriter == null) {
      datumWriter = createDatumWriter(conf);
    }
    ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream();
    binaryEncoder = EncoderFactory.get().binaryEncoder(byteOutStream, binaryEncoder);
    T target = createCopyTarget();
    try {
      datumWriter.write(source, binaryEncoder);
      binaryEncoder.flush();
      binaryDecoder =
          DecoderFactory.get().binaryDecoder(byteOutStream.toByteArray(), binaryDecoder);
      return datumReader.read(target, binaryDecoder);
    } catch (Exception e) {
      throw new CrunchRuntimeException("Error while deep copying avro value " + source, e);
    }
  }
Пример #6
0
  public byte[] serialize(T t) {

    byte[] bytes = null;

    SpecificDatumWriter<T> writer = new SpecificDatumWriter<>(t.getSchema());

    try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) {

      BinaryEncoder binaryEncoder = EncoderFactory.get().binaryEncoder(stream, null);
      writer.write(t, binaryEncoder);
      binaryEncoder.flush();

      bytes = stream.toByteArray();
    } catch (Exception e) {

      e.printStackTrace();
    }

    return bytes;
  }