@Override
 protected boolean doProcess(Record record) {
   if (preserveExisting
       && record.getFields().containsKey(fieldName)) {; // we must preserve the existing id
   } else {
     record.replaceValues(fieldName, generateUUID());
   }
   return super.doProcess(record);
 }
示例#2
0
    @Override
    protected boolean doProcess(Record inputRecord) {
      //
      // Preconditions.checkState(ReadAvroBuilder.AVRO_MEMORY_MIME_TYPE.equals(inputRecord.getFirstValue(Fields.ATTACHMENT_MIME_TYPE)));
      GenericContainer datum = (GenericContainer) inputRecord.getFirstValue(Fields.ATTACHMENT_BODY);
      Preconditions.checkNotNull(datum);
      Preconditions.checkNotNull(datum.getSchema());
      Record outputRecord = inputRecord.copy();

      extractTree(datum, datum.getSchema(), outputRecord, outputFieldPrefix);

      // pass record to next command in chain:
      return getChild().process(outputRecord);
    }
 @Override
 public void process(Event event) {
   Record record = new Record();
   for (Entry<String, String> entry : event.getHeaders().entrySet()) {
     record.put(entry.getKey(), entry.getValue());
   }
   byte[] bytes = event.getBody();
   if (bytes != null && bytes.length > 0) {
     record.put(Fields.ATTACHMENT_BODY, bytes);
   }
   try {
     Notifications.notifyStartSession(morphline);
     if (!morphline.process(record)) {
       LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record);
     }
   } catch (RuntimeException t) {
     morphlineContext.getExceptionHandler().handleException(t, record);
   }
 }
示例#4
0
    @Override
    protected boolean doProcess(Record inputRecord, InputStream stream) throws IOException {
      if (counter++ % 8192 == 0) {
        clob = new StringBuilder(); // periodically gc memory from large outlier strings
      }
      incrementNumRecords();
      Charset detectedCharset = detectCharset(inputRecord, charset);
      Reader reader = new InputStreamReader(stream, detectedCharset);
      clob.setLength(0);
      int len;
      while ((len = reader.read(buffer)) >= 0) {
        clob.append(buffer, 0, len);
      }
      Record outputRecord = inputRecord.copy();
      removeAttachments(outputRecord);
      outputRecord.replaceValues(Fields.MESSAGE, clob.toString());

      // pass record to next command in chain:
      return getChild().process(outputRecord);
    }
示例#5
0
 /** Writes the given Avro datum into the given record, using the given Avro schema */
 private void extractTree(Object datum, Schema schema, Record outputRecord, String prefix) {
   // RECORD, ENUM, ARRAY, MAP, UNION, FIXED, STRING, BYTES, INT, LONG, FLOAT,
   // DOUBLE, BOOLEAN, NULL
   switch (schema.getType()) {
     case RECORD:
       {
         IndexedRecord avroRecord = (IndexedRecord) datum;
         String prefix2 = prefix + "/";
         for (Field field : schema.getFields()) {
           extractTree(
               avroRecord.get(field.pos()),
               field.schema(),
               outputRecord,
               prefix2 + field.name());
         }
         break;
       }
     case ENUM:
       {
         GenericEnumSymbol symbol = (GenericEnumSymbol) datum;
         outputRecord.put(prefix, symbol.toString());
         break;
       }
     case ARRAY:
       {
         Iterator iter = ((Collection) datum).iterator();
         while (iter.hasNext()) {
           extractTree(iter.next(), schema.getElementType(), outputRecord, prefix);
         }
         break;
       }
     case MAP:
       {
         Map<CharSequence, ?> map = (Map<CharSequence, ?>) datum;
         for (Map.Entry<CharSequence, ?> entry : map.entrySet()) {
           extractTree(
               entry.getValue(),
               schema.getValueType(),
               outputRecord,
               prefix + "/" + entry.getKey().toString());
         }
         break;
       }
     case UNION:
       {
         int index = GenericData.get().resolveUnion(schema, datum);
         // String typeName = schema.getTypes().get(index).getName();
         // String prefix2 = prefix + "/" + typeName;
         String prefix2 = prefix;
         extractTree(datum, schema.getTypes().get(index), outputRecord, prefix2);
         break;
       }
     case FIXED:
       {
         GenericFixed fixed = (GenericFixed) datum;
         outputRecord.put(prefix, fixed.bytes());
         // outputRecord.put(prefix, utf8toString(fixed.bytes()));
         break;
       }
     case BYTES:
       {
         ByteBuffer buf = (ByteBuffer) datum;
         int pos = buf.position();
         byte[] bytes = new byte[buf.remaining()];
         buf.get(bytes);
         buf.position(pos); // undo relative read
         outputRecord.put(prefix, bytes);
         // outputRecord.put(prefix, utf8toString(bytes));
         break;
       }
     case STRING:
       {
         outputRecord.put(prefix, datum.toString());
         break;
       }
     case INT:
       {
         outputRecord.put(prefix, datum);
         break;
       }
     case LONG:
       {
         outputRecord.put(prefix, datum);
         break;
       }
     case FLOAT:
       {
         outputRecord.put(prefix, datum);
         break;
       }
     case DOUBLE:
       {
         outputRecord.put(prefix, datum);
         break;
       }
     case BOOLEAN:
       {
         outputRecord.put(prefix, datum);
         break;
       }
     case NULL:
       {
         break;
       }
     default:
       throw new MorphlineRuntimeException("Unknown Avro schema type: " + schema.getType());
   }
 }