Beispiel #1
0
 @Test
 public void testCollections() throws Exception {
   Collection<String> j = Lists.newArrayList();
   j.add("a");
   j.add("b");
   Schema collectionSchema = Schema.createArray(Avros.strings().getSchema());
   GenericData.Array<Utf8> w = new GenericData.Array<Utf8>(2, collectionSchema);
   w.add(new Utf8("a"));
   w.add(new Utf8("b"));
   testInputOutputFn(Avros.collections(Avros.strings()), j, w);
 }
  // Don't playa hate, avronate.
  public GenericArray<Column> avronateSubColumns(Collection<IColumn> columns) {
    if (columns == null || columns.isEmpty()) return EMPTY_SUBCOLUMNS;

    GenericData.Array<Column> avroColumns =
        new GenericData.Array<Column>(columns.size(), Column.SCHEMA$);

    for (IColumn column : columns) {
      if (column.isMarkedForDelete()) continue;

      Column avroColumn = newColumn(column.name(), column.value(), column.timestamp());
      avroColumns.add(avroColumn);
    }

    return avroColumns;
  }
  /** Parse the given string, return resulting data if appropriate. */
  ParseResult internalParse(
      String s, Map<String, Integer> targetUnionDecisions, boolean mustConsumeStr) {
    boolean hasData = false;
    Schema localSchema = getAvroSchema();
    GenericData.Array gda = new GenericData.Array(5, localSchema);
    Map<String, Integer> curUnionDecisions = new HashMap<String, Integer>();
    String currentStr = s;

    while (true) {
      ParseResult pr = bodyType.internalParse(currentStr, targetUnionDecisions, false);
      if (pr == null) {
        break;
      }
      assert (pr.hasData());

      gda.add(pr.getData());
      currentStr = pr.getRemainingString();
    }
    if (mustConsumeStr && currentStr.trim().length() != 0) {
      return null;
    }
    return new ParseResult(gda, true, currentStr);
  }
Beispiel #4
0
    private Object translate(Object value, DataSchema dataSchema, Schema avroSchema) {
      AvroOverride avroOverride = getAvroOverride(dataSchema);
      if (avroOverride != null) {
        return avroOverride
            .getCustomDataTranslator()
            .dataToAvroGeneric(this, value, dataSchema, avroSchema);
      }

      DataSchema dereferencedDataSchema = dataSchema.getDereferencedDataSchema();
      DataSchema.Type type = dereferencedDataSchema.getType();
      Object result;
      switch (type) {
        case NULL:
          if (value != Data.NULL) {
            appendMessage("value must be null for null schema");
            result = BAD_RESULT;
            break;
          }
          result = null;
          break;
        case BOOLEAN:
          result = ((Boolean) value).booleanValue();
          break;
        case INT:
          result = ((Number) value).intValue();
          break;
        case LONG:
          result = ((Number) value).longValue();
          break;
        case FLOAT:
          result = ((Number) value).floatValue();
          break;
        case DOUBLE:
          result = ((Number) value).doubleValue();
          break;
        case STRING:
          result = new Utf8((String) value);
          break;
        case BYTES:
          result = ByteBuffer.wrap(translateBytes(value));
          break;
        case ENUM:
          String enumValue = value.toString();
          EnumDataSchema enumDataSchema = (EnumDataSchema) dereferencedDataSchema;
          if (enumDataSchema.getSymbols().contains(enumValue) == false) {
            appendMessage(
                "enum value %1$s not one of %2$s", enumValue, enumDataSchema.getSymbols());
            result = BAD_RESULT;
            break;
          }
          result = _avroAdapter.createEnumSymbol(avroSchema, enumValue);
          break;
        case FIXED:
          byte[] bytes = translateBytes(value);
          FixedDataSchema fixedDataSchema = (FixedDataSchema) dereferencedDataSchema;
          if (fixedDataSchema.getSize() != bytes.length) {
            appendMessage(
                "ByteString size %1$d != FixedDataSchema size %2$d",
                bytes.length, fixedDataSchema.getSize());
            result = null;
            break;
          }
          GenericData.Fixed fixed = new GenericData.Fixed(avroSchema);
          fixed.bytes(bytes);
          result = fixed;
          break;
        case MAP:
          DataMap map = (DataMap) value;
          DataSchema valueDataSchema = ((MapDataSchema) dereferencedDataSchema).getValues();
          Schema valueAvroSchema = avroSchema.getValueType();
          Map<String, Object> avroMap = new HashMap<String, Object>(map.size());
          for (Map.Entry<String, Object> entry : map.entrySet()) {
            String key = entry.getKey();
            _path.addLast(key);
            Object entryAvroValue = translate(entry.getValue(), valueDataSchema, valueAvroSchema);
            _path.removeLast();
            avroMap.put(key, entryAvroValue);
          }
          result = avroMap;
          break;
        case ARRAY:
          DataList list = (DataList) value;
          DataSchema elementDataSchema = ((ArrayDataSchema) dereferencedDataSchema).getItems();
          Schema elementAvroSchema = avroSchema.getElementType();
          GenericData.Array<Object> avroList =
              new GenericData.Array<Object>(list.size(), avroSchema);
          for (int i = 0; i < list.size(); i++) {
            _path.addLast(i);
            Object entryAvroValue = translate(list.get(i), elementDataSchema, elementAvroSchema);
            _path.removeLast();
            avroList.add(entryAvroValue);
          }
          result = avroList;
          break;
        case RECORD:
          map = (DataMap) value;
          RecordDataSchema recordDataSchema = (RecordDataSchema) dereferencedDataSchema;
          GenericData.Record avroRecord = new GenericData.Record(avroSchema);
          for (RecordDataSchema.Field field : recordDataSchema.getFields()) {
            String fieldName = field.getName();
            DataSchema fieldDataSchema = field.getType();
            Schema.Field avroField = avroSchema.getField(fieldName);
            if (avroField == null) {
              // field present in input but there is no field for it in Avro schema.
              // TODO: Whether and how to indicate this condition to clients.
              continue;
            }
            _path.addLast(fieldName);
            Schema fieldAvroSchema = avroField.schema();
            Object fieldValue = map.get(fieldName);
            boolean isOptional = field.getOptional();
            if (isOptional) {
              if (fieldDataSchema.getDereferencedType() != DataSchema.Type.UNION) {
                if (fieldValue == null) {
                  fieldValue = Data.NULL;
                  fieldDataSchema = DataSchemaConstants.NULL_DATA_SCHEMA;
                }
                Map.Entry<String, Schema> fieldAvroEntry =
                    findUnionMember(fieldDataSchema, fieldAvroSchema);
                if (fieldAvroEntry == null) {
                  _path.removeLast();
                  continue;
                }
                fieldAvroSchema = fieldAvroEntry.getValue();
              } else {
                // already a union
                if (fieldValue == null) {
                  // field is not present
                  fieldValue = Data.NULL;
                  fieldDataSchema = DataSchemaConstants.NULL_DATA_SCHEMA;
                }
              }
            } else {
              if (fieldValue == null) {
                appendMessage("required field is absent");
                _path.removeLast();
                continue;
              }
            }
            Object fieldAvroValue = translate(fieldValue, fieldDataSchema, fieldAvroSchema);
            avroRecord.put(fieldName, fieldAvroValue);
            _path.removeLast();
          }
          result = avroRecord;
          break;
        case UNION:
          UnionDataSchema unionDataSchema = (UnionDataSchema) dereferencedDataSchema;
          String key;
          Object memberValue;
          if (value == Data.NULL) {
            key = DataSchemaConstants.NULL_TYPE;
            memberValue = Data.NULL;
          } else {
            map = (DataMap) value;
            Map.Entry<String, Object> entry = map.entrySet().iterator().next();
            key = entry.getKey();
            memberValue = entry.getValue();
          }
          DataSchema memberDataSchema = unionDataSchema.getType(key);
          Map.Entry<String, Schema> memberAvroEntry = findUnionMember(memberDataSchema, avroSchema);
          if (memberAvroEntry == null) {
            result = BAD_RESULT;
            break;
          }
          Schema memberAvroSchema = memberAvroEntry.getValue();
          _path.addLast(memberAvroEntry.getKey());
          Object memberAvroValue = translate(memberValue, memberDataSchema, memberAvroSchema);
          _path.removeLast();
          result = memberAvroValue;
          break;
        default:
          appendMessage("schema type unknown %1$s", dereferencedDataSchema.getType());
          result = BAD_RESULT;
          break;
      }
      return result;
    }