private static void handleMapField(
Record record,
Field field,
String fieldPath,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Map<String, Object> defaultValueMap,
Descriptors.FieldDescriptor fieldDescriptor,
DynamicMessage.Builder builder)
throws DataGeneratorException {
Descriptors.Descriptor mapEntryDescriptor = fieldDescriptor.getMessageType();
// MapEntry contains key and value fields
Map<String, Field> sdcMapField = field.getValueAsMap();
for (Map.Entry<String, Field> entry : sdcMapField.entrySet()) {
builder.addRepeatedField(
fieldDescriptor,
DynamicMessage.newBuilder(mapEntryDescriptor)
.setField(mapEntryDescriptor.findFieldByName(KEY), entry.getKey())
.setField(
mapEntryDescriptor.findFieldByName(VALUE),
getValue(
mapEntryDescriptor.findFieldByName(VALUE),
entry.getValue(),
record,
fieldPath + FORWARD_SLASH + entry.getKey(),
messageTypeToExtensionMap,
defaultValueMap))
.build());
}
}
private static void addDefaultsAndExtensions(
Map<String, Set<Descriptors.FieldDescriptor>> e,
Map<String, Object> defaultValueMap,
Descriptors.Descriptor d) {
for (Descriptors.FieldDescriptor fieldDescriptor : d.getExtensions()) {
String containingType = fieldDescriptor.getContainingType().getFullName();
Set<Descriptors.FieldDescriptor> fieldDescriptors = e.get(containingType);
if (fieldDescriptors == null) {
fieldDescriptors = new LinkedHashSet<>();
e.put(containingType, fieldDescriptors);
}
fieldDescriptors.add(fieldDescriptor);
if (fieldDescriptor.hasDefaultValue()) {
defaultValueMap.put(
fieldDescriptor.getContainingType().getFullName() + "." + fieldDescriptor.getName(),
fieldDescriptor.getDefaultValue());
}
}
for (Descriptors.FieldDescriptor fieldDescriptor : d.getFields()) {
if (fieldDescriptor.hasDefaultValue()) {
defaultValueMap.put(
d.getFullName() + "." + fieldDescriptor.getName(), fieldDescriptor.getDefaultValue());
}
}
for (Descriptors.Descriptor nestedType : d.getNestedTypes()) {
addDefaultsAndExtensions(e, defaultValueMap, nestedType);
}
}
/**
* Serializes a field path in a record to a protobuf message using the specified descriptor.
*
* @param record Record with the field to serialize
* @param field The field to serialize
* @param fieldPath The field path of the specified field
* @param desc Protobuf descriptor
* @param messageTypeToExtensionMap Protobuf extension map
* @param defaultValueMap Protobuf default field values
* @return serialized message
* @throws DataGeneratorException
*/
private static DynamicMessage sdcFieldToProtobufMsg(
Record record,
Field field,
String fieldPath,
Descriptors.Descriptor desc,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Map<String, Object> defaultValueMap)
throws DataGeneratorException {
if (field == null) {
return null;
}
// compute all fields to look for including extensions
DynamicMessage.Builder builder = DynamicMessage.newBuilder(desc);
List<Descriptors.FieldDescriptor> fields = new ArrayList<>();
fields.addAll(desc.getFields());
if (messageTypeToExtensionMap.containsKey(desc.getFullName())) {
fields.addAll(messageTypeToExtensionMap.get(desc.getFullName()));
}
// root field is always a Map in a record representing protobuf data
Map<String, Field> valueAsMap = field.getValueAsMap();
for (Descriptors.FieldDescriptor f : fields) {
Field mapField = valueAsMap.get(f.getName());
// Repeated field
if (f.isMapField()) {
handleMapField(
record, mapField, fieldPath, messageTypeToExtensionMap, defaultValueMap, f, builder);
} else if (f.isRepeated()) {
handleRepeatedField(
record, mapField, fieldPath, messageTypeToExtensionMap, defaultValueMap, f, builder);
} else {
// non repeated field
handleNonRepeatedField(
record,
valueAsMap,
fieldPath,
messageTypeToExtensionMap,
defaultValueMap,
desc,
f,
builder);
}
}
// if record has unknown fields for this field path, handle it
try {
handleUnknownFields(record, fieldPath, builder);
} catch (IOException e) {
throw new DataGeneratorException(Errors.PROTOBUF_05, e.toString(), e);
}
return builder.build();
}
private static void handleNonRepeatedField(
Record record,
Map<String, Field> valueAsMap,
String fieldPath,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Map<String, Object> defaultValueMap,
Descriptors.Descriptor desc,
Descriptors.FieldDescriptor f,
DynamicMessage.Builder builder)
throws DataGeneratorException {
Object val;
String keyName = f.getName();
if (valueAsMap.containsKey(keyName)) {
val =
getValue(
f,
valueAsMap.get(keyName),
record,
fieldPath + FORWARD_SLASH + f.getName(),
messageTypeToExtensionMap,
defaultValueMap);
} else {
// record does not contain field, look up default value
String key = desc.getFullName() + "." + f.getName();
if (!defaultValueMap.containsKey(key) && !f.isOptional()) {
throw new DataGeneratorException(Errors.PROTOBUF_04, record.getHeader().getSourceId(), key);
}
val = defaultValueMap.get(key);
}
if (val != null) {
builder.setField(f, val);
}
}
/**
* Converts a protobuf message to an SDC Record Field.
*
* @param record SDC Record to add field to
* @param fieldPath location in record where to insert field.
* @param descriptor protobuf descriptor instance
* @param messageTypeToExtensionMap protobuf extensions map
* @param message message to decode and insert into the specified field path
* @return new Field instance representing the decoded message
* @throws DataParserException
*/
public static Field protobufToSdcField(
Record record,
String fieldPath,
Descriptors.Descriptor descriptor,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Object message)
throws DataParserException {
Map<String, Field> sdcRecordMapFieldValue = new HashMap<>();
// get all the expected fields from the proto file
Map<String, Descriptors.FieldDescriptor> protobufFields = new LinkedHashMap<>();
for (Descriptors.FieldDescriptor fieldDescriptor : descriptor.getFields()) {
protobufFields.put(fieldDescriptor.getName(), fieldDescriptor);
}
// get all fields in the read message
Map<Descriptors.FieldDescriptor, Object> values = ((DynamicMessage) message).getAllFields();
// for every field present in the proto definition create an sdc field.
for (Descriptors.FieldDescriptor fieldDescriptor : protobufFields.values()) {
Object value = values.get(fieldDescriptor);
sdcRecordMapFieldValue.put(
fieldDescriptor.getName(),
createField(record, fieldPath, fieldDescriptor, messageTypeToExtensionMap, value));
}
// handle applicable extensions for this message type
if (messageTypeToExtensionMap.containsKey(descriptor.getFullName())) {
for (Descriptors.FieldDescriptor fieldDescriptor :
messageTypeToExtensionMap.get(descriptor.getFullName())) {
if (values.containsKey(fieldDescriptor)) {
Object value = values.get(fieldDescriptor);
sdcRecordMapFieldValue.put(
fieldDescriptor.getName(),
createField(record, fieldPath, fieldDescriptor, messageTypeToExtensionMap, value));
}
}
}
// handle unknown fields
// unknown fields can go into the record header
UnknownFieldSet unknownFields = ((DynamicMessage) message).getUnknownFields();
if (!unknownFields.asMap().isEmpty()) {
ByteArrayOutputStream bOut = new ByteArrayOutputStream();
try {
unknownFields.writeDelimitedTo(bOut);
bOut.flush();
bOut.close();
} catch (IOException e) {
throw new DataParserException(Errors.PROTOBUF_10, e.toString(), e);
}
String path = fieldPath.isEmpty() ? FORWARD_SLASH : fieldPath;
byte[] bytes = org.apache.commons.codec.binary.Base64.encodeBase64(bOut.toByteArray());
record
.getHeader()
.setAttribute(
PROTOBUF_UNKNOWN_FIELDS_PREFIX + path, new String(bytes, StandardCharsets.UTF_8));
}
return Field.create(sdcRecordMapFieldValue);
}
