/**
* Populates a map of protobuf extensions and map with the default values for each message field
* from a map of file descriptors.
*
* @param fileDescriptorMap Map of file descriptors
* @param typeToExtensionMap Map of extensions to populate
* @param defaultValueMap Map of default values to populate
*/
public static void populateDefaultsAndExtensions(
Map<String, Descriptors.FileDescriptor> fileDescriptorMap,
Map<String, Set<Descriptors.FieldDescriptor>> typeToExtensionMap,
Map<String, Object> defaultValueMap) {
for (Descriptors.FileDescriptor f : fileDescriptorMap.values()) {
// go over every file descriptor and look for extensions and default values of those
// extensions
for (Descriptors.FieldDescriptor fieldDescriptor : f.getExtensions()) {
String containingType = fieldDescriptor.getContainingType().getFullName();
Set<Descriptors.FieldDescriptor> fieldDescriptors = typeToExtensionMap.get(containingType);
if (fieldDescriptors == null) {
fieldDescriptors = new LinkedHashSet<>();
typeToExtensionMap.put(containingType, fieldDescriptors);
}
fieldDescriptors.add(fieldDescriptor);
if (fieldDescriptor.hasDefaultValue()) {
defaultValueMap.put(
containingType + "." + fieldDescriptor.getName(), fieldDescriptor.getDefaultValue());
}
}
// go over messages within file descriptor and look for all fields and extensions and their
// defaults
for (Descriptors.Descriptor d : f.getMessageTypes()) {
addDefaultsAndExtensions(typeToExtensionMap, defaultValueMap, d);
}
}
}
private static void handleNonRepeatedField(
Record record,
Map<String, Field> valueAsMap,
String fieldPath,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Map<String, Object> defaultValueMap,
Descriptors.Descriptor desc,
Descriptors.FieldDescriptor f,
DynamicMessage.Builder builder)
throws DataGeneratorException {
Object val;
String keyName = f.getName();
if (valueAsMap.containsKey(keyName)) {
val =
getValue(
f,
valueAsMap.get(keyName),
record,
fieldPath + FORWARD_SLASH + f.getName(),
messageTypeToExtensionMap,
defaultValueMap);
} else {
// record does not contain field, look up default value
String key = desc.getFullName() + "." + f.getName();
if (!defaultValueMap.containsKey(key) && !f.isOptional()) {
throw new DataGeneratorException(Errors.PROTOBUF_04, record.getHeader().getSourceId(), key);
}
val = defaultValueMap.get(key);
}
if (val != null) {
builder.setField(f, val);
}
}
private static void addDefaultsAndExtensions(
Map<String, Set<Descriptors.FieldDescriptor>> e,
Map<String, Object> defaultValueMap,
Descriptors.Descriptor d) {
for (Descriptors.FieldDescriptor fieldDescriptor : d.getExtensions()) {
String containingType = fieldDescriptor.getContainingType().getFullName();
Set<Descriptors.FieldDescriptor> fieldDescriptors = e.get(containingType);
if (fieldDescriptors == null) {
fieldDescriptors = new LinkedHashSet<>();
e.put(containingType, fieldDescriptors);
}
fieldDescriptors.add(fieldDescriptor);
if (fieldDescriptor.hasDefaultValue()) {
defaultValueMap.put(
fieldDescriptor.getContainingType().getFullName() + "." + fieldDescriptor.getName(),
fieldDescriptor.getDefaultValue());
}
}
for (Descriptors.FieldDescriptor fieldDescriptor : d.getFields()) {
if (fieldDescriptor.hasDefaultValue()) {
defaultValueMap.put(
d.getFullName() + "." + fieldDescriptor.getName(), fieldDescriptor.getDefaultValue());
}
}
for (Descriptors.Descriptor nestedType : d.getNestedTypes()) {
addDefaultsAndExtensions(e, defaultValueMap, nestedType);
}
}
private static Object getValue(
Descriptors.FieldDescriptor f,
Field field,
Record record,
String protoFieldPath,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Map<String, Object> defaultValueMap)
throws DataGeneratorException {
Object value = null;
try {
if (field.getValue() != null) {
switch (f.getJavaType()) {
case BOOLEAN:
value = field.getValueAsBoolean();
break;
case BYTE_STRING:
value = ByteString.copyFrom(field.getValueAsByteArray());
break;
case DOUBLE:
value = field.getValueAsDouble();
break;
case ENUM:
value = f.getEnumType().findValueByName(field.getValueAsString());
break;
case FLOAT:
value = field.getValueAsFloat();
break;
case INT:
value = field.getValueAsInteger();
break;
case LONG:
value = field.getValueAsLong();
break;
case STRING:
value = field.getValueAsString();
break;
case MESSAGE:
Descriptors.Descriptor messageType = f.getMessageType();
value =
sdcFieldToProtobufMsg(
record,
field,
protoFieldPath,
messageType,
messageTypeToExtensionMap,
defaultValueMap);
break;
default:
throw new DataGeneratorException(Errors.PROTOBUF_03, f.getJavaType().name());
}
}
} catch (IllegalArgumentException e) {
throw new DataGeneratorException(
Errors.PROTOBUF_11, field.getValue(), f.getJavaType().name(), e);
}
return value;
}
/**
* Serializes a field path in a record to a protobuf message using the specified descriptor.
*
* @param record Record with the field to serialize
* @param field The field to serialize
* @param fieldPath The field path of the specified field
* @param desc Protobuf descriptor
* @param messageTypeToExtensionMap Protobuf extension map
* @param defaultValueMap Protobuf default field values
* @return serialized message
* @throws DataGeneratorException
*/
private static DynamicMessage sdcFieldToProtobufMsg(
Record record,
Field field,
String fieldPath,
Descriptors.Descriptor desc,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Map<String, Object> defaultValueMap)
throws DataGeneratorException {
if (field == null) {
return null;
}
// compute all fields to look for including extensions
DynamicMessage.Builder builder = DynamicMessage.newBuilder(desc);
List<Descriptors.FieldDescriptor> fields = new ArrayList<>();
fields.addAll(desc.getFields());
if (messageTypeToExtensionMap.containsKey(desc.getFullName())) {
fields.addAll(messageTypeToExtensionMap.get(desc.getFullName()));
}
// root field is always a Map in a record representing protobuf data
Map<String, Field> valueAsMap = field.getValueAsMap();
for (Descriptors.FieldDescriptor f : fields) {
Field mapField = valueAsMap.get(f.getName());
// Repeated field
if (f.isMapField()) {
handleMapField(
record, mapField, fieldPath, messageTypeToExtensionMap, defaultValueMap, f, builder);
} else if (f.isRepeated()) {
handleRepeatedField(
record, mapField, fieldPath, messageTypeToExtensionMap, defaultValueMap, f, builder);
} else {
// non repeated field
handleNonRepeatedField(
record,
valueAsMap,
fieldPath,
messageTypeToExtensionMap,
defaultValueMap,
desc,
f,
builder);
}
}
// if record has unknown fields for this field path, handle it
try {
handleUnknownFields(record, fieldPath, builder);
} catch (IOException e) {
throw new DataGeneratorException(Errors.PROTOBUF_05, e.toString(), e);
}
return builder.build();
}
private static void handleMapField(
Record record,
Field field,
String fieldPath,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Map<String, Object> defaultValueMap,
Descriptors.FieldDescriptor fieldDescriptor,
DynamicMessage.Builder builder)
throws DataGeneratorException {
Descriptors.Descriptor mapEntryDescriptor = fieldDescriptor.getMessageType();
// MapEntry contains key and value fields
Map<String, Field> sdcMapField = field.getValueAsMap();
for (Map.Entry<String, Field> entry : sdcMapField.entrySet()) {
builder.addRepeatedField(
fieldDescriptor,
DynamicMessage.newBuilder(mapEntryDescriptor)
.setField(mapEntryDescriptor.findFieldByName(KEY), entry.getKey())
.setField(
mapEntryDescriptor.findFieldByName(VALUE),
getValue(
mapEntryDescriptor.findFieldByName(VALUE),
entry.getValue(),
record,
fieldPath + FORWARD_SLASH + entry.getKey(),
messageTypeToExtensionMap,
defaultValueMap))
.build());
}
}
private static Field createSdcField(
Record record,
String fieldPath,
Descriptors.FieldDescriptor fieldDescriptor,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Object value)
throws DataParserException {
Field f;
switch (fieldDescriptor.getJavaType()) {
case BOOLEAN:
f = Field.create(Field.Type.BOOLEAN, value);
break;
case BYTE_STRING:
f = Field.create(Field.Type.BYTE_ARRAY, ((ByteString) value).toByteArray());
break;
case DOUBLE:
f = Field.create(Field.Type.DOUBLE, value);
break;
case ENUM:
f = Field.create(Field.Type.STRING, ((Descriptors.EnumValueDescriptor) value).getName());
break;
case FLOAT:
f = Field.create(Field.Type.FLOAT, value);
break;
case INT:
f = Field.create(Field.Type.INTEGER, value);
break;
case LONG:
f = Field.create(Field.Type.LONG, value);
break;
case STRING:
f = Field.create(Field.Type.STRING, value);
break;
case MESSAGE:
f =
protobufToSdcField(
record,
fieldPath + FORWARD_SLASH + fieldDescriptor.getName(),
fieldDescriptor.getMessageType(),
messageTypeToExtensionMap,
value);
break;
default:
throw new DataParserException(Errors.PROTOBUF_03, fieldDescriptor.getJavaType().name());
}
return f;
}
private static void handleRepeatedField(
Record record,
Field field,
String fieldPath,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Map<String, Object> defaultValueMap,
Descriptors.FieldDescriptor f,
DynamicMessage.Builder builder)
throws DataGeneratorException {
List<Object> toReturn = new ArrayList<>();
List<Field> valueAsList = field.getValueAsList();
if (valueAsList != null) {
// According to proto 2 and 3 language guide repeated fields can have 0 elements.
// Also null is treated as empty in case of json mappings so I guess we can ignore if it is
// null.
for (int i = 0; i < valueAsList.size(); i++) {
if (f.getJavaType() == Descriptors.FieldDescriptor.JavaType.MESSAGE) {
// repeated field of type message
toReturn.add(
sdcFieldToProtobufMsg(
record,
valueAsList.get(i),
fieldPath + FORWARD_SLASH + f.getName() + "[" + i + "]",
f.getMessageType(),
messageTypeToExtensionMap,
defaultValueMap));
} else {
// repeated field of primitive types
toReturn.add(
getValue(
f,
valueAsList.get(i),
record,
fieldPath + FORWARD_SLASH + f.getName(),
messageTypeToExtensionMap,
defaultValueMap));
}
}
}
builder.setField(f, toReturn);
}
private static void handleRepeatedField(
Record record,
Field field,
String fieldPath,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Map<String, Object> defaultValueMap,
Descriptors.FieldDescriptor f,
DynamicMessage.Builder builder)
throws DataGeneratorException {
List<Object> toReturn = new ArrayList<>();
List<Field> valueAsList = field.getValueAsList();
for (int i = 0; i < valueAsList.size(); i++) {
if (f.getJavaType() == Descriptors.FieldDescriptor.JavaType.MESSAGE) {
// repeated field of type message
toReturn.add(
sdcFieldToProtobufMsg(
record,
valueAsList.get(i),
fieldPath + FORWARD_SLASH + f.getName() + "[" + i + "]",
f.getMessageType(),
messageTypeToExtensionMap,
defaultValueMap));
} else {
// repeated field of primitive types
toReturn.add(
getValue(
f,
valueAsList.get(i),
record,
fieldPath + FORWARD_SLASH + f.getName(),
messageTypeToExtensionMap,
defaultValueMap));
}
}
builder.setField(f, toReturn);
}
/**
* Creates an SDC Record Field from the provided protobuf message and descriptor.
*
* @param record record to be augmented
* @param fieldPath field path in the record to insert new Field
* @param fieldDescriptor descriptor for this message
* @param messageTypeToExtensionMap protobuf type extensions
* @param message protobuf message to decode
* @return reference to the Field added to the record.
* @throws DataParserException
*/
@SuppressWarnings("unchecked")
private static Field createField(
Record record,
String fieldPath,
Descriptors.FieldDescriptor fieldDescriptor,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Object message)
throws DataParserException {
Field newField;
if (message == null) {
// If the message does not contain required fields then builder.build() throws
// UninitializedMessageException
Object defaultValue = null;
// get default values only for optional fields and non-message types
if (fieldDescriptor.isOptional()
&& fieldDescriptor.getJavaType() != Descriptors.FieldDescriptor.JavaType.MESSAGE) {
defaultValue = fieldDescriptor.getDefaultValue();
}
newField = Field.create(getFieldType(fieldDescriptor.getJavaType()), defaultValue);
} else if (fieldDescriptor.isMapField()) {
// Map entry (protobuf 3 map)
Map<String, Field> sdcMapFieldValues = new HashMap<>();
Collection<DynamicMessage> mapEntries = (Collection<DynamicMessage>) message;
// MapEntry
for (DynamicMessage dynamicMessage : mapEntries) {
// MapEntry has 2 fields, key and value
Map<Descriptors.FieldDescriptor, Object> kv = dynamicMessage.getAllFields();
String key = null;
Object value = null;
Descriptors.FieldDescriptor valueDescriptor = null;
for (Map.Entry<Descriptors.FieldDescriptor, Object> entry : kv.entrySet()) {
switch (entry.getKey().getName()) {
case KEY:
key = entry.getValue().toString();
break;
case VALUE:
value = entry.getValue();
valueDescriptor = entry.getKey();
break;
default:
throw new DataParserException(Errors.PROTOBUF_09, entry.getKey().getName());
}
}
if (key != null && valueDescriptor != null) {
sdcMapFieldValues.put(
key,
createSdcField(record, fieldPath, valueDescriptor, messageTypeToExtensionMap, value));
}
}
newField = Field.create(sdcMapFieldValues);
} else if (fieldDescriptor.isRepeated()) {
// List entry (repeated)
List<?> list = (List<?>) message;
List<Field> listField = new ArrayList<>();
for (int i = 0; i < list.size(); i++) {
if (fieldDescriptor.getJavaType() == Descriptors.FieldDescriptor.JavaType.MESSAGE) {
listField.add(
protobufToSdcField(
record,
fieldPath + "[" + i + "]",
fieldDescriptor.getMessageType(),
messageTypeToExtensionMap,
list.get(i)));
} else {
listField.add(
createSdcField(
record,
fieldPath + "[" + i + "]",
fieldDescriptor,
messageTypeToExtensionMap,
list.get(i)));
}
}
newField = Field.create(listField);
} else {
// normal entry
newField =
createSdcField(record, fieldPath, fieldDescriptor, messageTypeToExtensionMap, message);
}
return newField;
}
/**
* Converts a protobuf message to an SDC Record Field.
*
* @param record SDC Record to add field to
* @param fieldPath location in record where to insert field.
* @param descriptor protobuf descriptor instance
* @param messageTypeToExtensionMap protobuf extensions map
* @param message message to decode and insert into the specified field path
* @return new Field instance representing the decoded message
* @throws DataParserException
*/
public static Field protobufToSdcField(
Record record,
String fieldPath,
Descriptors.Descriptor descriptor,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Object message)
throws DataParserException {
Map<String, Field> sdcRecordMapFieldValue = new HashMap<>();
// get all the expected fields from the proto file
Map<String, Descriptors.FieldDescriptor> protobufFields = new LinkedHashMap<>();
for (Descriptors.FieldDescriptor fieldDescriptor : descriptor.getFields()) {
protobufFields.put(fieldDescriptor.getName(), fieldDescriptor);
}
// get all fields in the read message
Map<Descriptors.FieldDescriptor, Object> values = ((DynamicMessage) message).getAllFields();
// for every field present in the proto definition create an sdc field.
for (Descriptors.FieldDescriptor fieldDescriptor : protobufFields.values()) {
Object value = values.get(fieldDescriptor);
sdcRecordMapFieldValue.put(
fieldDescriptor.getName(),
createField(record, fieldPath, fieldDescriptor, messageTypeToExtensionMap, value));
}
// handle applicable extensions for this message type
if (messageTypeToExtensionMap.containsKey(descriptor.getFullName())) {
for (Descriptors.FieldDescriptor fieldDescriptor :
messageTypeToExtensionMap.get(descriptor.getFullName())) {
if (values.containsKey(fieldDescriptor)) {
Object value = values.get(fieldDescriptor);
sdcRecordMapFieldValue.put(
fieldDescriptor.getName(),
createField(record, fieldPath, fieldDescriptor, messageTypeToExtensionMap, value));
}
}
}
// handle unknown fields
// unknown fields can go into the record header
UnknownFieldSet unknownFields = ((DynamicMessage) message).getUnknownFields();
if (!unknownFields.asMap().isEmpty()) {
ByteArrayOutputStream bOut = new ByteArrayOutputStream();
try {
unknownFields.writeDelimitedTo(bOut);
bOut.flush();
bOut.close();
} catch (IOException e) {
throw new DataParserException(Errors.PROTOBUF_10, e.toString(), e);
}
String path = fieldPath.isEmpty() ? FORWARD_SLASH : fieldPath;
byte[] bytes = org.apache.commons.codec.binary.Base64.encodeBase64(bOut.toByteArray());
record
.getHeader()
.setAttribute(
PROTOBUF_UNKNOWN_FIELDS_PREFIX + path, new String(bytes, StandardCharsets.UTF_8));
}
return Field.create(sdcRecordMapFieldValue);
}
/**
* Gets requirements from all configs. Merges whitelists of requirements with 'extends' equal to
* 'rule_id' of other rule.
*/
static List<Requirement> mergeRequirements(
AbstractCompiler compiler, List<ConformanceConfig> configs) {
List<Requirement.Builder> builders = new ArrayList<>();
Map<String, Requirement.Builder> extendable = new HashMap<>();
for (ConformanceConfig config : configs) {
for (Requirement requirement : config.getRequirementList()) {
Requirement.Builder builder = requirement.toBuilder();
if (requirement.hasRuleId()) {
if (requirement.getRuleId().isEmpty()) {
reportInvalidRequirement(compiler, requirement, "empty rule_id");
continue;
}
if (extendable.containsKey(requirement.getRuleId())) {
reportInvalidRequirement(
compiler,
requirement,
"two requirements with the same rule_id: " + requirement.getRuleId());
continue;
}
extendable.put(requirement.getRuleId(), builder);
}
if (!requirement.hasExtends()) {
builders.add(builder);
}
}
}
for (ConformanceConfig config : configs) {
for (Requirement requirement : config.getRequirementList()) {
if (requirement.hasExtends()) {
Requirement.Builder existing = extendable.get(requirement.getExtends());
if (existing == null) {
reportInvalidRequirement(
compiler, requirement, "no requirement with rule_id: " + requirement.getExtends());
continue;
}
for (Descriptors.FieldDescriptor field : requirement.getAllFields().keySet()) {
if (!EXTENDABLE_FIELDS.contains(field.getName())) {
reportInvalidRequirement(
compiler, requirement, "extending rules allow only " + EXTENDABLE_FIELDS);
}
}
existing.addAllWhitelist(requirement.getWhitelistList());
existing.addAllWhitelistRegexp(requirement.getWhitelistRegexpList());
existing.addAllOnlyApplyTo(requirement.getOnlyApplyToList());
existing.addAllOnlyApplyToRegexp(requirement.getOnlyApplyToRegexpList());
}
}
}
List<Requirement> requirements = new ArrayList<>(builders.size());
for (Requirement.Builder builder : builders) {
Requirement requirement = builder.build();
checkRequirementList(compiler, requirement, "whitelist");
checkRequirementList(compiler, requirement, "whitelist_regexp");
checkRequirementList(compiler, requirement, "only_apply_to");
checkRequirementList(compiler, requirement, "only_apply_to_regexp");
requirements.add(requirement);
}
return requirements;
}
