protected static void filterValue(
boolean clearEmpty,
Predicate<FieldDescriptor> filter,
Message.Builder builder,
Map.Entry<FieldDescriptor, Object> entry) {
FieldDescriptor fd = entry.getKey();
Object value = entry.getValue();
if (fd.getType() == FieldDescriptor.Type.MESSAGE) {
if (fd.isRepeated()) {
for (Object obj : ((Iterable<?>) value)) {
Message child = filter((Message) obj, clearEmpty, filter);
if (child != null) {
builder.addRepeatedField(fd, child);
}
}
} else {
Message child = filter((Message) value, clearEmpty, filter);
if (child != null) {
builder.setField(fd, child);
}
}
} else {
builder.setField(fd, value);
}
}
public int getSerializedSize() {
int size = memoizedSize;
if (size != -1) {
return size;
}
size = 0;
final boolean isMessageSet = getDescriptorForType().getOptions().getMessageSetWireFormat();
for (final Map.Entry<FieldDescriptor, Object> entry : getAllFields().entrySet()) {
final FieldDescriptor field = entry.getKey();
final Object value = entry.getValue();
if (isMessageSet
&& field.isExtension()
&& field.getType() == FieldDescriptor.Type.MESSAGE
&& !field.isRepeated()) {
size +=
CodedOutputStream.computeMessageSetExtensionSize(field.getNumber(), (Message) value);
} else {
size += FieldSet.computeFieldSize(field, value);
}
}
final UnknownFieldSet unknownFields = getUnknownFields();
if (isMessageSet) {
size += unknownFields.getSerializedSizeAsMessageSet();
} else {
size += unknownFields.getSerializedSize();
}
memoizedSize = size;
return size;
}
private static void recursivelyAddDescriptors(
FieldDescriptor field, Set<Descriptor> descriptors, ExtensionRegistry registry) {
switch (field.getType()) {
case BOOL:
case BYTES:
case DOUBLE:
case ENUM:
case FIXED32:
case FIXED64:
case FLOAT:
case INT32:
case INT64:
case SFIXED32:
case SFIXED64:
case SINT32:
case SINT64:
case STRING:
case UINT32:
case UINT64:
// Primitive types do not transitively access anything else.
break;
case GROUP:
case MESSAGE:
// Recursively adds all the fields from this nested Message.
recursivelyAddDescriptors(field.getMessageType(), descriptors, registry);
break;
default:
throw new UnsupportedOperationException(
"Unexpected Protocol Buffers field type: " + field.getType());
}
}
/**
* Assigns all fields in {@link JsonNode} instance to a {@link Builder}.
*
* @param builder the builder to be populated
* @param root the JSON object
* @return the original builder, populated with any fields that were discovered
*/
public static Builder toProto(final Builder builder, final JsonNode root) {
Descriptor type = builder.getDescriptorForType();
for (final FieldDescriptor fieldDesc : type.getFields()) {
final String attrName = fieldDesc.getName();
final JsonNode node = root.get(attrName);
if (node == null) {
continue;
}
if (node.isNull()) {
continue;
}
if (fieldDesc.isRepeated()) {
final Iterator<JsonNode> iter = node.getElements();
while (iter.hasNext()) {
builder.addRepeatedField(fieldDesc, toJavaObj(builder, fieldDesc, iter.next()));
}
} else {
builder.setField(fieldDesc, toJavaObj(builder, fieldDesc, node));
}
}
return builder;
}
@SuppressWarnings("unchecked")
public boolean isInitialized() {
// Check that all required fields are present.
for (final FieldDescriptor field : getDescriptorForType().getFields()) {
if (field.isRequired()) {
if (!hasField(field)) {
return false;
}
}
}
// Check that embedded messages are initialized.
for (final Map.Entry<FieldDescriptor, Object> entry : getAllFields().entrySet()) {
final FieldDescriptor field = entry.getKey();
if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
if (field.isRepeated()) {
for (final Message element : (List<Message>) entry.getValue()) {
if (!element.isInitialized()) {
return false;
}
}
} else {
if (!((Message) entry.getValue()).isInitialized()) {
return false;
}
}
}
}
return true;
}
private static void writeSingleFieldNoTag(
CodedOutputStream output, FieldDescriptor fd, Object value) throws IOException {
switch (fd.getType()) {
case DOUBLE:
output.writeDoubleNoTag((Double) value);
break;
case FLOAT:
output.writeFloatNoTag((Float) value);
break;
case INT64:
case UINT64:
output.writeInt64NoTag((Long) value);
break;
case INT32:
output.writeInt32NoTag((Integer) value);
break;
case FIXED64:
output.writeFixed64NoTag((Long) value);
break;
case FIXED32:
output.writeFixed32NoTag((Integer) value);
break;
case BOOL:
output.writeBoolNoTag((Boolean) value);
break;
case STRING:
output.writeStringNoTag((String) value);
break;
case GROUP:
case MESSAGE:
output.writeMessageNoTag((Message) value);
break;
case BYTES:
output.writeBytesNoTag((ByteString) value);
break;
case UINT32:
output.writeUInt32NoTag((Integer) value);
break;
case ENUM:
output.writeEnumNoTag(((ProtocolMessageEnum) value).getNumber());
break;
case SFIXED32:
output.writeSFixed32NoTag((Integer) value);
break;
case SFIXED64:
output.writeSFixed64NoTag((Long) value);
break;
case SINT32:
output.writeSInt32NoTag((Integer) value);
break;
case SINT64:
output.writeSInt64NoTag((Integer) value);
break;
default:
throw new IllegalArgumentException(
"Unknown type " + fd.getType() + " for " + fd.getFullName());
}
}
/**
* Converts a tupleField string to its corresponding protobuf enum type if necessary, otherwise
* returns the tupleField as is.
*
* @param fieldDescriptor the FieldDescriptor for the given tuple field
* @param tupleField the tupleField being converted to a protobuf field
* @return the protobuf type for the given tupleField. This will be the tupleField itself unless
* it's an enum, in which case this will return the enum type for the field.
*/
public Object tupleFieldToSingleField(FieldDescriptor fieldDescriptor, Object tupleField) {
if (fieldDescriptor.getType() == FieldDescriptor.Type.ENUM) {
// Convert tupleField to the enum value.
return fieldDescriptor.getEnumType().findValueByName((String) tupleField);
} else {
return tupleField;
}
}
public Builder newBuilderForField(FieldDescriptor field) {
verifyContainingType(field);
if (field.getJavaType() != FieldDescriptor.JavaType.MESSAGE) {
throw new IllegalArgumentException(
"newBuilderForField is only valid for fields with message type.");
}
return new Builder(field.getMessageType());
}
public Object getField(FieldDescriptor field) {
verifyContainingType(field);
Object result = fields.getField(field);
if (result == null) {
if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
result = getDefaultInstance(field.getMessageType());
} else {
result = field.getDefaultValue();
}
}
return result;
}
static boolean isInitialized(Descriptor type, FieldSet<FieldDescriptor> fields) {
// Check that all required fields are present.
for (final FieldDescriptor field : type.getFields()) {
if (field.isRequired()) {
if (!fields.hasField(field)) {
return false;
}
}
}
// Check that embedded messages are initialized.
return fields.isInitialized();
}
private static String subMessagePrefix(
final String prefix, final FieldDescriptor field, final int index) {
final StringBuilder result = new StringBuilder(prefix);
if (field.isExtension()) {
result.append('(').append(field.getFullName()).append(')');
} else {
result.append(field.getName());
}
if (index != -1) {
result.append('[').append(index).append(']');
}
result.append('.');
return result.toString();
}
private static void getRepeatedFieldNamesInternal(Descriptor d, Map<String, Set<String>> result) {
Set<String> repeatedFields = Sets.newHashSet();
for (FieldDescriptor fd : d.getFields()) {
if (fd.isRepeated()) {
repeatedFields.add(javaCase(fd.getName()));
}
if (fd.getType() == FieldDescriptor.Type.MESSAGE) {
getRepeatedFieldNamesInternal(fd.getMessageType(), result);
}
}
if (!repeatedFields.isEmpty()) {
result.put(javaCase(d.getName()), repeatedFields);
}
}
private static Object readSingleFieldNoTag(
CodedInputStream input, FieldDescriptor fd, Builder enclosingBuilder) throws IOException {
switch (fd.getType()) {
case DOUBLE:
return input.readDouble();
case FLOAT:
return input.readFloat();
case INT64:
case UINT64:
return input.readInt64();
case INT32:
return input.readInt32();
case FIXED64:
return input.readFixed64();
case FIXED32:
return input.readFixed32();
case BOOL:
return input.readBool();
case STRING:
return input.readString();
case GROUP:
case MESSAGE:
Builder fieldBuilder = enclosingBuilder.newBuilderForField(fd);
input.readMessage(fieldBuilder, null);
return fieldBuilder.build();
case BYTES:
return input.readBytes();
case UINT32:
return input.readUInt32();
case ENUM:
EnumValueDescriptor eVal = fd.getEnumType().findValueByNumber(input.readEnum());
// ideally if a given enum does not exist, we should search
// unknown fields. but we don't have access to that here. return default
return eVal != null ? eVal : fd.getDefaultValue();
case SFIXED32:
return input.readSFixed32();
case SFIXED64:
return input.readSFixed64();
case SINT32:
return input.readSInt32();
case SINT64:
return input.readSInt64();
default:
throw new IllegalArgumentException(
"Unknown type " + fd.getType() + " for " + fd.getFullName());
}
}
private static RequiredFieldList evenFields(List<FieldDescriptor> protoFields) {
RequiredFieldList reqList = new RequiredFieldList();
int i = 0;
for (FieldDescriptor fd : protoFields) {
if (i % 2 == 0) {
RequiredField field = new RequiredField();
field.setAlias(fd.getName());
field.setIndex(i);
// field.setType() type is not used
reqList.add(field);
}
i++;
}
return reqList;
}
/** Get a hash code for given fields and values, using the given seed. */
@SuppressWarnings("unchecked")
protected int hashFields(int hash, Map<FieldDescriptor, Object> map) {
for (Map.Entry<FieldDescriptor, Object> entry : map.entrySet()) {
FieldDescriptor field = entry.getKey();
Object value = entry.getValue();
hash = (37 * hash) + field.getNumber();
if (field.getType() != FieldDescriptor.Type.ENUM) {
hash = (53 * hash) + value.hashCode();
} else if (field.isRepeated()) {
List<? extends EnumLite> list = (List<? extends EnumLite>) value;
hash = (53 * hash) + hashEnumList(list);
} else {
hash = (53 * hash) + hashEnum((EnumLite) value);
}
}
return hash;
}
public Builder setField(FieldDescriptor field, Object value) {
verifyContainingType(field);
ensureIsMutable();
if (field.getType() == FieldDescriptor.Type.ENUM) {
verifyEnumType(field, value);
}
OneofDescriptor oneofDescriptor = field.getContainingOneof();
if (oneofDescriptor != null) {
int index = oneofDescriptor.getIndex();
FieldDescriptor oldField = oneofCases[index];
if ((oldField != null) && (oldField != field)) {
fields.clearField(oldField);
}
oneofCases[index] = field;
}
fields.setField(field, value);
return this;
}
@Override
public BuilderType mergeFrom(final Message other) {
if (other.getDescriptorForType() != getDescriptorForType()) {
throw new IllegalArgumentException(
"mergeFrom(Message) can only merge messages of the same type.");
}
// Note: We don't attempt to verify that other's fields have valid
// types. Doing so would be a losing battle. We'd have to verify
// all sub-messages as well, and we'd have to make copies of all of
// them to insure that they don't change after verification (since
// the Message interface itself cannot enforce immutability of
// implementations).
// TODO(kenton): Provide a function somewhere called makeDeepCopy()
// which allows people to make secure deep copies of messages.
for (final Map.Entry<FieldDescriptor, Object> entry : other.getAllFields().entrySet()) {
final FieldDescriptor field = entry.getKey();
if (field.isRepeated()) {
for (final Object element : (List) entry.getValue()) {
addRepeatedField(field, element);
}
} else if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
final Message existingValue = (Message) getField(field);
if (existingValue == existingValue.getDefaultInstanceForType()) {
setField(field, entry.getValue());
} else {
setField(
field,
existingValue
.newBuilderForType()
.mergeFrom(existingValue)
.mergeFrom((Message) entry.getValue())
.build());
}
} else {
setField(field, entry.getValue());
}
}
mergeUnknownFields(other.getUnknownFields());
return (BuilderType) this;
}
/**
* Turn a Tuple into a Message with the given type.
*
* @param builder a builder for the Message type the tuple will be converted to
* @param tuple the tuple
* @return a message representing the given tuple
*/
public Message tupleToMessage(Builder builder, Tuple tuple) {
List<FieldDescriptor> fieldDescriptors = builder.getDescriptorForType().getFields();
if (tuple == null) {
return builder.build();
}
for (int i = 0; i < fieldDescriptors.size() && i < tuple.size(); i++) {
Object tupleField = null;
FieldDescriptor fieldDescriptor = fieldDescriptors.get(i);
try {
tupleField = tuple.get(i);
} catch (ExecException e) {
LOG.warn(
"Could not convert tuple field "
+ tupleField
+ " to field with descriptor "
+ fieldDescriptor);
continue;
}
if (tupleField != null) {
if (fieldDescriptor.isRepeated()) {
// Repeated fields are set with Lists containing objects of the fields' Java type.
builder.setField(
fieldDescriptor,
dataBagToRepeatedField(builder, fieldDescriptor, (DataBag) tupleField));
} else {
if (fieldDescriptor.getType() == FieldDescriptor.Type.MESSAGE) {
Builder nestedMessageBuilder = builder.newBuilderForField(fieldDescriptor);
builder.setField(
fieldDescriptor,
tupleToMessage((Builder) nestedMessageBuilder, (Tuple) tupleField));
} else {
builder.setField(fieldDescriptor, tupleFieldToSingleField(fieldDescriptor, tupleField));
}
}
}
}
return builder.build();
}
/**
* Recursively checks whether the specified class uses any Protocol Buffers fields that cannot be
* deterministically encoded.
*
* @throws NonDeterministicException if the object cannot be encoded deterministically.
*/
static void verifyDeterministic(ProtoCoder<?> coder) throws NonDeterministicException {
Class<? extends Message> message = coder.getMessageType();
ExtensionRegistry registry = coder.getExtensionRegistry();
Set<Descriptor> descriptors = getRecursiveDescriptorsForClass(message, registry);
for (Descriptor d : descriptors) {
for (FieldDescriptor fd : d.getFields()) {
// If there is a transitively reachable Protocol Buffers map field, then this object cannot
// be encoded deterministically.
if (fd.isMapField()) {
String reason =
String.format(
"Protocol Buffers message %s transitively includes Map field %s (from file %s)."
+ " Maps cannot be deterministically encoded.",
message.getName(), fd.getFullName(), fd.getFile().getFullName());
throw new NonDeterministicException(coder, reason);
}
}
}
}
private void printField(final FieldDescriptor field, final Object value,
final TextGenerator generator) throws IOException {
if (field.isRepeated()) {
// Repeated field. Print each element.
for (Object element : (List<?>) value) {
printSingleField(field, element, generator);
}
} else {
printSingleField(field, value, generator);
}
}
private static Object toJavaObj(
final Builder builder, final FieldDescriptor fieldDesc, final JsonNode node) {
Object value;
switch (fieldDesc.getJavaType()) {
case MESSAGE:
final Builder subBuilder = builder.newBuilderForField(fieldDesc);
value = toProto(subBuilder, node).build();
break;
case BOOLEAN:
value = Boolean.valueOf(node.getBooleanValue());
break;
case BYTE_STRING:
try {
value = node.getBinaryValue();
} catch (IOException e) {
throw new RuntimeException(e);
}
break;
case DOUBLE:
value = Double.valueOf(node.getDoubleValue());
break;
case FLOAT:
value = Float.valueOf(Double.valueOf(node.getDoubleValue()).floatValue());
break;
case ENUM:
value = fieldDesc.getEnumType().findValueByName(node.getTextValue());
break;
case INT:
value = Integer.valueOf(node.getIntValue());
break;
case LONG:
value = Long.valueOf(node.getLongValue());
break;
case STRING:
value = node.getTextValue();
break;
default:
throw new IllegalArgumentException();
}
return value;
}
/** Verifies that the value is EnumValueDescriptor and matchs Enum Type. */
private void verifyEnumType(FieldDescriptor field, Object value) {
if (value == null) {
throw new NullPointerException();
}
if (!(value instanceof EnumValueDescriptor)) {
throw new IllegalArgumentException(
"DynamicMessage should use EnumValueDescriptor to set Enum Value.");
}
if (field.getEnumType() != ((EnumValueDescriptor) value).getType()) {
throw new IllegalArgumentException("EnumValueDescriptor doesn't much Enum Field.");
}
}
@Test
public void testLazyProtoToPig() throws ExecException {
Person personProto = Fixtures.buildPersonProto();
Tuple protoTuple = new ProtobufTuple(personProto);
Tuple normalTuple = Fixtures.buildPersonTuple();
List<FieldDescriptor> fieldDescs = personProto.getDescriptorForType().getFields();
TypeRef<Person> typeRef = PigUtil.getProtobufTypeRef(Person.class.getName());
Tuple projectedTuple =
new ProjectedProtobufTupleFactory<Person>(typeRef, evenFields(fieldDescs))
.newTuple(personProto);
int idx = 0;
for (FieldDescriptor fd : fieldDescs) {
// Skipping data bags. Data bags actually work; it' just our fixture is not good for this,
// since it tests "default value" functionality by leaving some elements as null, expecting
// protobuf conversion to fill the nulls in. Which is what happens. But that means converting
// back
// gives us non-null fields, which are not equal to the null fields...
if (normalTuple.get(fd.getIndex()) instanceof DataBag) {
continue;
}
assertEquals(protoTuple.get(fd.getIndex()), normalTuple.get(fd.getIndex()));
if (idx % 2 == 0) {
assertEquals(projectedTuple.get(fd.getIndex() / 2), normalTuple.get(fd.getIndex()));
}
idx++;
}
}
/** Recursive helper implementing {@link #findMissingFields(Message)}. */
private static void findMissingFields(
final MessageOrBuilder message, final String prefix, final List<String> results) {
for (final FieldDescriptor field : message.getDescriptorForType().getFields()) {
if (field.isRequired() && !message.hasField(field)) {
results.add(prefix + field.getName());
}
}
for (final Map.Entry<FieldDescriptor, Object> entry : message.getAllFields().entrySet()) {
final FieldDescriptor field = entry.getKey();
final Object value = entry.getValue();
if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
if (field.isRepeated()) {
int i = 0;
for (final Object element : (List) value) {
findMissingFields(
(MessageOrBuilder) element, subMessagePrefix(prefix, field, i++), results);
}
} else {
if (message.hasField(field)) {
findMissingFields(
(MessageOrBuilder) value, subMessagePrefix(prefix, field, -1), results);
}
}
}
}
}
public Builder clearField(FieldDescriptor field) {
verifyContainingType(field);
ensureIsMutable();
OneofDescriptor oneofDescriptor = field.getContainingOneof();
if (oneofDescriptor != null) {
int index = oneofDescriptor.getIndex();
if (oneofCases[index] == field) {
oneofCases[index] = null;
}
}
fields.clearField(field);
return this;
}
public void writeTo(final CodedOutputStream output) throws IOException {
final boolean isMessageSet = getDescriptorForType().getOptions().getMessageSetWireFormat();
for (final Map.Entry<FieldDescriptor, Object> entry : getAllFields().entrySet()) {
final FieldDescriptor field = entry.getKey();
final Object value = entry.getValue();
if (isMessageSet
&& field.isExtension()
&& field.getType() == FieldDescriptor.Type.MESSAGE
&& !field.isRepeated()) {
output.writeMessageSetExtension(field.getNumber(), (Message) value);
} else {
FieldSet.writeField(field, value, output);
}
}
final UnknownFieldSet unknownFields = getUnknownFields();
if (isMessageSet) {
unknownFields.writeAsMessageSetTo(output);
} else {
unknownFields.writeTo(output);
}
}
/**
* Deserializer for protobuf objects written with {@link #writeFieldNoTag(CodedOutputStream,
* FieldDescriptor, Object)}.
*
* @param input
* @param fd
* @param enclosingBuilder required to create a builder when the field is a message
* @throws IOException
*/
public static Object readFieldNoTag(
CodedInputStream input, FieldDescriptor fd, Builder enclosingBuilder) throws IOException {
if (!fd.isRepeated()) {
return readSingleFieldNoTag(input, fd, enclosingBuilder);
}
// repeated field
List<Object> values = Lists.newArrayList();
while (!input.isAtEnd()) {
values.add(readSingleFieldNoTag(input, fd, enclosingBuilder));
}
return values;
}
/**
* Serializes a single field. All the native fields are serialized using "NoTag" methods in {@link
* CodedOutputStream} e.g. <code>writeInt32NoTag()</code>. The field index is not written.
*
* @param output
* @param fd
* @param value
* @throws IOException
*/
public static void writeFieldNoTag(CodedOutputStream output, FieldDescriptor fd, Object value)
throws IOException {
if (value == null) {
return;
}
if (fd.isRepeated()) {
@SuppressWarnings("unchecked")
List<Object> values = (List<Object>) value;
for (Object obj : values) {
writeSingleFieldNoTag(output, fd, obj);
}
} else {
writeSingleFieldNoTag(output, fd, value);
}
}
/**
* Converts a DataBag into a List of objects with the type in the given FieldDescriptor. DataBags
* don't map cleanly to repeated protobuf types, so each Tuple has to be unwrapped (by taking the
* first element if the type is primitive or by converting the Tuple to a Message if the type is
* MESSAGE), and the contents have to be appended to a List.
*
* @param containingMessageBuilder a Message builder for the Message that contains this repeated
* field
* @param fieldDescriptor a FieldDescriptor for this repeated field
* @param bag the DataBag being serialized
* @return a protobuf-friendly List of fieldDescriptor-type objects
*/
public List<Object> dataBagToRepeatedField(
Builder containingMessageBuilder, FieldDescriptor fieldDescriptor, DataBag bag) {
ArrayList<Object> bagContents = new ArrayList<Object>((int) bag.size());
Iterator<Tuple> bagIter = bag.iterator();
while (bagIter.hasNext()) {
Tuple tuple = bagIter.next();
if (fieldDescriptor.getType() == FieldDescriptor.Type.MESSAGE) {
Builder nestedMessageBuilder = containingMessageBuilder.newBuilderForField(fieldDescriptor);
bagContents.add(tupleToMessage((Builder) nestedMessageBuilder, tuple));
} else {
try {
bagContents.add(tupleFieldToSingleField(fieldDescriptor, tuple.get(0)));
} catch (ExecException e) {
LOG.warn("Could not add a value for repeated field with descriptor " + fieldDescriptor);
}
}
}
return bagContents;
}
/** Verifies that the field is a field of this message. */
private void verifyContainingType(FieldDescriptor field) {
if (field.getContainingType() != type) {
throw new IllegalArgumentException("FieldDescriptor does not match message type.");
}
}
