/** * Exactly like skipRawBytes, but caller must have already checked the fast path: (size <= * (bufferSize - pos) && size >= 0) */ private void skipRawBytesSlowPath(final int size) throws IOException { if (size < 0) { throw InvalidProtocolBufferException.negativeSize(); } if (totalBytesRetired + bufferPos + size > currentLimit) { // Read to the end of the stream anyway. skipRawBytes(currentLimit - totalBytesRetired - bufferPos); // Then fail. throw InvalidProtocolBufferException.truncatedMessage(); } // Skipping more bytes than are in the buffer. First skip what we have. int pos = bufferSize - bufferPos; bufferPos = bufferSize; // Keep refilling the buffer until we get to the point we wanted to skip to. // This has the side effect of ensuring the limits are updated correctly. refillBuffer(1); while (size - pos > bufferSize) { pos += bufferSize; bufferPos = bufferSize; refillBuffer(1); } bufferPos = size - pos; }
/** * Like {@link #readRawVarint32(InputStream)}, but expects that the caller has already read one * byte. This allows the caller to determine if EOF has been reached before attempting to read. */ public static int readRawVarint32(final int firstByte, final InputStream input) throws IOException { if ((firstByte & 0x80) == 0) { return firstByte; } int result = firstByte & 0x7f; int offset = 7; for (; offset < 32; offset += 7) { final int b = input.read(); if (b == -1) { throw InvalidProtocolBufferException.truncatedMessage(); } result |= (b & 0x7f) << offset; if ((b & 0x80) == 0) { return result; } } // Keep reading up to 64 bits. for (; offset < 64; offset += 7) { final int b = input.read(); if (b == -1) { throw InvalidProtocolBufferException.truncatedMessage(); } if ((b & 0x80) == 0) { return result; } } throw InvalidProtocolBufferException.malformedVarint(); }
public void testParseRequiredStringWithBadUtf8() throws Exception { ByteString serialized = BytesWrapper.newBuilder().setReq(NON_UTF8_BYTE_STRING).build().toByteString(); try { StringWrapper.PARSER.parseFrom(serialized); fail("Expected InvalidProtocolBufferException for non UTF-8 byte string."); } catch (InvalidProtocolBufferException exception) { assertEquals("Protocol message had invalid UTF-8.", exception.getMessage()); } }
public void testInvalidTag() throws Exception { // Any tag number which corresponds to field number zero is invalid and // should throw InvalidProtocolBufferException. for (int i = 0; i < 8; i++) { try { CodedInputStream.newInstance(bytes(i)).readTag(); fail("Should have thrown an exception."); } catch (InvalidProtocolBufferException e) { assertEquals(InvalidProtocolBufferException.invalidTag().getMessage(), e.getMessage()); } } }
/** * Sets {@code currentLimit} to (current position) + {@code byteLimit}. This is called when * descending into a length-delimited embedded message. * * <p>Note that {@code pushLimit()} does NOT affect how many bytes the {@code CodedInputStream} * reads from an underlying {@code InputStream} when refreshing its buffer. If you need to prevent * reading past a certain point in the underlying {@code InputStream} (e.g. because you expect it * to contain more data after the end of the message which you need to handle differently) then * you must place a wrapper around your {@code InputStream} which limits the amount of data that * can be read from it. * * @return the old limit. */ public int pushLimit(int byteLimit) throws InvalidProtocolBufferException { if (byteLimit < 0) { throw InvalidProtocolBufferException.negativeSize(); } byteLimit += totalBytesRetired + bufferPos; final int oldLimit = currentLimit; if (byteLimit > oldLimit) { throw InvalidProtocolBufferException.truncatedMessage(); } currentLimit = byteLimit; recomputeBufferSizeAfterLimit(); return oldLimit; }
/** * Reads and discards a single field, given its tag value. * * @return {@code false} if the tag is an endgroup tag, in which case nothing is skipped. * Otherwise, returns {@code true}. */ public boolean skipField(final int tag) throws IOException { switch (WireFormat.getTagWireType(tag)) { case WireFormat.WIRETYPE_VARINT: skipRawVarint(); return true; case WireFormat.WIRETYPE_FIXED64: skipRawBytes(8); return true; case WireFormat.WIRETYPE_LENGTH_DELIMITED: skipRawBytes(readRawVarint32()); return true; case WireFormat.WIRETYPE_START_GROUP: skipMessage(); checkLastTagWas( WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP)); return true; case WireFormat.WIRETYPE_END_GROUP: return false; case WireFormat.WIRETYPE_FIXED32: skipRawBytes(4); return true; default: throw InvalidProtocolBufferException.invalidWireType(); } }
/** * Reads a varint from the input one byte at a time, so that it does not read any bytes after the * end of the varint. If you simply wrapped the stream in a CodedInputStream and used {@link * #readRawVarint32(InputStream)} then you would probably end up reading past the end of the * varint since CodedInputStream buffers its input. */ static int readRawVarint32(final InputStream input) throws IOException { final int firstByte = input.read(); if (firstByte == -1) { throw InvalidProtocolBufferException.truncatedMessage(); } return readRawVarint32(firstByte, input); }
/** Tests readRawVarint32() and readRawVarint64(). */ public void testReadVarint() throws Exception { assertReadVarint(bytes(0x00), 0); assertReadVarint(bytes(0x01), 1); assertReadVarint(bytes(0x7f), 127); // 14882 assertReadVarint(bytes(0xa2, 0x74), (0x22 << 0) | (0x74 << 7)); // 2961488830 assertReadVarint( bytes(0xbe, 0xf7, 0x92, 0x84, 0x0b), (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | (0x0bL << 28)); // 64-bit // 7256456126 assertReadVarint( bytes(0xbe, 0xf7, 0x92, 0x84, 0x1b), (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | (0x1bL << 28)); // 41256202580718336 assertReadVarint( bytes(0x80, 0xe6, 0xeb, 0x9c, 0xc3, 0xc9, 0xa4, 0x49), (0x00 << 0) | (0x66 << 7) | (0x6b << 14) | (0x1c << 21) | (0x43L << 28) | (0x49L << 35) | (0x24L << 42) | (0x49L << 49)); // 11964378330978735131 assertReadVarint( bytes(0x9b, 0xa8, 0xf9, 0xc2, 0xbb, 0xd6, 0x80, 0x85, 0xa6, 0x01), (0x1b << 0) | (0x28 << 7) | (0x79 << 14) | (0x42 << 21) | (0x3bL << 28) | (0x56L << 35) | (0x00L << 42) | (0x05L << 49) | (0x26L << 56) | (0x01L << 63)); // Failures assertReadVarintFailure( InvalidProtocolBufferException.malformedVarint(), bytes(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00)); assertReadVarintFailure(InvalidProtocolBufferException.truncatedMessage(), bytes(0x80)); }
private void skipRawVarintSlowPath() throws IOException { for (int i = 0; i < 10; i++) { if (readRawByte() >= 0) { return; } } throw InvalidProtocolBufferException.malformedVarint(); }
/** * Tests that if we readStringRequireUtf8 invalid UTF-8 bytes, an InvalidProtocolBufferException * is thrown. */ public void testReadStringRequireUtf8InvalidUtf8() throws Exception { ByteString.Output rawOutput = ByteString.newOutput(); CodedOutputStream output = CodedOutputStream.newInstance(rawOutput); int tag = WireFormat.makeTag(1, WireFormat.WIRETYPE_LENGTH_DELIMITED); output.writeRawVarint32(tag); output.writeRawVarint32(1); output.writeRawBytes(new byte[] {(byte) 0x80}); output.flush(); CodedInputStream input = rawOutput.toByteString().newCodedInput(); assertEquals(tag, input.readTag()); try { input.readStringRequireUtf8(); fail("Expected invalid UTF-8 exception."); } catch (InvalidProtocolBufferException exception) { assertEquals("Protocol message had invalid UTF-8.", exception.getMessage()); } }
/* Visible for testing */ long readRawVarint64SlowPath() throws IOException { long result = 0; for (int shift = 0; shift < 64; shift += 7) { final byte b = readRawByte(); result |= (long) (b & 0x7F) << shift; if ((b & 0x80) == 0) { return result; } } throw InvalidProtocolBufferException.malformedVarint(); }
/** Read a {@code group} field value from the stream. */ public <T extends MessageLite> T readGroup( final int fieldNumber, final Parser<T> parser, final ExtensionRegistryLite extensionRegistry) throws IOException { if (recursionDepth >= recursionLimit) { throw InvalidProtocolBufferException.recursionLimitExceeded(); } ++recursionDepth; T result = parser.parsePartialFrom(this, extensionRegistry); checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); --recursionDepth; return result; }
/** Read a {@code group} field value from the stream. */ public void readGroup( final int fieldNumber, final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) throws IOException { if (recursionDepth >= recursionLimit) { throw InvalidProtocolBufferException.recursionLimitExceeded(); } ++recursionDepth; builder.mergeFrom(this, extensionRegistry); checkLastTagWas(WireFormat.makeTag(fieldNumber, WireFormat.WIRETYPE_END_GROUP)); --recursionDepth; }
/** * Attempt to read a field tag, returning zero if we have reached EOF. Protocol message parsers * use this to read tags, since a protocol message may legally end wherever a tag occurs, and zero * is not a valid tag number. */ public int readTag() throws IOException { if (isAtEnd()) { lastTag = 0; return 0; } lastTag = readRawVarint32(); if (WireFormat.getTagFieldNumber(lastTag) == 0) { // If we actually read zero (or any tag number corresponding to field // number zero), that's not a valid tag. throw InvalidProtocolBufferException.invalidTag(); } return lastTag; }
/** Read an embedded message field value from the stream. */ public <T extends MessageLite> T readMessage( final Parser<T> parser, final ExtensionRegistryLite extensionRegistry) throws IOException { int length = readRawVarint32(); if (recursionDepth >= recursionLimit) { throw InvalidProtocolBufferException.recursionLimitExceeded(); } final int oldLimit = pushLimit(length); ++recursionDepth; T result = parser.parsePartialFrom(this, extensionRegistry); checkLastTagWas(0); --recursionDepth; popLimit(oldLimit); return result; }
/** Read an embedded message field value from the stream. */ public void readMessage( final MessageLite.Builder builder, final ExtensionRegistryLite extensionRegistry) throws IOException { final int length = readRawVarint32(); if (recursionDepth >= recursionLimit) { throw InvalidProtocolBufferException.recursionLimitExceeded(); } final int oldLimit = pushLimit(length); ++recursionDepth; builder.mergeFrom(this, extensionRegistry); checkLastTagWas(0); --recursionDepth; popLimit(oldLimit); }
/** * Reads a single field and writes it to output in wire format, given its tag value. * * @return {@code false} if the tag is an endgroup tag, in which case nothing is skipped. * Otherwise, returns {@code true}. */ public boolean skipField(final int tag, final CodedOutputStream output) throws IOException { switch (WireFormat.getTagWireType(tag)) { case WireFormat.WIRETYPE_VARINT: { long value = readInt64(); output.writeRawVarint32(tag); output.writeUInt64NoTag(value); return true; } case WireFormat.WIRETYPE_FIXED64: { long value = readRawLittleEndian64(); output.writeRawVarint32(tag); output.writeFixed64NoTag(value); return true; } case WireFormat.WIRETYPE_LENGTH_DELIMITED: { ByteString value = readBytes(); output.writeRawVarint32(tag); output.writeBytesNoTag(value); return true; } case WireFormat.WIRETYPE_START_GROUP: { output.writeRawVarint32(tag); skipMessage(output); int endtag = WireFormat.makeTag(WireFormat.getTagFieldNumber(tag), WireFormat.WIRETYPE_END_GROUP); checkLastTagWas(endtag); output.writeRawVarint32(endtag); return true; } case WireFormat.WIRETYPE_END_GROUP: { return false; } case WireFormat.WIRETYPE_FIXED32: { int value = readRawLittleEndian32(); output.writeRawVarint32(tag); output.writeFixed32NoTag(value); return true; } default: throw InvalidProtocolBufferException.invalidWireType(); } }
/** * Tries to read more bytes from the input, making at least {@code n} bytes available in the * buffer. Caller must ensure that the requested space is not yet available, and that the * requested space is less than BUFFER_SIZE. * * @return {@code true} if the bytes could be made available; {@code false} if the end of the * stream or the current limit was reached. */ private boolean tryRefillBuffer(int n) throws IOException { if (bufferPos + n <= bufferSize) { throw new IllegalStateException( "refillBuffer() called when " + n + " bytes were already available in buffer"); } if (totalBytesRetired + bufferPos + n > currentLimit) { // Oops, we hit a limit. return false; } if (refillCallback != null) { refillCallback.onRefill(); } if (input != null) { int pos = bufferPos; if (pos > 0) { if (bufferSize > pos) { System.arraycopy(buffer, pos, buffer, 0, bufferSize - pos); } totalBytesRetired += pos; bufferSize -= pos; bufferPos = 0; } int bytesRead = input.read(buffer, bufferSize, buffer.length - bufferSize); if (bytesRead == 0 || bytesRead < -1 || bytesRead > buffer.length) { throw new IllegalStateException( "InputStream#read(byte[]) returned invalid result: " + bytesRead + "\nThe InputStream implementation is buggy."); } if (bytesRead > 0) { bufferSize += bytesRead; // Integer-overflow-conscious check against sizeLimit if (totalBytesRetired + n - sizeLimit > 0) { throw InvalidProtocolBufferException.sizeLimitExceeded(); } recomputeBufferSizeAfterLimit(); return (bufferSize >= n) ? true : tryRefillBuffer(n); } } return false; }
/** * Read a {@code string} field value from the stream. If the stream contains malformed UTF-8, * throw exception {@link InvalidProtocolBufferException}. */ public String readStringRequireUtf8() throws IOException { final int size = readRawVarint32(); final byte[] bytes; int pos = bufferPos; if (size <= (bufferSize - pos) && size > 0) { // Fast path: We already have the bytes in a contiguous buffer, so // just copy directly from it. bytes = buffer; bufferPos = pos + size; } else if (size == 0) { return ""; } else { // Slow path: Build a byte array first then copy it. bytes = readRawBytesSlowPath(size); pos = 0; } // TODO(martinrb): We could save a pass by validating while decoding. if (!Utf8.isValidUtf8(bytes, pos, pos + size)) { throw InvalidProtocolBufferException.invalidUtf8(); } return new String(bytes, pos, size, Internal.UTF_8); }
/** * Exactly like readRawBytes, but caller must have already checked the fast path: (size <= * (bufferSize - pos) && size > 0) */ private byte[] readRawBytesSlowPath(final int size) throws IOException { if (size <= 0) { if (size == 0) { return Internal.EMPTY_BYTE_ARRAY; } else { throw InvalidProtocolBufferException.negativeSize(); } } if (totalBytesRetired + bufferPos + size > currentLimit) { // Read to the end of the stream anyway. skipRawBytes(currentLimit - totalBytesRetired - bufferPos); // Then fail. throw InvalidProtocolBufferException.truncatedMessage(); } if (size < BUFFER_SIZE) { // Reading more bytes than are in the buffer, but not an excessive number // of bytes. We can safely allocate the resulting array ahead of time. // First copy what we have. final byte[] bytes = new byte[size]; int pos = bufferSize - bufferPos; System.arraycopy(buffer, bufferPos, bytes, 0, pos); bufferPos = bufferSize; // We want to refill the buffer and then copy from the buffer into our // byte array rather than reading directly into our byte array because // the input may be unbuffered. ensureAvailable(size - pos); System.arraycopy(buffer, 0, bytes, pos, size - pos); bufferPos = size - pos; return bytes; } else { // The size is very large. For security reasons, we can't allocate the // entire byte array yet. The size comes directly from the input, so a // maliciously-crafted message could provide a bogus very large size in // order to trick the app into allocating a lot of memory. We avoid this // by allocating and reading only a small chunk at a time, so that the // malicious message must actually *be* extremely large to cause // problems. Meanwhile, we limit the allowed size of a message elsewhere. // Remember the buffer markers since we'll have to copy the bytes out of // it later. final int originalBufferPos = bufferPos; final int originalBufferSize = bufferSize; // Mark the current buffer consumed. totalBytesRetired += bufferSize; bufferPos = 0; bufferSize = 0; // Read all the rest of the bytes we need. int sizeLeft = size - (originalBufferSize - originalBufferPos); final List<byte[]> chunks = new ArrayList<byte[]>(); while (sizeLeft > 0) { final byte[] chunk = new byte[Math.min(sizeLeft, BUFFER_SIZE)]; int pos = 0; while (pos < chunk.length) { final int n = (input == null) ? -1 : input.read(chunk, pos, chunk.length - pos); if (n == -1) { throw InvalidProtocolBufferException.truncatedMessage(); } totalBytesRetired += n; pos += n; } sizeLeft -= chunk.length; chunks.add(chunk); } // OK, got everything. Now concatenate it all into one buffer. final byte[] bytes = new byte[size]; // Start by copying the leftover bytes from this.buffer. int pos = originalBufferSize - originalBufferPos; System.arraycopy(buffer, originalBufferPos, bytes, 0, pos); // And now all the chunks. for (final byte[] chunk : chunks) { System.arraycopy(chunk, 0, bytes, pos, chunk.length); pos += chunk.length; } // Done. return bytes; } }
/** * Verifies that the last call to readTag() returned the given tag value. This is used to verify * that a nested group ended with the correct end tag. * * @throws InvalidProtocolBufferException {@code value} does not match the last tag. */ public void checkLastTagWas(final int value) throws InvalidProtocolBufferException { if (lastTag != value) { throw InvalidProtocolBufferException.invalidEndTag(); } }
private void checkSizeLimitExceeded(InvalidProtocolBufferException e) { assertEquals(InvalidProtocolBufferException.sizeLimitExceeded().getMessage(), e.getMessage()); }
/** * Parses the given bytes using readRawVarint32() and readRawVarint64() and expects them to fail * with an InvalidProtocolBufferException whose description matches the given one. */ private void assertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data) throws Exception { CodedInputStream input = CodedInputStream.newInstance(data); try { input.readRawVarint32(); fail("Should have thrown an exception."); } catch (InvalidProtocolBufferException e) { assertEquals(expected.getMessage(), e.getMessage()); } input = CodedInputStream.newInstance(data); try { input.readRawVarint64(); fail("Should have thrown an exception."); } catch (InvalidProtocolBufferException e) { assertEquals(expected.getMessage(), e.getMessage()); } input = CodedInputStream.newInstance(data); try { input.readRawVarint64SlowPath(); fail("Should have thrown an exception."); } catch (InvalidProtocolBufferException e) { assertEquals(expected.getMessage(), e.getMessage()); } // Make sure we get the same error when reading direct from an InputStream. try { CodedInputStream.readRawVarint32(new ByteArrayInputStream(data)); fail("Should have thrown an exception."); } catch (InvalidProtocolBufferException e) { assertEquals(expected.getMessage(), e.getMessage()); } }
/** * Reads more bytes from the input, making at least {@code n} bytes available in the buffer. * Caller must ensure that the requested space is not yet available, and that the requested space * is less than BUFFER_SIZE. * * @throws InvalidProtocolBufferException The end of the stream or the current limit was reached. */ private void refillBuffer(int n) throws IOException { if (!tryRefillBuffer(n)) { throw InvalidProtocolBufferException.truncatedMessage(); } }