/**
* Serializes the {@code messages} into the stream using the given schema with the supplied
* buffer.
*/
public static <T> void writeListTo(
OutputStream out, List<T> messages, Schema<T> schema, boolean numeric, LinkedBuffer buffer)
throws IOException {
if (buffer.start != buffer.offset)
throw new IllegalArgumentException("Buffer previously used and had not been reset.");
if (messages.isEmpty()) {
System.arraycopy(EMPTY_ARRAY, 0, buffer.buffer, buffer.offset, EMPTY_ARRAY.length);
buffer.offset += EMPTY_ARRAY.length;
return;
}
final JsonXOutput output = new JsonXOutput(buffer, out, numeric, schema);
output.writeStartArray();
boolean first = true;
for (T m : messages) {
if (first) {
first = false;
output.writeStartObject();
} else output.writeCommaAndStartObject();
schema.writeTo(output, m);
if (output.isLastRepeated()) output.writeEndArray();
output.writeEndObject().clear(false);
}
output.writeEndArray();
LinkedBuffer.writeTo(out, buffer);
}
public void testMultipleLargeStringsExceedingBufferSizeDelimited() throws Exception {
LinkedBuffer buffer = LinkedBuffer.allocate(256);
ByteArrayOutputStream out = new ByteArrayOutputStream();
WriteSession session = new WriteSession(buffer, out);
String utf8OneByte = repeatChar('a', 1024);
String utf8TwoBytes = repeatChar((char) 0x7ff, 1024 / 2);
String utf8ThreeBytes = repeatChar((char) 0x800, 1024 / 3);
writeToSession(utf8OneByte, utf8TwoBytes, utf8ThreeBytes, session, true);
assertTrue(session.tail == session.head);
// temporary buffers will remain
assertTrue(session.tail.next != null);
// flush remaining
LinkedBuffer.writeTo(out, buffer);
// clear
buffer.clear();
byte[] data = out.toByteArray();
LinkedBuffer buffer2 = LinkedBuffer.allocate(256);
WriteSession session2 = new WriteSession(buffer2);
writeToSession(utf8OneByte, utf8TwoBytes, utf8ThreeBytes, session2, true);
byte[] data2 = session2.toByteArray();
assertEquals(STRING.deser(data), STRING.deser(data2));
}
public void testInt() throws Exception {
for (int i : int_targets) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
LinkedBuffer lb = new LinkedBuffer(BUF_SIZE);
WriteSession session = new WriteSession(lb);
StreamedStringSerializer.writeInt(i, session, out, lb);
LinkedBuffer.writeTo(out, lb);
ByteArrayOutputStream out2 = new ByteArrayOutputStream();
LinkedBuffer lb2 = new LinkedBuffer(NUM_BUF_SIZE);
WriteSession session2 = new WriteSession(lb2);
StreamedStringSerializer.writeInt(i, session2, out2, lb2);
LinkedBuffer.writeTo(out2, lb2);
byte[] buffered = out.toByteArray();
byte[] buffered_needed_to_flush = out2.toByteArray();
byte[] builtin = STRING.ser(Integer.toString(i));
assertEquals(builtin, buffered);
assertEquals(builtin, buffered_needed_to_flush);
}
}
static void checkVarDelimited(String str, int size, int stringLen) throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
LinkedBuffer lb = new LinkedBuffer(BUF_SIZE);
WriteSession session = new WriteSession(lb);
StreamedStringSerializer.writeUTF8VarDelimited(str, session, out, lb);
LinkedBuffer.writeTo(out, lb);
byte[] buf = out.toByteArray();
assertTrue(buf.length == stringLen + size);
int len = readRawVarint32(buf, 0);
assertTrue(len == stringLen);
print("total len: " + buf.length);
}
/**
* Serializes the {@code message} into an {@link OutputStream} via {@link JsonXOutput} using the
* given {@code schema}.
*/
public static <T> void writeTo(
OutputStream out, T message, Schema<T> schema, boolean numeric, LinkedBuffer buffer)
throws IOException {
if (buffer.start != buffer.offset)
throw new IllegalArgumentException("Buffer previously used and had not been reset.");
final JsonXOutput output = new JsonXOutput(buffer, out, numeric, schema);
output.writeStartObject();
schema.writeTo(output, message);
if (output.isLastRepeated()) output.writeEndArray();
output.writeEndObject();
LinkedBuffer.writeTo(out, buffer);
}
static void checkFixedDelimited(String str) throws Exception {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
DataOutputStream dout = new DataOutputStream(bout);
dout.writeUTF(str);
dout.close();
ByteArrayOutputStream out = new ByteArrayOutputStream();
LinkedBuffer lb = new LinkedBuffer(BUF_SIZE);
WriteSession session = new WriteSession(lb);
StreamedStringSerializer.writeUTF8FixedDelimited(str, session, out, lb);
LinkedBuffer.writeTo(out, lb);
byte[] b1 = bout.toByteArray();
byte[] b2 = out.toByteArray();
assertEquals(b1, b2);
}
static void testStream(String str, String prefix, LinkedBuffer tail) throws IOException {
byte[] data = str.getBytes();
ByteArrayOutputStream out = new ByteArrayOutputStream();
WriteSession session = new WriteSession(tail, out);
session.size += (tail.offset - tail.start);
tail = B64Code.encode(data, 0, data.length, session, out, tail);
assertTrue(tail == session.head);
LinkedBuffer.writeTo(out, tail);
byte[] result = out.toByteArray();
// System.err.println(new String(result, prefix.length(), result.length - prefix.length()));
byte[] decoded = B64Code.decode(result, prefix.length(), result.length - prefix.length());
String strd = new String(decoded);
assertEquals(str, strd);
// System.err.println(gg.length + " == " + decoded.length + " | " +
// str.equals(strd) + " | " + str + " == " + strd);
}
static void check(String str) throws Exception {
byte[] builtin = BUILT_IN_SERIALIZER.serialize(str);
ByteArrayOutputStream out = new ByteArrayOutputStream();
LinkedBuffer lb = new LinkedBuffer(BUF_SIZE);
WriteSession session = new WriteSession(lb);
StreamedStringSerializer.writeUTF8(str, session, out, lb);
LinkedBuffer.writeTo(out, lb);
assertTrue(builtin.length == session.size);
byte[] buffered = out.toByteArray();
assertTrue(builtin.length == buffered.length);
String strBuiltin = new String(builtin, "UTF-8");
String strBuffered = new String(buffered, "UTF-8");
assertEquals(strBuiltin, strBuffered);
print(strBuiltin);
print("len: " + builtin.length);
}
private static LinkedBuffer writeUTF8Escaped(
final String str, final WriteSink sink, final WriteSession session, LinkedBuffer lb)
throws IOException {
final int len = str.length();
if (len == 0) return lb;
byte[] buffer = lb.buffer;
int limit = buffer.length, offset = lb.offset, size = len;
for (int i = 0; i < len; i++) {
final char c = str.charAt(i);
if (c < 0x0080) {
final int escape = sOutputEscapes[c];
// System.out.print(c + "|" + escape + " ");
if (escape == 0) {
// nothing to escape
if (offset == limit) {
lb.offset = offset;
lb = sink.drain(session, lb);
offset = lb.offset;
buffer = lb.buffer;
limit = buffer.length;
}
// ascii
buffer[offset++] = (byte) c;
} else if (escape < 0) {
// hex escape
if (offset + 6 > limit) {
lb.offset = offset;
lb = sink.drain(session, lb);
offset = lb.offset;
buffer = lb.buffer;
limit = buffer.length;
}
final int value = -(escape + 1);
buffer[offset++] = (byte) '\\';
buffer[offset++] = (byte) 'u';
buffer[offset++] = (byte) '0';
buffer[offset++] = (byte) '0';
buffer[offset++] = HEX_BYTES[value >> 4];
buffer[offset++] = HEX_BYTES[value & 0x0F];
size += 5;
} else {
if (offset + 2 > limit) {
lb.offset = offset;
lb = sink.drain(session, lb);
offset = lb.offset;
buffer = lb.buffer;
limit = buffer.length;
}
buffer[offset++] = (byte) '\\';
buffer[offset++] = (byte) escape;
size++;
}
} else if (c < 0x0800) {
if (offset + 2 > limit) {
lb.offset = offset;
lb = sink.drain(session, lb);
offset = lb.offset;
buffer = lb.buffer;
limit = buffer.length;
}
buffer[offset++] = (byte) (0xC0 | ((c >> 6) & 0x1F));
buffer[offset++] = (byte) (0x80 | ((c >> 0) & 0x3F));
size++;
} else {
if (offset + 3 > limit) {
lb.offset = offset;
lb = sink.drain(session, lb);
offset = lb.offset;
buffer = lb.buffer;
limit = buffer.length;
}
buffer[offset++] = (byte) (0xE0 | ((c >> 12) & 0x0F));
buffer[offset++] = (byte) (0x80 | ((c >> 6) & 0x3F));
buffer[offset++] = (byte) (0x80 | ((c >> 0) & 0x3F));
size += 2;
}
}
session.size += size;
lb.offset = offset;
return lb;
}
