public byte[] serializeBatch(BatchTuple batch) {
if (batch == null || batch.currBatchSize() == 0) return null;
_kryoOut.clear();
for (Tuple tuple : batch.getTuples()) {
/*
* byte structure:
* 1st tuple: length + tuple bytes
* 2nd tuple: length + tuple bytes
* ......
*/
int startPos = _kryoOut.position();
// Set initial value of tuple length, which will be updated accordingly after serialization
_kryoOut.writeInt(0);
serializeTuple(_kryoOut, tuple);
// Update the tuple length
int endPos = _kryoOut.position();
_kryoOut.setPosition(startPos);
_kryoOut.writeInt(endPos - startPos - 4);
_kryoOut.setPosition(endPos);
}
return _kryoOut.toBytes();
}
private void writeIndex() {
Output indexOutput;
try {
indexOutput = new Output(new FileOutputStream(getIndexFile()));
} catch (FileNotFoundException e) {
throw new UncheckedIOException(e);
}
try {
indexOutput.writeInt(index.size(), true);
for (Map.Entry<Long, Map<Long, TestCaseRegion>> classEntry : index.entrySet()) {
Long classId = classEntry.getKey();
Map<Long, TestCaseRegion> regions = classEntry.getValue();
indexOutput.writeLong(classId, true);
indexOutput.writeInt(regions.size(), true);
for (Map.Entry<Long, TestCaseRegion> testCaseEntry : regions.entrySet()) {
long id = testCaseEntry.getKey();
TestCaseRegion region = testCaseEntry.getValue();
indexOutput.writeLong(id, true);
indexOutput.writeLong(region.stdOutRegion.start);
indexOutput.writeLong(region.stdOutRegion.stop);
indexOutput.writeLong(region.stdErrRegion.start);
indexOutput.writeLong(region.stdErrRegion.stop);
}
}
} finally {
indexOutput.close();
}
}
@Override
public void write(Kryo kryo, Output out) {
byte[] bytes = getBytes();
out.writeInt(bytes.length);
out.writeInt(this.numFields);
out.write(bytes);
}
@Override
public void write(Kryo kryo, Output output) {
output.writeInt(objectID, true);
int methodClassID = kryo.getRegistration(method.getDeclaringClass()).getId();
output.writeInt(methodClassID, true);
CachedMethod[] cachedMethods = getMethods(kryo, method.getDeclaringClass());
CachedMethod cachedMethod = null;
for (int i = 0, n = cachedMethods.length; i < n; i++) {
cachedMethod = cachedMethods[i];
if (cachedMethod.method.equals(method)) {
output.writeByte(i);
break;
}
}
for (int i = 0, n = cachedMethod.serializers.length; i < n; i++) {
Serializer serializer = cachedMethod.serializers[i];
if (serializer != null) {
kryo.writeObjectOrNull(output, args[i], serializer);
} else {
kryo.writeClassAndObject(output, args[i]);
}
}
output.writeByte(responseID);
}
public void write(Kryo kryo, Output output, Calendar object) {
timeZoneSerializer.write(kryo, output, object.getTimeZone()); // can't be null
output.writeLong(object.getTimeInMillis(), true);
output.writeBoolean(object.isLenient());
output.writeInt(object.getFirstDayOfWeek(), true);
output.writeInt(object.getMinimalDaysInFirstWeek(), true);
if (object instanceof GregorianCalendar)
output.writeLong(((GregorianCalendar) object).getGregorianChange().getTime(), false);
else output.writeLong(DEFAULT_GREGORIAN_CUTOVER, false);
}
public void write(Kryo kryo, Output output, BigDecimal object) {
if (object == null) {
output.writeVarInt(NULL, true);
return;
}
BigDecimal value = (BigDecimal) object;
// fast-path optimizations for BigDecimal constants
if (value == BigDecimal.ZERO) {
bigIntegerSerializer.write(kryo, output, BigInteger.ZERO);
output.writeInt(0, false); // for backwards compatibility
return;
}
// default behaviour
bigIntegerSerializer.write(kryo, output, value.unscaledValue());
output.writeInt(value.scale(), false);
}
@Override
public void write(Kryo kryo, Output output) {
output.writeInt(player.getId());
output.writeInt(target.getId());
output.writeLong(time);
kryo.writeObject(output, meanOfDeath);
}
@Override
protected void writeObject(Kryo kryo, Output output, VideoChunk video) throws Exception {
output.writeLong(video.getDuration());
output.writeInt(video.getVideo().length);
output.writeBytes(video.getVideo());
output.writeString(video.getContainer());
}
/**
* @@@ in the furture, it will skill serialize 'targetTask' through check some flag
*
* @see ITupleSerializer#serialize(int, Tuple)
*/
private void serializeTuple(Output output, Tuple tuple) {
try {
if (tuple instanceof TupleExt) {
output.writeInt(((TupleExt) tuple).getTargetTaskId());
output.writeLong(((TupleExt) tuple).getCreationTimeStamp());
}
output.writeInt(tuple.getSourceTask(), true);
output.writeInt(
_ids.getStreamId(tuple.getSourceComponent(), tuple.getSourceStreamId()), true);
tuple.getMessageId().serialize(output);
_kryo.serializeInto(tuple.getValues(), output);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private static void prepatePackageData(PackageVersion version, List<Instance> members) {
LOG.info("Preparing Package Data file for package '" + version);
String jarName = JarUtils.getPackagePath(version, false);
// Prepare jar for this package version
DataStore dataStore = InvocationContext.get().getDataStore();
try (GZIPOutputStream out = new GZIPOutputStream(new FileOutputStream(jarName))) {
Kryo kryo = new Kryo();
Output output = new Output(out);
// Write Package Members
List<Instance> collect = members.stream().collect(Collectors.toList());
long id = version.getId();
SerializationUtil.writeInstances(kryo, output, collect);
// Write QueryPlans
List<ReplicationSQLQueryPlan> qPlans = dataStore.getAllQueryPlansByPackageVersion(id);
output.writeInt(qPlans.size());
qPlans.forEach(m -> m.write(kryo, output));
// Write ModelHierarchyInfos
List<ModelHierarchyInfo> mHierarchies = dataStore.getAllModelHierachyInfoByPackageVersion(id);
output.writeInt(mHierarchies.size());
mHierarchies.forEach(
m -> {
output.writeInt(m.getParent());
output.writeInt(m.getSubModel());
});
// Write ClassHierarchy infos
List<ClassHierarchyInfo> clsHierarchies =
dataStore.getAllClassHierachyInfoByPackageVersion(id);
output.writeInt(clsHierarchies.size());
clsHierarchies.forEach(
m -> {
output.writeString(m.getCls());
output.writeString(m.getSubClass());
});
output.close();
} catch (Exception e) {
LOG.info("Package Data not prepared for " + jarName);
throw new RuntimeException(e);
}
}
@Override
public void write(Kryo kryo, Output output) {
output.writeString(this.firstname);
output.writeString(this.lastname);
kryo.writeObjectOrNull(output, this.emailAddress, SerializableEmailAddress.class);
output.writeInt(this.addresses.size());
for (Address address : this.addresses) {
kryo.writeObject(output, address);
}
}
public void testCanReadInt() throws IOException {
Output write = new Output(new ByteArrayOutputStream());
Input read = new Input(write.toBytes());
assertEquals(false, read.canReadInt());
write.writeInt(400, true);
read = new Input(write.toBytes());
assertEquals(true, read.canReadInt());
read.setLimit(read.limit() - 1);
assertEquals(false, read.canReadInt());
}
public void write(Kryo kryo, Output output, EnumSet object) {
Serializer serializer;
if (object.isEmpty()) {
EnumSet tmp = EnumSet.complementOf(object);
if (tmp.isEmpty())
throw new KryoException("An EnumSet must have a defined Enum to be serialized.");
serializer = kryo.writeClass(output, tmp.iterator().next().getClass()).getSerializer();
} else {
serializer = kryo.writeClass(output, object.iterator().next().getClass()).getSerializer();
}
output.writeInt(object.size(), true);
for (Object element : object) serializer.write(kryo, output, element);
}
public void onOutput(long classId, long testId, TestOutputEvent outputEvent) {
boolean stdout = outputEvent.getDestination() == TestOutputEvent.Destination.StdOut;
mark(classId, testId, stdout);
output.writeBoolean(stdout);
output.writeLong(classId);
output.writeLong(testId);
byte[] bytes;
try {
bytes = outputEvent.getMessage().getBytes(messageStorageCharset.name());
} catch (UnsupportedEncodingException e) {
throw UncheckedException.throwAsUncheckedException(e);
}
output.writeInt(bytes.length);
output.writeBytes(bytes);
}
@Override
public void write(final Kryo kryo, final Output output, final List<?> obj) {
try {
final Object[] array = (Object[]) _arrayField.get(obj);
output.writeInt(array.length, true);
final Class<?> componentType = array.getClass().getComponentType();
kryo.writeClass(output, componentType);
for (final Object item : array) {
kryo.writeClassAndObject(output, item);
}
} catch (final RuntimeException e) {
// Don't eat and wrap RuntimeExceptions because the ObjectBuffer.write...
// handles SerializationException specifically (resizing the buffer)...
throw e;
} catch (final Exception e) {
throw new RuntimeException(e);
}
}
@Override
public void write(Kryo kryo, Output output, MastershipTerm object) {
kryo.writeClassAndObject(output, object.master());
output.writeInt(object.termNumber());
}
public void write(Kryo kryo, Output output, Integer object) {
output.writeInt(object, false);
}
@Override
public void write(Kryo kryo, Output output) {
super.write(kryo, output);
output.writeInt(code);
}
@Override
public void write(Kryo kryo, Output output, Timestamp ts) {
output.writeLong(ts.getTime());
output.writeInt(ts.getNanos());
}
private void runIntTest(Output write) throws IOException {
write.writeInt(0);
write.writeInt(63);
write.writeInt(64);
write.writeInt(127);
write.writeInt(128);
write.writeInt(8192);
write.writeInt(16384);
write.writeInt(2097151);
write.writeInt(1048575);
write.writeInt(134217727);
write.writeInt(268435455);
write.writeInt(134217728);
write.writeInt(268435456);
write.writeInt(-2097151);
write.writeInt(-1048575);
write.writeInt(-134217727);
write.writeInt(-268435455);
write.writeInt(-134217728);
write.writeInt(-268435456);
assertEquals(1, write.writeInt(0, true));
assertEquals(1, write.writeInt(0, false));
assertEquals(1, write.writeInt(63, true));
assertEquals(1, write.writeInt(63, false));
assertEquals(1, write.writeInt(64, true));
assertEquals(2, write.writeInt(64, false));
assertEquals(1, write.writeInt(127, true));
assertEquals(2, write.writeInt(127, false));
assertEquals(2, write.writeInt(128, true));
assertEquals(2, write.writeInt(128, false));
assertEquals(2, write.writeInt(8191, true));
assertEquals(2, write.writeInt(8191, false));
assertEquals(2, write.writeInt(8192, true));
assertEquals(3, write.writeInt(8192, false));
assertEquals(2, write.writeInt(16383, true));
assertEquals(3, write.writeInt(16383, false));
assertEquals(3, write.writeInt(16384, true));
assertEquals(3, write.writeInt(16384, false));
assertEquals(3, write.writeInt(2097151, true));
assertEquals(4, write.writeInt(2097151, false));
assertEquals(3, write.writeInt(1048575, true));
assertEquals(3, write.writeInt(1048575, false));
assertEquals(4, write.writeInt(134217727, true));
assertEquals(4, write.writeInt(134217727, false));
assertEquals(4, write.writeInt(268435455, true));
assertEquals(5, write.writeInt(268435455, false));
assertEquals(4, write.writeInt(134217728, true));
assertEquals(5, write.writeInt(134217728, false));
assertEquals(5, write.writeInt(268435456, true));
assertEquals(5, write.writeInt(268435456, false));
assertEquals(1, write.writeInt(-64, false));
assertEquals(5, write.writeInt(-64, true));
assertEquals(2, write.writeInt(-65, false));
assertEquals(5, write.writeInt(-65, true));
assertEquals(2, write.writeInt(-8192, false));
assertEquals(5, write.writeInt(-8192, true));
assertEquals(3, write.writeInt(-1048576, false));
assertEquals(5, write.writeInt(-1048576, true));
assertEquals(4, write.writeInt(-134217728, false));
assertEquals(5, write.writeInt(-134217728, true));
assertEquals(5, write.writeInt(-134217729, false));
assertEquals(5, write.writeInt(-134217729, true));
Input read = new Input(write.toBytes());
assertEquals(0, read.readInt());
assertEquals(63, read.readInt());
assertEquals(64, read.readInt());
assertEquals(127, read.readInt());
assertEquals(128, read.readInt());
assertEquals(8192, read.readInt());
assertEquals(16384, read.readInt());
assertEquals(2097151, read.readInt());
assertEquals(1048575, read.readInt());
assertEquals(134217727, read.readInt());
assertEquals(268435455, read.readInt());
assertEquals(134217728, read.readInt());
assertEquals(268435456, read.readInt());
assertEquals(-2097151, read.readInt());
assertEquals(-1048575, read.readInt());
assertEquals(-134217727, read.readInt());
assertEquals(-268435455, read.readInt());
assertEquals(-134217728, read.readInt());
assertEquals(-268435456, read.readInt());
assertEquals(true, read.canReadInt());
assertEquals(true, read.canReadInt());
assertEquals(true, read.canReadInt());
assertEquals(0, read.readInt(true));
assertEquals(0, read.readInt(false));
assertEquals(63, read.readInt(true));
assertEquals(63, read.readInt(false));
assertEquals(64, read.readInt(true));
assertEquals(64, read.readInt(false));
assertEquals(127, read.readInt(true));
assertEquals(127, read.readInt(false));
assertEquals(128, read.readInt(true));
assertEquals(128, read.readInt(false));
assertEquals(8191, read.readInt(true));
assertEquals(8191, read.readInt(false));
assertEquals(8192, read.readInt(true));
assertEquals(8192, read.readInt(false));
assertEquals(16383, read.readInt(true));
assertEquals(16383, read.readInt(false));
assertEquals(16384, read.readInt(true));
assertEquals(16384, read.readInt(false));
assertEquals(2097151, read.readInt(true));
assertEquals(2097151, read.readInt(false));
assertEquals(1048575, read.readInt(true));
assertEquals(1048575, read.readInt(false));
assertEquals(134217727, read.readInt(true));
assertEquals(134217727, read.readInt(false));
assertEquals(268435455, read.readInt(true));
assertEquals(268435455, read.readInt(false));
assertEquals(134217728, read.readInt(true));
assertEquals(134217728, read.readInt(false));
assertEquals(268435456, read.readInt(true));
assertEquals(268435456, read.readInt(false));
assertEquals(-64, read.readInt(false));
assertEquals(-64, read.readInt(true));
assertEquals(-65, read.readInt(false));
assertEquals(-65, read.readInt(true));
assertEquals(-8192, read.readInt(false));
assertEquals(-8192, read.readInt(true));
assertEquals(-1048576, read.readInt(false));
assertEquals(-1048576, read.readInt(true));
assertEquals(-134217728, read.readInt(false));
assertEquals(-134217728, read.readInt(true));
assertEquals(-134217729, read.readInt(false));
assertEquals(-134217729, read.readInt(true));
assertEquals(false, read.canReadInt());
Random random = new Random();
for (int i = 0; i < 10000; i++) {
int value = random.nextInt();
write.clear();
write.writeInt(value);
write.writeInt(value, true);
write.writeInt(value, false);
read.setBuffer(write.toBytes());
assertEquals(value, read.readInt());
assertEquals(value, read.readInt(true));
assertEquals(value, read.readInt(false));
}
}
@Override
public void write(Kryo kryo, Output output) {
if (this.ic != null) this.icID = this.ic.getID();
output.writeInt(this.icID);
}
public void writeBinary(byte[] bytes, int offset, int count) throws IOException {
output.writeInt(count, true);
output.writeBytes(bytes, offset, count);
}
public void writeSizeInt(int value) throws IOException {
output.writeInt(value, true);
}