/*
* (non-Javadoc)
* @see com.esotericsoftware.kryo.KryoSerializable#read(com.esotericsoftware.kryo.Kryo,
* com.esotericsoftware.kryo.io.Input)
*/
@SuppressWarnings("unchecked")
@Override
public void read(final Kryo kryo, final Input input) {
this.mode = kryo.readObject(input, ExecutionMode.class);
final ArrayList<String> requiredPackages = kryo.readObject(input, ArrayList.class);
final JobID dummId = JobID.generate();
final ClassLoader oldClassLoader = kryo.getClassLoader();
try {
LibraryCacheManager.register(
dummId, requiredPackages.toArray(new String[requiredPackages.size()]));
kryo.setClassLoader(LibraryCacheManager.getClassLoader(dummId));
this.query = kryo.readObject(input, SopremoPlan.class);
} catch (final Exception e) {
SopremoUtil.LOG.error(e.getMessage());
throw new KryoException(e);
} finally {
kryo.setClassLoader(oldClassLoader);
try {
LibraryCacheManager.unregister(dummId);
} catch (final Throwable e) {
SopremoUtil.LOG.error(e.getMessage());
}
}
}
private Vertex readVertex(
final Direction directionRequested,
final Function<DetachedVertex, Vertex> vertexMaker,
final Function<DetachedEdge, Edge> edgeMaker,
final Input input)
throws IOException {
if (null != directionRequested && null == edgeMaker)
throw new IllegalArgumentException(
"If a directionRequested is specified then an edgeAdder function should also be specified");
this.headerReader.read(kryo, input);
final DetachedVertex detachedVertex = (DetachedVertex) kryo.readClassAndObject(input);
final Vertex v = vertexMaker.apply(detachedVertex);
final boolean streamContainsEdgesInSomeDirection = input.readBoolean();
if (!streamContainsEdgesInSomeDirection && directionRequested != null)
throw new IllegalStateException(
String.format(
"The direction %s was requested but no attempt was made to serialize edges into this stream",
directionRequested));
// if there are edges in the stream and the direction is not present then the rest of the stream
// is
// simply ignored
if (directionRequested != null) {
final Direction directionsInStream = kryo.readObject(input, Direction.class);
if (directionsInStream != Direction.BOTH && directionsInStream != directionRequested)
throw new IllegalStateException(
String.format(
"Stream contains %s edges, but requesting %s",
directionsInStream, directionRequested));
final Direction firstDirection = kryo.readObject(input, Direction.class);
if (firstDirection == Direction.OUT
&& (directionRequested == Direction.BOTH || directionRequested == Direction.OUT))
readEdges(input, edgeMaker);
else {
// requested direction in, but BOTH must be serialized so skip this. the
// illegalstateexception
// prior to this IF should have caught a problem where IN is not supported at all
if (firstDirection == Direction.OUT && directionRequested == Direction.IN) skipEdges(input);
}
if (directionRequested == Direction.BOTH || directionRequested == Direction.IN) {
// if the first direction was OUT then it was either read or skipped. in that case, the
// marker
// of the stream is currently ready to read the IN direction. otherwise it's in the perfect
// place
// to start reading edges
if (firstDirection == Direction.OUT) kryo.readObject(input, Direction.class);
readEdges(input, edgeMaker);
}
}
return v;
}
// Required by Kryo serialization.
public void read(Kryo kryo, Input in) {
intVal = kryo.readObject(in, Integer.class);
floatVal = kryo.readObject(in, Float.class);
shortVal = kryo.readObject(in, Short.class);
byteArr = kryo.readObject(in, byte[].class);
longArr = kryo.readObject(in, long[].class);
dblArr = kryo.readObject(in, double[].class);
str = kryo.readObject(in, String.class);
}
public SerializablePage loadPage(InputStream in) {
synchronized (serializeLock) {
Input input = null;
try {
input = new Input(in);
input.readString();
kryo.readObject(input, UrlPageModel.class);
return kryo.readObject(input, SerializablePage.class);
} finally {
IOUtils.closeQuietly(input);
}
}
}
@Override
public VectorSet read(Kryo kryo, Input input, Class<VectorSet> type) {
String key = kryo.readObject(input, String.class);
float accumuFactor = kryo.readObject(input, float.class);
int sparseFactor = kryo.readObject(input, int.class);
VectorSet vectorSet = new VectorSet(key, basis, accumuFactor, sparseFactor);
int sizeVector = kryo.readObject(input, int.class);
int vecid, start, length;
while (sizeVector > 0) {
vecid = kryo.readObject(input, int.class);
start = kryo.readObject(input, int.class);
length = kryo.readObject(input, int.class);
vectorSet.indexer.put(vecid, start);
vectorSet.lengths.put(vecid, length);
sizeVector--;
}
int sizeData = kryo.readObject(input, int.class);
while (sizeData > 0) {
float val = kryo.readObject(input, float.class);
vectorSet.data.add(val);
sizeData--;
}
return vectorSet;
}
private static <T> T deserializeObjectByKryo(Kryo kryo, InputStream in, Class<T> clazz) {
Input inp = new Input(in);
kryo.setClassLoader(Utilities.getSessionSpecifiedClassLoader());
T t = kryo.readObject(inp, clazz);
inp.close();
return t;
}
public Object read(Kryo kryo, Input input, Class type) {
// The inflater would read from input beyond the compressed bytes if chunked enoding wasn't
// used.
InflaterInputStream inflaterStream =
new InflaterInputStream(new InputChunked(input, 256), new Inflater(noHeaders));
return kryo.readObject(new Input(inflaterStream, 256), type, serializer);
}
@Override
public void read(Kryo kryo, Input input) {
playerId = input.readInt();
targetId = input.readInt();
time = input.readLong();
meanOfDeath = kryo.readObject(input, EnumMeanOfDeath.class);
}
@Override
public <T> T decode(byte[] data, Class<T> classOfT) {
Input input = null;
input = new Input(data);
T res = kryo.readObject(input, classOfT);
input.close();
return res;
}
public Map<String, Recommendation> readRecommendations(
Input input, Map<String, VectorSet> vectorSets) {
Map<String, Recommendation> recs = new HashMap<String, Recommendation>();
int size = kryo.readObject(input, int.class);
while (size > 0) {
String srcKey = kryo.readObject(input, String.class);
String tgtKey = kryo.readObject(input, String.class);
VectorSet src = vectorSets.get(srcKey);
VectorSet tgt = vectorSets.get(tgtKey);
Recommendation rec = readR(src, tgt, input);
recs.put(srcKey + '_' + tgtKey, rec);
src.addListener(rec);
if (src != tgt) {
tgt.addListener(rec);
}
size--;
}
return recs;
}
public Map<String, VectorSet> readVectorSets(Input input, Basis base) {
Map<String, VectorSet> vectorSets = new HashMap<String, VectorSet>();
int size = kryo.readObject(input, int.class);
while (size > 0) {
VectorSet vectorSet = readSVS(base, input);
vectorSets.put(vectorSet.key(), vectorSet);
size--;
}
return vectorSets;
}
@Override
public void read(Kryo kryo, Input input) {
this.firstname = input.readString();
this.lastname = input.readString();
this.emailAddress = kryo.readObjectOrNull(input, SerializableEmailAddress.class);
this.addresses = new HashSet<Address>();
int numAddresses = input.readInt();
for (int i = 0; i < numAddresses; i++) {
this.add(kryo.readObject(input, SerializableAddress.class));
}
}
public Pair<String, UrlPageModel> loadPageInfo(InputStream in) {
synchronized (serializeLock) {
Input input = null;
try {
input = new Input(in);
String title = input.readString();
UrlPageModel pageModel = kryo.readObject(input, UrlPageModel.class);
return Pair.of(title, pageModel);
} finally {
IOUtils.closeQuietly(input);
}
}
}
private static <T extends Serializable> T deserializeObjectFromKryo(
byte[] bytes, Class<T> clazz) {
Input inp = new Input(new ByteArrayInputStream(bytes));
Kryo kryo = borrowKryo();
T func = null;
try {
func = kryo.readObject(inp, clazz);
} finally {
releaseKryo(kryo);
}
inp.close();
return func;
}
@Test
public void testSerDeserPerf2() throws Exception {
Kryo kryo = new Kryo();
String outputPath = FilenameUtils.concat(getTmpPath(), "file2.bin");
Output output = new Output(new FileOutputStream(outputPath));
for (int i = 0; i < 1000; i++) {
kryo.writeObject(output, constructNewPE());
}
output.close();
Input input = new Input(new FileInputStream(outputPath));
NewPartitionedEvent someObject = kryo.readObject(input, NewPartitionedEvent.class);
input.close();
Assert.assertTrue(someObject.getData().length == 1);
}
/**
* @param sopremoRecord
* @return
*/
private SopremoRecord serializeAndDeserialize(SopremoRecord sopremoRecord) throws IOException {
Kryo kryo = new Kryo();
kryo.setReferences(false);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
Output output = new Output(baos);
kryo.writeObject(output, sopremoRecord);
output.close();
baos.close();
ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
Input input = new Input(bais);
SopremoRecord deserialized = kryo.readObject(input, SopremoRecord.class);
deserialized.setLayout(sopremoRecord.getLayout());
input.close();
return deserialized;
}
@Override
public Object read(final Kryo kryo, final Input input, final Class<Object> type) {
final InvocationHandler invocationHandler = (InvocationHandler) kryo.readClassAndObject(input);
final Class<?>[] interfaces = kryo.readObject(input, Class[].class);
final ClassLoader classLoader = kryo.getClass().getClassLoader(); // TODO: can we do this?
try {
return Proxy.newProxyInstance(classLoader, interfaces, invocationHandler);
} catch (final RuntimeException e) {
System.err.println(
getClass().getName()
+ ".read:\n"
+ "Could not create proxy using classLoader "
+ classLoader
+ ","
+ " have invocationhandler.classloader: "
+ invocationHandler.getClass().getClassLoader()
+ " have contextclassloader: "
+ Thread.currentThread().getContextClassLoader());
throw e;
}
}
private <E> E readTableFile(String key, File originalFile) {
try {
final Input i = new Input(new FileInputStream(originalFile));
final Kryo kryo = getKryo();
//noinspection unchecked
final ObjectEntity<E> objectEntity = kryo.readObject(i, ObjectEntity.class);
objectEntity.restoreReferences();
i.close();
return objectEntity.getContent();
} catch (FileNotFoundException | KryoException e) {
// Clean up an unsuccessfully written file
if (originalFile.exists()) {
if (!originalFile.delete()) {
throw new PaperDbException(
"Couldn't clean up broken/unserializable file " + originalFile, e);
}
}
throw new PaperDbException(
"Couldn't read/deserialize file " + originalFile + " for table " + key, e);
}
}
private <T> T deserialize(File file, Class<T> type) {
if (file == null || !file.exists()) {
return null;
}
synchronized (serializeLock) {
Input input = null;
try {
input = new Input(new FileInputStream(file));
return kryo.readObject(input, type);
} catch (Exception e) {
Logger.e(TAG, e);
} catch (OutOfMemoryError oom) {
MainApplication.freeMemory();
Logger.e(TAG, oom);
} finally {
IOUtils.closeQuietly(input);
}
}
return null;
}
@Override
public Collection<Partition<AbstractFileInputOperator<T>>> definePartitions(
Collection<Partition<AbstractFileInputOperator<T>>> partitions, PartitioningContext context) {
lastRepartition = System.currentTimeMillis();
int totalCount = getNewPartitionCount(partitions, context);
LOG.debug("Computed new partitions: {}", totalCount);
if (totalCount == partitions.size()) {
return partitions;
}
AbstractFileInputOperator<T> tempOperator =
partitions.iterator().next().getPartitionedInstance();
MutableLong tempGlobalNumberOfRetries = tempOperator.globalNumberOfRetries;
MutableLong tempGlobalNumberOfFailures = tempOperator.globalNumberOfRetries;
/*
* Build collective state from all instances of the operator.
*/
Set<String> totalProcessedFiles = Sets.newHashSet();
Set<FailedFile> currentFiles = Sets.newHashSet();
List<DirectoryScanner> oldscanners = Lists.newLinkedList();
List<FailedFile> totalFailedFiles = Lists.newLinkedList();
List<String> totalPendingFiles = Lists.newLinkedList();
Set<Integer> deletedOperators = Sets.newHashSet();
for (Partition<AbstractFileInputOperator<T>> partition : partitions) {
AbstractFileInputOperator<T> oper = partition.getPartitionedInstance();
totalProcessedFiles.addAll(oper.processedFiles);
totalFailedFiles.addAll(oper.failedFiles);
totalPendingFiles.addAll(oper.pendingFiles);
currentFiles.addAll(unfinishedFiles);
tempGlobalNumberOfRetries.add(oper.localNumberOfRetries);
tempGlobalNumberOfFailures.add(oper.localNumberOfFailures);
if (oper.currentFile != null) {
currentFiles.add(new FailedFile(oper.currentFile, oper.offset));
}
oldscanners.add(oper.getScanner());
deletedOperators.add(oper.operatorId);
}
/*
* Create partitions of scanners, scanner's partition method will do state
* transfer for DirectoryScanner objects.
*/
List<DirectoryScanner> scanners = scanner.partition(totalCount, oldscanners);
Kryo kryo = new Kryo();
Collection<Partition<AbstractFileInputOperator<T>>> newPartitions =
Lists.newArrayListWithExpectedSize(totalCount);
Collection<IdempotentStorageManager> newManagers =
Lists.newArrayListWithExpectedSize(totalCount);
for (int i = 0; i < scanners.size(); i++) {
// Kryo.copy fails as it attempts to clone transient fields
ByteArrayOutputStream bos = new ByteArrayOutputStream();
Output loutput = new Output(bos);
kryo.writeObject(loutput, this);
loutput.close();
Input lInput = new Input(bos.toByteArray());
@SuppressWarnings("unchecked")
AbstractFileInputOperator<T> oper = kryo.readObject(lInput, this.getClass());
lInput.close();
DirectoryScanner scn = scanners.get(i);
oper.setScanner(scn);
// Do state transfer for processed files.
oper.processedFiles.addAll(totalProcessedFiles);
oper.globalNumberOfFailures = tempGlobalNumberOfRetries;
oper.localNumberOfFailures.setValue(0);
oper.globalNumberOfRetries = tempGlobalNumberOfFailures;
oper.localNumberOfRetries.setValue(0);
/* redistribute unfinished files properly */
oper.unfinishedFiles.clear();
oper.currentFile = null;
oper.offset = 0;
Iterator<FailedFile> unfinishedIter = currentFiles.iterator();
while (unfinishedIter.hasNext()) {
FailedFile unfinishedFile = unfinishedIter.next();
if (scn.acceptFile(unfinishedFile.path)) {
oper.unfinishedFiles.add(unfinishedFile);
unfinishedIter.remove();
}
}
/* transfer failed files */
oper.failedFiles.clear();
Iterator<FailedFile> iter = totalFailedFiles.iterator();
while (iter.hasNext()) {
FailedFile ff = iter.next();
if (scn.acceptFile(ff.path)) {
oper.failedFiles.add(ff);
iter.remove();
}
}
/* redistribute pending files properly */
oper.pendingFiles.clear();
Iterator<String> pendingFilesIterator = totalPendingFiles.iterator();
while (pendingFilesIterator.hasNext()) {
String pathString = pendingFilesIterator.next();
if (scn.acceptFile(pathString)) {
oper.pendingFiles.add(pathString);
pendingFilesIterator.remove();
}
}
newPartitions.add(new DefaultPartition<AbstractFileInputOperator<T>>(oper));
newManagers.add(oper.idempotentStorageManager);
}
idempotentStorageManager.partitioned(newManagers, deletedOperators);
LOG.info("definePartitions called returning {} partitions", newPartitions.size());
return newPartitions;
}
@Override
public void read(Kryo kryo, Input input) {
setUser(kryo.readObject(input, KryoUser.class));
setPassword(input.readString());
setFriends((List<User>) kryo.readClassAndObject(input));
}
@Override
public Basis read(Kryo kryo, Input input, Class<Basis> type) {
String key = kryo.readObject(input, String.class);
String[] schema = kryo.readObject(input, String[].class);
return new Basis(key, schema);
}
@Override
public void readGraph(final InputStream inputStream, final Graph graphToWriteTo)
throws IOException {
this.counter.set(0);
final Input input = new Input(inputStream);
this.headerReader.read(kryo, input);
final BatchGraph graph;
try {
// will throw an exception if not constructed properly
graph =
BatchGraph.build(graphToWriteTo)
.vertexIdKey(vertexIdKey)
.edgeIdKey(edgeIdKey)
.bufferSize(batchSize)
.create();
} catch (Exception ex) {
throw new IOException("Could not instantiate BatchGraph wrapper", ex);
}
try (final Output output = new Output(new FileOutputStream(tempFile))) {
final boolean supportedMemory = input.readBoolean();
if (supportedMemory) {
// if the graph that serialized the data supported sideEffects then the sideEffects needs to
// be read
// to advance the reader forward. if the graph being read into doesn't support the
// sideEffects
// then we just setting the data to sideEffects.
final Map<String, Object> memMap =
(Map<String, Object>) kryo.readObject(input, HashMap.class);
if (graphToWriteTo.features().graph().variables().supportsVariables()) {
final Graph.Variables variables = graphToWriteTo.variables();
memMap.forEach(variables::set);
}
}
final boolean hasSomeVertices = input.readBoolean();
if (hasSomeVertices) {
while (!input.eof()) {
final List<Object> vertexArgs = new ArrayList<>();
final DetachedVertex current = (DetachedVertex) kryo.readClassAndObject(input);
appendToArgList(vertexArgs, T.id, current.id());
appendToArgList(vertexArgs, T.label, current.label());
final Vertex v = graph.addVertex(vertexArgs.toArray());
current
.iterators()
.propertyIterator()
.forEachRemaining(p -> createVertexProperty(graphToWriteTo, v, p, false));
// current.iterators().hiddenPropertyIterator().forEachRemaining(p ->
// createVertexProperty(graphToWriteTo, v, p, true));
// the gio file should have been written with a direction specified
final boolean hasDirectionSpecified = input.readBoolean();
final Direction directionInStream = kryo.readObject(input, Direction.class);
final Direction directionOfEdgeBatch = kryo.readObject(input, Direction.class);
// graph serialization requires that a direction be specified in the stream and that the
// direction of the edges be OUT
if (!hasDirectionSpecified
|| directionInStream != Direction.OUT
|| directionOfEdgeBatch != Direction.OUT)
throw new IllegalStateException(
String.format(
"Stream must specify edge direction and that direction must be %s",
Direction.OUT));
// if there are edges then read them to end and write to temp, otherwise read what should
// be
// the vertex terminator
if (!input.readBoolean()) kryo.readClassAndObject(input);
else readToEndOfEdgesAndWriteToTemp(input, output);
}
}
} catch (Exception ex) {
throw new IOException(ex);
}
// done writing to temp
// start reading in the edges now from the temp file
try (final Input edgeInput = new Input(new FileInputStream(tempFile))) {
readFromTempEdges(edgeInput, graph);
graph.tx().commit();
} catch (Exception ex) {
ex.printStackTrace();
throw new IOException(ex);
} finally {
deleteTempFileSilently();
}
}
@Override
public Recommendation read(Kryo kryo, Input input, Class<Recommendation> type) {
kryo.readObject(input, String.class);
CosineScore scoring = new CosineScore();
int limits = kryo.readObject(input, int.class);
int sortersSize = kryo.readObject(input, int.class);
Recommendation rec = new Recommendation(source, target, scoring, limits);
while (sortersSize > 0) {
int size = kryo.readObject(input, int.class);
long srcId = kryo.readObject(input, long.class);
float waterline = kryo.readObject(input, float.class);
while (size > 0) {
long tgtId = kryo.readObject(input, long.class);
float score = kryo.readObject(input, float.class);
rec.add(srcId, tgtId, score);
size--;
}
if (rec.sorters.containsKey(srcId)) {
rec.sorters.get(srcId).waterline = waterline;
} else {
logger.warn(String.format("vecid[%d] not in sorters", srcId));
}
sortersSize--;
}
return rec;
}