public Object read(int token, DataInputStream inputStream) throws IOException {
String className = tupleSerialization.getClassNameFor(token);
if (className == null) className = WritableUtils.readString(inputStream);
Deserializer deserializer = deserializers.get(className);
if (deserializer == null) {
deserializer = tupleSerialization.getNewDeserializer(className);
deserializer.open(inputStream);
deserializers.put(className, deserializer);
}
Object foundObject = null;
Object object = null;
try {
object = deserializer.deserialize(foundObject);
} catch (IOException exception) {
LOG.error(
"failed deserializing token: " + token + " with classname: " + className, exception);
throw exception;
}
return object;
}
@SuppressWarnings("unchecked")
public void readFields(DataInput is) throws IOException {
disableCounter = is.readBoolean();
isMultiInputs = is.readBoolean();
totalSplits = is.readInt();
splitIndex = is.readInt();
inputIndex = is.readInt();
targetOps = (ArrayList<OperatorKey>) readObject(is);
int splitLen = is.readInt();
String splitClassName = is.readUTF();
try {
Class splitClass = conf.getClassByName(splitClassName);
SerializationFactory sf = new SerializationFactory(conf);
// The correct call sequence for Deserializer is, we shall open, then deserialize, but we
// shall not close
Deserializer d = sf.getDeserializer(splitClass);
d.open((InputStream) is);
wrappedSplits = new InputSplit[splitLen];
for (int i = 0; i < splitLen; i++) {
wrappedSplits[i] = (InputSplit) ReflectionUtils.newInstance(splitClass, conf);
d.deserialize(wrappedSplits[i]);
}
} catch (ClassNotFoundException e) {
throw new IOException(e);
}
}
public void close() {
if (deserializers.size() == 0) return;
Collection<Deserializer> clone = new ArrayList<Deserializer>(deserializers.values());
deserializers.clear();
for (Deserializer deserializer : clone) {
try {
deserializer.close();
} catch (IOException exception) {
// do nothing
}
}
}
@SuppressWarnings("unchecked")
public List<KeyData<K>> getKeyData() throws IOException, ClassNotFoundException {
if (keyData != null) return keyData;
final FileSystem fs = FileSystem.get(conf);
keyData = new ArrayList<KeyData<K>>();
final long filesize = fs.getFileStatus(path).getLen();
FSDataInputStream in = fs.open(path);
/* The last long in the file is the start position of the trailer section */
in.seek(filesize - 8);
long metaDataStartPos = in.readLong();
in.seek(metaDataStartPos);
ObjectMapper mapper = new ObjectMapper();
metadataJson = mapper.readValue(in.readUTF(), JsonNode.class);
int keySectionSize = in.readInt();
// load the key section
byte[] keySection = new byte[keySectionSize];
in.seek(filesize - keySectionSize - 8);
in.read(keySection, 0, keySectionSize);
in.close();
ByteArrayInputStream bis = new ByteArrayInputStream(keySection);
DataInput dataInput = new DataInputStream(bis);
int numberOfBlocks = metadataJson.get("numberOfBlocks").getIntValue();
// load the key section
keyClass = (Class<K>) ClassCache.forName(JsonUtils.getText(metadataJson, "keyClass"));
valueClass = (Class<V>) ClassCache.forName(JsonUtils.getText(metadataJson, "valueClass"));
SerializationFactory serializationFactory = new SerializationFactory(conf);
Deserializer<K> deserializer = serializationFactory.getDeserializer(keyClass);
deserializer.open(bis);
while (bis.available() > 0 && numberOfBlocks > 0) {
K key = deserializer.deserialize(null);
long offset = dataInput.readLong();
long blockId = dataInput.readLong();
long numRecords = dataInput.readLong();
keyData.add(new KeyData<K>(key, offset, 0, numRecords, blockId));
numberOfBlocks--;
}
// Assign length to each keydata entry
int numEntries = keyData.size();
for (int i = 1; i < numEntries; i++) {
KeyData<K> prev = keyData.get(i - 1);
KeyData<K> current = keyData.get(i);
prev.setLength(current.getOffset() - prev.getOffset());
}
if (numEntries > 0) {
KeyData<K> last = keyData.get(numEntries - 1);
last.setLength(metaDataStartPos - last.offset);
}
return keyData;
}
