public Object read(int token, DataInputStream inputStream) throws IOException {
    String className = tupleSerialization.getClassNameFor(token);

    if (className == null) className = WritableUtils.readString(inputStream);

    Deserializer deserializer = deserializers.get(className);

    if (deserializer == null) {
      deserializer = tupleSerialization.getNewDeserializer(className);
      deserializer.open(inputStream);
      deserializers.put(className, deserializer);
    }

    Object foundObject = null;
    Object object = null;

    try {
      object = deserializer.deserialize(foundObject);
    } catch (IOException exception) {
      LOG.error(
          "failed deserializing token: " + token + " with classname: " + className, exception);

      throw exception;
    }

    return object;
  }
예제 #2
0
 @SuppressWarnings("unchecked")
 public void readFields(DataInput is) throws IOException {
   disableCounter = is.readBoolean();
   isMultiInputs = is.readBoolean();
   totalSplits = is.readInt();
   splitIndex = is.readInt();
   inputIndex = is.readInt();
   targetOps = (ArrayList<OperatorKey>) readObject(is);
   int splitLen = is.readInt();
   String splitClassName = is.readUTF();
   try {
     Class splitClass = conf.getClassByName(splitClassName);
     SerializationFactory sf = new SerializationFactory(conf);
     // The correct call sequence for Deserializer is, we shall open, then deserialize, but we
     // shall not close
     Deserializer d = sf.getDeserializer(splitClass);
     d.open((InputStream) is);
     wrappedSplits = new InputSplit[splitLen];
     for (int i = 0; i < splitLen; i++) {
       wrappedSplits[i] = (InputSplit) ReflectionUtils.newInstance(splitClass, conf);
       d.deserialize(wrappedSplits[i]);
     }
   } catch (ClassNotFoundException e) {
     throw new IOException(e);
   }
 }
예제 #3
0
  @SuppressWarnings("unchecked")
  public List<KeyData<K>> getKeyData() throws IOException, ClassNotFoundException {
    if (keyData != null) return keyData;

    final FileSystem fs = FileSystem.get(conf);
    keyData = new ArrayList<KeyData<K>>();

    final long filesize = fs.getFileStatus(path).getLen();
    FSDataInputStream in = fs.open(path);

    /* The last long in the file is the start position of the trailer section */
    in.seek(filesize - 8);
    long metaDataStartPos = in.readLong();

    in.seek(metaDataStartPos);

    ObjectMapper mapper = new ObjectMapper();
    metadataJson = mapper.readValue(in.readUTF(), JsonNode.class);

    int keySectionSize = in.readInt();

    // load the key section
    byte[] keySection = new byte[keySectionSize];

    in.seek(filesize - keySectionSize - 8);
    in.read(keySection, 0, keySectionSize);
    in.close();

    ByteArrayInputStream bis = new ByteArrayInputStream(keySection);
    DataInput dataInput = new DataInputStream(bis);

    int numberOfBlocks = metadataJson.get("numberOfBlocks").getIntValue();

    // load the key section
    keyClass = (Class<K>) ClassCache.forName(JsonUtils.getText(metadataJson, "keyClass"));
    valueClass = (Class<V>) ClassCache.forName(JsonUtils.getText(metadataJson, "valueClass"));

    SerializationFactory serializationFactory = new SerializationFactory(conf);
    Deserializer<K> deserializer = serializationFactory.getDeserializer(keyClass);

    deserializer.open(bis);

    while (bis.available() > 0 && numberOfBlocks > 0) {
      K key = deserializer.deserialize(null);

      long offset = dataInput.readLong();
      long blockId = dataInput.readLong();
      long numRecords = dataInput.readLong();

      keyData.add(new KeyData<K>(key, offset, 0, numRecords, blockId));
      numberOfBlocks--;
    }

    // Assign length to each keydata entry
    int numEntries = keyData.size();
    for (int i = 1; i < numEntries; i++) {
      KeyData<K> prev = keyData.get(i - 1);
      KeyData<K> current = keyData.get(i);

      prev.setLength(current.getOffset() - prev.getOffset());
    }

    if (numEntries > 0) {
      KeyData<K> last = keyData.get(numEntries - 1);
      last.setLength(metaDataStartPos - last.offset);
    }

    return keyData;
  }