Example #1
0
 @Override
 public void readFields(DataInput in) throws IOException {
   first.readFields(in);
   second.readFields(in);
 }
 @Override
 public void readFields(DataInput arg0) throws IOException {
   _dval.readFields(arg0);
   _lval.readFields(arg0);
 }
Example #3
0
 /** Read an integer from an input stream */
 private static long readLongWritable(DataInputStream in) throws IOException {
   synchronized (longWritable) {
     longWritable.readFields(in);
     return longWritable.get();
   }
 }
Example #4
0
  /** modified by tony */
  @SuppressWarnings("deprecation")
  int loadEditRecords(int logVersion, DataInputStream in, boolean closeOnExit) throws IOException {

    FSNamesystem.LOG.info("logversion: " + logVersion);

    FSDirectory fsDir = fsNamesys.dir;
    int numEdits = 0;
    String clientName = null;
    String clientMachine = null;
    String path = null;
    int numOpAdd = 0,
        numOpClose = 0,
        numOpDelete = 0,
        numOpRenameOld = 0,
        numOpSetRepl = 0,
        numOpMkDir = 0,
        numOpSetPerm = 0,
        numOpSetOwner = 0,
        numOpSetGenStamp = 0,
        numOpTimes = 0,
        numOpRename = 0,
        numOpConcatDelete = 0,
        numOpSymlink = 0,
        numOpGetDelegationToken = 0,
        numOpRenewDelegationToken = 0,
        numOpCancelDelegationToken = 0,
        numOpUpdateMasterKey = 0,
        numOpOther = 0;

    try {
      while (true) {
        long timestamp = 0;
        long mtime = 0;
        long atime = 0;
        long blockSize = 0;
        byte opcode = -1;
        try {
          in.mark(1);
          opcode = in.readByte();
          if (opcode == Ops.OP_INVALID) {
            in.reset(); // reset back to end of file if somebody reads it again
            break; // no more transactions
          }
        } catch (EOFException e) {
          break; // no more transactions
        }
        numEdits++;
        switch (opcode) {
          case Ops.OP_ADD:
          case Ops.OP_CLOSE:
            {
              // versions > 0 support per file replication
              // get name and replication
              int length = in.readInt();
              // modified by tony
              if (-7 == logVersion && length != 3
                  || -17 < logVersion && logVersion < -7 && length != 4
                  || logVersion <= -17 && length != 7) {
                throw new IOException(
                    "Incorrect data format."
                        + " logVersion is "
                        + logVersion
                        + " but writables.length is "
                        + length
                        + ". ");
              }
              path = FSImageSerialization.readString(in);
              short replication = fsNamesys.adjustReplication(readShort(in));
              mtime = readLong(in);
              if (LayoutVersion.supports(Feature.FILE_ACCESS_TIME, logVersion)) {
                atime = readLong(in);
              }
              if (logVersion < -7) {
                blockSize = readLong(in);
              }

              long fileSize = readLong(in);
              byte type = (byte) readLong(in);

              // get blocks
              boolean isFileUnderConstruction = (opcode == Ops.OP_ADD);
              BlockInfo blocks[] = readBlocks(in, logVersion, isFileUnderConstruction, replication);

              // Older versions of HDFS does not store the block size in inode.
              // If the file has more than one block, use the size of the
              // first block as the blocksize. Otherwise use the default
              // block size.
              if (-8 <= logVersion && blockSize == 0) {
                if (blocks.length > 1) {
                  blockSize = blocks[0].getNumBytes();
                } else {
                  long first = ((blocks.length == 1) ? blocks[0].getNumBytes() : 0);
                  blockSize = Math.max(fsNamesys.getDefaultBlockSize(), first);
                }
              }

              PermissionStatus permissions = fsNamesys.getUpgradePermission();
              if (logVersion <= -11) {
                permissions = PermissionStatus.read(in);
              }
              CodingMatrix codingMatrix = CodingMatrix.getMatrixofCertainType(type);
              codingMatrix.readFields(in);

              /** added by tony* */
              LongWritable offset = new LongWritable();
              offset.readFields(in);
              long headeroffset = offset.get();
              // clientname, clientMachine and block locations of last block.
              if (opcode == Ops.OP_ADD && logVersion <= -12) {
                clientName = FSImageSerialization.readString(in);
                clientMachine = FSImageSerialization.readString(in);
                if (-13 <= logVersion) {
                  readDatanodeDescriptorArray(in);
                }
              } else {
                clientName = "";
                clientMachine = "";
              }

              // The open lease transaction re-creates a file if necessary.
              // Delete the file if it already exists.
              if (FSNamesystem.LOG.isDebugEnabled()) {
                FSNamesystem.LOG.debug(
                    opcode
                        + ": "
                        + path
                        + " numblocks : "
                        + blocks.length
                        + " clientHolder "
                        + clientName
                        + " clientMachine "
                        + clientMachine);
              }

              fsDir.unprotectedDelete(path, mtime);

              /** modified by tony add to the file tree */
              INodeFile node =
                  (INodeFile)
                      fsDir.unprotectedAddFile(
                          path,
                          permissions,
                          codingMatrix,
                          headeroffset,
                          fileSize,
                          blocks,
                          replication,
                          mtime,
                          atime,
                          blockSize);
              if (isFileUnderConstruction) {
                numOpAdd++;
                //
                // Replace current node with a INodeUnderConstruction.
                // Recreate in-memory lease record.
                //
                //            INodeFileUnderConstruction cons = new INodeFileUnderConstruction(
                //                                      node.getLocalNameBytes(),
                //                                      node.getReplication(),
                //                                      node.getModificationTime(),
                //                                      node.getPreferredBlockSize(),
                //                                      node.getBlocks(),
                //                                      node.getPermissionStatus(),
                //                                      clientName,
                //                                      clientMachine,
                //                                      null);
                // TODO:
                INodeFileUnderConstruction cons = null;
                fsDir.replaceNode(path, node, cons);
                fsNamesys.leaseManager.addLease(cons.getClientName(), path);
              }
              break;
            }
          case Ops.OP_SET_REPLICATION:
            {
              numOpSetRepl++;
              path = FSImageSerialization.readString(in);
              short replication = fsNamesys.adjustReplication(readShort(in));
              fsDir.unprotectedSetReplication(path, replication, null);
              break;
            }
          case Ops.OP_CONCAT_DELETE:
            {
              numOpConcatDelete++;
              int length = in.readInt();
              if (length < 3) { // trg, srcs.., timestam
                throw new IOException("Incorrect data format. " + "Mkdir operation.");
              }
              String trg = FSImageSerialization.readString(in);
              int srcSize = length - 1 - 1; // trg and timestamp
              String[] srcs = new String[srcSize];
              for (int i = 0; i < srcSize; i++) {
                srcs[i] = FSImageSerialization.readString(in);
              }
              timestamp = readLong(in);
              fsDir.unprotectedConcat(trg, srcs);
              break;
            }
          case Ops.OP_RENAME_OLD:
            {
              numOpRenameOld++;
              int length = in.readInt();
              if (length != 3) {
                throw new IOException("Incorrect data format. " + "Mkdir operation.");
              }
              String s = FSImageSerialization.readString(in);
              String d = FSImageSerialization.readString(in);
              timestamp = readLong(in);
              HdfsFileStatus dinfo = fsDir.getFileInfo(d, false);
              fsDir.unprotectedRenameTo(s, d, timestamp);
              fsNamesys.changeLease(s, d, dinfo);
              break;
            }
          case Ops.OP_DELETE:
            {
              numOpDelete++;
              int length = in.readInt();
              if (length != 2) {
                throw new IOException("Incorrect data format. " + "delete operation.");
              }
              path = FSImageSerialization.readString(in);
              timestamp = readLong(in);
              fsDir.unprotectedDelete(path, timestamp);
              break;
            }
          case Ops.OP_MKDIR:
            {
              numOpMkDir++;
              PermissionStatus permissions = fsNamesys.getUpgradePermission();
              int length = in.readInt();
              if (-17 < logVersion && length != 2 || logVersion <= -17 && length != 3) {
                throw new IOException("Incorrect data format. " + "Mkdir operation.");
              }
              path = FSImageSerialization.readString(in);
              timestamp = readLong(in);

              // The disk format stores atimes for directories as well.
              // However, currently this is not being updated/used because of
              // performance reasons.
              if (LayoutVersion.supports(Feature.FILE_ACCESS_TIME, logVersion)) {
                atime = readLong(in);
              }

              if (logVersion <= -11) {
                permissions = PermissionStatus.read(in);
              }
              fsDir.unprotectedMkdir(path, permissions, timestamp);
              break;
            }
          case Ops.OP_SET_GENSTAMP:
            {
              numOpSetGenStamp++;
              long lw = in.readLong();
              fsNamesys.setGenerationStamp(lw);
              break;
            }
          case Ops.OP_DATANODE_ADD:
            {
              numOpOther++;
              // Datanodes are not persistent any more.
              FSImageSerialization.DatanodeImage.skipOne(in);
              break;
            }
          case Ops.OP_DATANODE_REMOVE:
            {
              numOpOther++;
              DatanodeID nodeID = new DatanodeID();
              nodeID.readFields(in);
              // Datanodes are not persistent any more.
              break;
            }
          case Ops.OP_SET_PERMISSIONS:
            {
              numOpSetPerm++;
              fsDir.unprotectedSetPermission(
                  FSImageSerialization.readString(in), FsPermission.read(in));
              break;
            }
          case Ops.OP_SET_OWNER:
            {
              numOpSetOwner++;
              fsDir.unprotectedSetOwner(
                  FSImageSerialization.readString(in),
                  FSImageSerialization.readString_EmptyAsNull(in),
                  FSImageSerialization.readString_EmptyAsNull(in));
              break;
            }
          case Ops.OP_SET_NS_QUOTA:
            {
              fsDir.unprotectedSetQuota(
                  FSImageSerialization.readString(in),
                  readLongWritable(in),
                  FSConstants.QUOTA_DONT_SET);
              break;
            }
          case Ops.OP_CLEAR_NS_QUOTA:
            {
              fsDir.unprotectedSetQuota(
                  FSImageSerialization.readString(in),
                  FSConstants.QUOTA_RESET,
                  FSConstants.QUOTA_DONT_SET);
              break;
            }

          case Ops.OP_SET_QUOTA:
            fsDir.unprotectedSetQuota(
                FSImageSerialization.readString(in), readLongWritable(in), readLongWritable(in));

            break;

          case Ops.OP_TIMES:
            {
              numOpTimes++;
              int length = in.readInt();
              if (length != 3) {
                throw new IOException("Incorrect data format. " + "times operation.");
              }
              path = FSImageSerialization.readString(in);
              mtime = readLong(in);
              atime = readLong(in);
              fsDir.unprotectedSetTimes(path, mtime, atime, true);
              break;
            }
          case Ops.OP_SYMLINK:
            {
              numOpSymlink++;
              int length = in.readInt();
              if (length != 4) {
                throw new IOException("Incorrect data format. " + "symlink operation.");
              }
              path = FSImageSerialization.readString(in);
              String value = FSImageSerialization.readString(in);
              mtime = readLong(in);
              atime = readLong(in);
              PermissionStatus perm = PermissionStatus.read(in);
              fsDir.unprotectedSymlink(path, value, mtime, atime, perm);
              break;
            }
          case Ops.OP_RENAME:
            {
              numOpRename++;
              int length = in.readInt();
              if (length != 3) {
                throw new IOException("Incorrect data format. " + "Mkdir operation.");
              }
              String s = FSImageSerialization.readString(in);
              String d = FSImageSerialization.readString(in);
              timestamp = readLong(in);
              Rename[] options = readRenameOptions(in);
              HdfsFileStatus dinfo = fsDir.getFileInfo(d, false);
              fsDir.unprotectedRenameTo(s, d, timestamp, options);
              fsNamesys.changeLease(s, d, dinfo);
              break;
            }
          case Ops.OP_GET_DELEGATION_TOKEN:
            {
              numOpGetDelegationToken++;
              DelegationTokenIdentifier delegationTokenId = new DelegationTokenIdentifier();
              delegationTokenId.readFields(in);
              long expiryTime = readLong(in);
              fsNamesys
                  .getDelegationTokenSecretManager()
                  .addPersistedDelegationToken(delegationTokenId, expiryTime);
              break;
            }
          case Ops.OP_RENEW_DELEGATION_TOKEN:
            {
              numOpRenewDelegationToken++;
              DelegationTokenIdentifier delegationTokenId = new DelegationTokenIdentifier();
              delegationTokenId.readFields(in);
              long expiryTime = readLong(in);
              fsNamesys
                  .getDelegationTokenSecretManager()
                  .updatePersistedTokenRenewal(delegationTokenId, expiryTime);
              break;
            }
          case Ops.OP_CANCEL_DELEGATION_TOKEN:
            {
              numOpCancelDelegationToken++;
              DelegationTokenIdentifier delegationTokenId = new DelegationTokenIdentifier();
              delegationTokenId.readFields(in);
              fsNamesys
                  .getDelegationTokenSecretManager()
                  .updatePersistedTokenCancellation(delegationTokenId);
              break;
            }
          case Ops.OP_UPDATE_MASTER_KEY:
            {
              numOpUpdateMasterKey++;
              DelegationKey delegationKey = new DelegationKey();
              delegationKey.readFields(in);
              fsNamesys.getDelegationTokenSecretManager().updatePersistedMasterKey(delegationKey);
              break;
            }
          default:
            {
              throw new IOException("Never seen opcode " + opcode);
            }
        }
      }
    } catch (IOException ex) {
      check203UpgradeFailure(logVersion, ex);
    } finally {
      if (closeOnExit) in.close();
    }
    if (FSImage.LOG.isDebugEnabled()) {
      FSImage.LOG.debug(
          "numOpAdd = "
              + numOpAdd
              + " numOpClose = "
              + numOpClose
              + " numOpDelete = "
              + numOpDelete
              + " numOpRenameOld = "
              + numOpRenameOld
              + " numOpSetRepl = "
              + numOpSetRepl
              + " numOpMkDir = "
              + numOpMkDir
              + " numOpSetPerm = "
              + numOpSetPerm
              + " numOpSetOwner = "
              + numOpSetOwner
              + " numOpSetGenStamp = "
              + numOpSetGenStamp
              + " numOpTimes = "
              + numOpTimes
              + " numOpConcatDelete  = "
              + numOpConcatDelete
              + " numOpRename = "
              + numOpRename
              + " numOpGetDelegationToken = "
              + numOpGetDelegationToken
              + " numOpRenewDelegationToken = "
              + numOpRenewDelegationToken
              + " numOpCancelDelegationToken = "
              + numOpCancelDelegationToken
              + " numOpUpdateMasterKey = "
              + numOpUpdateMasterKey
              + " numOpOther = "
              + numOpOther);
    }
    return numEdits;
  }
Example #5
0
 @Override
 public void readFields(DataInput in) throws IOException {
   __term.readFields(in);
   __documentId.readFields(in);
 }
Example #6
0
 @Override
 public void readFields(DataInput in) throws IOException {
   task = new BSPTask();
   task.readFields(in);
   superstepNumber.readFields(in);
 }
Example #7
0
 public void readFields(DataInput in) throws IOException {
   super.readFields(in);
   throw new RuntimeException(ERR_MSG);
 }