Пример #1
0
 private int loadRMDTSecretManagerKeys(RMState state) throws IOException {
   int numKeys = 0;
   LeveldbIterator iter = null;
   try {
     iter = new LeveldbIterator(db);
     iter.seek(bytes(RM_DT_MASTER_KEY_KEY_PREFIX));
     while (iter.hasNext()) {
       Entry<byte[], byte[]> entry = iter.next();
       String key = asString(entry.getKey());
       if (!key.startsWith(RM_DT_MASTER_KEY_KEY_PREFIX)) {
         break;
       }
       DelegationKey masterKey = loadDelegationKey(entry.getValue());
       state.rmSecretManagerState.masterKeyState.add(masterKey);
       ++numKeys;
       if (LOG.isDebugEnabled()) {
         LOG.debug(
             "Loaded RM delegation key from "
                 + key
                 + ": keyId="
                 + masterKey.getKeyId()
                 + ", expirationDate="
                 + masterKey.getExpiryDate());
       }
     }
   } catch (DBException e) {
     throw new IOException(e);
   } finally {
     if (iter != null) {
       iter.close();
     }
   }
   return numKeys;
 }
 @Override
 public synchronized void removeRMDTMasterKeyState(DelegationKey masterKey) throws Exception {
   Path nodeCreatePath =
       getNodePath(rmDTSecretManagerRoot, DELEGATION_KEY_PREFIX + masterKey.getKeyId());
   LOG.info("Removing RMDelegationKey_" + masterKey.getKeyId());
   deleteFileWithRetries(nodeCreatePath);
 }
  private void loadRMDTSecretManagerState(RMState rmState) throws Exception {
    FileStatus[] childNodes = fs.listStatus(rmDTSecretManagerRoot);

    for (FileStatus childNodeStatus : childNodes) {
      assert childNodeStatus.isFile();
      String childNodeName = childNodeStatus.getPath().getName();
      if (childNodeName.startsWith(DELEGATION_TOKEN_SEQUENCE_NUMBER_PREFIX)) {
        rmState.rmSecretManagerState.dtSequenceNumber =
            Integer.parseInt(childNodeName.split("_")[1]);
        continue;
      }

      Path childNodePath = getNodePath(rmDTSecretManagerRoot, childNodeName);
      byte[] childData = readFile(childNodePath, childNodeStatus.getLen());
      ByteArrayInputStream is = new ByteArrayInputStream(childData);
      DataInputStream fsIn = new DataInputStream(is);
      if (childNodeName.startsWith(DELEGATION_KEY_PREFIX)) {
        DelegationKey key = new DelegationKey();
        key.readFields(fsIn);
        rmState.rmSecretManagerState.masterKeyState.add(key);
      } else if (childNodeName.startsWith(DELEGATION_TOKEN_PREFIX)) {
        RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier();
        identifier.readFields(fsIn);
        long renewDate = fsIn.readLong();
        rmState.rmSecretManagerState.delegationTokenState.put(identifier, renewDate);
      } else {
        LOG.warn("Unknown file for recovering RMDelegationTokenSecretManager");
      }
      fsIn.close();
    }
  }
Пример #4
0
 private DelegationKey loadDelegationKey(byte[] data) throws IOException {
   DelegationKey key = new DelegationKey();
   DataInputStream in = new DataInputStream(new ByteArrayInputStream(data));
   try {
     key.readFields(in);
   } finally {
     IOUtils.cleanup(LOG, in);
   }
   return key;
 }
 @Override
 public synchronized void storeRMDTMasterKeyState(DelegationKey masterKey) throws Exception {
   Path nodeCreatePath =
       getNodePath(rmDTSecretManagerRoot, DELEGATION_KEY_PREFIX + masterKey.getKeyId());
   ByteArrayOutputStream os = new ByteArrayOutputStream();
   try (DataOutputStream fsOut = new DataOutputStream(os)) {
     LOG.info("Storing RMDelegationKey_" + masterKey.getKeyId());
     masterKey.write(fsOut);
     writeFileWithRetries(nodeCreatePath, os.toByteArray(), true);
   }
 }
  private void loadRMDTSecretManagerState(RMState rmState) throws Exception {
    checkAndResumeUpdateOperation(rmDTSecretManagerRoot);
    FileStatus[] childNodes = listStatusWithRetries(rmDTSecretManagerRoot);

    for (FileStatus childNodeStatus : childNodes) {
      assert childNodeStatus.isFile();
      String childNodeName = childNodeStatus.getPath().getName();
      if (checkAndRemovePartialRecordWithRetries(childNodeStatus.getPath())) {
        continue;
      }
      if (childNodeName.startsWith(DELEGATION_TOKEN_SEQUENCE_NUMBER_PREFIX)) {
        rmState.rmSecretManagerState.dtSequenceNumber =
            Integer.parseInt(childNodeName.split("_")[1]);
        continue;
      }

      Path childNodePath = getNodePath(rmDTSecretManagerRoot, childNodeName);
      byte[] childData = readFileWithRetries(childNodePath, childNodeStatus.getLen());
      ByteArrayInputStream is = new ByteArrayInputStream(childData);
      try (DataInputStream fsIn = new DataInputStream(is)) {
        if (childNodeName.startsWith(DELEGATION_KEY_PREFIX)) {
          DelegationKey key = new DelegationKey();
          key.readFields(fsIn);
          rmState.rmSecretManagerState.masterKeyState.add(key);
          if (LOG.isDebugEnabled()) {
            LOG.debug(
                "Loaded delegation key: keyId="
                    + key.getKeyId()
                    + ", expirationDate="
                    + key.getExpiryDate());
          }
        } else if (childNodeName.startsWith(DELEGATION_TOKEN_PREFIX)) {
          RMDelegationTokenIdentifierData identifierData = new RMDelegationTokenIdentifierData();
          identifierData.readFields(fsIn);
          RMDelegationTokenIdentifier identifier = identifierData.getTokenIdentifier();
          long renewDate = identifierData.getRenewDate();

          rmState.rmSecretManagerState.delegationTokenState.put(identifier, renewDate);
          if (LOG.isDebugEnabled()) {
            LOG.debug(
                "Loaded RMDelegationTokenIdentifier: " + identifier + " renewDate=" + renewDate);
          }
        } else {
          LOG.warn("Unknown file for recovering RMDelegationTokenSecretManager");
        }
      }
    }
  }
Пример #7
0
 @Override
 protected void storeRMDTMasterKeyState(DelegationKey masterKey) throws IOException {
   String dbKey = getRMDTMasterKeyNodeKey(masterKey);
   if (LOG.isDebugEnabled()) {
     LOG.debug("Storing token master key to " + dbKey);
   }
   ByteArrayOutputStream os = new ByteArrayOutputStream();
   DataOutputStream out = new DataOutputStream(os);
   try {
     masterKey.write(out);
   } finally {
     out.close();
   }
   try {
     db.put(bytes(dbKey), os.toByteArray());
   } catch (DBException e) {
     throw new IOException(e);
   }
 }
Пример #8
0
  /** modified by tony */
  @SuppressWarnings("deprecation")
  int loadEditRecords(int logVersion, DataInputStream in, boolean closeOnExit) throws IOException {

    FSNamesystem.LOG.info("logversion: " + logVersion);

    FSDirectory fsDir = fsNamesys.dir;
    int numEdits = 0;
    String clientName = null;
    String clientMachine = null;
    String path = null;
    int numOpAdd = 0,
        numOpClose = 0,
        numOpDelete = 0,
        numOpRenameOld = 0,
        numOpSetRepl = 0,
        numOpMkDir = 0,
        numOpSetPerm = 0,
        numOpSetOwner = 0,
        numOpSetGenStamp = 0,
        numOpTimes = 0,
        numOpRename = 0,
        numOpConcatDelete = 0,
        numOpSymlink = 0,
        numOpGetDelegationToken = 0,
        numOpRenewDelegationToken = 0,
        numOpCancelDelegationToken = 0,
        numOpUpdateMasterKey = 0,
        numOpOther = 0;

    try {
      while (true) {
        long timestamp = 0;
        long mtime = 0;
        long atime = 0;
        long blockSize = 0;
        byte opcode = -1;
        try {
          in.mark(1);
          opcode = in.readByte();
          if (opcode == Ops.OP_INVALID) {
            in.reset(); // reset back to end of file if somebody reads it again
            break; // no more transactions
          }
        } catch (EOFException e) {
          break; // no more transactions
        }
        numEdits++;
        switch (opcode) {
          case Ops.OP_ADD:
          case Ops.OP_CLOSE:
            {
              // versions > 0 support per file replication
              // get name and replication
              int length = in.readInt();
              // modified by tony
              if (-7 == logVersion && length != 3
                  || -17 < logVersion && logVersion < -7 && length != 4
                  || logVersion <= -17 && length != 7) {
                throw new IOException(
                    "Incorrect data format."
                        + " logVersion is "
                        + logVersion
                        + " but writables.length is "
                        + length
                        + ". ");
              }
              path = FSImageSerialization.readString(in);
              short replication = fsNamesys.adjustReplication(readShort(in));
              mtime = readLong(in);
              if (LayoutVersion.supports(Feature.FILE_ACCESS_TIME, logVersion)) {
                atime = readLong(in);
              }
              if (logVersion < -7) {
                blockSize = readLong(in);
              }

              long fileSize = readLong(in);
              byte type = (byte) readLong(in);

              // get blocks
              boolean isFileUnderConstruction = (opcode == Ops.OP_ADD);
              BlockInfo blocks[] = readBlocks(in, logVersion, isFileUnderConstruction, replication);

              // Older versions of HDFS does not store the block size in inode.
              // If the file has more than one block, use the size of the
              // first block as the blocksize. Otherwise use the default
              // block size.
              if (-8 <= logVersion && blockSize == 0) {
                if (blocks.length > 1) {
                  blockSize = blocks[0].getNumBytes();
                } else {
                  long first = ((blocks.length == 1) ? blocks[0].getNumBytes() : 0);
                  blockSize = Math.max(fsNamesys.getDefaultBlockSize(), first);
                }
              }

              PermissionStatus permissions = fsNamesys.getUpgradePermission();
              if (logVersion <= -11) {
                permissions = PermissionStatus.read(in);
              }
              CodingMatrix codingMatrix = CodingMatrix.getMatrixofCertainType(type);
              codingMatrix.readFields(in);

              /** added by tony* */
              LongWritable offset = new LongWritable();
              offset.readFields(in);
              long headeroffset = offset.get();
              // clientname, clientMachine and block locations of last block.
              if (opcode == Ops.OP_ADD && logVersion <= -12) {
                clientName = FSImageSerialization.readString(in);
                clientMachine = FSImageSerialization.readString(in);
                if (-13 <= logVersion) {
                  readDatanodeDescriptorArray(in);
                }
              } else {
                clientName = "";
                clientMachine = "";
              }

              // The open lease transaction re-creates a file if necessary.
              // Delete the file if it already exists.
              if (FSNamesystem.LOG.isDebugEnabled()) {
                FSNamesystem.LOG.debug(
                    opcode
                        + ": "
                        + path
                        + " numblocks : "
                        + blocks.length
                        + " clientHolder "
                        + clientName
                        + " clientMachine "
                        + clientMachine);
              }

              fsDir.unprotectedDelete(path, mtime);

              /** modified by tony add to the file tree */
              INodeFile node =
                  (INodeFile)
                      fsDir.unprotectedAddFile(
                          path,
                          permissions,
                          codingMatrix,
                          headeroffset,
                          fileSize,
                          blocks,
                          replication,
                          mtime,
                          atime,
                          blockSize);
              if (isFileUnderConstruction) {
                numOpAdd++;
                //
                // Replace current node with a INodeUnderConstruction.
                // Recreate in-memory lease record.
                //
                //            INodeFileUnderConstruction cons = new INodeFileUnderConstruction(
                //                                      node.getLocalNameBytes(),
                //                                      node.getReplication(),
                //                                      node.getModificationTime(),
                //                                      node.getPreferredBlockSize(),
                //                                      node.getBlocks(),
                //                                      node.getPermissionStatus(),
                //                                      clientName,
                //                                      clientMachine,
                //                                      null);
                // TODO:
                INodeFileUnderConstruction cons = null;
                fsDir.replaceNode(path, node, cons);
                fsNamesys.leaseManager.addLease(cons.getClientName(), path);
              }
              break;
            }
          case Ops.OP_SET_REPLICATION:
            {
              numOpSetRepl++;
              path = FSImageSerialization.readString(in);
              short replication = fsNamesys.adjustReplication(readShort(in));
              fsDir.unprotectedSetReplication(path, replication, null);
              break;
            }
          case Ops.OP_CONCAT_DELETE:
            {
              numOpConcatDelete++;
              int length = in.readInt();
              if (length < 3) { // trg, srcs.., timestam
                throw new IOException("Incorrect data format. " + "Mkdir operation.");
              }
              String trg = FSImageSerialization.readString(in);
              int srcSize = length - 1 - 1; // trg and timestamp
              String[] srcs = new String[srcSize];
              for (int i = 0; i < srcSize; i++) {
                srcs[i] = FSImageSerialization.readString(in);
              }
              timestamp = readLong(in);
              fsDir.unprotectedConcat(trg, srcs);
              break;
            }
          case Ops.OP_RENAME_OLD:
            {
              numOpRenameOld++;
              int length = in.readInt();
              if (length != 3) {
                throw new IOException("Incorrect data format. " + "Mkdir operation.");
              }
              String s = FSImageSerialization.readString(in);
              String d = FSImageSerialization.readString(in);
              timestamp = readLong(in);
              HdfsFileStatus dinfo = fsDir.getFileInfo(d, false);
              fsDir.unprotectedRenameTo(s, d, timestamp);
              fsNamesys.changeLease(s, d, dinfo);
              break;
            }
          case Ops.OP_DELETE:
            {
              numOpDelete++;
              int length = in.readInt();
              if (length != 2) {
                throw new IOException("Incorrect data format. " + "delete operation.");
              }
              path = FSImageSerialization.readString(in);
              timestamp = readLong(in);
              fsDir.unprotectedDelete(path, timestamp);
              break;
            }
          case Ops.OP_MKDIR:
            {
              numOpMkDir++;
              PermissionStatus permissions = fsNamesys.getUpgradePermission();
              int length = in.readInt();
              if (-17 < logVersion && length != 2 || logVersion <= -17 && length != 3) {
                throw new IOException("Incorrect data format. " + "Mkdir operation.");
              }
              path = FSImageSerialization.readString(in);
              timestamp = readLong(in);

              // The disk format stores atimes for directories as well.
              // However, currently this is not being updated/used because of
              // performance reasons.
              if (LayoutVersion.supports(Feature.FILE_ACCESS_TIME, logVersion)) {
                atime = readLong(in);
              }

              if (logVersion <= -11) {
                permissions = PermissionStatus.read(in);
              }
              fsDir.unprotectedMkdir(path, permissions, timestamp);
              break;
            }
          case Ops.OP_SET_GENSTAMP:
            {
              numOpSetGenStamp++;
              long lw = in.readLong();
              fsNamesys.setGenerationStamp(lw);
              break;
            }
          case Ops.OP_DATANODE_ADD:
            {
              numOpOther++;
              // Datanodes are not persistent any more.
              FSImageSerialization.DatanodeImage.skipOne(in);
              break;
            }
          case Ops.OP_DATANODE_REMOVE:
            {
              numOpOther++;
              DatanodeID nodeID = new DatanodeID();
              nodeID.readFields(in);
              // Datanodes are not persistent any more.
              break;
            }
          case Ops.OP_SET_PERMISSIONS:
            {
              numOpSetPerm++;
              fsDir.unprotectedSetPermission(
                  FSImageSerialization.readString(in), FsPermission.read(in));
              break;
            }
          case Ops.OP_SET_OWNER:
            {
              numOpSetOwner++;
              fsDir.unprotectedSetOwner(
                  FSImageSerialization.readString(in),
                  FSImageSerialization.readString_EmptyAsNull(in),
                  FSImageSerialization.readString_EmptyAsNull(in));
              break;
            }
          case Ops.OP_SET_NS_QUOTA:
            {
              fsDir.unprotectedSetQuota(
                  FSImageSerialization.readString(in),
                  readLongWritable(in),
                  FSConstants.QUOTA_DONT_SET);
              break;
            }
          case Ops.OP_CLEAR_NS_QUOTA:
            {
              fsDir.unprotectedSetQuota(
                  FSImageSerialization.readString(in),
                  FSConstants.QUOTA_RESET,
                  FSConstants.QUOTA_DONT_SET);
              break;
            }

          case Ops.OP_SET_QUOTA:
            fsDir.unprotectedSetQuota(
                FSImageSerialization.readString(in), readLongWritable(in), readLongWritable(in));

            break;

          case Ops.OP_TIMES:
            {
              numOpTimes++;
              int length = in.readInt();
              if (length != 3) {
                throw new IOException("Incorrect data format. " + "times operation.");
              }
              path = FSImageSerialization.readString(in);
              mtime = readLong(in);
              atime = readLong(in);
              fsDir.unprotectedSetTimes(path, mtime, atime, true);
              break;
            }
          case Ops.OP_SYMLINK:
            {
              numOpSymlink++;
              int length = in.readInt();
              if (length != 4) {
                throw new IOException("Incorrect data format. " + "symlink operation.");
              }
              path = FSImageSerialization.readString(in);
              String value = FSImageSerialization.readString(in);
              mtime = readLong(in);
              atime = readLong(in);
              PermissionStatus perm = PermissionStatus.read(in);
              fsDir.unprotectedSymlink(path, value, mtime, atime, perm);
              break;
            }
          case Ops.OP_RENAME:
            {
              numOpRename++;
              int length = in.readInt();
              if (length != 3) {
                throw new IOException("Incorrect data format. " + "Mkdir operation.");
              }
              String s = FSImageSerialization.readString(in);
              String d = FSImageSerialization.readString(in);
              timestamp = readLong(in);
              Rename[] options = readRenameOptions(in);
              HdfsFileStatus dinfo = fsDir.getFileInfo(d, false);
              fsDir.unprotectedRenameTo(s, d, timestamp, options);
              fsNamesys.changeLease(s, d, dinfo);
              break;
            }
          case Ops.OP_GET_DELEGATION_TOKEN:
            {
              numOpGetDelegationToken++;
              DelegationTokenIdentifier delegationTokenId = new DelegationTokenIdentifier();
              delegationTokenId.readFields(in);
              long expiryTime = readLong(in);
              fsNamesys
                  .getDelegationTokenSecretManager()
                  .addPersistedDelegationToken(delegationTokenId, expiryTime);
              break;
            }
          case Ops.OP_RENEW_DELEGATION_TOKEN:
            {
              numOpRenewDelegationToken++;
              DelegationTokenIdentifier delegationTokenId = new DelegationTokenIdentifier();
              delegationTokenId.readFields(in);
              long expiryTime = readLong(in);
              fsNamesys
                  .getDelegationTokenSecretManager()
                  .updatePersistedTokenRenewal(delegationTokenId, expiryTime);
              break;
            }
          case Ops.OP_CANCEL_DELEGATION_TOKEN:
            {
              numOpCancelDelegationToken++;
              DelegationTokenIdentifier delegationTokenId = new DelegationTokenIdentifier();
              delegationTokenId.readFields(in);
              fsNamesys
                  .getDelegationTokenSecretManager()
                  .updatePersistedTokenCancellation(delegationTokenId);
              break;
            }
          case Ops.OP_UPDATE_MASTER_KEY:
            {
              numOpUpdateMasterKey++;
              DelegationKey delegationKey = new DelegationKey();
              delegationKey.readFields(in);
              fsNamesys.getDelegationTokenSecretManager().updatePersistedMasterKey(delegationKey);
              break;
            }
          default:
            {
              throw new IOException("Never seen opcode " + opcode);
            }
        }
      }
    } catch (IOException ex) {
      check203UpgradeFailure(logVersion, ex);
    } finally {
      if (closeOnExit) in.close();
    }
    if (FSImage.LOG.isDebugEnabled()) {
      FSImage.LOG.debug(
          "numOpAdd = "
              + numOpAdd
              + " numOpClose = "
              + numOpClose
              + " numOpDelete = "
              + numOpDelete
              + " numOpRenameOld = "
              + numOpRenameOld
              + " numOpSetRepl = "
              + numOpSetRepl
              + " numOpMkDir = "
              + numOpMkDir
              + " numOpSetPerm = "
              + numOpSetPerm
              + " numOpSetOwner = "
              + numOpSetOwner
              + " numOpSetGenStamp = "
              + numOpSetGenStamp
              + " numOpTimes = "
              + numOpTimes
              + " numOpConcatDelete  = "
              + numOpConcatDelete
              + " numOpRename = "
              + numOpRename
              + " numOpGetDelegationToken = "
              + numOpGetDelegationToken
              + " numOpRenewDelegationToken = "
              + numOpRenewDelegationToken
              + " numOpCancelDelegationToken = "
              + numOpCancelDelegationToken
              + " numOpUpdateMasterKey = "
              + numOpUpdateMasterKey
              + " numOpOther = "
              + numOpOther);
    }
    return numEdits;
  }
Пример #9
0
 private String getRMDTMasterKeyNodeKey(DelegationKey masterKey) {
   return RM_DT_MASTER_KEY_KEY_PREFIX + masterKey.getKeyId();
 }