/**
* The function will verify the token with NameNode if available and will create a
* UserGroupInformation.
*
* <p>Code in this function is copied from JspHelper.getTokenUGI
*
* @param identifier Delegation token identifier
* @param password Delegation token password
* @param kind the kind of token
* @param service the service for this token
* @param servletContext Jetty servlet context which contains the NN address
* @throws SecurityException Thrown when authentication fails
*/
private static void verifyToken(
byte[] identifier, byte[] password, Text kind, Text service, ServletContext servletContext) {
try {
Token<DelegationTokenIdentifier> token =
new Token<DelegationTokenIdentifier>(identifier, password, kind, service);
ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
DataInputStream in = new DataInputStream(buf);
DelegationTokenIdentifier id = new DelegationTokenIdentifier();
id.readFields(in);
final NameNode nn = NameNodeHttpServer.getNameNodeFromContext(servletContext);
if (nn != null) {
nn.getNamesystem().verifyToken(id, token.getPassword());
}
UserGroupInformation userGroupInformation = id.getUser();
userGroupInformation.addToken(token);
LOG.debug(
"user "
+ userGroupInformation.getUserName()
+ " ("
+ userGroupInformation.getShortUserName()
+ ") authenticated");
// re-login if necessary
userGroupInformation.checkTGTAndReloginFromKeytab();
} catch (IOException e) {
throw new SecurityException("Failed to verify delegation token " + e, e);
}
}
/** @return a string representation of the token */
public static String stringifyToken(final Token<?> token) throws IOException {
DelegationTokenIdentifier ident = new DelegationTokenIdentifier();
ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
DataInputStream in = new DataInputStream(buf);
ident.readFields(in);
if (token.getService().getLength() > 0) {
return ident + " on " + token.getService();
} else {
return ident.toString();
}
}
private void printTokens(JobID jobId, Credentials credentials) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Printing tokens for job: " + jobId);
for (Token<?> token : credentials.getAllTokens()) {
if (token.getKind().toString().equals("HDFS_DELEGATION_TOKEN")) {
LOG.debug(
"Submitting with "
+ org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier
.stringifyToken(token));
}
}
}
}
/** modified by tony */
@SuppressWarnings("deprecation")
int loadEditRecords(int logVersion, DataInputStream in, boolean closeOnExit) throws IOException {
FSNamesystem.LOG.info("logversion: " + logVersion);
FSDirectory fsDir = fsNamesys.dir;
int numEdits = 0;
String clientName = null;
String clientMachine = null;
String path = null;
int numOpAdd = 0,
numOpClose = 0,
numOpDelete = 0,
numOpRenameOld = 0,
numOpSetRepl = 0,
numOpMkDir = 0,
numOpSetPerm = 0,
numOpSetOwner = 0,
numOpSetGenStamp = 0,
numOpTimes = 0,
numOpRename = 0,
numOpConcatDelete = 0,
numOpSymlink = 0,
numOpGetDelegationToken = 0,
numOpRenewDelegationToken = 0,
numOpCancelDelegationToken = 0,
numOpUpdateMasterKey = 0,
numOpOther = 0;
try {
while (true) {
long timestamp = 0;
long mtime = 0;
long atime = 0;
long blockSize = 0;
byte opcode = -1;
try {
in.mark(1);
opcode = in.readByte();
if (opcode == Ops.OP_INVALID) {
in.reset(); // reset back to end of file if somebody reads it again
break; // no more transactions
}
} catch (EOFException e) {
break; // no more transactions
}
numEdits++;
switch (opcode) {
case Ops.OP_ADD:
case Ops.OP_CLOSE:
{
// versions > 0 support per file replication
// get name and replication
int length = in.readInt();
// modified by tony
if (-7 == logVersion && length != 3
|| -17 < logVersion && logVersion < -7 && length != 4
|| logVersion <= -17 && length != 7) {
throw new IOException(
"Incorrect data format."
+ " logVersion is "
+ logVersion
+ " but writables.length is "
+ length
+ ". ");
}
path = FSImageSerialization.readString(in);
short replication = fsNamesys.adjustReplication(readShort(in));
mtime = readLong(in);
if (LayoutVersion.supports(Feature.FILE_ACCESS_TIME, logVersion)) {
atime = readLong(in);
}
if (logVersion < -7) {
blockSize = readLong(in);
}
long fileSize = readLong(in);
byte type = (byte) readLong(in);
// get blocks
boolean isFileUnderConstruction = (opcode == Ops.OP_ADD);
BlockInfo blocks[] = readBlocks(in, logVersion, isFileUnderConstruction, replication);
// Older versions of HDFS does not store the block size in inode.
// If the file has more than one block, use the size of the
// first block as the blocksize. Otherwise use the default
// block size.
if (-8 <= logVersion && blockSize == 0) {
if (blocks.length > 1) {
blockSize = blocks[0].getNumBytes();
} else {
long first = ((blocks.length == 1) ? blocks[0].getNumBytes() : 0);
blockSize = Math.max(fsNamesys.getDefaultBlockSize(), first);
}
}
PermissionStatus permissions = fsNamesys.getUpgradePermission();
if (logVersion <= -11) {
permissions = PermissionStatus.read(in);
}
CodingMatrix codingMatrix = CodingMatrix.getMatrixofCertainType(type);
codingMatrix.readFields(in);
/** added by tony* */
LongWritable offset = new LongWritable();
offset.readFields(in);
long headeroffset = offset.get();
// clientname, clientMachine and block locations of last block.
if (opcode == Ops.OP_ADD && logVersion <= -12) {
clientName = FSImageSerialization.readString(in);
clientMachine = FSImageSerialization.readString(in);
if (-13 <= logVersion) {
readDatanodeDescriptorArray(in);
}
} else {
clientName = "";
clientMachine = "";
}
// The open lease transaction re-creates a file if necessary.
// Delete the file if it already exists.
if (FSNamesystem.LOG.isDebugEnabled()) {
FSNamesystem.LOG.debug(
opcode
+ ": "
+ path
+ " numblocks : "
+ blocks.length
+ " clientHolder "
+ clientName
+ " clientMachine "
+ clientMachine);
}
fsDir.unprotectedDelete(path, mtime);
/** modified by tony add to the file tree */
INodeFile node =
(INodeFile)
fsDir.unprotectedAddFile(
path,
permissions,
codingMatrix,
headeroffset,
fileSize,
blocks,
replication,
mtime,
atime,
blockSize);
if (isFileUnderConstruction) {
numOpAdd++;
//
// Replace current node with a INodeUnderConstruction.
// Recreate in-memory lease record.
//
// INodeFileUnderConstruction cons = new INodeFileUnderConstruction(
// node.getLocalNameBytes(),
// node.getReplication(),
// node.getModificationTime(),
// node.getPreferredBlockSize(),
// node.getBlocks(),
// node.getPermissionStatus(),
// clientName,
// clientMachine,
// null);
// TODO:
INodeFileUnderConstruction cons = null;
fsDir.replaceNode(path, node, cons);
fsNamesys.leaseManager.addLease(cons.getClientName(), path);
}
break;
}
case Ops.OP_SET_REPLICATION:
{
numOpSetRepl++;
path = FSImageSerialization.readString(in);
short replication = fsNamesys.adjustReplication(readShort(in));
fsDir.unprotectedSetReplication(path, replication, null);
break;
}
case Ops.OP_CONCAT_DELETE:
{
numOpConcatDelete++;
int length = in.readInt();
if (length < 3) { // trg, srcs.., timestam
throw new IOException("Incorrect data format. " + "Mkdir operation.");
}
String trg = FSImageSerialization.readString(in);
int srcSize = length - 1 - 1; // trg and timestamp
String[] srcs = new String[srcSize];
for (int i = 0; i < srcSize; i++) {
srcs[i] = FSImageSerialization.readString(in);
}
timestamp = readLong(in);
fsDir.unprotectedConcat(trg, srcs);
break;
}
case Ops.OP_RENAME_OLD:
{
numOpRenameOld++;
int length = in.readInt();
if (length != 3) {
throw new IOException("Incorrect data format. " + "Mkdir operation.");
}
String s = FSImageSerialization.readString(in);
String d = FSImageSerialization.readString(in);
timestamp = readLong(in);
HdfsFileStatus dinfo = fsDir.getFileInfo(d, false);
fsDir.unprotectedRenameTo(s, d, timestamp);
fsNamesys.changeLease(s, d, dinfo);
break;
}
case Ops.OP_DELETE:
{
numOpDelete++;
int length = in.readInt();
if (length != 2) {
throw new IOException("Incorrect data format. " + "delete operation.");
}
path = FSImageSerialization.readString(in);
timestamp = readLong(in);
fsDir.unprotectedDelete(path, timestamp);
break;
}
case Ops.OP_MKDIR:
{
numOpMkDir++;
PermissionStatus permissions = fsNamesys.getUpgradePermission();
int length = in.readInt();
if (-17 < logVersion && length != 2 || logVersion <= -17 && length != 3) {
throw new IOException("Incorrect data format. " + "Mkdir operation.");
}
path = FSImageSerialization.readString(in);
timestamp = readLong(in);
// The disk format stores atimes for directories as well.
// However, currently this is not being updated/used because of
// performance reasons.
if (LayoutVersion.supports(Feature.FILE_ACCESS_TIME, logVersion)) {
atime = readLong(in);
}
if (logVersion <= -11) {
permissions = PermissionStatus.read(in);
}
fsDir.unprotectedMkdir(path, permissions, timestamp);
break;
}
case Ops.OP_SET_GENSTAMP:
{
numOpSetGenStamp++;
long lw = in.readLong();
fsNamesys.setGenerationStamp(lw);
break;
}
case Ops.OP_DATANODE_ADD:
{
numOpOther++;
// Datanodes are not persistent any more.
FSImageSerialization.DatanodeImage.skipOne(in);
break;
}
case Ops.OP_DATANODE_REMOVE:
{
numOpOther++;
DatanodeID nodeID = new DatanodeID();
nodeID.readFields(in);
// Datanodes are not persistent any more.
break;
}
case Ops.OP_SET_PERMISSIONS:
{
numOpSetPerm++;
fsDir.unprotectedSetPermission(
FSImageSerialization.readString(in), FsPermission.read(in));
break;
}
case Ops.OP_SET_OWNER:
{
numOpSetOwner++;
fsDir.unprotectedSetOwner(
FSImageSerialization.readString(in),
FSImageSerialization.readString_EmptyAsNull(in),
FSImageSerialization.readString_EmptyAsNull(in));
break;
}
case Ops.OP_SET_NS_QUOTA:
{
fsDir.unprotectedSetQuota(
FSImageSerialization.readString(in),
readLongWritable(in),
FSConstants.QUOTA_DONT_SET);
break;
}
case Ops.OP_CLEAR_NS_QUOTA:
{
fsDir.unprotectedSetQuota(
FSImageSerialization.readString(in),
FSConstants.QUOTA_RESET,
FSConstants.QUOTA_DONT_SET);
break;
}
case Ops.OP_SET_QUOTA:
fsDir.unprotectedSetQuota(
FSImageSerialization.readString(in), readLongWritable(in), readLongWritable(in));
break;
case Ops.OP_TIMES:
{
numOpTimes++;
int length = in.readInt();
if (length != 3) {
throw new IOException("Incorrect data format. " + "times operation.");
}
path = FSImageSerialization.readString(in);
mtime = readLong(in);
atime = readLong(in);
fsDir.unprotectedSetTimes(path, mtime, atime, true);
break;
}
case Ops.OP_SYMLINK:
{
numOpSymlink++;
int length = in.readInt();
if (length != 4) {
throw new IOException("Incorrect data format. " + "symlink operation.");
}
path = FSImageSerialization.readString(in);
String value = FSImageSerialization.readString(in);
mtime = readLong(in);
atime = readLong(in);
PermissionStatus perm = PermissionStatus.read(in);
fsDir.unprotectedSymlink(path, value, mtime, atime, perm);
break;
}
case Ops.OP_RENAME:
{
numOpRename++;
int length = in.readInt();
if (length != 3) {
throw new IOException("Incorrect data format. " + "Mkdir operation.");
}
String s = FSImageSerialization.readString(in);
String d = FSImageSerialization.readString(in);
timestamp = readLong(in);
Rename[] options = readRenameOptions(in);
HdfsFileStatus dinfo = fsDir.getFileInfo(d, false);
fsDir.unprotectedRenameTo(s, d, timestamp, options);
fsNamesys.changeLease(s, d, dinfo);
break;
}
case Ops.OP_GET_DELEGATION_TOKEN:
{
numOpGetDelegationToken++;
DelegationTokenIdentifier delegationTokenId = new DelegationTokenIdentifier();
delegationTokenId.readFields(in);
long expiryTime = readLong(in);
fsNamesys
.getDelegationTokenSecretManager()
.addPersistedDelegationToken(delegationTokenId, expiryTime);
break;
}
case Ops.OP_RENEW_DELEGATION_TOKEN:
{
numOpRenewDelegationToken++;
DelegationTokenIdentifier delegationTokenId = new DelegationTokenIdentifier();
delegationTokenId.readFields(in);
long expiryTime = readLong(in);
fsNamesys
.getDelegationTokenSecretManager()
.updatePersistedTokenRenewal(delegationTokenId, expiryTime);
break;
}
case Ops.OP_CANCEL_DELEGATION_TOKEN:
{
numOpCancelDelegationToken++;
DelegationTokenIdentifier delegationTokenId = new DelegationTokenIdentifier();
delegationTokenId.readFields(in);
fsNamesys
.getDelegationTokenSecretManager()
.updatePersistedTokenCancellation(delegationTokenId);
break;
}
case Ops.OP_UPDATE_MASTER_KEY:
{
numOpUpdateMasterKey++;
DelegationKey delegationKey = new DelegationKey();
delegationKey.readFields(in);
fsNamesys.getDelegationTokenSecretManager().updatePersistedMasterKey(delegationKey);
break;
}
default:
{
throw new IOException("Never seen opcode " + opcode);
}
}
}
} catch (IOException ex) {
check203UpgradeFailure(logVersion, ex);
} finally {
if (closeOnExit) in.close();
}
if (FSImage.LOG.isDebugEnabled()) {
FSImage.LOG.debug(
"numOpAdd = "
+ numOpAdd
+ " numOpClose = "
+ numOpClose
+ " numOpDelete = "
+ numOpDelete
+ " numOpRenameOld = "
+ numOpRenameOld
+ " numOpSetRepl = "
+ numOpSetRepl
+ " numOpMkDir = "
+ numOpMkDir
+ " numOpSetPerm = "
+ numOpSetPerm
+ " numOpSetOwner = "
+ numOpSetOwner
+ " numOpSetGenStamp = "
+ numOpSetGenStamp
+ " numOpTimes = "
+ numOpTimes
+ " numOpConcatDelete = "
+ numOpConcatDelete
+ " numOpRename = "
+ numOpRename
+ " numOpGetDelegationToken = "
+ numOpGetDelegationToken
+ " numOpRenewDelegationToken = "
+ numOpRenewDelegationToken
+ " numOpCancelDelegationToken = "
+ numOpCancelDelegationToken
+ " numOpUpdateMasterKey = "
+ numOpUpdateMasterKey
+ " numOpOther = "
+ numOpOther);
}
return numEdits;
}
/**
* Get {@link UserGroupInformation} and possibly the delegation token out of the request.
*
* @param context the Servlet context
* @param request the http request
* @param conf configuration
* @param secureAuthMethod the AuthenticationMethod used in secure mode.
* @param tryUgiParameter Should it try the ugi parameter?
* @return a new user from the request
* @throws AccessControlException if the request has no token
*/
public static UserGroupInformation getUGI(
ServletContext context,
HttpServletRequest request,
Configuration conf,
final AuthenticationMethod secureAuthMethod,
final boolean tryUgiParameter)
throws IOException {
final UserGroupInformation ugi;
final String usernameFromQuery = getUsernameFromQuery(request, tryUgiParameter);
final String doAsUserFromQuery = request.getParameter(DoAsParam.NAME);
if (UserGroupInformation.isSecurityEnabled()) {
final String remoteUser = request.getRemoteUser();
String tokenString = request.getParameter(DELEGATION_PARAMETER_NAME);
if (tokenString != null) {
Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>();
token.decodeFromUrlString(tokenString);
SecurityUtil.setTokenService(token, NameNode.getAddress(conf));
token.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
DataInputStream in = new DataInputStream(buf);
DelegationTokenIdentifier id = new DelegationTokenIdentifier();
id.readFields(in);
if (context != null) {
NameNode nn = (NameNode) context.getAttribute("name.node");
if (nn != null) {
// Verify the token.
nn.getNamesystem()
.getDelegationTokenSecretManager()
.verifyToken(id, token.getPassword());
}
}
ugi = id.getUser();
if (ugi.getRealUser() == null) {
// non-proxy case
checkUsername(ugi.getShortUserName(), usernameFromQuery);
checkUsername(null, doAsUserFromQuery);
} else {
// proxy case
checkUsername(ugi.getRealUser().getShortUserName(), usernameFromQuery);
checkUsername(ugi.getShortUserName(), doAsUserFromQuery);
ProxyUsers.authorize(ugi, request.getRemoteAddr(), conf);
}
ugi.addToken(token);
ugi.setAuthenticationMethod(AuthenticationMethod.TOKEN);
} else {
if (remoteUser == null) {
throw new IOException("Security enabled but user not " + "authenticated by filter");
}
final UserGroupInformation realUgi = UserGroupInformation.createRemoteUser(remoteUser);
checkUsername(realUgi.getShortUserName(), usernameFromQuery);
// This is not necessarily true, could have been auth'ed by user-facing
// filter
realUgi.setAuthenticationMethod(secureAuthMethod);
ugi = initUGI(realUgi, doAsUserFromQuery, request, true, conf);
}
} else { // Security's not on, pull from url
final UserGroupInformation realUgi =
usernameFromQuery == null
? getDefaultWebUser(conf) // not specified in request
: UserGroupInformation.createRemoteUser(usernameFromQuery);
realUgi.setAuthenticationMethod(AuthenticationMethod.SIMPLE);
ugi = initUGI(realUgi, doAsUserFromQuery, request, false, conf);
}
if (LOG.isDebugEnabled()) LOG.debug("getUGI is returning: " + ugi.getShortUserName());
return ugi;
}