@Test(timeout = 60000)
public void testAppRejectionWithCancelledDelegationToken() throws Exception {
MyFS dfs = (MyFS) FileSystem.get(conf);
LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + conf.hashCode());
MyToken token = dfs.getDelegationToken(new Text("user1"));
token.cancelToken();
Credentials ts = new Credentials();
ts.addToken(token.getKind(), token);
// register the tokens for renewal
ApplicationId appId = BuilderUtils.newApplicationId(0, 0);
delegationTokenRenewer.addApplication(appId, ts, true, false);
int waitCnt = 20;
while (waitCnt-- > 0) {
if (!eventQueue.isEmpty()) {
Event evt = eventQueue.take();
if (evt.getType() == RMAppEventType.APP_REJECTED) {
Assert.assertTrue(((RMAppEvent) evt).getApplicationId().equals(appId));
return;
}
} else {
Thread.sleep(500);
}
}
fail("App submission with a cancelled token should have failed");
}
/**
* Basic idea of the test: 1. register a token for 2 seconds with no cancel at the end 2. cancel
* it immediately 3. Sleep and check that the 2 seconds renew didn't happen (totally 5 renewals)
* 4. check cancellation
*
* @throws IOException
* @throws URISyntaxException
*/
@Test(timeout = 60000)
public void testDTRenewalWithNoCancel() throws Exception {
MyFS dfs = (MyFS) FileSystem.get(conf);
LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + conf.hashCode());
Credentials ts = new Credentials();
MyToken token1 = dfs.getDelegationToken(new Text("user1"));
// to cause this one to be set for renew in 2 secs
Renewer.tokenToRenewIn2Sec = token1;
LOG.info("token=" + token1 + " should be renewed for 2 secs");
String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
ts.addToken(new Text(nn1), token1);
ApplicationId applicationId_1 = BuilderUtils.newApplicationId(0, 1);
delegationTokenRenewer.addApplication(applicationId_1, ts, false, false);
waitForEventsToGetProcessed(delegationTokenRenewer);
delegationTokenRenewer.applicationFinished(applicationId_1);
waitForEventsToGetProcessed(delegationTokenRenewer);
int numberOfExpectedRenewals = Renewer.counter; // number of renewals so far
try {
Thread.sleep(6 * 1000); // sleep 6 seconds, so it has time to renew
} catch (InterruptedException e) {
}
LOG.info("Counter = " + Renewer.counter + ";t=" + Renewer.lastRenewed);
// counter and the token should still be the old ones
assertEquals(
"renew wasn't called as many times as expected", numberOfExpectedRenewals, Renewer.counter);
// also renewing of the canceled token should not fail, because it has not
// been canceled
token1.renew(conf);
}
/**
* Basic idea of the test: 0. Setup token KEEP_ALIVE 1. create tokens. 2. register them for
* renewal - to be cancelled on app complete 3. Complete app. 4. Verify token is alive within the
* KEEP_ALIVE time 5. Verify token has been cancelled after the KEEP_ALIVE_TIME
*
* @throws IOException
* @throws URISyntaxException
*/
@Test(timeout = 60000)
public void testDTKeepAlive1() throws Exception {
Configuration lconf = new Configuration(conf);
lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
// Keep tokens alive for 6 seconds.
lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
// Try removing tokens every second.
lconf.setLong(YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS, 1000l);
DelegationTokenRenewer localDtr = createNewDelegationTokenRenewer(lconf, counter);
localDtr.init(lconf);
RMContext mockContext = mock(RMContext.class);
ClientRMService mockClientRMService = mock(ClientRMService.class);
when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
when(mockContext.getDelegationTokenRenewer()).thenReturn(localDtr);
when(mockContext.getDispatcher()).thenReturn(dispatcher);
InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
localDtr.setRMContext(mockContext);
localDtr.start();
MyFS dfs = (MyFS) FileSystem.get(lconf);
LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + lconf.hashCode());
Credentials ts = new Credentials();
// get the delegation tokens
MyToken token1 = dfs.getDelegationToken(new Text("user1"));
String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
ts.addToken(new Text(nn1), token1);
// register the tokens for renewal
ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
localDtr.addApplication(applicationId_0, ts, true, false);
waitForEventsToGetProcessed(localDtr);
if (!eventQueue.isEmpty()) {
Event evt = eventQueue.take();
if (evt instanceof RMAppEvent) {
Assert.assertEquals(((RMAppEvent) evt).getType(), RMAppEventType.START);
} else {
fail("RMAppEvent.START was expected!!");
}
}
localDtr.applicationFinished(applicationId_0);
waitForEventsToGetProcessed(localDtr);
// Token should still be around. Renewal should not fail.
token1.renew(lconf);
// Allow the keepalive time to run out
Thread.sleep(10000l);
// The token should have been cancelled at this point. Renewal will fail.
try {
token1.renew(lconf);
fail("Renewal of cancelled token should have failed");
} catch (InvalidToken ite) {
}
}
// checks (3) and (1)
private Expression genObjOperation(Context ctx, MClass srcClass, Expression srcExpr)
throws SemanticException {
Expression res = null;
// find operation
String opname = fOp.getText();
MOperation op = srcClass.operation(opname, true);
if (op != null) {
// check for isQuery operation with OCL body
if (!op.hasExpression())
throw new SemanticException(
fOp,
"Operation `"
+ srcClass.name()
+ "::"
+ op.signature()
+ "' cannot be used in OCL expression "
+ "(only side effect-free operations with a return type and an OCL expression as body may be used).");
try {
// constructor performs additional checks
res = new ExpObjOp(op, fArgExprs);
} catch (ExpInvalidException ex) {
throw new SemanticException(
fOp, "In operation call `" + srcClass.name() + "::" + opname + "': " + ex.getMessage());
}
} else {
// try standard operation
res = genStdOperation(ctx, fOp, opname, fArgExprs);
}
return res;
}
/**
* Auxiliary - create token
*
* @param renewer
* @return
* @throws IOException
*/
static MyToken createTokens(Text renewer) throws IOException {
Text user1 = new Text("user1");
MyDelegationTokenSecretManager sm =
new MyDelegationTokenSecretManager(
DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT,
3600000,
null);
sm.startThreads();
DelegationTokenIdentifier dtId1 = new DelegationTokenIdentifier(user1, renewer, user1);
MyToken token1 = new MyToken(dtId1, sm);
token1.setService(new Text("localhost:0"));
return token1;
}
public Expression gen(Context ctx) throws SemanticException {
Expression res = null;
Expression srcExpr = null;
String opname = fOp.getText();
if (fSrcExpr != null) {
srcExpr = fSrcExpr.gen(ctx);
res = gen1(ctx, srcExpr);
} else {
// if no source expression is given, it is either a
// variable or a reference to expression determined by
// context. In the latter case we use the context as
// source expression and proceed as if it has been given
// explicitly.
if (!fHasParentheses) {
// variable?
// (7) check for variable
Type type = ctx.varTable().lookup(opname);
if (type != null) res = new ExpVariable(opname, type);
}
// do we have a context expression that is implicitly
// assumed to be the source expression?
if (res == null) {
ExprContext ec = ctx.exprContext();
if (!ec.isEmpty()) {
// construct source expression
ExprContext.Entry e = ec.peek();
srcExpr = new ExpVariable(e.fName, e.fType);
if (e.fType.isCollection()) fFollowsArrow = true;
res = gen1(ctx, srcExpr);
} else
throw new SemanticException(
fOp,
"Undefined " + (fHasParentheses ? "operation" : "variable") + " `" + opname + "'.");
}
}
if (fIsPre) {
if (!ctx.insidePostCondition())
throw new SemanticException(fOp, "Modifier @pre is only allowed in postconditions.");
res.setIsPre();
}
if (opname.equals("oclIsNew")) {
if (!ctx.insidePostCondition())
throw new SemanticException(fOp, "Operation oclIsNew is only allowed in postconditions.");
}
return res;
}
@Override
public long renew(Token<?> t, Configuration conf) throws IOException {
MyToken token = (MyToken) t;
if (token.isCanceled()) {
throw new InvalidToken("token has been canceled");
}
lastRenewed = token;
counter++;
LOG.info(
"Called MYDFS.renewdelegationtoken "
+ token
+ ";this dfs="
+ this.hashCode()
+ ";c="
+ counter);
if (tokenToRenewIn2Sec == token) {
// this token first renewal in 2 seconds
LOG.info("RENEW in 2 seconds");
tokenToRenewIn2Sec = null;
return 2 * 1000 + System.currentTimeMillis();
} else {
return 86400 * 1000 + System.currentTimeMillis();
}
}
public Expression gen(Context ctx) throws SemanticException {
String opname = fOp.getText();
Expression res = null;
Expression range, expr;
// check for empty range: do we have a context expression that
// is implicitly assumed to be the source expression?
if (fRange != null) range = fRange.gen(ctx);
else {
ExprContext ec = ctx.exprContext();
if (!ec.isEmpty()) {
// construct source expression
ExprContext.Entry e = ec.peek();
range = new ExpVariable(e.fName, e.fType);
} else throw new SemanticException(fOp, "Need a collection to apply `" + opname + "'.");
}
if (!range.type().isCollection())
throw new SemanticException(
fOp,
"Source of `"
+ opname
+ "' expression must be a collection, "
+ "found source expression of type `"
+ range.type()
+ "'.");
VarDeclList declList = new VarDeclList(true);
if (fDeclList.isEmpty()) {
// when there are no explicit var decls, we declare an
// internal variable with the element type
ExprContext ec = ctx.exprContext();
CollectionType ct = (CollectionType) range.type();
String var = ec.push(ct.elemType());
expr = fExpr.gen(ctx);
ec.pop();
// use the generated element variable
VarDecl decl = new VarDecl(var, ct.elemType());
declList.add(decl);
} else {
declList = fDeclList.gen(ctx, range);
// enter declared variable into scope before evaluating
// the argument expression
Symtable vars = ctx.varTable();
vars.enterScope();
fDeclList.addVariablesToSymtable(vars, declList.varDecl(0).type());
expr = fExpr.gen(ctx);
vars.exitScope();
}
try {
Integer id = (Integer) GParser.queryIdentMap.get(opname);
if (id == null)
throw new SemanticException(
fOp, "Internal error: unknown query operation `" + opname + "'.");
int idval = id.intValue();
switch (idval) {
case GParser.Q_SELECT_ID:
case GParser.Q_COLLECT_ID:
case GParser.Q_REJECT_ID:
case GParser.Q_ISUNIQUE_ID:
case GParser.Q_SORTEDBY_ID:
case GParser.Q_ANY_ID:
case GParser.Q_ONE_ID:
VarDecl decl;
if (declList.isEmpty()) decl = null;
else if (declList.size() == 1) decl = declList.varDecl(0);
else
throw new SemanticException(
fOp, "Only one element variable in " + opname + " expression allowed.");
switch (idval) {
case GParser.Q_SELECT_ID:
res = new ExpSelect(decl, range, expr);
break;
case GParser.Q_COLLECT_ID:
res = new ExpCollect(decl, range, expr);
break;
case GParser.Q_REJECT_ID:
res = new ExpReject(decl, range, expr);
break;
case GParser.Q_ISUNIQUE_ID:
res = new ExpIsUnique(decl, range, expr);
break;
case GParser.Q_SORTEDBY_ID:
res = new ExpSortedBy(decl, range, expr);
break;
case GParser.Q_ANY_ID:
res = new ExpAny(decl, range, expr);
break;
case GParser.Q_ONE_ID:
res = new ExpOne(decl, range, expr);
break;
}
break;
case GParser.Q_EXISTS_ID:
res = new ExpExists(declList, range, expr);
break;
case GParser.Q_FORALL_ID:
res = new ExpForAll(declList, range, expr);
break;
default:
// internal error
throw new SemanticException(
fOp, "Internal error: unknown query operation `" + opname + "'.");
}
} catch (ExpInvalidException ex) {
throw new SemanticException(fOp, ex);
}
return res;
}
public String toString() {
return super.toString() + fOp.toString();
}
/**
* Basic idea of the test: 1. create tokens. 2. Mark one of them to be renewed in 2 seconds
* (instead of 24 hours) 3. register them for renewal 4. sleep for 3 seconds 5. count number of
* renewals (should 3 initial ones + one extra) 6. register another token for 2 seconds 7. cancel
* it immediately 8. Sleep and check that the 2 seconds renew didn't happen (totally 5 renewals)
* 9. check cancellation
*
* @throws IOException
* @throws URISyntaxException
*/
@Test(timeout = 60000)
public void testDTRenewal() throws Exception {
MyFS dfs = (MyFS) FileSystem.get(conf);
LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + conf.hashCode());
// Test 1. - add three tokens - make sure exactly one get's renewed
// get the delegation tokens
MyToken token1, token2, token3;
token1 = dfs.getDelegationToken(new Text("user1"));
token2 = dfs.getDelegationToken(new Text("user2"));
token3 = dfs.getDelegationToken(new Text("user3"));
// to cause this one to be set for renew in 2 secs
Renewer.tokenToRenewIn2Sec = token1;
LOG.info("token=" + token1 + " should be renewed for 2 secs");
// three distinct Namenodes
String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
String nn2 = DelegationTokenRenewer.SCHEME + "://host2:0";
String nn3 = DelegationTokenRenewer.SCHEME + "://host3:0";
Credentials ts = new Credentials();
// register the token for renewal
ts.addToken(new Text(nn1), token1);
ts.addToken(new Text(nn2), token2);
ts.addToken(new Text(nn3), token3);
// register the tokens for renewal
ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
delegationTokenRenewer.addApplication(applicationId_0, ts, true, false);
waitForEventsToGetProcessed(delegationTokenRenewer);
// first 3 initial renewals + 1 real
int numberOfExpectedRenewals = 3 + 1;
int attempts = 10;
while (attempts-- > 0) {
try {
Thread.sleep(3 * 1000); // sleep 3 seconds, so it has time to renew
} catch (InterruptedException e) {
}
// since we cannot guarantee timely execution - let's give few chances
if (Renewer.counter == numberOfExpectedRenewals) break;
}
LOG.info(
"dfs=" + dfs.hashCode() + ";Counter = " + Renewer.counter + ";t=" + Renewer.lastRenewed);
assertEquals(
"renew wasn't called as many times as expected(4):",
numberOfExpectedRenewals,
Renewer.counter);
assertEquals("most recently renewed token mismatch", Renewer.lastRenewed, token1);
// Test 2.
// add another token ( that expires in 2 secs). Then remove it, before
// time is up.
// Wait for 3 secs , and make sure no renews were called
ts = new Credentials();
MyToken token4 = dfs.getDelegationToken(new Text("user4"));
// to cause this one to be set for renew in 2 secs
Renewer.tokenToRenewIn2Sec = token4;
LOG.info("token=" + token4 + " should be renewed for 2 secs");
String nn4 = DelegationTokenRenewer.SCHEME + "://host4:0";
ts.addToken(new Text(nn4), token4);
ApplicationId applicationId_1 = BuilderUtils.newApplicationId(0, 1);
delegationTokenRenewer.addApplication(applicationId_1, ts, true, false);
waitForEventsToGetProcessed(delegationTokenRenewer);
delegationTokenRenewer.applicationFinished(applicationId_1);
waitForEventsToGetProcessed(delegationTokenRenewer);
numberOfExpectedRenewals = Renewer.counter; // number of renewals so far
try {
Thread.sleep(6 * 1000); // sleep 6 seconds, so it has time to renew
} catch (InterruptedException e) {
}
LOG.info("Counter = " + Renewer.counter + ";t=" + Renewer.lastRenewed);
// counter and the token should stil be the old ones
assertEquals(
"renew wasn't called as many times as expected", numberOfExpectedRenewals, Renewer.counter);
// also renewing of the cancelled token should fail
try {
token4.renew(conf);
fail("Renewal of cancelled token should have failed");
} catch (InvalidToken ite) {
// expected
}
}
@Override
public void cancel(Token<?> t, Configuration conf) {
MyToken token = (MyToken) t;
LOG.info("Cancel token " + token);
token.cancelToken();
}
private Expression gen1(Context ctx, Expression srcExpr) throws SemanticException {
Expression res = null;
String opname = fOp.getText();
Type srcType = srcExpr.type();
// generate argument expressions
fArgExprs = new Expression[fArgs.size() + 1];
fArgExprs[0] = srcExpr;
Iterator it = fArgs.iterator();
int i = 1;
while (it.hasNext()) {
ASTExpression astExpr = (ASTExpression) it.next();
fArgExprs[i++] = astExpr.gen(ctx);
}
// flags for various cases
final int SRC_SIMPLE_TYPE = 0x100;
final int SRC_OBJECT_TYPE = 0x200;
final int SRC_COLLECTION_TYPE = 0x400;
final int DOT = 0x010;
final int ARROW = 0x020;
final int NO_PARENTHESES = 0x000;
final int PARENTHESES = 0x001;
int opcase;
if (srcType.isObjectType()) opcase = SRC_OBJECT_TYPE;
else if (srcType.isCollection()) opcase = SRC_COLLECTION_TYPE;
else opcase = SRC_SIMPLE_TYPE;
opcase += fFollowsArrow ? ARROW : DOT;
opcase += fHasParentheses ? PARENTHESES : NO_PARENTHESES;
switch (opcase) {
case SRC_SIMPLE_TYPE + DOT + NO_PARENTHESES:
case SRC_SIMPLE_TYPE + DOT + PARENTHESES:
case SRC_COLLECTION_TYPE + ARROW + PARENTHESES:
case SRC_COLLECTION_TYPE + ARROW + NO_PARENTHESES:
// (1) predefined OCL operation
res = genStdOperation(ctx, fOp, opname, fArgExprs);
break;
case SRC_SIMPLE_TYPE + ARROW + NO_PARENTHESES:
case SRC_SIMPLE_TYPE + ARROW + PARENTHESES:
ctx.reportWarning(
fOp,
"application of `"
+ opname
+ "' to a single value should be done with `.' "
+ "instead of `->'.");
// (1) predefined OCL operation
res = genStdOperation(ctx, fOp, opname, fArgExprs);
break;
case SRC_OBJECT_TYPE + DOT + NO_PARENTHESES:
MClass srcClass = ((ObjectType) srcType).cls();
MAttribute attr = srcClass.attribute(opname, true);
if (attr != null) {
// (2) attribute operation on object type (no arguments)
res = new ExpAttrOp(attr, srcExpr);
} else {
// (4) navigation operation on object type
// must be a role name
MAssociationEnd dstEnd = srcClass.navigableEnd(opname);
if (dstEnd != null) res = genNavigation(fOp, srcClass, srcExpr, dstEnd);
else {
// (1) predefined OCL operation
res = genStdOperation(ctx, fOp, opname, fArgExprs);
}
}
break;
case SRC_OBJECT_TYPE + DOT + PARENTHESES:
// (3) "isQuery" operation on object type (possibly with
// arguments) or (1)
MClass srcClass2 = ((ObjectType) srcType).cls();
res = genObjOperation(ctx, srcClass2, srcExpr);
break;
case SRC_OBJECT_TYPE + ARROW + NO_PARENTHESES:
case SRC_OBJECT_TYPE + ARROW + PARENTHESES:
// (6) set operation on single object resulting from
// navigation over associations with multiplicity zero
// or one (p. 7-13 of OMG UML 1.3)
if (srcExpr instanceof ExpNavigation) {
// first map object to set with internal operation
Expression mappedSrcExpr = new ExpObjAsSet(srcExpr);
// replace receiver in arg list
fArgExprs[0] = mappedSrcExpr;
try {
// lookup collection operation
res = ExpStdOp.create(opname, fArgExprs);
} catch (ExpInvalidException ex) {
throw new SemanticException(fOp, ex);
}
} else {
throw new SemanticException(
fOp,
"An arrow operation treating a single object as a "
+ "set may only be applied, if the object results "
+ "from a navigation to an association end with "
+ "multiplicity 0..1.");
}
break;
case SRC_COLLECTION_TYPE + DOT + NO_PARENTHESES:
// c.op 200 (5) with implicit (2,4,1)
if (Options.disableCollectShorthand)
throw new SemanticException(fOp, MSG_DISABLE_COLLECT_SHORTHAND);
res = collectShorthandWithOutArgs(opname, srcExpr);
break;
case SRC_COLLECTION_TYPE + DOT + PARENTHESES:
// c.op() 201 (5) with implicit (3,1)
if (Options.disableCollectShorthand)
throw new SemanticException(fOp, MSG_DISABLE_COLLECT_SHORTHAND);
res = collectShorthandWithArgs(opname, srcExpr);
break;
// throw new SemanticException(fOp,
// "If you want to apply an operation to a collection, please use an `->' instead of a
// `.'. If you are trying to apply the shorthand notation for `collect' - sorry, this is not
// yet supported. Please use the explicit notation.");
default:
throw new RuntimeException("case " + Integer.toHexString(opcase) + " not handled");
}
if (fIsPre) res.setIsPre();
return res;
}