private static ArrayList<String> convertToStringArrayList(Collection<TezTaskID> collection) {
ArrayList<String> list = new ArrayList<String>(collection.size());
for (TezTaskID t : collection) {
list.add(t.toString());
}
return list;
}
@Test(timeout = 10000)
public void testBasicSpeculationPerVertexConf() throws Exception {
DAG dag = DAG.create("test");
String vNameNoSpec = "A";
String vNameSpec = "B";
Vertex vA = Vertex.create(vNameNoSpec, ProcessorDescriptor.create("Proc.class"), 5);
Vertex vB = Vertex.create(vNameSpec, ProcessorDescriptor.create("Proc.class"), 5);
vA.setConf(TezConfiguration.TEZ_AM_SPECULATION_ENABLED, "false");
dag.addVertex(vA);
dag.addVertex(vB);
// min/max src fraction is set to 1. So vertices will run sequentially
dag.addEdge(
Edge.create(
vA,
vB,
EdgeProperty.create(
DataMovementType.SCATTER_GATHER,
DataSourceType.PERSISTED,
SchedulingType.SEQUENTIAL,
OutputDescriptor.create("O"),
InputDescriptor.create("I"))));
MockTezClient tezClient = createTezSession();
DAGClient dagClient = tezClient.submitDAG(dag);
DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
TezVertexID vertexId = dagImpl.getVertex(vNameSpec).getVertexId();
TezVertexID vertexIdNoSpec = dagImpl.getVertex(vNameNoSpec).getVertexId();
// original attempt is killed and speculative one is successful
TezTaskAttemptID killedTaId =
TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
TezTaskAttemptID noSpecTaId =
TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexIdNoSpec, 0), 0);
// cause speculation trigger for both
mockLauncher.setStatusUpdatesForTask(killedTaId, 100);
mockLauncher.setStatusUpdatesForTask(noSpecTaId, 100);
mockLauncher.startScheduling(true);
dagClient.waitForCompletion();
Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
org.apache.tez.dag.app.dag.Vertex vSpec = dagImpl.getVertex(vertexId);
org.apache.tez.dag.app.dag.Vertex vNoSpec = dagImpl.getVertex(vertexIdNoSpec);
// speculation for vA but not for vB
Assert.assertTrue(
vSpec.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue() > 0);
Assert.assertEquals(
0, vNoSpec.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
tezClient.stop();
}
public void testBasicSpeculation(boolean withProgress) throws Exception {
DAG dag = DAG.create("test");
Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5);
dag.addVertex(vA);
MockTezClient tezClient = createTezSession();
DAGClient dagClient = tezClient.submitDAG(dag);
DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0);
// original attempt is killed and speculative one is successful
TezTaskAttemptID killedTaId =
TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
TezTaskAttemptID successTaId =
TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 1);
mockLauncher.updateProgress(withProgress);
// cause speculation trigger
mockLauncher.setStatusUpdatesForTask(killedTaId, 100);
mockLauncher.startScheduling(true);
dagClient.waitForCompletion();
Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
Task task = dagImpl.getTask(killedTaId.getTaskID());
Assert.assertEquals(2, task.getAttempts().size());
Assert.assertEquals(successTaId, task.getSuccessfulAttempt().getID());
TaskAttempt killedAttempt = task.getAttempt(killedTaId);
Joiner.on(",").join(killedAttempt.getDiagnostics()).contains("Killed as speculative attempt");
Assert.assertEquals(
TaskAttemptTerminationCause.TERMINATED_EFFECTIVE_SPECULATION,
killedAttempt.getTerminationCause());
if (withProgress) {
// without progress updates occasionally more than 1 task speculates
Assert.assertEquals(
1, task.getCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
Assert.assertEquals(
1, dagImpl.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
org.apache.tez.dag.app.dag.Vertex v = dagImpl.getVertex(killedTaId.getTaskID().getVertexID());
Assert.assertEquals(
1, v.getAllCounters().findCounter(TaskCounter.NUM_SPECULATIONS).getValue());
}
tezClient.stop();
}
/**
* Create a local reduce input file name.
*
* @param mapId a map task id
* @param size the size of the file
* @return path
* @throws IOException
*/
@Override
public Path getInputFileForWrite(TezTaskID mapId, long size) throws IOException {
return lDirAlloc.getLocalPathForWrite(
String.format(
Constants.TEZ_ENGINE_TASK_INPUT_FILE_FORMAT_STRING,
Constants.TASK_OUTPUT_DIR,
mapId.getId()),
size,
getConf());
}
@Test(timeout = 5000)
public void testDagNumber() {
String[] localDirs = new String[] {"dummyLocalDir"};
int appAttemptNumber = 1;
TezUmbilical tezUmbilical = mock(TezUmbilical.class);
String dagName = "DAG_NAME";
String vertexName = "VERTEX_NAME";
int vertexParallelism = 20;
int dagNumber = 52;
ApplicationId appId = ApplicationId.newInstance(10000, 13);
TezDAGID dagId = TezDAGID.getInstance(appId, dagNumber);
TezVertexID vertexId = TezVertexID.getInstance(dagId, 6);
TezTaskID taskId = TezTaskID.getInstance(vertexId, 4);
TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 2);
LogicalIOProcessorRuntimeTask runtimeTask = mock(LogicalIOProcessorRuntimeTask.class);
doReturn(new TezCounters()).when(runtimeTask).addAndGetTezCounter(any(String.class));
Map<String, ByteBuffer> serviceConsumerMetadata = Maps.newHashMap();
Map<String, String> auxServiceEnv = Maps.newHashMap();
MemoryDistributor memDist = mock(MemoryDistributor.class);
ProcessorDescriptor processorDesc = mock(ProcessorDescriptor.class);
InputReadyTracker inputReadyTracker = mock(InputReadyTracker.class);
ObjectRegistry objectRegistry = new ObjectRegistryImpl();
ExecutionContext execContext = new ExecutionContextImpl("localhost");
long memAvailable = 10000l;
TezProcessorContextImpl procContext =
new TezProcessorContextImpl(
new Configuration(),
localDirs,
appAttemptNumber,
tezUmbilical,
dagName,
vertexName,
vertexParallelism,
taskAttemptId,
null,
runtimeTask,
serviceConsumerMetadata,
auxServiceEnv,
memDist,
processorDesc,
inputReadyTracker,
objectRegistry,
execContext,
memAvailable);
assertEquals(dagNumber, procContext.getDagIdentifier());
assertEquals(appAttemptNumber, procContext.getDAGAttemptNumber());
assertEquals(appId, procContext.getApplicationId());
assertEquals(dagName, procContext.getDAGName());
assertEquals(vertexName, procContext.getTaskVertexName());
assertEquals(vertexId.getId(), procContext.getTaskVertexIndex());
assertTrue(Arrays.equals(localDirs, procContext.getWorkDirs()));
}
@Before
public void setup() {
applicationId = ApplicationId.newInstance(9999l, 1);
applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
tezDAGID = TezDAGID.getInstance(applicationId, random.nextInt());
tezVertexID = TezVertexID.getInstance(tezDAGID, random.nextInt());
tezTaskID = TezTaskID.getInstance(tezVertexID, random.nextInt());
tezTaskAttemptID = TezTaskAttemptID.getInstance(tezTaskID, random.nextInt());
dagPlan = DAGPlan.newBuilder().setName("DAGPlanMock").build();
containerId = ContainerId.newInstance(applicationAttemptId, 111);
nodeId = NodeId.newInstance("node", 13435);
}
public void fromProto(TaskFinishedProto proto) {
this.taskID = TezTaskID.fromString(proto.getTaskId());
this.finishTime = proto.getFinishTime();
this.state = TaskState.values()[proto.getState()];
if (proto.hasDiagnostics()) {
this.diagnostics = proto.getDiagnostics();
}
if (proto.hasCounters()) {
this.tezCounters = DagTypeConverters.convertTezCountersFromProto(proto.getCounters());
}
if (proto.hasSuccessfulTaskAttemptId()) {
this.successfulAttemptID = TezTaskAttemptID.fromString(proto.getSuccessfulTaskAttemptId());
}
}
public TaskFinishedProto toProto() {
TaskFinishedProto.Builder builder = TaskFinishedProto.newBuilder();
builder.setTaskId(taskID.toString()).setState(state.ordinal()).setFinishTime(finishTime);
if (diagnostics != null) {
builder.setDiagnostics(diagnostics);
}
if (tezCounters != null) {
builder.setCounters(DagTypeConverters.convertTezCountersToProto(tezCounters));
}
if (successfulAttemptID != null) {
builder.setSuccessfulTaskAttemptId(successfulAttemptID.toString());
}
return builder.build();
}
public WrappedContainer(boolean shouldProfile, String profileString) {
applicationID = ApplicationId.newInstance(rmIdentifier, 1);
appAttemptID = ApplicationAttemptId.newInstance(applicationID, 1);
containerID = ContainerId.newInstance(appAttemptID, 1);
nodeID = NodeId.newInstance("host", 12500);
nodeHttpAddress = "host:12501";
resource = Resource.newInstance(1024, 1);
priority = Priority.newInstance(1);
container =
Container.newInstance(containerID, nodeID, nodeHttpAddress, resource, priority, null);
chh = mock(ContainerHeartbeatHandler.class);
InetSocketAddress addr = new InetSocketAddress("localhost", 0);
tal = mock(TaskAttemptListener.class);
doReturn(addr).when(tal).getAddress();
dagID = TezDAGID.getInstance(applicationID, 1);
vertexID = TezVertexID.getInstance(dagID, 1);
taskID = TezTaskID.getInstance(vertexID, 1);
taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);
eventHandler = mock(EventHandler.class);
historyEventHandler = mock(HistoryEventHandler.class);
Configuration conf = new Configuration(false);
appContext = mock(AppContext.class);
doReturn(new HashMap<ApplicationAccessType, String>()).when(appContext).getApplicationACLs();
doReturn(eventHandler).when(appContext).getEventHandler();
doReturn(appAttemptID).when(appContext).getApplicationAttemptId();
doReturn(applicationID).when(appContext).getApplicationID();
doReturn(new SystemClock()).when(appContext).getClock();
doReturn(historyEventHandler).when(appContext).getHistoryHandler();
doReturn(conf).when(appContext).getAMConf();
mockDAGID();
taskSpec = mock(TaskSpec.class);
doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID();
amContainer =
new AMContainerImpl(container, chh, tal, new ContainerContextMatcher(), appContext);
}
@SuppressWarnings("unchecked")
@Test
public void testCredentialsTransfer() {
WrappedContainerMultipleDAGs wc = new WrappedContainerMultipleDAGs();
TezDAGID dagID2 = TezDAGID.getInstance("800", 500, 2);
TezDAGID dagID3 = TezDAGID.getInstance("800", 500, 3);
TezVertexID vertexID2 = TezVertexID.getInstance(dagID2, 1);
TezVertexID vertexID3 = TezVertexID.getInstance(dagID3, 1);
TezTaskID taskID2 = TezTaskID.getInstance(vertexID2, 1);
TezTaskID taskID3 = TezTaskID.getInstance(vertexID3, 1);
TezTaskAttemptID attempt11 = TezTaskAttemptID.getInstance(wc.taskID, 200);
TezTaskAttemptID attempt12 = TezTaskAttemptID.getInstance(wc.taskID, 300);
TezTaskAttemptID attempt21 = TezTaskAttemptID.getInstance(taskID2, 200);
TezTaskAttemptID attempt22 = TezTaskAttemptID.getInstance(taskID2, 300);
TezTaskAttemptID attempt31 = TezTaskAttemptID.getInstance(taskID3, 200);
TezTaskAttemptID attempt32 = TezTaskAttemptID.getInstance(taskID3, 300);
Map<String, LocalResource> LRs = new HashMap<String, LocalResource>();
AMContainerTask fetchedTask = null;
Token<TokenIdentifier> amGenToken = mock(Token.class);
Token<TokenIdentifier> token1 = mock(Token.class);
Token<TokenIdentifier> token3 = mock(Token.class);
Credentials containerCredentials = new Credentials();
TokenCache.setSessionToken(amGenToken, containerCredentials);
Text token1Name = new Text("tokenDag1");
Text token3Name = new Text("tokenDag3");
Credentials dag1Credentials = new Credentials();
dag1Credentials.addToken(new Text(token1Name), token1);
Credentials dag3Credentials = new Credentials();
dag3Credentials.addToken(new Text(token3Name), token3);
wc.launchContainer(new HashMap<String, LocalResource>(), containerCredentials);
wc.containerLaunched();
wc.assignTaskAttempt(attempt11, LRs, dag1Credentials);
fetchedTask = wc.pullTaskToRun();
assertTrue(fetchedTask.haveCredentialsChanged());
assertNotNull(fetchedTask.getCredentials());
assertNotNull(fetchedTask.getCredentials().getToken(token1Name));
wc.taskAttemptSucceeded(attempt11);
wc.assignTaskAttempt(attempt12, LRs, dag1Credentials);
fetchedTask = wc.pullTaskToRun();
assertFalse(fetchedTask.haveCredentialsChanged());
assertNull(fetchedTask.getCredentials());
wc.taskAttemptSucceeded(attempt12);
// Move to running a second DAG, with no credentials.
wc.setNewDAGID(dagID2);
wc.assignTaskAttempt(attempt21, LRs, null);
fetchedTask = wc.pullTaskToRun();
assertTrue(fetchedTask.haveCredentialsChanged());
assertNull(fetchedTask.getCredentials());
wc.taskAttemptSucceeded(attempt21);
wc.assignTaskAttempt(attempt22, LRs, null);
fetchedTask = wc.pullTaskToRun();
assertFalse(fetchedTask.haveCredentialsChanged());
assertNull(fetchedTask.getCredentials());
wc.taskAttemptSucceeded(attempt22);
// Move to running a third DAG, with Credentials this time
wc.setNewDAGID(dagID3);
wc.assignTaskAttempt(attempt31, LRs, dag3Credentials);
fetchedTask = wc.pullTaskToRun();
assertTrue(fetchedTask.haveCredentialsChanged());
assertNotNull(fetchedTask.getCredentials());
assertNotNull(fetchedTask.getCredentials().getToken(token3Name));
assertNull(fetchedTask.getCredentials().getToken(token1Name));
wc.taskAttemptSucceeded(attempt31);
wc.assignTaskAttempt(attempt32, LRs, dag1Credentials);
fetchedTask = wc.pullTaskToRun();
assertFalse(fetchedTask.haveCredentialsChanged());
assertNull(fetchedTask.getCredentials());
wc.taskAttemptSucceeded(attempt32);
}
@Override
public Vertex getVertex() {
return appContext.getCurrentDAG().getVertex(taskId.getVertexID());
}