/**
* simple test JobPriorityChangeEvent and JobPriorityChange
*
* @throws Exception
*/
@Test(timeout = 10000)
public void testJobPriorityChange() throws Exception {
org.apache.hadoop.mapreduce.JobID jid = new JobID("001", 1);
JobPriorityChangeEvent test = new JobPriorityChangeEvent(jid, JobPriority.LOW);
assertEquals(test.getJobId().toString(), jid.toString());
assertEquals(test.getPriority(), JobPriority.LOW);
}
@Test(timeout = 10000)
public void testJobQueueChange() throws Exception {
org.apache.hadoop.mapreduce.JobID jid = new JobID("001", 1);
JobQueueChangeEvent test = new JobQueueChangeEvent(jid, "newqueue");
assertEquals(test.getJobId().toString(), jid.toString());
assertEquals(test.getJobQueueName(), "newqueue");
}
private static void setJobID(Job job, JobID jobID, String namedOutput) {
JobID newJobID =
jobID == null || jobID.getJtIdentifier().contains(namedOutput)
? jobID
: new JobID(jobID.getJtIdentifier() + "_" + namedOutput, jobID.getId());
job.setJobID(newJobID);
}
private static StubbedJob createStubbedJob(
Configuration conf, Dispatcher dispatcher, int numSplits, AppContext appContext) {
JobID jobID = JobID.forName("job_1234567890000_0001");
JobId jobId = TypeConverter.toYarn(jobID);
if (appContext == null) {
appContext = mock(AppContext.class);
when(appContext.hasSuccessfullyUnregistered()).thenReturn(true);
}
StubbedJob job =
new StubbedJob(
jobId,
ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0),
conf,
dispatcher.getEventHandler(),
true,
"somebody",
numSplits,
appContext);
dispatcher.register(JobEventType.class, job);
EventHandler mockHandler = mock(EventHandler.class);
dispatcher.register(TaskEventType.class, mockHandler);
dispatcher.register(org.apache.hadoop.mapreduce.jobhistory.EventType.class, mockHandler);
dispatcher.register(JobFinishEvent.Type.class, mockHandler);
return job;
}
private boolean testUberDecision(Configuration conf) {
JobID jobID = JobID.forName("job_1234567890000_0001");
JobId jobId = TypeConverter.toYarn(jobID);
MRAppMetrics mrAppMetrics = MRAppMetrics.create();
JobImpl job =
new JobImpl(
jobId,
ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0),
conf,
mock(EventHandler.class),
null,
new JobTokenSecretManager(),
new Credentials(),
null,
null,
mrAppMetrics,
null,
true,
null,
0,
null,
null,
null,
null);
InitTransition initTransition = getInitTransition(2);
JobEvent mockJobEvent = mock(JobEvent.class);
initTransition.transition(job, mockJobEvent);
boolean isUber = job.isUber();
return isUber;
}
@Test
public void testReportDiagnostics() throws Exception {
JobID jobID = JobID.forName("job_1234567890000_0001");
JobId jobId = TypeConverter.toYarn(jobID);
final String diagMsg = "some diagnostic message";
final JobDiagnosticsUpdateEvent diagUpdateEvent = new JobDiagnosticsUpdateEvent(jobId, diagMsg);
MRAppMetrics mrAppMetrics = MRAppMetrics.create();
AppContext mockContext = mock(AppContext.class);
when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
JobImpl job =
new JobImpl(
jobId,
Records.newRecord(ApplicationAttemptId.class),
new Configuration(),
mock(EventHandler.class),
null,
mock(JobTokenSecretManager.class),
null,
new SystemClock(),
null,
mrAppMetrics,
null,
true,
null,
0,
null,
mockContext,
null,
null);
job.handle(diagUpdateEvent);
String diagnostics = job.getReport().getDiagnostics();
Assert.assertNotNull(diagnostics);
Assert.assertTrue(diagnostics.contains(diagMsg));
job =
new JobImpl(
jobId,
Records.newRecord(ApplicationAttemptId.class),
new Configuration(),
mock(EventHandler.class),
null,
mock(JobTokenSecretManager.class),
null,
new SystemClock(),
null,
mrAppMetrics,
null,
true,
null,
0,
null,
mockContext,
null,
null);
job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
job.handle(diagUpdateEvent);
diagnostics = job.getReport().getDiagnostics();
Assert.assertNotNull(diagnostics);
Assert.assertTrue(diagnostics.contains(diagMsg));
}
public static String getJobFile(
Configuration conf, String user, org.apache.hadoop.mapreduce.JobID jobId) {
Path jobFile =
new Path(
MRApps.getStagingAreaDir(conf, user),
jobId.toString() + Path.SEPARATOR + MRJobConfig.JOB_CONF_FILE);
return jobFile.toString();
}
@Override
protected Job createJob(Configuration conf, JobStateInternal forcedState, String diagnostic) {
JobImpl jobImpl = mock(JobImpl.class);
when(jobImpl.getInternalState()).thenReturn(this.jobStateInternal);
JobID jobID = JobID.forName("job_1234567890000_0001");
JobId jobId = TypeConverter.toYarn(jobID);
when(jobImpl.getID()).thenReturn(jobId);
((AppContext) getContext()).getAllJobs().put(jobImpl.getID(), jobImpl);
return jobImpl;
}
/**
* Tests if {@link TraceBuilder} can correctly identify and parse jobhistory filenames. The
* testcase checks if {@link TraceBuilder} - correctly identifies a jobhistory filename -
* correctly parses a jobhistory filename to extract out the jobid - correctly identifies a
* job-configuration filename stored along with the jobhistory files
*/
@Test
public void testJobHistoryFilenameParsing() throws IOException {
final Configuration conf = new Configuration();
final FileSystem lfs = FileSystem.getLocal(conf);
String user = "******";
org.apache.hadoop.mapred.JobID jid = new org.apache.hadoop.mapred.JobID("12345", 1);
final Path rootInputDir =
new Path(System.getProperty("test.tools.input.dir", "")).makeQualified(lfs);
// Check if jobhistory filename are detected properly
Path jhFilename = new Path(jid + "_1234_user_jobname");
JobID extractedJID = JobID.forName(TraceBuilder.extractJobID(jhFilename.getName()));
assertEquals("TraceBuilder failed to parse the current JH filename", jid, extractedJID);
// Check if the conf filename in jobhistory are detected properly
Path jhConfFilename = new Path(jid + "_conf.xml");
assertTrue(
"TraceBuilder failed to parse the current JH conf filename",
TraceBuilder.isJobConfXml(jhConfFilename.getName(), null));
}
/**
* Get the user log directory for the job jobid.
*
* @param jobid the jobid object
* @return user log directory for the job
*/
public static File getJobDir(JobID jobid) {
return getJobDir(jobid.toString());
}
/** Tests for ClientServiceDelegate.java */
@RunWith(value = Parameterized.class)
public class TestClientServiceDelegate {
private JobID oldJobId = JobID.forName("job_1315895242400_2");
private org.apache.hadoop.mapreduce.v2.api.records.JobId jobId = TypeConverter.toYarn(oldJobId);
private boolean isAMReachableFromClient;
public TestClientServiceDelegate(boolean isAMReachableFromClient) {
this.isAMReachableFromClient = isAMReachableFromClient;
}
@Parameters
public static Collection<Object[]> data() {
Object[][] data = new Object[][] {{true}, {false}};
return Arrays.asList(data);
}
@Test
public void testUnknownAppInRM() throws Exception {
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(getJobReportResponse());
ClientServiceDelegate clientServiceDelegate =
getClientServiceDelegate(historyServerProxy, getRMDelegate());
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
}
@Test
public void testRemoteExceptionFromHistoryServer() throws Exception {
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(getJobReportRequest()))
.thenThrow(new IOException("Job ID doesnot Exist"));
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);
try {
clientServiceDelegate.getJobStatus(oldJobId);
Assert.fail("Invoke should throw exception after retries.");
} catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("Job ID doesnot Exist"));
}
}
@Test
public void testRetriesOnConnectionFailure() throws Exception {
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(getJobReportRequest()))
.thenThrow(new RuntimeException("1"))
.thenThrow(new RuntimeException("2"))
.thenReturn(getJobReportResponse());
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
verify(historyServerProxy, times(3)).getJobReport(any(GetJobReportRequest.class));
}
@Test
public void testRetriesOnAMConnectionFailures() throws Exception {
if (!isAMReachableFromClient) {
return;
}
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
.thenReturn(getRunningApplicationReport("am1", 78));
// throw exception in 1st, 2nd, 3rd and 4th call of getJobReport, and
// succeed in the 5th call.
final MRClientProtocol amProxy = mock(MRClientProtocol.class);
when(amProxy.getJobReport(any(GetJobReportRequest.class)))
.thenThrow(new RuntimeException("11"))
.thenThrow(new RuntimeException("22"))
.thenThrow(new RuntimeException("33"))
.thenThrow(new RuntimeException("44"))
.thenReturn(getJobReportResponse());
Configuration conf = new YarnConfiguration();
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
ClientServiceDelegate clientServiceDelegate =
new ClientServiceDelegate(conf, rm, oldJobId, null) {
@Override
MRClientProtocol instantiateAMProxy(final InetSocketAddress serviceAddr)
throws IOException {
super.instantiateAMProxy(serviceAddr);
return amProxy;
}
};
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
// assert maxClientRetry is not decremented.
Assert.assertEquals(
conf.getInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES),
clientServiceDelegate.getMaxClientRetry());
verify(amProxy, times(5)).getJobReport(any(GetJobReportRequest.class));
}
@Test
public void testHistoryServerNotConfigured() throws Exception {
// RM doesn't have app report and job History Server is not configured
ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(null, getRMDelegate());
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertEquals("N/A", jobStatus.getUsername());
Assert.assertEquals(JobStatus.State.PREP, jobStatus.getState());
// RM has app report and job History Server is not configured
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
ApplicationReport applicationReport = getFinishedApplicationReport();
when(rm.getApplicationReport(jobId.getAppId())).thenReturn(applicationReport);
clientServiceDelegate = getClientServiceDelegate(null, rm);
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertEquals(applicationReport.getUser(), jobStatus.getUsername());
Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
}
@Test
public void testJobReportFromHistoryServer() throws Exception {
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(getJobReportRequest()))
.thenReturn(getJobReportResponseFromHistoryServer());
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile());
Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl());
Assert.assertEquals(1.0f, jobStatus.getMapProgress(), 0.0f);
Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f);
}
@Test
public void testCountersFromHistoryServer() throws Exception {
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getCounters(getCountersRequest()))
.thenReturn(getCountersResponseFromHistoryServer());
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())).thenReturn(null);
ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(historyServerProxy, rm);
Counters counters = TypeConverter.toYarn(clientServiceDelegate.getJobCounters(oldJobId));
Assert.assertNotNull(counters);
Assert.assertEquals(
1001, counters.getCounterGroup("dummyCounters").getCounter("dummyCounter").getValue());
}
@Test
public void testReconnectOnAMRestart() throws IOException {
// test not applicable when AM not reachable
// as instantiateAMProxy is not called at all
if (!isAMReachableFromClient) {
return;
}
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
// RM returns AM1 url, null, null and AM2 url on invocations.
// Nulls simulate the time when AM2 is in the process of restarting.
ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
try {
when(rmDelegate.getApplicationReport(jobId.getAppId()))
.thenReturn(getRunningApplicationReport("am1", 78))
.thenReturn(getRunningApplicationReport(null, 0))
.thenReturn(getRunningApplicationReport(null, 0))
.thenReturn(getRunningApplicationReport("am2", 90));
} catch (YarnException e) {
throw new IOException(e);
}
GetJobReportResponse jobReportResponse1 = mock(GetJobReportResponse.class);
when(jobReportResponse1.getJobReport())
.thenReturn(
MRBuilderUtils.newJobReport(
jobId,
"jobName-firstGen",
"user",
JobState.RUNNING,
0,
0,
0,
0,
0,
0,
0,
"anything",
null,
false,
""));
// First AM returns a report with jobName firstGen and simulates AM shutdown
// on second invocation.
MRClientProtocol firstGenAMProxy = mock(MRClientProtocol.class);
when(firstGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
.thenReturn(jobReportResponse1)
.thenThrow(new RuntimeException("AM is down!"));
GetJobReportResponse jobReportResponse2 = mock(GetJobReportResponse.class);
when(jobReportResponse2.getJobReport())
.thenReturn(
MRBuilderUtils.newJobReport(
jobId,
"jobName-secondGen",
"user",
JobState.RUNNING,
0,
0,
0,
0,
0,
0,
0,
"anything",
null,
false,
""));
// Second AM generation returns a report with jobName secondGen
MRClientProtocol secondGenAMProxy = mock(MRClientProtocol.class);
when(secondGenAMProxy.getJobReport(any(GetJobReportRequest.class)))
.thenReturn(jobReportResponse2);
ClientServiceDelegate clientServiceDelegate =
spy(getClientServiceDelegate(historyServerProxy, rmDelegate));
// First time, connection should be to AM1, then to AM2. Further requests
// should use the same proxy to AM2 and so instantiateProxy shouldn't be
// called.
doReturn(firstGenAMProxy)
.doReturn(secondGenAMProxy)
.when(clientServiceDelegate)
.instantiateAMProxy(any(InetSocketAddress.class));
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("jobName-firstGen", jobStatus.getJobName());
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("jobName-secondGen", jobStatus.getJobName());
verify(clientServiceDelegate, times(2)).instantiateAMProxy(any(InetSocketAddress.class));
}
@Test
public void testAMAccessDisabled() throws IOException {
// test only applicable when AM not reachable
if (isAMReachableFromClient) {
return;
}
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(getJobReportRequest()))
.thenReturn(getJobReportResponseFromHistoryServer());
ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
try {
when(rmDelegate.getApplicationReport(jobId.getAppId()))
.thenReturn(getRunningApplicationReport("am1", 78))
.thenReturn(getRunningApplicationReport("am1", 78))
.thenReturn(getRunningApplicationReport("am1", 78))
.thenReturn(getFinishedApplicationReport());
} catch (YarnException e) {
throw new IOException(e);
}
ClientServiceDelegate clientServiceDelegate =
spy(getClientServiceDelegate(historyServerProxy, rmDelegate));
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("N/A", jobStatus.getJobName());
verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
// Should not reach AM even for second and third times too.
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("N/A", jobStatus.getJobName());
verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus);
Assert.assertEquals("N/A", jobStatus.getJobName());
verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
// The third time around, app is completed, so should go to JHS
JobStatus jobStatus1 = clientServiceDelegate.getJobStatus(oldJobId);
Assert.assertNotNull(jobStatus1);
Assert.assertEquals("TestJobFilePath", jobStatus1.getJobFile());
Assert.assertEquals("http://TestTrackingUrl", jobStatus1.getTrackingUrl());
Assert.assertEquals(1.0f, jobStatus1.getMapProgress(), 0.0f);
Assert.assertEquals(1.0f, jobStatus1.getReduceProgress(), 0.0f);
verify(clientServiceDelegate, times(0)).instantiateAMProxy(any(InetSocketAddress.class));
}
@Test
public void testRMDownForJobStatusBeforeGetAMReport() throws IOException {
Configuration conf = new YarnConfiguration();
testRMDownForJobStatusBeforeGetAMReport(conf, MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES);
}
@Test
public void testRMDownForJobStatusBeforeGetAMReportWithRetryTimes() throws IOException {
Configuration conf = new YarnConfiguration();
conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 2);
testRMDownForJobStatusBeforeGetAMReport(
conf,
conf.getInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES));
}
@Test
public void testRMDownRestoreForJobStatusBeforeGetAMReport() throws IOException {
Configuration conf = new YarnConfiguration();
conf.setInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, 3);
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
when(historyServerProxy.getJobReport(any(GetJobReportRequest.class)))
.thenReturn(getJobReportResponse());
ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
try {
when(rmDelegate.getApplicationReport(jobId.getAppId()))
.thenThrow(
new java.lang.reflect.UndeclaredThrowableException(
new IOException("Connection refuced1")))
.thenThrow(
new java.lang.reflect.UndeclaredThrowableException(
new IOException("Connection refuced2")))
.thenReturn(getFinishedApplicationReport());
ClientServiceDelegate clientServiceDelegate =
new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy);
JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
verify(rmDelegate, times(3)).getApplicationReport(any(ApplicationId.class));
Assert.assertNotNull(jobStatus);
} catch (YarnException e) {
throw new IOException(e);
}
}
private void testRMDownForJobStatusBeforeGetAMReport(Configuration conf, int noOfRetries)
throws IOException {
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
ResourceMgrDelegate rmDelegate = mock(ResourceMgrDelegate.class);
try {
when(rmDelegate.getApplicationReport(jobId.getAppId()))
.thenThrow(
new java.lang.reflect.UndeclaredThrowableException(
new IOException("Connection refuced1")))
.thenThrow(
new java.lang.reflect.UndeclaredThrowableException(
new IOException("Connection refuced2")))
.thenThrow(
new java.lang.reflect.UndeclaredThrowableException(
new IOException("Connection refuced3")));
ClientServiceDelegate clientServiceDelegate =
new ClientServiceDelegate(conf, rmDelegate, oldJobId, historyServerProxy);
try {
clientServiceDelegate.getJobStatus(oldJobId);
Assert.fail("It should throw exception after retries");
} catch (IOException e) {
System.out.println("fail to get job status,and e=" + e.toString());
}
verify(rmDelegate, times(noOfRetries)).getApplicationReport(any(ApplicationId.class));
} catch (YarnException e) {
throw new IOException(e);
}
}
private GetJobReportRequest getJobReportRequest() {
GetJobReportRequest request = Records.newRecord(GetJobReportRequest.class);
request.setJobId(jobId);
return request;
}
private GetJobReportResponse getJobReportResponse() {
GetJobReportResponse jobReportResponse = Records.newRecord(GetJobReportResponse.class);
JobReport jobReport = Records.newRecord(JobReport.class);
jobReport.setJobId(jobId);
jobReport.setJobState(JobState.SUCCEEDED);
jobReportResponse.setJobReport(jobReport);
return jobReportResponse;
}
private GetCountersRequest getCountersRequest() {
GetCountersRequest request = Records.newRecord(GetCountersRequest.class);
request.setJobId(jobId);
return request;
}
private ApplicationReport getFinishedApplicationReport() {
ApplicationId appId = ApplicationId.newInstance(1234, 5);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0);
return ApplicationReport.newInstance(
appId,
attemptId,
"user",
"queue",
"appname",
"host",
124,
null,
YarnApplicationState.FINISHED,
"diagnostics",
"url",
0,
0,
FinalApplicationStatus.SUCCEEDED,
null,
"N/A",
0.0f,
YarnConfiguration.DEFAULT_APPLICATION_TYPE,
null);
}
private ApplicationReport getRunningApplicationReport(String host, int port) {
ApplicationId appId = ApplicationId.newInstance(1234, 5);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0);
return ApplicationReport.newInstance(
appId,
attemptId,
"user",
"queue",
"appname",
host,
port,
null,
YarnApplicationState.RUNNING,
"diagnostics",
"url",
0,
0,
FinalApplicationStatus.UNDEFINED,
null,
"N/A",
0.0f,
YarnConfiguration.DEFAULT_APPLICATION_TYPE,
null);
}
private ResourceMgrDelegate getRMDelegate() throws IOException {
ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
try {
ApplicationId appId = jobId.getAppId();
when(rm.getApplicationReport(appId))
.thenThrow(new ApplicationNotFoundException(appId + " not found"));
} catch (YarnException e) {
throw new IOException(e);
}
return rm;
}
private ClientServiceDelegate getClientServiceDelegate(
MRClientProtocol historyServerProxy, ResourceMgrDelegate rm) {
Configuration conf = new YarnConfiguration();
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, !isAMReachableFromClient);
ClientServiceDelegate clientServiceDelegate =
new ClientServiceDelegate(conf, rm, oldJobId, historyServerProxy);
return clientServiceDelegate;
}
private GetJobReportResponse getJobReportResponseFromHistoryServer() {
GetJobReportResponse jobReportResponse = Records.newRecord(GetJobReportResponse.class);
JobReport jobReport = Records.newRecord(JobReport.class);
jobReport.setJobId(jobId);
jobReport.setJobState(JobState.SUCCEEDED);
jobReport.setMapProgress(1.0f);
jobReport.setReduceProgress(1.0f);
jobReport.setJobFile("TestJobFilePath");
jobReport.setTrackingUrl("http://TestTrackingUrl");
jobReportResponse.setJobReport(jobReport);
return jobReportResponse;
}
private GetCountersResponse getCountersResponseFromHistoryServer() {
GetCountersResponse countersResponse = Records.newRecord(GetCountersResponse.class);
Counter counter = Records.newRecord(Counter.class);
CounterGroup counterGroup = Records.newRecord(CounterGroup.class);
Counters counters = Records.newRecord(Counters.class);
counter.setDisplayName("dummyCounter");
counter.setName("dummyCounter");
counter.setValue(1001);
counterGroup.setName("dummyCounters");
counterGroup.setDisplayName("dummyCounters");
counterGroup.setCounter("dummyCounter", counter);
counters.setCounterGroup("dummyCounters", counterGroup);
countersResponse.setCounters(counters);
return countersResponse;
}
}
@Override
public TaskAttemptID newTaskAttemptID(JobID jobId, boolean isMap, int taskId, int id) {
return new TaskAttemptID(
jobId.getJtIdentifier(), jobId.getId(), isMap ? TaskType.MAP : TaskType.REDUCE, taskId, id);
}
public void test(
String workflowId,
String workflowName,
String workflowNodeName,
Map<String, String[]> adjacencies) {
Configuration conf = new Configuration();
setProperties(conf, workflowId, workflowName, workflowNodeName, adjacencies);
String log =
log(
"JOB",
new String[] {ID, NAME, NODE, ADJ},
new String[] {
conf.get(ID_PROP),
conf.get(NAME_PROP),
conf.get(NODE_PROP),
JobHistory.JobInfo.getWorkflowAdjacencies(conf)
});
ParsedLine line = new ParsedLine(log);
JobID jobid = new JobID("id", 1);
JobSubmittedEvent event =
new JobSubmittedEvent(
jobid,
workflowName,
"",
0l,
"",
null,
"",
line.get(ID),
line.get(NAME),
line.get(NODE),
line.get(ADJ));
WorkflowContext context = MapReduceJobHistoryUpdater.buildWorkflowContext(event);
String resultingWorkflowId = workflowId;
if (workflowId.isEmpty()) resultingWorkflowId = jobid.toString().replace("job_", "mr_");
assertEquals("Didn't recover workflowId", resultingWorkflowId, context.getWorkflowId());
assertEquals("Didn't recover workflowName", workflowName, context.getWorkflowName());
assertEquals(
"Didn't recover workflowNodeName", workflowNodeName, context.getWorkflowEntityName());
Map<String, String[]> resultingAdjacencies = adjacencies;
if (resultingAdjacencies.size() == 0) {
resultingAdjacencies = new HashMap<String, String[]>();
resultingAdjacencies.put(workflowNodeName, new String[] {});
}
assertEquals(
"Got incorrect number of adjacencies",
resultingAdjacencies.size(),
context.getWorkflowDag().getEntries().size());
for (WorkflowDagEntry entry : context.getWorkflowDag().getEntries()) {
String[] sTargets = resultingAdjacencies.get(entry.getSource());
assertNotNull("No original targets for " + entry.getSource(), sTargets);
List<String> dTargets = entry.getTargets();
assertEquals(
"Got incorrect number of targets for " + entry.getSource(),
sTargets.length,
dTargets.size());
for (int i = 0; i < sTargets.length; i++) {
assertEquals("Got incorrect target for " + entry.getSource(), sTargets[i], dTargets.get(i));
}
}
}
public static Path getJobPath(JobID jobID, Path workingDirectory) {
return new Path(workingDirectory, jobID.toString());
}
@Test
public void testCheckAccess() {
// Create two unique users
String user1 = System.getProperty("user.name");
String user2 = user1 + "1234";
UserGroupInformation ugi1 = UserGroupInformation.createRemoteUser(user1);
UserGroupInformation ugi2 = UserGroupInformation.createRemoteUser(user2);
// Create the job
JobID jobID = JobID.forName("job_1234567890000_0001");
JobId jobId = TypeConverter.toYarn(jobID);
// Setup configuration access only to user1 (owner)
Configuration conf1 = new Configuration();
conf1.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
conf1.set(MRJobConfig.JOB_ACL_VIEW_JOB, "");
// Verify access
JobImpl job1 =
new JobImpl(
jobId, null, conf1, null, null, null, null, null, null, null, null, true, null, 0, null,
null, null, null);
Assert.assertTrue(job1.checkAccess(ugi1, JobACL.VIEW_JOB));
Assert.assertFalse(job1.checkAccess(ugi2, JobACL.VIEW_JOB));
// Setup configuration access to the user1 (owner) and user2
Configuration conf2 = new Configuration();
conf2.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
conf2.set(MRJobConfig.JOB_ACL_VIEW_JOB, user2);
// Verify access
JobImpl job2 =
new JobImpl(
jobId, null, conf2, null, null, null, null, null, null, null, null, true, null, 0, null,
null, null, null);
Assert.assertTrue(job2.checkAccess(ugi1, JobACL.VIEW_JOB));
Assert.assertTrue(job2.checkAccess(ugi2, JobACL.VIEW_JOB));
// Setup configuration access with security enabled and access to all
Configuration conf3 = new Configuration();
conf3.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
conf3.set(MRJobConfig.JOB_ACL_VIEW_JOB, "*");
// Verify access
JobImpl job3 =
new JobImpl(
jobId, null, conf3, null, null, null, null, null, null, null, null, true, null, 0, null,
null, null, null);
Assert.assertTrue(job3.checkAccess(ugi1, JobACL.VIEW_JOB));
Assert.assertTrue(job3.checkAccess(ugi2, JobACL.VIEW_JOB));
// Setup configuration access without security enabled
Configuration conf4 = new Configuration();
conf4.setBoolean(MRConfig.MR_ACLS_ENABLED, false);
conf4.set(MRJobConfig.JOB_ACL_VIEW_JOB, "");
// Verify access
JobImpl job4 =
new JobImpl(
jobId, null, conf4, null, null, null, null, null, null, null, null, true, null, 0, null,
null, null, null);
Assert.assertTrue(job4.checkAccess(ugi1, JobACL.VIEW_JOB));
Assert.assertTrue(job4.checkAccess(ugi2, JobACL.VIEW_JOB));
// Setup configuration access without security enabled
Configuration conf5 = new Configuration();
conf5.setBoolean(MRConfig.MR_ACLS_ENABLED, true);
conf5.set(MRJobConfig.JOB_ACL_VIEW_JOB, "");
// Verify access
JobImpl job5 =
new JobImpl(
jobId, null, conf5, null, null, null, null, null, null, null, null, true, null, 0, null,
null, null, null);
Assert.assertTrue(job5.checkAccess(ugi1, null));
Assert.assertTrue(job5.checkAccess(ugi2, null));
}
