@Test // First attempt is failed and second attempt is passed // The job succeeds. public void testFailTask() throws Exception { MRApp app = new MockFirstFailingAttemptMRApp(1, 0); Configuration conf = new Configuration(); // this test requires two task attempts, but uberization overrides max to 1 conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); Job job = app.submit(conf); app.waitForState(job, JobState.SUCCEEDED); Map<TaskId, Task> tasks = job.getTasks(); Assert.assertEquals("Num tasks is not correct", 1, tasks.size()); Task task = tasks.values().iterator().next(); Assert.assertEquals( "Task state not correct", TaskState.SUCCEEDED, task.getReport().getTaskState()); Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator().next().getAttempts(); Assert.assertEquals("Num attempts is not correct", 2, attempts.size()); // one attempt must be failed // and another must have succeeded Iterator<TaskAttempt> it = attempts.values().iterator(); Assert.assertEquals( "Attempt state not correct", TaskAttemptState.FAILED, it.next().getReport().getTaskAttemptState()); Assert.assertEquals( "Attempt state not correct", TaskAttemptState.SUCCEEDED, it.next().getReport().getTaskAttemptState()); }
public void verifyTaskGeneric( Task task, String id, String state, String type, String successfulAttempt, long startTime, long finishTime, long elapsedTime, float progress) { TaskId taskid = task.getID(); String tid = MRApps.toString(taskid); TaskReport report = task.getReport(); WebServicesTestUtils.checkStringMatch("id", tid, id); WebServicesTestUtils.checkStringMatch("type", task.getType().toString(), type); WebServicesTestUtils.checkStringMatch("state", report.getTaskState().toString(), state); // not easily checked without duplicating logic, just make sure its here assertNotNull("successfulAttempt null", successfulAttempt); assertEquals("startTime wrong", report.getStartTime(), startTime); assertEquals("finishTime wrong", report.getFinishTime(), finishTime); assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime); assertEquals("progress wrong", report.getProgress() * 100, progress, 1e-3f); }
@Test // All Task attempts are timed out, leading to Job failure public void testTimedOutTask() throws Exception { MRApp app = new TimeOutTaskMRApp(1, 0); Configuration conf = new Configuration(); int maxAttempts = 2; conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, maxAttempts); // disable uberization (requires entire job to be reattempted, so max for // subtask attempts is overridden to 1) conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); Job job = app.submit(conf); app.waitForState(job, JobState.FAILED); Map<TaskId, Task> tasks = job.getTasks(); Assert.assertEquals("Num tasks is not correct", 1, tasks.size()); Task task = tasks.values().iterator().next(); Assert.assertEquals( "Task state not correct", TaskState.FAILED, task.getReport().getTaskState()); Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator().next().getAttempts(); Assert.assertEquals("Num attempts is not correct", maxAttempts, attempts.size()); for (TaskAttempt attempt : attempts.values()) { Assert.assertEquals( "Attempt state not correct", TaskAttemptState.FAILED, attempt.getReport().getTaskAttemptState()); } }
// @Test public void testCompletedMapsForReduceSlowstart() throws Exception { MRApp app = new MRApp(2, 1, false, this.getClass().getName(), true); Configuration conf = new Configuration(); // after half of the map completion, reduce will start conf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.5f); // uberization forces full slowstart (1.0), so disable that conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false); Job job = app.submit(conf); app.waitForState(job, JobState.RUNNING); // all maps would be running Assert.assertEquals("Num tasks not correct", 3, job.getTasks().size()); Iterator<Task> it = job.getTasks().values().iterator(); Task mapTask1 = it.next(); Task mapTask2 = it.next(); Task reduceTask = it.next(); // all maps must be running app.waitForState(mapTask1, TaskState.RUNNING); app.waitForState(mapTask2, TaskState.RUNNING); TaskAttempt task1Attempt = mapTask1.getAttempts().values().iterator().next(); TaskAttempt task2Attempt = mapTask2.getAttempts().values().iterator().next(); // before sending the TA_DONE, event make sure attempt has come to // RUNNING state app.waitForState(task1Attempt, TaskAttemptState.RUNNING); app.waitForState(task2Attempt, TaskAttemptState.RUNNING); // reduces must be in NEW state Assert.assertEquals( "Reduce Task state not correct", TaskState.NEW, reduceTask.getReport().getTaskState()); // send the done signal to the 1st map task app.getContext() .getEventHandler() .handle( new TaskAttemptEvent( mapTask1.getAttempts().values().iterator().next().getID(), TaskAttemptEventType.TA_DONE)); // wait for first map task to complete app.waitForState(mapTask1, TaskState.SUCCEEDED); // Once the first map completes, it will schedule the reduces // now reduce must be running app.waitForState(reduceTask, TaskState.RUNNING); // send the done signal to 2nd map and the reduce to complete the job app.getContext() .getEventHandler() .handle( new TaskAttemptEvent( mapTask2.getAttempts().values().iterator().next().getID(), TaskAttemptEventType.TA_DONE)); app.getContext() .getEventHandler() .handle( new TaskAttemptEvent( reduceTask.getAttempts().values().iterator().next().getID(), TaskAttemptEventType.TA_DONE)); app.waitForState(job, JobState.SUCCEEDED); }