private void testMapOnlyNoOutputInternal(int version) throws Exception {
JobConf conf = new JobConf();
// This is not set on purpose. FileOutputFormat.setOutputPath(conf, outDir);
conf.set(JobContext.TASK_ATTEMPT_ID, attempt);
conf.setInt(
org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION,
version);
JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// setup
committer.setupJob(jContext);
committer.setupTask(tContext);
if (committer.needsTaskCommit(tContext)) {
// do commit
committer.commitTask(tContext);
}
committer.commitJob(jContext);
// validate output
FileUtil.fullyDelete(new File(outDir.toString()));
}
private void testAbortInternal(int version) throws IOException, InterruptedException {
JobConf conf = new JobConf();
FileOutputFormat.setOutputPath(conf, outDir);
conf.set(JobContext.TASK_ATTEMPT_ID, attempt);
conf.setInt(
org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION,
version);
JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// do setup
committer.setupJob(jContext);
committer.setupTask(tContext);
// write output
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(null, conf, partFile, null);
writeOutput(theRecordWriter, tContext);
// do abort
committer.abortTask(tContext);
File out = new File(outDir.toUri().getPath());
Path workPath = committer.getWorkPath(tContext, outDir);
File wp = new File(workPath.toUri().getPath());
File expectedFile = new File(wp, partFile);
assertFalse("task temp dir still exists", expectedFile.exists());
committer.abortJob(jContext, JobStatus.State.FAILED);
expectedFile = new File(out, FileOutputCommitter.TEMP_DIR_NAME);
assertFalse("job temp dir still exists", expectedFile.exists());
assertEquals("Output directory not empty", 0, out.listFiles().length);
FileUtil.fullyDelete(out);
}
private void testMapFileOutputCommitterInternal(int version) throws Exception {
JobConf conf = new JobConf();
FileOutputFormat.setOutputPath(conf, outDir);
conf.set(JobContext.TASK_ATTEMPT_ID, attempt);
conf.setInt(
org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION,
version);
JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// setup
committer.setupJob(jContext);
committer.setupTask(tContext);
// write output
MapFileOutputFormat theOutputFormat = new MapFileOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(null, conf, partFile, null);
writeMapFileOutput(theRecordWriter, tContext);
// do commit
if (committer.needsTaskCommit(tContext)) {
committer.commitTask(tContext);
}
committer.commitJob(jContext);
// validate output
validateMapFileOutputContent(FileSystem.get(conf), outDir);
FileUtil.fullyDelete(new File(outDir.toString()));
}
/**
* Creates the configuration for mumak simulation. This is kept modular mostly for testing
* purposes. so that the standard configuration can be modified before passing it to the init()
* function.
*
* @return JobConf: the configuration for the SimulatorJobTracker
*/
JobConf createMumakConf() {
JobConf jobConf = new JobConf(getConf());
jobConf.setClass(
"topology.node.switch.mapping.impl", StaticMapping.class, DNSToSwitchMapping.class);
jobConf.set("fs.default.name", "file:///");
jobConf.set("mapred.job.tracker", "localhost:8012");
jobConf.setInt("mapred.jobtracker.job.history.block.size", 512);
jobConf.setInt("mapred.jobtracker.job.history.buffer.size", 512);
jobConf.setLong("mapred.tasktracker.expiry.interval", 5000);
jobConf.setInt("mapred.reduce.copy.backoff", 4);
jobConf.setLong("mapred.job.reuse.jvm.num.tasks", -1);
jobConf.setUser("mumak");
jobConf.set(
"mapred.system.dir",
jobConf.get("hadoop.log.dir", "/tmp/hadoop-" + jobConf.getUser()) + "/mapred/system");
return jobConf;
}
/**
* Initialize DFSCopyFileMapper specific job-configuration.
*
* @param conf : The dfs/mapred configuration.
* @param jobConf : The handle to the jobConf object to be initialized.
* @param args Arguments
*/
private static void setup(Configuration conf, JobConf jobConf, final Arguments args)
throws IOException {
jobConf.set(DST_DIR_LABEL, args.dst.toUri().toString());
// set boolean values
final boolean update = args.flags.contains(Options.UPDATE);
final boolean overwrite = !update && args.flags.contains(Options.OVERWRITE);
jobConf.setBoolean(Options.UPDATE.propertyname, update);
jobConf.setBoolean(Options.OVERWRITE.propertyname, overwrite);
jobConf.setBoolean(
Options.IGNORE_READ_FAILURES.propertyname,
args.flags.contains(Options.IGNORE_READ_FAILURES));
jobConf.setBoolean(
Options.PRESERVE_STATUS.propertyname, args.flags.contains(Options.PRESERVE_STATUS));
final String randomId = getRandomId();
JobClient jClient = new JobClient(jobConf);
Path jobDirectory = new Path(jClient.getSystemDir(), NAME + "_" + randomId);
jobConf.set(JOB_DIR_LABEL, jobDirectory.toString());
FileSystem dstfs = args.dst.getFileSystem(conf);
boolean dstExists = dstfs.exists(args.dst);
boolean dstIsDir = false;
if (dstExists) {
dstIsDir = dstfs.getFileStatus(args.dst).isDir();
}
// default logPath
Path logPath = args.log;
if (logPath == null) {
String filename = "_distcp_logs_" + randomId;
if (!dstExists || !dstIsDir) {
Path parent = args.dst.getParent();
if (!dstfs.exists(parent)) {
dstfs.mkdirs(parent);
}
logPath = new Path(parent, filename);
} else {
logPath = new Path(args.dst, filename);
}
}
FileOutputFormat.setOutputPath(jobConf, logPath);
// create src list, dst list
FileSystem jobfs = jobDirectory.getFileSystem(jobConf);
Path srcfilelist = new Path(jobDirectory, "_distcp_src_files");
jobConf.set(SRC_LIST_LABEL, srcfilelist.toString());
SequenceFile.Writer src_writer =
SequenceFile.createWriter(
jobfs,
jobConf,
srcfilelist,
LongWritable.class,
FilePair.class,
SequenceFile.CompressionType.NONE);
Path dstfilelist = new Path(jobDirectory, "_distcp_dst_files");
SequenceFile.Writer dst_writer =
SequenceFile.createWriter(
jobfs, jobConf, dstfilelist, Text.class, Text.class, SequenceFile.CompressionType.NONE);
Path dstdirlist = new Path(jobDirectory, "_distcp_dst_dirs");
jobConf.set(DST_DIR_LIST_LABEL, dstdirlist.toString());
SequenceFile.Writer dir_writer =
SequenceFile.createWriter(
jobfs,
jobConf,
dstdirlist,
Text.class,
FilePair.class,
SequenceFile.CompressionType.NONE);
// handle the case where the destination directory doesn't exist
// and we've only a single src directory OR we're updating/overwriting
// the contents of the destination directory.
final boolean special = (args.srcs.size() == 1 && !dstExists) || update || overwrite;
int srcCount = 0, cnsyncf = 0, dirsyn = 0;
long fileCount = 0L, byteCount = 0L, cbsyncs = 0L;
try {
for (Iterator<Path> srcItr = args.srcs.iterator(); srcItr.hasNext(); ) {
final Path src = srcItr.next();
FileSystem srcfs = src.getFileSystem(conf);
FileStatus srcfilestat = srcfs.getFileStatus(src);
Path root = special && srcfilestat.isDir() ? src : src.getParent();
if (srcfilestat.isDir()) {
++srcCount;
}
Stack<FileStatus> pathstack = new Stack<FileStatus>();
for (pathstack.push(srcfilestat); !pathstack.empty(); ) {
FileStatus cur = pathstack.pop();
FileStatus[] children = srcfs.listStatus(cur.getPath());
for (int i = 0; i < children.length; i++) {
boolean skipfile = false;
final FileStatus child = children[i];
final String dst = makeRelative(root, child.getPath());
++srcCount;
if (child.isDir()) {
pathstack.push(child);
} else {
// skip file if the src and the dst files are the same.
skipfile = update && sameFile(srcfs, child, dstfs, new Path(args.dst, dst));
// skip file if it exceed file limit or size limit
skipfile |=
fileCount == args.filelimit || byteCount + child.getLen() > args.sizelimit;
if (!skipfile) {
++fileCount;
byteCount += child.getLen();
if (LOG.isTraceEnabled()) {
LOG.trace("adding file " + child.getPath());
}
++cnsyncf;
cbsyncs += child.getLen();
if (cnsyncf > SYNC_FILE_MAX || cbsyncs > BYTES_PER_MAP) {
src_writer.sync();
dst_writer.sync();
cnsyncf = 0;
cbsyncs = 0L;
}
}
}
if (!skipfile) {
src_writer.append(
new LongWritable(child.isDir() ? 0 : child.getLen()), new FilePair(child, dst));
}
dst_writer.append(new Text(dst), new Text(child.getPath().toString()));
}
if (cur.isDir()) {
String dst = makeRelative(root, cur.getPath());
dir_writer.append(new Text(dst), new FilePair(cur, dst));
if (++dirsyn > SYNC_FILE_MAX) {
dirsyn = 0;
dir_writer.sync();
}
}
}
}
} finally {
checkAndClose(src_writer);
checkAndClose(dst_writer);
checkAndClose(dir_writer);
}
FileStatus dststatus = null;
try {
dststatus = dstfs.getFileStatus(args.dst);
} catch (FileNotFoundException fnfe) {
LOG.info(args.dst + " does not exist.");
}
// create dest path dir if copying > 1 file
if (dststatus == null) {
if (srcCount > 1 && !dstfs.mkdirs(args.dst)) {
throw new IOException("Failed to create" + args.dst);
}
}
final Path sorted = new Path(jobDirectory, "_distcp_sorted");
checkDuplication(jobfs, dstfilelist, sorted, conf);
if (dststatus != null && args.flags.contains(Options.DELETE)) {
deleteNonexisting(dstfs, dststatus, sorted, jobfs, jobDirectory, jobConf, conf);
}
Path tmpDir =
new Path(
(dstExists && !dstIsDir) || (!dstExists && srcCount == 1)
? args.dst.getParent()
: args.dst,
"_distcp_tmp_" + randomId);
jobConf.set(TMP_DIR_LABEL, tmpDir.toUri().toString());
LOG.info("srcCount=" + srcCount);
jobConf.setInt(SRC_COUNT_LABEL, srcCount);
jobConf.setLong(TOTAL_SIZE_LABEL, byteCount);
setMapCount(byteCount, jobConf);
}
/**
* The main driver for word count map/reduce program. Invoke this method to submit the map/reduce
* job.
*
* @throws IOException When there is communication problems with the job tracker.
*/
public void testNewJobWithMapAndReducers() throws Exception {
JobConf conf = new JobConf(TestJobCounters.class);
conf.setInt("io.sort.mb", 1);
conf.setInt("io.sort.factor", 2);
conf.set("io.sort.record.percent", "0.05");
conf.set("io.sort.spill.percent", "0.80");
FileSystem fs = FileSystem.get(conf);
Path testDir = new Path(TEST_ROOT_DIR, "countertest2");
conf.set("test.build.data", testDir.toString());
try {
if (fs.exists(testDir)) {
fs.delete(testDir, true);
}
if (!fs.mkdirs(testDir)) {
throw new IOException("Mkdirs failed to create " + testDir.toString());
}
String inDir = testDir + File.separator + "genins" + File.separator;
Path wordsIns = new Path(inDir);
if (!fs.mkdirs(wordsIns)) {
throw new IOException("Mkdirs failed to create " + wordsIns.toString());
}
String outDir = testDir + File.separator;
long inputSize = 0;
// create 3 input files each with 5*2k words
File inpFile = new File(inDir + "input5_2k_1");
createWordsFile(inpFile);
inputSize += inpFile.length();
inpFile = new File(inDir + "input5_2k_2");
createWordsFile(inpFile);
inputSize += inpFile.length();
inpFile = new File(inDir + "input5_2k_3");
createWordsFile(inpFile);
inputSize += inpFile.length();
FileInputFormat.setInputPaths(conf, inDir);
Path outputPath1 = new Path(outDir, "output5_2k_3");
FileOutputFormat.setOutputPath(conf, outputPath1);
Job job = new Job(conf);
job.setJobName("wordcount-map-reducers");
// the keys are words (strings)
job.setOutputKeyClass(Text.class);
// the values are counts (ints)
job.setOutputValueClass(IntWritable.class);
job.setMapperClass(NewMapTokenizer.class);
job.setCombinerClass(NewIdentityReducer.class);
job.setReducerClass(NewIdentityReducer.class);
job.setNumReduceTasks(1);
job.waitForCompletion(false);
org.apache.hadoop.mapreduce.Counters c1 = job.getCounters();
LogFactory.getLog(this.getClass()).info(c1);
// 3maps & in each map, 4 first level spills --- So total 12.
// spilled records count:
// Each Map: 1st level:2k+2k+2k+2k=8k;2ndlevel=4k+4k=8k;
// 3rd level=2k(4k from 1st level & 4k from 2nd level & combineAndSpill)
// So total 8k+8k+2k=18k
// For 3 Maps, total = 3*18=54k
// Reduce: each of the 3 map o/p's(2k each) will be spilled in shuffleToDisk()
// So 3*2k=6k in 1st level; 2nd level:4k(2k+2k);
// 3rd level directly given to reduce(4k+2k --- combineAndSpill => 2k.
// So 0 records spilled to disk in 3rd level)
// So total of 6k+4k=10k
// Total job counter will be 54k+10k = 64k
// 3 maps and 2.5k lines --- So total 7.5k map input records
// 3 maps and 10k words in each --- So total of 30k map output recs
validateCounters(c1, 64000, 7500, 30000);
validateFileCounters(c1, inputSize, 0, 0, 0);
// create 4th input file each with 5*2k words and test with 4 maps
inpFile = new File(inDir + "input5_2k_4");
createWordsFile(inpFile);
inputSize += inpFile.length();
JobConf newJobConf = new JobConf(job.getConfiguration());
Path outputPath2 = new Path(outDir, "output5_2k_4");
FileOutputFormat.setOutputPath(newJobConf, outputPath2);
Job newJob = new Job(newJobConf);
newJob.waitForCompletion(false);
c1 = newJob.getCounters();
LogFactory.getLog(this.getClass()).info(c1);
// 4maps & in each map 4 first level spills --- So total 16.
// spilled records count:
// Each Map: 1st level:2k+2k+2k+2k=8k;2ndlevel=4k+4k=8k;
// 3rd level=2k(4k from 1st level & 4k from 2nd level & combineAndSpill)
// So total 8k+8k+2k=18k
// For 3 Maps, total = 4*18=72k
// Reduce: each of the 4 map o/p's(2k each) will be spilled in shuffleToDisk()
// So 4*2k=8k in 1st level; 2nd level:4k+4k=8k;
// 3rd level directly given to reduce(4k+4k --- combineAndSpill => 2k.
// So 0 records spilled to disk in 3rd level)
// So total of 8k+8k=16k
// Total job counter will be 72k+16k = 88k
// 4 maps and 2.5k words in each --- So 10k map input records
// 4 maps and 10k unique words --- So 40k map output records
validateCounters(c1, 88000, 10000, 40000);
validateFileCounters(c1, inputSize, 0, 0, 0);
JobConf newJobConf2 = new JobConf(newJob.getConfiguration());
Path outputPath3 = new Path(outDir, "output5_2k_5");
FileOutputFormat.setOutputPath(newJobConf2, outputPath3);
Job newJob2 = new Job(newJobConf2);
newJob2.setNumReduceTasks(0);
newJob2.waitForCompletion(false);
c1 = newJob2.getCounters();
LogFactory.getLog(this.getClass()).info(c1);
// 4 maps and 2.5k words in each --- So 10k map input records
// 4 maps and 10k unique words --- So 40k map output records
validateCounters(c1, 0, 10000, 40000);
validateFileCounters(c1, inputSize, 0, -1, -1);
} finally {
// clean up the input and output files
if (fs.exists(testDir)) {
fs.delete(testDir, true);
}
}
}
private void testRecoveryInternal(int commitVersion, int recoveryVersion) throws Exception {
JobConf conf = new JobConf();
FileOutputFormat.setOutputPath(conf, outDir);
conf.set(JobContext.TASK_ATTEMPT_ID, attempt);
conf.setInt(MRConstants.APPLICATION_ATTEMPT_ID, 1);
conf.setInt(
org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION,
commitVersion);
JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// setup
committer.setupJob(jContext);
committer.setupTask(tContext);
// write output
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(null, conf, partFile, null);
writeOutput(theRecordWriter, tContext);
// do commit
if (committer.needsTaskCommit(tContext)) {
committer.commitTask(tContext);
}
Path jobTempDir1 = committer.getCommittedTaskPath(tContext);
File jtd1 = new File(jobTempDir1.toUri().getPath());
if (commitVersion == 1) {
assertTrue("Version 1 commits to temporary dir " + jtd1, jtd1.exists());
validateContent(jobTempDir1);
} else {
assertFalse("Version 2 commits to output dir " + jtd1, jtd1.exists());
}
// now while running the second app attempt,
// recover the task output from first attempt
JobConf conf2 = new JobConf(conf);
conf2.set(JobContext.TASK_ATTEMPT_ID, attempt);
conf2.setInt(MRConstants.APPLICATION_ATTEMPT_ID, 2);
conf2.setInt(
org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION,
recoveryVersion);
JobContext jContext2 = new JobContextImpl(conf2, taskID.getJobID());
TaskAttemptContext tContext2 = new TaskAttemptContextImpl(conf2, taskID);
FileOutputCommitter committer2 = new FileOutputCommitter();
committer2.setupJob(jContext2);
committer2.recoverTask(tContext2);
Path jobTempDir2 = committer2.getCommittedTaskPath(tContext2);
File jtd2 = new File(jobTempDir2.toUri().getPath());
if (recoveryVersion == 1) {
assertTrue("Version 1 recovers to " + jtd2, jtd2.exists());
validateContent(jobTempDir2);
} else {
assertFalse("Version 2 commits to output dir " + jtd2, jtd2.exists());
if (commitVersion == 1) {
assertTrue("Version 2 recovery moves to output dir from " + jtd1, jtd1.list().length == 0);
}
}
committer2.commitJob(jContext2);
validateContent(outDir);
FileUtil.fullyDelete(new File(outDir.toString()));
}
private void testFailAbortInternal(int version) throws IOException, InterruptedException {
JobConf conf = new JobConf();
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "faildel:///");
conf.setClass("fs.faildel.impl", FakeFileSystem.class, FileSystem.class);
conf.set(JobContext.TASK_ATTEMPT_ID, attempt);
conf.setInt(
org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION,
version);
conf.setInt(MRConstants.APPLICATION_ATTEMPT_ID, 1);
FileOutputFormat.setOutputPath(conf, outDir);
JobContext jContext = new JobContextImpl(conf, taskID.getJobID());
TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
// do setup
committer.setupJob(jContext);
committer.setupTask(tContext);
// write output
File jobTmpDir =
new File(
new Path(
outDir,
FileOutputCommitter.TEMP_DIR_NAME
+ Path.SEPARATOR
+ conf.getInt(MRConstants.APPLICATION_ATTEMPT_ID, 0)
+ Path.SEPARATOR
+ FileOutputCommitter.TEMP_DIR_NAME)
.toString());
File taskTmpDir = new File(jobTmpDir, "_" + taskID);
File expectedFile = new File(taskTmpDir, partFile);
TextOutputFormat<?, ?> theOutputFormat = new TextOutputFormat();
RecordWriter<?, ?> theRecordWriter =
theOutputFormat.getRecordWriter(null, conf, expectedFile.getAbsolutePath(), null);
writeOutput(theRecordWriter, tContext);
// do abort
Throwable th = null;
try {
committer.abortTask(tContext);
} catch (IOException ie) {
th = ie;
}
assertNotNull(th);
assertTrue(th instanceof IOException);
assertTrue(th.getMessage().contains("fake delete failed"));
assertTrue(expectedFile + " does not exists", expectedFile.exists());
th = null;
try {
committer.abortJob(jContext, JobStatus.State.FAILED);
} catch (IOException ie) {
th = ie;
}
assertNotNull(th);
assertTrue(th instanceof IOException);
assertTrue(th.getMessage().contains("fake delete failed"));
assertTrue("job temp dir does not exists", jobTmpDir.exists());
FileUtil.fullyDelete(new File(outDir.toString()));
}
private void ttRestarted() throws IOException, InterruptedException {
JobConf conf = new JobConf();
conf.setInt(JobContext.USER_LOG_RETAIN_HOURS, 3);
startTT(conf);
}
private void ttReinited() throws IOException {
JobConf conf = new JobConf();
conf.setInt(JobContext.USER_LOG_RETAIN_HOURS, 3);
userLogManager.clearOldUserLogs(conf);
}
/**
* Test job retire with tasks that report their *first* status only after the job retires. Steps :
* - Start a mini-mr cluster with 1 task-tracker having only map slots. Note that this
* task-tracker will take care of setup/cleanup and the map tasks. - Submit a job with 1 map task
* and 1 reduce task - Wait for the job to finish the map task - Start a 2nd tracker that waits
* for a long time after contacting the JT. - Wait for the 2nd tracker to get stuck - Kill the job
* - Wait for the job to retire - Check if the tip mappings are cleaned up.
*/
public void testJobRetireWithUnreportedTasks() throws Exception {
MiniMRCluster mr = null;
try {
JobConf conf = new JobConf();
// set the num-map-slots to 1 so that no reduce tasks but setup/cleanup
// can run on it
conf.setInt("mapred.tasktracker.map.tasks.maximum", 1);
conf.setInt("mapred.tasktracker.reduce.tasks.maximum", 0);
mr = startCluster(conf, 1);
JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker();
RunningJob job =
UtilsForTests.runJob(
mr.createJobConf(), new Path(testDir, "in-1"), new Path(testDir, "out-1"), 1, 1);
JobID id = JobID.downgrade(job.getID());
JobInProgress jip = jobtracker.getJob(id);
// wait 100 secs for the job to complete its map task
for (int i = 0; i < 1000 && jip.finishedMaps() < 1; i++) {
UtilsForTests.waitFor(100);
}
assertEquals(jip.finishedMaps(), 1);
// start a tracker that will wait
LOG.info("Adding a waiting tracker");
TaskTrackerRunner testTrackerRunner =
mr.new TaskTrackerRunner(1, 1, null, mr.createJobConf()) {
@Override
TaskTracker createTaskTracker(JobConf conf) throws InterruptedException, IOException {
return new WaitingTaskTracker(conf);
}
};
mr.addTaskTracker(testTrackerRunner);
LOG.info("Waiting tracker added");
WaitingTaskTracker testTT = (WaitingTaskTracker) testTrackerRunner.getTaskTracker();
// wait 100 secs for the newly started task-tracker to join
for (int i = 0; i < 1000 && jobtracker.taskTrackers().size() < 2; i++) {
UtilsForTests.waitFor(100);
}
assertEquals(jobtracker.taskTrackers().size(), 2);
LOG.info("Cluster is now up with 2 trackers");
// stop the test-tt as its no longer required
mr.stopTaskTracker(mr.getTaskTrackerID(testTT.getName()));
// 1 reduce task should be scheduled
assertEquals("TestTT contacted but no reduce task scheduled on it", 1, jip.runningReduces());
// kill the job
LOG.info("Killing job " + id);
job.killJob();
// check if the reduce task attempt status is missing
TaskInProgress tip = jip.getTasks(TaskType.REDUCE)[0];
assertNull(tip.getTaskStatus(tip.getAllTaskAttemptIDs()[0]));
// wait for the job to retire
waitTillRetire(id, jobtracker);
// check the taskidToTIPMap
for (TaskAttemptID tid : jobtracker.taskidToTIPMap.keySet()) {
LOG.info("TaskidToTIP : " + tid);
}
assertEquals("'taskid' to TIP mapping still exists", 0, jobtracker.taskidToTIPMap.size());
} finally {
if (mr != null) {
mr.shutdown();
}
// cleanup
FileUtil.fullyDelete(new File(testDir.toString()));
}
}
public CoronaJobHistory(JobConf conf, JobID jobId, String logPath) {
try {
this.conf = conf;
this.jobId = jobId;
if (logPath == null) {
logPath =
"file:///"
+ new File(System.getProperty("hadoop.log.dir", "/tmp")).getAbsolutePath()
+ File.separator
+ "history";
}
logDir = new Path(logPath);
logDirFs = logDir.getFileSystem(conf);
if (!logDirFs.exists(logDir)) {
LOG.info("Creating history folder at " + logDir);
if (!logDirFs.mkdirs(logDir, new FsPermission(HISTORY_DIR_PERMISSION))) {
throw new IOException("Mkdirs failed to create " + logDir.toString());
}
}
conf.set("hadoop.job.history.location", logDir.toString());
disableHistory = false;
// set the job history block size (default is 3MB)
jobHistoryBlockSize =
conf.getLong("mapred.jobtracker.job.history.block.size", 3 * 1024 * 1024);
doneDir = new Path(logDir, "done");
doneDirFs = logDirFs;
if (!doneDirFs.exists(doneDir)) {
LOG.info("Creating DONE folder at " + doneDir);
if (!doneDirFs.mkdirs(doneDir, new FsPermission(HISTORY_DIR_PERMISSION))) {
throw new IOException("Mkdirs failed to create " + doneDir);
}
}
String logFileName =
encodeJobHistoryFileName(CoronaJobHistoryFilesManager.getHistoryFilename(jobId));
logFile = new Path(logDir, logFileName);
doneFile = new Path(doneDir, logFileName);
// initialize the file manager
conf.setInt("mapred.jobtracker.historythreads.maximum", 1);
fileManager =
new CoronaJobHistoryFilesManager(
conf,
new JobHistoryObserver() {
public void historyFileCopied(JobID jobid, String historyFile) {}
},
logDir);
fileManager.setDoneDir(doneDir);
// sleeping with the past means tolerating two start methods instead of one
fileManager.start();
fileManager.startIOExecutor();
} catch (IOException e) {
LOG.error("Failed to initialize JobHistory log file", e);
disableHistory = true;
}
}
static void checkRecords(Configuration defaults, Path sortInput, Path sortOutput)
throws IOException {
FileSystem inputfs = sortInput.getFileSystem(defaults);
FileSystem outputfs = sortOutput.getFileSystem(defaults);
FileSystem defaultfs = FileSystem.get(defaults);
JobConf jobConf = new JobConf(defaults, RecordStatsChecker.class);
jobConf.setJobName("sortvalidate-recordstats-checker");
int noSortReduceTasks = outputfs.listStatus(sortOutput, sortPathsFilter).length;
jobConf.setInt("sortvalidate.sort.reduce.tasks", noSortReduceTasks);
int noSortInputpaths = inputfs.listStatus(sortInput).length;
jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class);
jobConf.setOutputFormat(SequenceFileOutputFormat.class);
jobConf.setOutputKeyClass(IntWritable.class);
jobConf.setOutputValueClass(RecordStatsChecker.RecordStatsWritable.class);
jobConf.setMapperClass(Map.class);
jobConf.setCombinerClass(Reduce.class);
jobConf.setReducerClass(Reduce.class);
jobConf.setNumMapTasks(noSortReduceTasks);
jobConf.setNumReduceTasks(1);
FileInputFormat.setInputPaths(jobConf, sortInput);
FileInputFormat.addInputPath(jobConf, sortOutput);
Path outputPath = new Path("/tmp/sortvalidate/recordstatschecker");
if (defaultfs.exists(outputPath)) {
defaultfs.delete(outputPath, true);
}
FileOutputFormat.setOutputPath(jobConf, outputPath);
// Uncomment to run locally in a single process
// job_conf.set("mapred.job.tracker", "local");
Path[] inputPaths = FileInputFormat.getInputPaths(jobConf);
System.out.println(
"\nSortValidator.RecordStatsChecker: Validate sort "
+ "from "
+ inputPaths[0]
+ " ("
+ noSortInputpaths
+ " files), "
+ inputPaths[1]
+ " ("
+ noSortReduceTasks
+ " files) into "
+ FileOutputFormat.getOutputPath(jobConf)
+ " with 1 reducer.");
Date startTime = new Date();
System.out.println("Job started: " + startTime);
JobClient.runJob(jobConf);
Date end_time = new Date();
System.out.println("Job ended: " + end_time);
System.out.println(
"The job took " + (end_time.getTime() - startTime.getTime()) / 1000 + " seconds.");
// Check to ensure that the statistics of the
// framework's sort-input and sort-output match
SequenceFile.Reader stats =
new SequenceFile.Reader(defaultfs, new Path(outputPath, "part-00000"), defaults);
IntWritable k1 = new IntWritable();
IntWritable k2 = new IntWritable();
RecordStatsWritable v1 = new RecordStatsWritable();
RecordStatsWritable v2 = new RecordStatsWritable();
if (!stats.next(k1, v1)) {
throw new IOException("Failed to read record #1 from reduce's output");
}
if (!stats.next(k2, v2)) {
throw new IOException("Failed to read record #2 from reduce's output");
}
if ((v1.getBytes() != v2.getBytes())
|| (v1.getRecords() != v2.getRecords())
|| v1.getChecksum() != v2.getChecksum()) {
throw new IOException(
"("
+ v1.getBytes()
+ ", "
+ v1.getRecords()
+ ", "
+ v1.getChecksum()
+ ") v/s ("
+ v2.getBytes()
+ ", "
+ v2.getRecords()
+ ", "
+ v2.getChecksum()
+ ")");
}
}
