public void testJobRetire() throws Exception {
MiniMRCluster mr = null;
try {
JobConf conf = new JobConf();
mr = startCluster(conf, 1);
JobConf jobConf = mr.createJobConf();
JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker();
Path inDir = new Path(testDir, "input1");
Path outDir = new Path(testDir, "output1");
JobID id1 = validateJobRetire(jobConf, inDir, outDir, jobtracker);
outDir = new Path(testDir, "output2");
JobID id2 = validateJobRetire(jobConf, inDir, outDir, jobtracker);
assertNull("Job not removed from cache", jobtracker.getJobStatus(id1));
assertEquals("Total job in cache not correct", 1, jobtracker.getAllJobs().length);
} finally {
if (mr != null) {
mr.shutdown();
}
FileUtil.fullyDelete(new File(testDir.toString()));
}
}
public static void main(String[] args) throws IOException {
LOG.info("Bringing up Jobtracker and tasktrackers.");
MiniMRCluster mr = new MiniMRCluster(4, "file:///", 1);
LOG.info("JobTracker and TaskTrackers are up.");
mr.shutdown();
LOG.info("JobTracker and TaskTrackers brought down.");
}
/**
* run a distributed job and verify that TokenCache is available
*
* @throws IOException
*/
@Test
public void testTokenCache() throws IOException {
System.out.println("running dist job");
// make sure JT starts
jConf = mrCluster.createJobConf();
// provide namenodes names for the job to get the delegation tokens for
String nnUri = dfsCluster.getURI(0).toString();
jConf.set(MRJobConfig.JOB_NAMENODES, nnUri + "," + nnUri);
// job tracker principla id..
jConf.set(JTConfig.JT_USER_NAME, "jt_id/foo@BAR");
// using argument to pass the file name
String[] args = {
"-tokenCacheFile", tokenFileName.toString(), "-m", "1", "-r", "1", "-mt", "1", "-rt", "1"
};
int res = -1;
try {
res = ToolRunner.run(jConf, new MySleepJob(), args);
} catch (Exception e) {
System.out.println("Job failed with" + e.getLocalizedMessage());
e.printStackTrace(System.out);
fail("Job failed");
}
assertEquals("dist job res is not 0", res, 0);
}
@AfterClass
public static void tearDown() throws Exception {
if (mrCluster != null) mrCluster.shutdown();
mrCluster = null;
if (dfsCluster != null) dfsCluster.shutdown();
dfsCluster = null;
}
@BeforeClass
public static void setup() throws Exception {
String testDir = System.getProperty("test.data.dir", "./");
testDir = testDir + "/test_multitable_" + Math.abs(new Random().nextLong()) + "/";
workDir = new File(new File(testDir).getCanonicalPath());
FileUtil.fullyDelete(workDir);
workDir.mkdirs();
warehousedir = new Path(workDir + "/warehouse");
// Run hive metastore server
t = new Thread(new RunMS());
t.start();
// LocalJobRunner does not work with mapreduce OutputCommitter. So need
// to use MiniMRCluster. MAPREDUCE-2350
Configuration conf = new Configuration(true);
conf.set("yarn.scheduler.capacity.root.queues", "default");
conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
FileSystem fs = FileSystem.get(conf);
System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath());
mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, new JobConf(conf));
mrConf = mrCluster.createJobConf();
fs.mkdirs(warehousedir);
initializeSetup();
}
@Override
public void setupConfiguration(Configuration conf) {
JobConf jConf = mr.createJobConf();
for (Map.Entry<String, String> pair : jConf) {
conf.set(pair.getKey(), pair.getValue());
}
}
@AfterClass
public static void tearDown() throws IOException {
FileUtil.fullyDelete(workDir);
FileSystem fs = FileSystem.get(mrConf);
if (fs.exists(warehousedir)) {
fs.delete(warehousedir, true);
}
if (mrCluster != null) {
mrCluster.shutdown();
}
}
private void myTearDown() throws Exception {
if (cnode != null) {
cnode.stop();
cnode.join();
}
if (mr != null) {
mr.shutdown();
}
if (dfsCluster != null) {
dfsCluster.shutdown();
}
}
/** (Mock)Test JobTracker.removeJobTasks() which is called only when the job retires. */
public void testJobRemoval() throws Exception {
MiniMRCluster mr = null;
try {
JobConf conf = new JobConf();
mr = startCluster(conf, 0);
JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker();
// test map task removal
testRemoveJobTasks(jobtracker, conf, TaskType.MAP);
// test reduce task removal
testRemoveJobTasks(jobtracker, conf, TaskType.REDUCE);
// test job setup removal
testRemoveJobTasks(jobtracker, conf, TaskType.JOB_SETUP);
// test job cleanup removal
testRemoveJobTasks(jobtracker, conf, TaskType.JOB_CLEANUP);
} finally {
if (mr != null) {
mr.shutdown();
}
// cleanup
FileUtil.fullyDelete(new File(testDir.toString()));
}
}
private void mySetup(int stripeLength) throws Exception {
if (System.getProperty("hadoop.log.dir") == null) {
String base = new File(".").getAbsolutePath();
System.setProperty("hadoop.log.dir", new Path(base).toString() + "/logs");
}
new File(TEST_DIR).mkdirs(); // Make sure data directory exists
conf = new Configuration();
conf.set("raid.config.file", CONFIG_FILE);
conf.setBoolean("raid.config.reload", true);
conf.setLong("raid.config.reload.interval", RELOAD_INTERVAL);
// scan all policies once every 5 second
conf.setLong("raid.policy.rescan.interval", 5000);
// do not use map-reduce cluster for Raiding
conf.set("raid.classname", "org.apache.hadoop.raid.LocalRaidNode");
conf.set("raid.server.address", "localhost:" + MiniDFSCluster.getFreePort());
conf.set("mapred.raid.http.address", "localhost:0");
Utils.loadTestCodecs(
conf, stripeLength, stripeLength, 1, 3, "/destraid", "/destraidrs", false, true);
conf.setBoolean("dfs.permissions", false);
// Make sure initial repl is smaller than NUM_DATANODES
conf.setInt(RaidNode.RAID_PARITY_INITIAL_REPL_KEY, 1);
dfsCluster = new MiniDFSCluster(conf, NUM_DATANODES, true, null);
dfsCluster.waitActive();
fileSys = dfsCluster.getFileSystem();
namenode = fileSys.getUri().toString();
FileSystem.setDefaultUri(conf, namenode);
mr = new MiniMRCluster(4, namenode, 3);
jobTrackerName = "localhost:" + mr.getJobTrackerPort();
hftp = "hftp://localhost.localdomain:" + dfsCluster.getNameNodePort();
FileSystem.setDefaultUri(conf, namenode);
conf.set("mapred.job.tracker", jobTrackerName);
conf.set(RaidNode.RAID_CHECKSUM_STORE_CLASS_KEY, "org.apache.hadoop.raid.LocalChecksumStore");
conf.setBoolean(RaidNode.RAID_CHECKSUM_STORE_REQUIRED_KEY, true);
conf.set(LocalChecksumStore.LOCAL_CHECK_STORE_DIR_KEY, CHECKSUM_STORE_DIR);
conf.set(RaidNode.RAID_STRIPE_STORE_CLASS_KEY, "org.apache.hadoop.raid.LocalStripeStore");
conf.set(LocalStripeStore.LOCAL_STRIPE_STORE_DIR_KEY, STRIPE_STORE_DIR);
ConfigBuilder cb = new ConfigBuilder(CONFIG_FILE);
cb.addPolicy("RaidTest1", "/user/dhruba/raidtest", 1, 1);
cb.addPolicy("RaidTest2", "/user/dhruba/raidtestrs", 1, 1, "rs");
cb.persist();
}
/** Check refreshNodes for decommissioning blacklisted nodes. */
public void testBlacklistedNodeDecommissioning() throws Exception {
LOG.info("Testing blacklisted node decommissioning");
MiniMRCluster mr = null;
JobTracker jt = null;
try {
// start mini mr
JobConf jtConf = new JobConf();
jtConf.set("mapred.max.tracker.blacklists", "1");
mr = new MiniMRCluster(0, 0, 2, "file:///", 1, null, null, null, jtConf);
jt = mr.getJobTrackerRunner().getJobTracker();
assertEquals("Trackers not up", 2, jt.taskTrackers().size());
// validate the total tracker count
assertEquals(
"Active tracker count mismatch", 2, jt.getClusterStatus(false).getTaskTrackers());
// validate blacklisted count
assertEquals(
"Blacklisted tracker count mismatch",
0,
jt.getClusterStatus(false).getBlacklistedTrackers());
// run a failing job to blacklist the tracker
JobConf jConf = mr.createJobConf();
jConf.set("mapred.max.tracker.failures", "1");
jConf.setJobName("test-job-fail-once");
jConf.setMapperClass(FailOnceMapper.class);
jConf.setReducerClass(IdentityReducer.class);
jConf.setNumMapTasks(1);
jConf.setNumReduceTasks(0);
RunningJob job =
UtilsForTests.runJob(jConf, new Path(TEST_DIR, "in"), new Path(TEST_DIR, "out"));
job.waitForCompletion();
// validate the total tracker count
assertEquals(
"Active tracker count mismatch", 1, jt.getClusterStatus(false).getTaskTrackers());
// validate blacklisted count
assertEquals(
"Blacklisted tracker count mismatch",
1,
jt.getClusterStatus(false).getBlacklistedTrackers());
// find the blacklisted tracker
String trackerName = null;
for (TaskTrackerStatus status : jt.taskTrackers()) {
if (jt.isBlacklisted(status.getTrackerName())) {
trackerName = status.getTrackerName();
break;
}
}
// get the hostname
String hostToDecommission = JobInProgress.convertTrackerNameToHostName(trackerName);
LOG.info("Decommissioning tracker " + hostToDecommission);
// decommission the node
HashSet<String> decom = new HashSet<String>(1);
decom.add(hostToDecommission);
jt.decommissionNodes(decom);
// validate
// check the cluster status and tracker size
assertEquals(
"Tracker is not lost upon host decommissioning",
1,
jt.getClusterStatus(false).getTaskTrackers());
assertEquals(
"Blacklisted tracker count incorrect in cluster status " + "after decommissioning",
0,
jt.getClusterStatus(false).getBlacklistedTrackers());
assertEquals("Tracker is not lost upon host decommissioning", 1, jt.taskTrackers().size());
} finally {
if (mr != null) {
mr.shutdown();
mr = null;
jt = null;
FileUtil.fullyDelete(new File(TEST_DIR.toString()));
}
}
}
/** Check that we can reach a NameNode or a JobTracker using a specific socket factory */
public void testSocketFactory() throws IOException {
// Create a standard mini-cluster
Configuration sconf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster(sconf, 1, true, null);
final int nameNodePort = cluster.getNameNodePort();
// Get a reference to its DFS directly
FileSystem fs = cluster.getFileSystem();
assertTrue(fs instanceof DistributedFileSystem);
DistributedFileSystem directDfs = (DistributedFileSystem) fs;
// Get another reference via network using a specific socket factory
Configuration cconf = new Configuration();
FileSystem.setDefaultUri(cconf, String.format("hdfs://localhost:%s/", nameNodePort + 10));
cconf.set(
"hadoop.rpc.socket.factory.class.default", "org.apache.hadoop.ipc.DummySocketFactory");
cconf.set(
"hadoop.rpc.socket.factory.class.ClientProtocol",
"org.apache.hadoop.ipc.DummySocketFactory");
cconf.set(
"hadoop.rpc.socket.factory.class.JobSubmissionProtocol",
"org.apache.hadoop.ipc.DummySocketFactory");
fs = FileSystem.get(cconf);
assertTrue(fs instanceof DistributedFileSystem);
DistributedFileSystem dfs = (DistributedFileSystem) fs;
JobClient client = null;
MiniMRCluster mr = null;
try {
// This will test RPC to the NameNode only.
// could we test Client-DataNode connections?
Path filePath = new Path("/dir");
assertFalse(directDfs.exists(filePath));
assertFalse(dfs.exists(filePath));
directDfs.mkdirs(filePath);
assertTrue(directDfs.exists(filePath));
assertTrue(dfs.exists(filePath));
// This will test TPC to a JobTracker
mr = new MiniMRCluster(1, fs.getUri().toString(), 1);
final int jobTrackerPort = mr.getJobTrackerPort();
JobConf jconf = new JobConf(cconf);
jconf.set("mapred.job.tracker", String.format("localhost:%d", jobTrackerPort + 10));
client = new JobClient(jconf);
JobStatus[] jobs = client.jobsToComplete();
assertTrue(jobs.length == 0);
} finally {
try {
if (client != null) client.close();
} catch (Exception ignored) {
// nothing we can do
ignored.printStackTrace();
}
try {
if (dfs != null) dfs.close();
} catch (Exception ignored) {
// nothing we can do
ignored.printStackTrace();
}
try {
if (directDfs != null) directDfs.close();
} catch (Exception ignored) {
// nothing we can do
ignored.printStackTrace();
}
try {
if (cluster != null) cluster.shutdown();
} catch (Exception ignored) {
// nothing we can do
ignored.printStackTrace();
}
if (mr != null) {
try {
mr.shutdown();
} catch (Exception ignored) {
ignored.printStackTrace();
}
}
}
}
/**
* Test job retire with tasks that report their *first* status only after the job retires. Steps :
* - Start a mini-mr cluster with 1 task-tracker having only map slots. Note that this
* task-tracker will take care of setup/cleanup and the map tasks. - Submit a job with 1 map task
* and 1 reduce task - Wait for the job to finish the map task - Start a 2nd tracker that waits
* for a long time after contacting the JT. - Wait for the 2nd tracker to get stuck - Kill the job
* - Wait for the job to retire - Check if the tip mappings are cleaned up.
*/
public void testJobRetireWithUnreportedTasks() throws Exception {
MiniMRCluster mr = null;
try {
JobConf conf = new JobConf();
// set the num-map-slots to 1 so that no reduce tasks but setup/cleanup
// can run on it
conf.setInt("mapred.tasktracker.map.tasks.maximum", 1);
conf.setInt("mapred.tasktracker.reduce.tasks.maximum", 0);
mr = startCluster(conf, 1);
JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker();
RunningJob job =
UtilsForTests.runJob(
mr.createJobConf(), new Path(testDir, "in-1"), new Path(testDir, "out-1"), 1, 1);
JobID id = JobID.downgrade(job.getID());
JobInProgress jip = jobtracker.getJob(id);
// wait 100 secs for the job to complete its map task
for (int i = 0; i < 1000 && jip.finishedMaps() < 1; i++) {
UtilsForTests.waitFor(100);
}
assertEquals(jip.finishedMaps(), 1);
// start a tracker that will wait
LOG.info("Adding a waiting tracker");
TaskTrackerRunner testTrackerRunner =
mr.new TaskTrackerRunner(1, 1, null, mr.createJobConf()) {
@Override
TaskTracker createTaskTracker(JobConf conf) throws InterruptedException, IOException {
return new WaitingTaskTracker(conf);
}
};
mr.addTaskTracker(testTrackerRunner);
LOG.info("Waiting tracker added");
WaitingTaskTracker testTT = (WaitingTaskTracker) testTrackerRunner.getTaskTracker();
// wait 100 secs for the newly started task-tracker to join
for (int i = 0; i < 1000 && jobtracker.taskTrackers().size() < 2; i++) {
UtilsForTests.waitFor(100);
}
assertEquals(jobtracker.taskTrackers().size(), 2);
LOG.info("Cluster is now up with 2 trackers");
// stop the test-tt as its no longer required
mr.stopTaskTracker(mr.getTaskTrackerID(testTT.getName()));
// 1 reduce task should be scheduled
assertEquals("TestTT contacted but no reduce task scheduled on it", 1, jip.runningReduces());
// kill the job
LOG.info("Killing job " + id);
job.killJob();
// check if the reduce task attempt status is missing
TaskInProgress tip = jip.getTasks(TaskType.REDUCE)[0];
assertNull(tip.getTaskStatus(tip.getAllTaskAttemptIDs()[0]));
// wait for the job to retire
waitTillRetire(id, jobtracker);
// check the taskidToTIPMap
for (TaskAttemptID tid : jobtracker.taskidToTIPMap.keySet()) {
LOG.info("TaskidToTIP : " + tid);
}
assertEquals("'taskid' to TIP mapping still exists", 0, jobtracker.taskidToTIPMap.size());
} finally {
if (mr != null) {
mr.shutdown();
}
// cleanup
FileUtil.fullyDelete(new File(testDir.toString()));
}
}
/** Test if {@link CurrentJHParser} can read events from current JH files. */
@Test
public void testCurrentJHParser() throws Exception {
final Configuration conf = new Configuration();
final FileSystem lfs = FileSystem.getLocal(conf);
final Path rootTempDir =
new Path(System.getProperty("test.build.data", "/tmp")).makeQualified(lfs);
final Path tempDir = new Path(rootTempDir, "TestCurrentJHParser");
lfs.delete(tempDir, true);
String queueName = "testQueue";
// Run a MR job
// create a MR cluster
conf.setInt("mapred.tasktracker.map.tasks.maximum", 1);
conf.setInt("mapred.tasktracker.reduce.tasks.maximum", 1);
conf.set("mapred.queue.names", queueName);
MiniMRCluster mrCluster = new MiniMRCluster(1, "file:///", 1, null, null, new JobConf(conf));
// run a job
Path inDir = new Path(tempDir, "input");
Path outDir = new Path(tempDir, "output");
JobHistoryParser parser = null;
RewindableInputStream ris = null;
ArrayList<String> seenEvents = new ArrayList<String>(10);
RunningJob rJob = null;
try {
JobConf jobConf = mrCluster.createJobConf();
jobConf.setQueueName(queueName);
// construct a job with 1 map and 1 reduce task.
rJob = UtilsForTests.runJob(jobConf, inDir, outDir, 1, 1);
rJob.waitForCompletion();
assertTrue("Job failed", rJob.isSuccessful());
JobID id = rJob.getID();
// get the jobhistory filepath
Path inputPath =
new Path(JobHistory.getHistoryFilePath(org.apache.hadoop.mapred.JobID.downgrade(id)));
// wait for 10 secs for the jobhistory file to move into the done folder
for (int i = 0; i < 100; ++i) {
if (lfs.exists(inputPath)) {
break;
}
TimeUnit.MILLISECONDS.wait(100);
}
assertTrue("Missing job history file", lfs.exists(inputPath));
InputDemuxer inputDemuxer = new DefaultInputDemuxer();
inputDemuxer.bindTo(inputPath, conf);
Pair<String, InputStream> filePair = inputDemuxer.getNext();
assertNotNull(filePair);
ris = new RewindableInputStream(filePair.second());
// Test if the JobHistoryParserFactory can detect the parser correctly
parser = JobHistoryParserFactory.getParser(ris);
// Get ParsedJob
String jobId = TraceBuilder.extractJobID(filePair.first());
JobBuilder builder = new JobBuilder(jobId);
HistoryEvent e;
while ((e = parser.nextEvent()) != null) {
String eventString = e.getEventType().toString();
System.out.println(eventString);
seenEvents.add(eventString);
if (builder != null) {
builder.process(e);
}
}
ParsedJob parsedJob = builder.build();
// validate the obtainXXX api of ParsedJob, ParsedTask and
// ParsedTaskAttempt.
validateParsedJob(parsedJob, 1, 1, queueName);
} finally {
// stop the MR cluster
mrCluster.shutdown();
if (ris != null) {
ris.close();
}
if (parser != null) {
parser.close();
}
// cleanup the filesystem
lfs.delete(tempDir, true);
}
// Check against the gold standard
System.out.println("testCurrentJHParser validating using gold std ");
String[] goldLines =
new String[] {
"JOB_SUBMITTED",
"JOB_PRIORITY_CHANGED",
"JOB_STATUS_CHANGED",
"JOB_INITED",
"JOB_INFO_CHANGED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"MAP_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"JOB_STATUS_CHANGED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"MAP_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"REDUCE_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"TASK_STARTED",
"MAP_ATTEMPT_STARTED",
"MAP_ATTEMPT_FINISHED",
"MAP_ATTEMPT_FINISHED",
"TASK_UPDATED",
"TASK_FINISHED",
"JOB_STATUS_CHANGED",
"JOB_FINISHED"
};
// Check the output with gold std
assertEquals("Size mismatch", goldLines.length, seenEvents.size());
int index = 0;
for (String goldLine : goldLines) {
assertEquals("Content mismatch", goldLine, seenEvents.get(index++));
}
}
@Override
public void shutdown() throws IOException {
mr.shutdown();
}
public void testSymLink() {
try {
boolean mayExit = false;
MiniMRCluster mr = null;
MiniDFSCluster dfs = null;
try {
Configuration conf = new Configuration();
dfs = new MiniDFSCluster(conf, 1, true, null);
FileSystem fileSys = dfs.getFileSystem();
String namenode = fileSys.getUri().toString();
mr = new MiniMRCluster(1, namenode, 3);
// During tests, the default Configuration will use a local mapred
// So don't specify -config or -cluster
String strJobtracker = "mapred.job.tracker=" + "localhost:" + mr.getJobTrackerPort();
String strNamenode = "fs.default.name=" + namenode;
String argv[] =
new String[] {
"-input", INPUT_FILE,
"-output", OUTPUT_DIR,
"-mapper", map,
"-reducer", reduce,
// "-verbose",
// "-jobconf", "stream.debug=set"
"-jobconf", strNamenode,
"-jobconf", strJobtracker,
"-jobconf", "stream.tmpdir=" + System.getProperty("test.build.data", "/tmp"),
"-jobconf",
"mapred.child.java.opts=-Dcontrib.name="
+ System.getProperty("contrib.name")
+ " "
+ "-Dbuild.test="
+ System.getProperty("build.test")
+ " "
+ conf.get("mapred.child.java.opts", ""),
"-cacheFile", fileSys.getUri() + CACHE_FILE + "#testlink"
};
fileSys.delete(new Path(OUTPUT_DIR), true);
DataOutputStream file = fileSys.create(new Path(INPUT_FILE));
file.writeBytes(mapString);
file.close();
file = fileSys.create(new Path(CACHE_FILE));
file.writeBytes(cacheString);
file.close();
job = new StreamJob(argv, mayExit);
job.go();
fileSys = dfs.getFileSystem();
String line = null;
Path[] fileList =
FileUtil.stat2Paths(fileSys.listStatus(new Path(OUTPUT_DIR), new OutputLogFilter()));
for (int i = 0; i < fileList.length; i++) {
System.out.println(fileList[i].toString());
BufferedReader bread =
new BufferedReader(new InputStreamReader(fileSys.open(fileList[i])));
line = bread.readLine();
System.out.println(line);
}
assertEquals(cacheString + "\t", line);
} finally {
if (dfs != null) {
dfs.shutdown();
}
if (mr != null) {
mr.shutdown();
}
}
} catch (Exception e) {
failTrace(e);
}
}
