/**
* Test if the {@link JobConfigurationParser} can correctly extract out key-value pairs from the
* job configuration.
*/
@Test
public void testJobConfigurationParsing() throws Exception {
final FileSystem lfs = FileSystem.getLocal(new Configuration());
final Path rootTempDir =
new Path(System.getProperty("test.build.data", "/tmp"))
.makeQualified(lfs.getUri(), lfs.getWorkingDirectory());
final Path tempDir = new Path(rootTempDir, "TestJobConfigurationParser");
lfs.delete(tempDir, true);
// Add some configuration parameters to the conf
JobConf jConf = new JobConf(false);
String key = "test.data";
String value = "hello world";
jConf.set(key, value);
// create the job conf file
Path jobConfPath = new Path(tempDir.toString(), "job.xml");
lfs.delete(jobConfPath, false);
DataOutputStream jobConfStream = lfs.create(jobConfPath);
jConf.writeXml(jobConfStream);
jobConfStream.close();
// now read the job conf file using the job configuration parser
Properties properties = JobConfigurationParser.parse(lfs.open(jobConfPath));
// check if the required parameter is loaded
assertEquals(
"Total number of extracted properties ("
+ properties.size()
+ ") doesn't match the expected size of 1 ["
+ "JobConfigurationParser]",
1,
properties.size());
// check if the key is present in the extracted configuration
assertTrue(
"Key " + key + " is missing in the configuration extracted " + "[JobConfigurationParser]",
properties.keySet().contains(key));
// check if the desired property has the correct value
assertEquals(
"JobConfigurationParser couldn't recover the parameters" + " correctly",
value,
properties.get(key));
// Test ZombieJob
LoggedJob job = new LoggedJob();
job.setJobProperties(properties);
ZombieJob zjob = new ZombieJob(job, null);
Configuration zconf = zjob.getJobConf();
// check if the required parameter is loaded
assertEquals("ZombieJob couldn't recover the parameters correctly", value, zconf.get(key));
}
/** Testing {@link ResourceUsageMetrics} using {@link HadoopLogsAnalyzer}. */
@Test
@SuppressWarnings("deprecation")
public void testResourceUsageMetricsWithHadoopLogsAnalyzer() throws IOException {
Configuration conf = new Configuration();
// get the input trace file
Path rootInputDir = new Path(System.getProperty("test.tools.input.dir", ""));
Path rootInputSubFolder = new Path(rootInputDir, "rumen/small-trace-test");
Path traceFile = new Path(rootInputSubFolder, "v20-resource-usage-log.gz");
FileSystem lfs = FileSystem.getLocal(conf);
// define the root test directory
Path rootTempDir = new Path(System.getProperty("test.build.data", "/tmp"));
// define output directory
Path outputDir = new Path(rootTempDir, "testResourceUsageMetricsWithHadoopLogsAnalyzer");
lfs.delete(outputDir, true);
lfs.deleteOnExit(outputDir);
// run HadoopLogsAnalyzer
HadoopLogsAnalyzer analyzer = new HadoopLogsAnalyzer();
analyzer.setConf(conf);
Path traceOutput = new Path(outputDir, "trace.json");
analyzer.run(
new String[] {
"-write-job-trace", traceOutput.toString(),
"-v1", traceFile.toString()
});
// test HadoopLogsAnalyzer's output w.r.t ResourceUsageMetrics
// get the logged job
JsonObjectMapperParser<LoggedJob> traceParser =
new JsonObjectMapperParser<LoggedJob>(traceOutput, LoggedJob.class, conf);
// get the logged job from the output trace file
LoggedJob job = traceParser.getNext();
LoggedTaskAttempt attempt = job.getMapTasks().get(0).getAttempts().get(0);
ResourceUsageMetrics metrics = attempt.getResourceUsageMetrics();
// test via deepCompare()
testResourceUsageMetricViaDeepCompare(metrics, 200, 100, 75, 50, true);
}
