/** * Test if the {@link JobConfigurationParser} can correctly extract out key-value pairs from the * job configuration. */ @Test public void testJobConfigurationParsing() throws Exception { final FileSystem lfs = FileSystem.getLocal(new Configuration()); final Path rootTempDir = new Path(System.getProperty("test.build.data", "/tmp")) .makeQualified(lfs.getUri(), lfs.getWorkingDirectory()); final Path tempDir = new Path(rootTempDir, "TestJobConfigurationParser"); lfs.delete(tempDir, true); // Add some configuration parameters to the conf JobConf jConf = new JobConf(false); String key = "test.data"; String value = "hello world"; jConf.set(key, value); // create the job conf file Path jobConfPath = new Path(tempDir.toString(), "job.xml"); lfs.delete(jobConfPath, false); DataOutputStream jobConfStream = lfs.create(jobConfPath); jConf.writeXml(jobConfStream); jobConfStream.close(); // now read the job conf file using the job configuration parser Properties properties = JobConfigurationParser.parse(lfs.open(jobConfPath)); // check if the required parameter is loaded assertEquals( "Total number of extracted properties (" + properties.size() + ") doesn't match the expected size of 1 [" + "JobConfigurationParser]", 1, properties.size()); // check if the key is present in the extracted configuration assertTrue( "Key " + key + " is missing in the configuration extracted " + "[JobConfigurationParser]", properties.keySet().contains(key)); // check if the desired property has the correct value assertEquals( "JobConfigurationParser couldn't recover the parameters" + " correctly", value, properties.get(key)); // Test ZombieJob LoggedJob job = new LoggedJob(); job.setJobProperties(properties); ZombieJob zjob = new ZombieJob(job, null); Configuration zconf = zjob.getJobConf(); // check if the required parameter is loaded assertEquals("ZombieJob couldn't recover the parameters correctly", value, zconf.get(key)); }
/** Testing {@link ResourceUsageMetrics} using {@link HadoopLogsAnalyzer}. */ @Test @SuppressWarnings("deprecation") public void testResourceUsageMetricsWithHadoopLogsAnalyzer() throws IOException { Configuration conf = new Configuration(); // get the input trace file Path rootInputDir = new Path(System.getProperty("test.tools.input.dir", "")); Path rootInputSubFolder = new Path(rootInputDir, "rumen/small-trace-test"); Path traceFile = new Path(rootInputSubFolder, "v20-resource-usage-log.gz"); FileSystem lfs = FileSystem.getLocal(conf); // define the root test directory Path rootTempDir = new Path(System.getProperty("test.build.data", "/tmp")); // define output directory Path outputDir = new Path(rootTempDir, "testResourceUsageMetricsWithHadoopLogsAnalyzer"); lfs.delete(outputDir, true); lfs.deleteOnExit(outputDir); // run HadoopLogsAnalyzer HadoopLogsAnalyzer analyzer = new HadoopLogsAnalyzer(); analyzer.setConf(conf); Path traceOutput = new Path(outputDir, "trace.json"); analyzer.run( new String[] { "-write-job-trace", traceOutput.toString(), "-v1", traceFile.toString() }); // test HadoopLogsAnalyzer's output w.r.t ResourceUsageMetrics // get the logged job JsonObjectMapperParser<LoggedJob> traceParser = new JsonObjectMapperParser<LoggedJob>(traceOutput, LoggedJob.class, conf); // get the logged job from the output trace file LoggedJob job = traceParser.getNext(); LoggedTaskAttempt attempt = job.getMapTasks().get(0).getAttempts().get(0); ResourceUsageMetrics metrics = attempt.getResourceUsageMetrics(); // test via deepCompare() testResourceUsageMetricViaDeepCompare(metrics, 200, 100, 75, 50, true); }