/** * Generate the data in a REPLAY submission policy with RoundRobinUserResolver mode and verify * whether the generated data matches with the given input size or not. * * @throws Exception */ @Test public void testGenerateDataWithREPLAYSubmission() throws Exception { conf = rtClient.getDaemonConf(); final long inputSizeInMB = cSize * 300; String[] runtimeValues = { "LOADJOB", RoundRobinUserResolver.class.getName(), "REPLAY", inputSizeInMB + "m", "file://" + UtilsForGridmix.getProxyUsersFile(conf), "file:///dev/null" }; String[] otherArgs = { "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; int exitCode = UtilsForGridmix.runGridmixJob( gridmixDir, conf, GridMixRunMode.DATA_GENERATION.getValue(), runtimeValues, otherArgs); Assert.assertEquals("Data generation has failed.", 0, exitCode); checkGeneratedDataAndJobStatus(inputSizeInMB); }
/** * Generate the data in a SERIAL submission policy with EchoUserResolver mode and also set the * no.of bytes per file in the data.Verify whether each file size matches with given per file size * or not and also verify the overall size of generated data. * * @throws Exception */ @Test public void testGenerateDataWithSERIALSubmission() throws Exception { conf = rtClient.getDaemonConf(); long perNodeSizeInMB = 500; // 500 mb per node data final long inputSizeInMB = cSize * perNodeSizeInMB; String[] runtimeValues = { "LOADJOB", EchoUserResolver.class.getName(), "SERIAL", inputSizeInMB + "m", "file:///dev/null" }; long bytesPerFile = 200 * 1024 * 1024; // 200 mb per file of data String[] otherArgs = { "-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile, "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false", "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false" }; int exitCode = UtilsForGridmix.runGridmixJob( gridmixDir, conf, GridMixRunMode.DATA_GENERATION.getValue(), runtimeValues, otherArgs); Assert.assertEquals("Data generation has failed.", 0, exitCode); LOG.info("Verify the eache file size in a generate data."); verifyEachNodeSize(new Path(gridmixDir, "input"), perNodeSizeInMB); verifyNumOfFilesGeneratedInEachNode( new Path(gridmixDir, "input"), perNodeSizeInMB, bytesPerFile); checkGeneratedDataAndJobStatus(inputSizeInMB); }
@BeforeClass public static void before() throws Exception { String[] excludeExpList = {"java.net.ConnectException", "java.io.IOException"}; cluster = MRCluster.createCluster(conf); cluster.setExcludeExpList(excludeExpList); cluster.setUp(); cSize = cluster.getTTClients().size(); jtClient = cluster.getJTClient(); rtClient = jtClient.getProxy(); gridmixDir = new Path("herriot-gridmix"); UtilsForGridmix.createDirs(gridmixDir, rtClient.getDaemonConf()); }
@AfterClass public static void after() throws Exception { UtilsForGridmix.cleanup(gridmixDir, conf); cluster.tearDown(); }