/**
* Generate the data in a REPLAY submission policy with RoundRobinUserResolver mode and verify
* whether the generated data matches with the given input size or not.
*
* @throws Exception
*/
@Test
public void testGenerateDataWithREPLAYSubmission() throws Exception {
conf = rtClient.getDaemonConf();
final long inputSizeInMB = cSize * 300;
String[] runtimeValues = {
"LOADJOB",
RoundRobinUserResolver.class.getName(),
"REPLAY",
inputSizeInMB + "m",
"file://" + UtilsForGridmix.getProxyUsersFile(conf),
"file:///dev/null"
};
String[] otherArgs = {
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};
int exitCode =
UtilsForGridmix.runGridmixJob(
gridmixDir, conf, GridMixRunMode.DATA_GENERATION.getValue(), runtimeValues, otherArgs);
Assert.assertEquals("Data generation has failed.", 0, exitCode);
checkGeneratedDataAndJobStatus(inputSizeInMB);
}
/**
* Generate the data in a SERIAL submission policy with EchoUserResolver mode and also set the
* no.of bytes per file in the data.Verify whether each file size matches with given per file size
* or not and also verify the overall size of generated data.
*
* @throws Exception
*/
@Test
public void testGenerateDataWithSERIALSubmission() throws Exception {
conf = rtClient.getDaemonConf();
long perNodeSizeInMB = 500; // 500 mb per node data
final long inputSizeInMB = cSize * perNodeSizeInMB;
String[] runtimeValues = {
"LOADJOB", EchoUserResolver.class.getName(), "SERIAL", inputSizeInMB + "m", "file:///dev/null"
};
long bytesPerFile = 200 * 1024 * 1024; // 200 mb per file of data
String[] otherArgs = {
"-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile,
"-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
"-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
};
int exitCode =
UtilsForGridmix.runGridmixJob(
gridmixDir, conf, GridMixRunMode.DATA_GENERATION.getValue(), runtimeValues, otherArgs);
Assert.assertEquals("Data generation has failed.", 0, exitCode);
LOG.info("Verify the eache file size in a generate data.");
verifyEachNodeSize(new Path(gridmixDir, "input"), perNodeSizeInMB);
verifyNumOfFilesGeneratedInEachNode(
new Path(gridmixDir, "input"), perNodeSizeInMB, bytesPerFile);
checkGeneratedDataAndJobStatus(inputSizeInMB);
}
@BeforeClass
public static void before() throws Exception {
String[] excludeExpList = {"java.net.ConnectException", "java.io.IOException"};
cluster = MRCluster.createCluster(conf);
cluster.setExcludeExpList(excludeExpList);
cluster.setUp();
cSize = cluster.getTTClients().size();
jtClient = cluster.getJTClient();
rtClient = jtClient.getProxy();
gridmixDir = new Path("herriot-gridmix");
UtilsForGridmix.createDirs(gridmixDir, rtClient.getDaemonConf());
}
@AfterClass
public static void after() throws Exception {
UtilsForGridmix.cleanup(gridmixDir, conf);
cluster.tearDown();
}