Exemple #1
0
  /**
   * Generate the data in a REPLAY submission policy with RoundRobinUserResolver mode and verify
   * whether the generated data matches with the given input size or not.
   *
   * @throws Exception
   */
  @Test
  public void testGenerateDataWithREPLAYSubmission() throws Exception {
    conf = rtClient.getDaemonConf();
    final long inputSizeInMB = cSize * 300;
    String[] runtimeValues = {
      "LOADJOB",
      RoundRobinUserResolver.class.getName(),
      "REPLAY",
      inputSizeInMB + "m",
      "file://" + UtilsForGridmix.getProxyUsersFile(conf),
      "file:///dev/null"
    };

    String[] otherArgs = {
      "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
      "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
      "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
    };

    int exitCode =
        UtilsForGridmix.runGridmixJob(
            gridmixDir, conf, GridMixRunMode.DATA_GENERATION.getValue(), runtimeValues, otherArgs);
    Assert.assertEquals("Data generation has failed.", 0, exitCode);
    checkGeneratedDataAndJobStatus(inputSizeInMB);
  }
Exemple #2
0
 /**
  * Generate the data in a SERIAL submission policy with EchoUserResolver mode and also set the
  * no.of bytes per file in the data.Verify whether each file size matches with given per file size
  * or not and also verify the overall size of generated data.
  *
  * @throws Exception
  */
 @Test
 public void testGenerateDataWithSERIALSubmission() throws Exception {
   conf = rtClient.getDaemonConf();
   long perNodeSizeInMB = 500; // 500 mb per node data
   final long inputSizeInMB = cSize * perNodeSizeInMB;
   String[] runtimeValues = {
     "LOADJOB", EchoUserResolver.class.getName(), "SERIAL", inputSizeInMB + "m", "file:///dev/null"
   };
   long bytesPerFile = 200 * 1024 * 1024; // 200 mb per file of data
   String[] otherArgs = {
     "-D", GridMixConfig.GRIDMIX_BYTES_PER_FILE + "=" + bytesPerFile,
     "-D", GridmixJob.GRIDMIX_HIGHRAM_EMULATION_ENABLE + "=false",
     "-D", GridMixConfig.GRIDMIX_DISTCACHE_ENABLE + "=false",
     "-D", GridMixConfig.GRIDMIX_COMPRESSION_ENABLE + "=false"
   };
   int exitCode =
       UtilsForGridmix.runGridmixJob(
           gridmixDir, conf, GridMixRunMode.DATA_GENERATION.getValue(), runtimeValues, otherArgs);
   Assert.assertEquals("Data generation has failed.", 0, exitCode);
   LOG.info("Verify the eache file size in a generate data.");
   verifyEachNodeSize(new Path(gridmixDir, "input"), perNodeSizeInMB);
   verifyNumOfFilesGeneratedInEachNode(
       new Path(gridmixDir, "input"), perNodeSizeInMB, bytesPerFile);
   checkGeneratedDataAndJobStatus(inputSizeInMB);
 }
Exemple #3
0
 @BeforeClass
 public static void before() throws Exception {
   String[] excludeExpList = {"java.net.ConnectException", "java.io.IOException"};
   cluster = MRCluster.createCluster(conf);
   cluster.setExcludeExpList(excludeExpList);
   cluster.setUp();
   cSize = cluster.getTTClients().size();
   jtClient = cluster.getJTClient();
   rtClient = jtClient.getProxy();
   gridmixDir = new Path("herriot-gridmix");
   UtilsForGridmix.createDirs(gridmixDir, rtClient.getDaemonConf());
 }
Exemple #4
0
 @AfterClass
 public static void after() throws Exception {
   UtilsForGridmix.cleanup(gridmixDir, conf);
   cluster.tearDown();
 }