Exemple #1
0
  @AfterClass
  public static void teardown() throws Exception {
    // Unset FLINK_CONF_DIR, as it might change the behavior of other tests
    Map<String, String> map = new HashMap<>(System.getenv());
    map.remove(CliFrontend.ENV_CONFIG_DIRECTORY);
    TestBaseUtils.setEnv(map);

    // When we are on travis, we copy the tmp files of JUnit (containing the MiniYARNCluster log
    // files)
    // to <flinkRoot>/target/flink-yarn-tests-*.
    // The files from there are picked up by the ./tools/travis_watchdog.sh script
    // to upload them to Amazon S3.
    if (isOnTravis()) {
      File target = new File("../target" + yarnConfiguration.get(TEST_CLUSTER_NAME_KEY));
      if (!target.mkdirs()) {
        LOG.warn("Error creating dirs to {}", target);
      }
      File src = tmp.getRoot();
      LOG.info(
          "copying the final files from {} to {}", src.getAbsolutePath(), target.getAbsolutePath());
      try {
        FileUtils.copyDirectoryToDirectory(src, target);
      } catch (IOException e) {
        LOG.warn(
            "Error copying the final files from {} to {}: msg: {}",
            src.getAbsolutePath(),
            target.getAbsolutePath(),
            e.getMessage(),
            e);
      }
    }
  }
  @Test
  public void testDynamicProperties() throws IOException {

    Map<String, String> map = new HashMap<String, String>(System.getenv());
    File tmpFolder = tmp.newFolder();
    File fakeConf = new File(tmpFolder, "flink-conf.yaml");
    fakeConf.createNewFile();
    map.put(ConfigConstants.ENV_FLINK_CONF_DIR, tmpFolder.getAbsolutePath());
    TestBaseUtils.setEnv(map);
    FlinkYarnSessionCli cli = new FlinkYarnSessionCli("", "", false);
    Options options = new Options();
    cli.addGeneralOptions(options);
    cli.addRunOptions(options);

    CommandLineParser parser = new DefaultParser();
    CommandLine cmd = null;
    try {
      cmd =
          parser.parse(
              options,
              new String[] {"run", "-j", "fake.jar", "-n", "15", "-D", "akka.ask.timeout=5 min"});
    } catch (Exception e) {
      e.printStackTrace();
      Assert.fail("Parsing failed with " + e.getMessage());
    }

    AbstractYarnClusterDescriptor flinkYarnDescriptor = cli.createDescriptor(null, cmd);

    Assert.assertNotNull(flinkYarnDescriptor);

    Map<String, String> dynProperties =
        FlinkYarnSessionCli.getDynamicProperties(flinkYarnDescriptor.getDynamicPropertiesEncoded());
    Assert.assertEquals(1, dynProperties.size());
    Assert.assertEquals("5 min", dynProperties.get("akka.ask.timeout"));
  }
Exemple #3
0
  private static void start(Configuration conf, String principal, String keytab) {
    // set the home directory to a temp directory. Flink on YARN is using the home dir to distribute
    // the file
    File homeDir = null;
    try {
      homeDir = tmp.newFolder();
    } catch (IOException e) {
      e.printStackTrace();
      Assert.fail(e.getMessage());
    }
    System.setProperty("user.home", homeDir.getAbsolutePath());
    String uberjarStartLoc = "..";
    LOG.info("Trying to locate uberjar in {}", new File(uberjarStartLoc));
    flinkUberjar = findFile(uberjarStartLoc, new RootDirFilenameFilter());
    Assert.assertNotNull("Flink uberjar not found", flinkUberjar);
    String flinkDistRootDir = flinkUberjar.getParentFile().getParent();
    flinkLibFolder = flinkUberjar.getParentFile(); // the uberjar is located in lib/
    Assert.assertNotNull("Flink flinkLibFolder not found", flinkLibFolder);
    Assert.assertTrue("lib folder not found", flinkLibFolder.exists());
    Assert.assertTrue("lib folder not found", flinkLibFolder.isDirectory());

    if (!flinkUberjar.exists()) {
      Assert.fail("Unable to locate yarn-uberjar.jar");
    }

    try {
      LOG.info("Starting up MiniYARNCluster");
      if (yarnCluster == null) {
        yarnCluster =
            new MiniYARNCluster(
                conf.get(YarnTestBase.TEST_CLUSTER_NAME_KEY), NUM_NODEMANAGERS, 1, 1);

        yarnCluster.init(conf);
        yarnCluster.start();
      }

      Map<String, String> map = new HashMap<String, String>(System.getenv());

      File flinkConfDirPath =
          findFile(flinkDistRootDir, new ContainsName(new String[] {"flink-conf.yaml"}));
      Assert.assertNotNull(flinkConfDirPath);

      if (!StringUtils.isBlank(principal) && !StringUtils.isBlank(keytab)) {
        // copy conf dir to test temporary workspace location
        tempConfPathForSecureRun = tmp.newFolder("conf");

        String confDirPath = flinkConfDirPath.getParentFile().getAbsolutePath();
        FileUtils.copyDirectory(new File(confDirPath), tempConfPathForSecureRun);

        try (FileWriter fw =
                new FileWriter(new File(tempConfPathForSecureRun, "flink-conf.yaml"), true);
            BufferedWriter bw = new BufferedWriter(fw);
            PrintWriter out = new PrintWriter(bw)) {
          LOG.info(
              "writing keytab: " + keytab + " and principal: " + principal + " to config file");
          out.println("");
          out.println("#Security Configurations Auto Populated ");
          out.println(ConfigConstants.SECURITY_KEYTAB_KEY + ": " + keytab);
          out.println(ConfigConstants.SECURITY_PRINCIPAL_KEY + ": " + principal);
          out.println("");
        } catch (IOException e) {
          throw new RuntimeException(
              "Exception occured while trying to append the security configurations.", e);
        }

        String configDir = tempConfPathForSecureRun.getAbsolutePath();

        LOG.info(
            "Temporary Flink configuration directory to be used for secure test: {}", configDir);

        Assert.assertNotNull(configDir);

        map.put(ConfigConstants.ENV_FLINK_CONF_DIR, configDir);

      } else {
        map.put(ConfigConstants.ENV_FLINK_CONF_DIR, flinkConfDirPath.getParent());
      }

      File yarnConfFile = writeYarnSiteConfigXML(conf);
      map.put("YARN_CONF_DIR", yarnConfFile.getParentFile().getAbsolutePath());
      map.put("IN_TESTS", "yes we are in tests"); // see YarnClusterDescriptor() for more infos
      TestBaseUtils.setEnv(map);

      Assert.assertTrue(yarnCluster.getServiceState() == Service.STATE.STARTED);

      // wait for the nodeManagers to connect
      while (!yarnCluster.waitForNodeManagersToConnect(500)) {
        LOG.info("Waiting for Nodemanagers to connect");
      }
    } catch (Exception ex) {
      ex.printStackTrace();
      LOG.error("setup failure", ex);
      Assert.fail();
    }
  }
Exemple #4
0
  public static void startYARNWithConfig(Configuration conf) {
    // set the home directory to a tmp directory. Flink on YARN is using the home dir to distribute
    // the file
    File homeDir = null;
    try {
      homeDir = tmp.newFolder();
    } catch (IOException e) {
      e.printStackTrace();
      Assert.fail(e.getMessage());
    }
    System.setProperty("user.home", homeDir.getAbsolutePath());
    String uberjarStartLoc = "..";
    LOG.info("Trying to locate uberjar in {}", new File(uberjarStartLoc));
    flinkUberjar = findFile(uberjarStartLoc, new RootDirFilenameFilter());
    Assert.assertNotNull("Flink uberjar not found", flinkUberjar);
    String flinkDistRootDir = flinkUberjar.getParentFile().getParent();
    flinkLibFolder = flinkUberjar.getParentFile(); // the uberjar is located in lib/
    Assert.assertNotNull("Flink flinkLibFolder not found", flinkLibFolder);
    Assert.assertTrue("lib folder not found", flinkLibFolder.exists());
    Assert.assertTrue("lib folder not found", flinkLibFolder.isDirectory());

    if (!flinkUberjar.exists()) {
      Assert.fail("Unable to locate yarn-uberjar.jar");
    }

    try {
      LOG.info("Starting up MiniYARNCluster");
      if (yarnCluster == null) {
        yarnCluster =
            new MiniYARNCluster(
                conf.get(YarnTestBase.TEST_CLUSTER_NAME_KEY), NUM_NODEMANAGERS, 1, 1);

        yarnCluster.init(conf);
        yarnCluster.start();
      }

      Map<String, String> map = new HashMap<String, String>(System.getenv());

      File flinkConfDirPath =
          findFile(flinkDistRootDir, new ContainsName(new String[] {"flink-conf.yaml"}));
      Assert.assertNotNull(flinkConfDirPath);

      map.put(CliFrontend.ENV_CONFIG_DIRECTORY, flinkConfDirPath.getParent());

      File yarnConfFile = writeYarnSiteConfigXML(conf);
      map.put("YARN_CONF_DIR", yarnConfFile.getParentFile().getAbsolutePath());
      map.put("IN_TESTS", "yes we are in tests"); // see YarnClusterDescriptor() for more infos
      TestBaseUtils.setEnv(map);

      Assert.assertTrue(yarnCluster.getServiceState() == Service.STATE.STARTED);

      // wait for the nodeManagers to connect
      while (!yarnCluster.waitForNodeManagersToConnect(500)) {
        LOG.info("Waiting for Nodemanagers to connect");
      }
    } catch (Exception ex) {
      ex.printStackTrace();
      LOG.error("setup failure", ex);
      Assert.fail();
    }
  }