/**
   * Ensure that we will never have the following error.
   *
   * <p>The test works as follows: - Use the CliFrontend to invoke a jar file that loads a class
   * which is only available in the jarfile itself (via a custom classloader) - Change the Usercode
   * classloader of the PackagedProgram to a special classloader for this test - the classloader
   * will accept the special class (and return a String.class)
   *
   * <p>org.apache.flink.client.program.ProgramInvocationException: The main method caused an error.
   * at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:398) at
   * org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:301)
   * at org.apache.flink.client.program.Client.getOptimizedPlan(Client.java:140) at
   * org.apache.flink.client.program.Client.getOptimizedPlanAsJson(Client.java:125) at
   * org.apache.flink.client.CliFrontend.info(CliFrontend.java:439) at
   * org.apache.flink.client.CliFrontend.parseParameters(CliFrontend.java:931) at
   * org.apache.flink.client.CliFrontend.main(CliFrontend.java:951) Caused by: java.io.IOException:
   * java.lang.RuntimeException: java.lang.ClassNotFoundException:
   * org.apache.hadoop.hive.ql.io.RCFileInputFormat at
   * org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:102) at
   * org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:54) at
   * tlabs.CDR_In_Report.createHCatInputFormat(CDR_In_Report.java:322) at
   * tlabs.CDR_Out_Report.main(CDR_Out_Report.java:380) at
   * sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
   * sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at
   * sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at
   * java.lang.reflect.Method.invoke(Method.java:622) at
   * org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:383)
   */
  @Test
  public void testPlanWithExternalClass() throws CompilerException, ProgramInvocationException {
    final Boolean callme[] = {
      false
    }; // create a final object reference, to be able to change its val later
    try {
      String[] parameters = {
        getTestJarPath(), "-c", TEST_JAR_CLASSLOADERTEST_CLASS, "some", "program"
      };
      CommandLine line =
          new PosixParser()
              .parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false);

      CliFrontend frontend = new CliFrontend();
      Object result = frontend.buildProgram(line);
      assertTrue(result instanceof PackagedProgram);

      PackagedProgram prog = spy((PackagedProgram) result);

      ClassLoader testClassLoader =
          new ClassLoader(prog.getUserCodeClassLoader()) {
            @Override
            public Class<?> loadClass(String name) throws ClassNotFoundException {
              assertTrue(name.equals("org.apache.hadoop.hive.ql.io.RCFileInputFormat"));
              callme[0] = true;
              return String.class; // Intentionally return the wrong class.
            }
          };
      when(prog.getUserCodeClassLoader()).thenReturn(testClassLoader);

      Assert.assertArrayEquals(new String[] {"some", "program"}, prog.getArguments());
      Assert.assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, prog.getMainClassName());

      Configuration c = new Configuration();
      c.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "devil");
      Client cli = new Client(c, getClass().getClassLoader());

      cli.getOptimizedPlanAsJson(prog, 666);
    } catch (ProgramInvocationException pie) {
      assertTrue("Classloader was not called", callme[0]);
      // class not found exception is expected as some point
      if (!(pie.getCause() instanceof ClassNotFoundException)) {
        System.err.println(pie.getMessage());
        pie.printStackTrace();
        fail("Program caused an exception: " + pie.getMessage());
      }
    } catch (Exception e) {
      assertTrue("Classloader was not called", callme[0]);
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail("Program caused an exception: " + e.getMessage());
    }
  }
Пример #2
0
  @Test
  public void testCorrectSettingOfMaxSlots() throws Exception {

    File confFile = tmp.newFile("flink-conf.yaml");
    File jarFile = tmp.newFile("test.jar");
    new CliFrontend(tmp.getRoot().getAbsolutePath());

    String[] params = new String[] {"-yn", "2", "-ys", "3", jarFile.getAbsolutePath()};

    RunOptions runOptions = CliFrontendParser.parseRunCommand(params);

    FlinkYarnSessionCli yarnCLI = new TestCLI("y", "yarn");

    AbstractYarnClusterDescriptor descriptor =
        yarnCLI.createDescriptor("", runOptions.getCommandLine());

    // each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
    Assert.assertEquals(3, descriptor.getTaskManagerSlots());
    Assert.assertEquals(2, descriptor.getTaskManagerCount());

    Configuration config = new Configuration();
    CliFrontend.setJobManagerAddressInConfig(config, new InetSocketAddress("test", 9000));
    ClusterClient client = new TestingYarnClusterClient(descriptor, config);
    Assert.assertEquals(6, client.getMaxSlots());
  }
  @Test
  public void testFileNotJarFile() {
    try {
      String[] arguments = {"-j", getNonJarFilePath(), "-a", "plenty", "of", "arguments"};
      CommandLine line =
          new PosixParser()
              .parse(CliFrontend.getProgramSpecificOptions(new Options()), arguments, false);

      CliFrontend frontend = new CliFrontend();
      Object result = frontend.buildProgram(line);
      assertTrue(result == null);
    } catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail("Program caused an exception: " + e.getMessage());
    }
  }
  @Test
  public void testNonExistingFileWithoutArguments() {
    try {
      String[] parameters = {"/some/none/existing/path"};
      CommandLine line =
          new PosixParser()
              .parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false);

      CliFrontend frontend = new CliFrontend();
      Object result = frontend.buildProgram(line);
      assertTrue(result == null);
    } catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail("Program caused an exception: " + e.getMessage());
    }
  }
  @Test
  public void testVariantWithExplicitJarAndArgumentsOption() {
    try {
      String[] parameters = {"-j", getTestJarPath(), "-a", "some", "program", "arguments"};
      CommandLine line =
          new PosixParser()
              .parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false);

      CliFrontend frontend = new CliFrontend();
      Object result = frontend.buildProgram(line);
      assertTrue(result instanceof PackagedProgram);

      PackagedProgram prog = (PackagedProgram) result;

      Assert.assertArrayEquals(new String[] {"some", "program", "arguments"}, prog.getArguments());
      Assert.assertEquals(TEST_JAR_MAIN_CLASS, prog.getMainClassName());
    } catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail("Program caused an exception: " + e.getMessage());
    }
  }
Пример #6
0
    @Override
    public void run() {
      switch (type) {
        case YARN_SESSION:
          yCli = new FlinkYarnSessionCli("", "", false);
          returnValue = yCli.run(args);
          break;
        case CLI_FRONTEND:
          try {
            CliFrontend cli = new CliFrontend();
            returnValue = cli.parseParameters(args);
          } catch (Exception e) {
            throw new RuntimeException(e);
          }
          break;
        default:
          throw new RuntimeException("Unknown type " + type);
      }

      if (returnValue != 0) {
        Assert.fail("The YARN session returned with non-null value=" + returnValue);
      }
    }