@BeforeClass
  public static void startActorSystem() {
    config = new Configuration();
    config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "5 s");
    config.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_INTERVAL, "200 ms");
    config.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "2 s");
    config.setDouble(ConfigConstants.AKKA_WATCH_THRESHOLD, 2.0);

    actorSystem = AkkaUtils.createLocalActorSystem(config);
  }
  @Test
  public void testSavepointBackendFileSystemButCheckpointBackendJobManager() throws Exception {
    Configuration config = new Configuration();

    // This combination does not make sense, because the checkpoints will be
    // lost after the job manager shuts down.
    config.setString(ConfigConstants.STATE_BACKEND, "jobmanager");
    config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "filesystem");
    SavepointStore store = SavepointStoreFactory.createFromConfig(config);
    assertTrue(store.getStateStore() instanceof HeapStateStore);
  }
Пример #3
0
  static Configuration createReporterConfig(Configuration config, TimeUnit timeunit, long period) {
    Configuration reporterConfig = new Configuration();
    reporterConfig.setLong("period", period);
    reporterConfig.setString("timeunit", timeunit.name());

    String[] arguments = config.getString(KEY_METRICS_REPORTER_ARGUMENTS, "").split(" ");
    if (arguments.length > 1) {
      for (int x = 0; x < arguments.length; x += 2) {
        reporterConfig.setString(arguments[x].replace("--", ""), arguments[x + 1]);
      }
    }
    return reporterConfig;
  }
  @Test
  public void testSavepointBackendFileSystem() throws Exception {
    Configuration config = new Configuration();
    String rootPath = System.getProperty("java.io.tmpdir");
    config.setString(ConfigConstants.STATE_BACKEND, "filesystem");
    config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "filesystem");
    config.setString(SavepointStoreFactory.SAVEPOINT_DIRECTORY_KEY, rootPath);

    SavepointStore store = SavepointStoreFactory.createFromConfig(config);
    assertTrue(store.getStateStore() instanceof FileSystemStateStore);

    FileSystemStateStore<CompletedCheckpoint> stateStore =
        (FileSystemStateStore<CompletedCheckpoint>) store.getStateStore();
    assertEquals(new Path(rootPath), stateStore.getRootPath());
  }
Пример #5
0
  @Override
  public void open() {
    out = new ByteArrayOutputStream();
    flinkConf = new org.apache.flink.configuration.Configuration();
    Properties intpProperty = getProperty();
    for (Object k : intpProperty.keySet()) {
      String key = (String) k;
      String val = toString(intpProperty.get(key));
      flinkConf.setString(key, val);
    }

    if (localMode()) {
      startFlinkMiniCluster();
    }

    flinkIloop = new FlinkILoop(getHost(), getPort(), (BufferedReader) null, new PrintWriter(out));
    flinkIloop.settings_$eq(createSettings());
    flinkIloop.createInterpreter();

    imain = flinkIloop.intp();

    // prepare bindings
    imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
    binder = (Map<String, Object>) getValue("_binder");

    // import libraries
    imain.interpret("import scala.tools.nsc.io._");
    imain.interpret("import Properties.userHome");
    imain.interpret("import scala.compat.Platform.EOL");

    imain.interpret("import org.apache.flink.api.scala._");
    imain.interpret("import org.apache.flink.api.common.functions._");
    imain.bindValue("env", flinkIloop.scalaEnv());
  }
  /**
   * Checks if the local instance manager reads the default correctly from the configuration file.
   */
  @Test
  public void testInstanceTypeFromConfiguration() {

    try {
      Configuration cfg = new Configuration();
      cfg.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "127.0.0.1");
      cfg.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 6123);
      cfg.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 1);
      cfg.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);

      GlobalConfiguration.includeConfiguration(cfg);

      // start JobManager
      ExecutionMode executionMode = ExecutionMode.LOCAL;
      JobManager jm = new JobManager(executionMode);

      final TestInstanceListener testInstanceListener = new TestInstanceListener();

      InstanceManager im = jm.getInstanceManager();
      try {
        im.setInstanceListener(testInstanceListener);

      } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Instantiation of LocalInstanceManager failed: " + e.getMessage());
      } finally {
        jm.shutdown();
      }
    } catch (Exception e) {
      System.err.println(e.getMessage());
      e.printStackTrace();
      Assert.fail("Test caused an error: " + e.getMessage());
    }
  }
 @Test
 public void testSavepointBackendFileSystemButNoDirectory() throws Exception {
   Configuration config = new Configuration();
   config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "filesystem");
   SavepointStore store = SavepointStoreFactory.createFromConfig(config);
   assertTrue(store.getStateStore() instanceof HeapStateStore);
 }
Пример #8
0
  @BeforeClass
  public static void setupJobManager() {
    Configuration config = new Configuration();

    int port = NetUtils.getAvailablePort();

    config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost");
    config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, port);

    scala.Option<Tuple2<String, Object>> listeningAddress =
        scala.Option.apply(new Tuple2<String, Object>("localhost", port));
    jobManagerSystem = AkkaUtils.createActorSystem(config, listeningAddress);
    ActorRef jobManagerActorRef =
        JobManager.startJobManagerActors(
                config,
                jobManagerSystem,
                StreamingMode.BATCH_ONLY,
                JobManager.class,
                MemoryArchivist.class)
            ._1();

    try {
      LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(config);

      jmGateway = LeaderRetrievalUtils.retrieveLeaderGateway(lrs, jobManagerSystem, timeout);
    } catch (Exception e) {
      fail("Could not retrieve the JobManager gateway. " + e.getMessage());
    }
  }
 @Test
 public void testUnexpectedSavepointBackend() throws Exception {
   Configuration config = new Configuration();
   config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "unexpected");
   SavepointStore store = SavepointStoreFactory.createFromConfig(config);
   assertTrue(store.getStateStore() instanceof HeapStateStore);
 }
Пример #10
0
  public void killTopologyWithOpts(final String name, final KillOptions options)
      throws NotAliveException {
    final JobID jobId = this.getTopologyJobId(name);
    if (jobId == null) {
      throw new NotAliveException("Storm topology with name " + name + " not found.");
    }

    if (options != null) {
      try {
        Thread.sleep(1000 * options.get_wait_secs());
      } catch (final InterruptedException e) {
        throw new RuntimeException(e);
      }
    }

    final Configuration configuration = GlobalConfiguration.getConfiguration();
    configuration.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, this.jobManagerHost);
    configuration.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, this.jobManagerPort);

    final Client client;
    try {
      client = new Client(configuration);
    } catch (final IOException e) {
      throw new RuntimeException("Could not establish a connection to the job manager", e);
    }

    try {
      client.cancel(jobId);
    } catch (final Exception e) {
      throw new RuntimeException("Cannot stop job.", e);
    }
  }
Пример #11
0
  private FiniteDuration getTimeout() {
    final Configuration configuration = GlobalConfiguration.getConfiguration();
    if (this.timeout != null) {
      configuration.setString(ConfigConstants.AKKA_ASK_TIMEOUT, this.timeout);
    }

    return AkkaUtils.getClientTimeout(configuration);
  }
Пример #12
0
  @BeforeClass
  public static void setup() throws Exception {
    Configuration config = new Configuration();
    config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, numTaskManagers);
    config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, slotsPerTaskManager);

    final File checkpointDir = temporaryFolder.newFolder();
    final File savepointDir = temporaryFolder.newFolder();

    config.setString(ConfigConstants.STATE_BACKEND, "filesystem");
    config.setString(
        FsStateBackendFactory.CHECKPOINT_DIRECTORY_URI_CONF_KEY, checkpointDir.toURI().toString());
    config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "filesystem");
    config.setString(
        SavepointStoreFactory.SAVEPOINT_DIRECTORY_KEY, savepointDir.toURI().toString());

    cluster = new TestingCluster(config);
    cluster.start();
  }
  @BeforeClass
  public static void startTestCluster() {
    Configuration config = new Configuration();
    config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
    config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM / 2);
    config.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 48);
    config.setString(ConfigConstants.EXECUTION_RETRY_DELAY_KEY, "0 ms");

    cluster = new ForkableFlinkMiniCluster(config, false);
    cluster.start();
  }
Пример #14
0
  /**
   * Parameter {@code uploadedJarLocation} is actually used to point to the local jar, because Flink
   * does not support uploading a jar file before hand. Jar files are always uploaded directly when
   * a program is submitted.
   */
  public void submitTopologyWithOpts(
      final String name, final String uploadedJarLocation, final FlinkTopology topology)
      throws AlreadyAliveException, InvalidTopologyException {

    if (this.getTopologyJobId(name) != null) {
      throw new AlreadyAliveException();
    }

    final URI uploadedJarUri;
    final URL uploadedJarUrl;
    try {
      uploadedJarUri = new File(uploadedJarLocation).getAbsoluteFile().toURI();
      uploadedJarUrl = uploadedJarUri.toURL();
      JobWithJars.checkJarFile(uploadedJarUrl);
    } catch (final IOException e) {
      throw new RuntimeException("Problem with jar file " + uploadedJarLocation, e);
    }

    try {
      FlinkClient.addStormConfigToTopology(topology, conf);
    } catch (ClassNotFoundException e) {
      LOG.error("Could not register class for Kryo serialization.", e);
      throw new InvalidTopologyException("Could not register class for Kryo serialization.");
    }

    final StreamGraph streamGraph = topology.getExecutionEnvironment().getStreamGraph();
    streamGraph.setJobName(name);

    final JobGraph jobGraph = streamGraph.getJobGraph();
    jobGraph.addJar(new Path(uploadedJarUri));

    final Configuration configuration = jobGraph.getJobConfiguration();
    configuration.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, jobManagerHost);
    configuration.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, jobManagerPort);

    final Client client;
    try {
      client = new Client(configuration);
    } catch (IOException e) {
      throw new RuntimeException("Could not establish a connection to the job manager", e);
    }

    try {
      ClassLoader classLoader =
          JobWithJars.buildUserCodeClassLoader(
              Lists.newArrayList(uploadedJarUrl),
              Collections.<URL>emptyList(),
              this.getClass().getClassLoader());
      client.runDetached(jobGraph, classLoader);
    } catch (final ProgramInvocationException e) {
      throw new RuntimeException("Cannot execute job due to ProgramInvocationException", e);
    }
  }
  /**
   * Ensure that we will never have the following error.
   *
   * <p>The test works as follows: - Use the CliFrontend to invoke a jar file that loads a class
   * which is only available in the jarfile itself (via a custom classloader) - Change the Usercode
   * classloader of the PackagedProgram to a special classloader for this test - the classloader
   * will accept the special class (and return a String.class)
   *
   * <p>org.apache.flink.client.program.ProgramInvocationException: The main method caused an error.
   * at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:398) at
   * org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:301)
   * at org.apache.flink.client.program.Client.getOptimizedPlan(Client.java:140) at
   * org.apache.flink.client.program.Client.getOptimizedPlanAsJson(Client.java:125) at
   * org.apache.flink.client.CliFrontend.info(CliFrontend.java:439) at
   * org.apache.flink.client.CliFrontend.parseParameters(CliFrontend.java:931) at
   * org.apache.flink.client.CliFrontend.main(CliFrontend.java:951) Caused by: java.io.IOException:
   * java.lang.RuntimeException: java.lang.ClassNotFoundException:
   * org.apache.hadoop.hive.ql.io.RCFileInputFormat at
   * org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:102) at
   * org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:54) at
   * tlabs.CDR_In_Report.createHCatInputFormat(CDR_In_Report.java:322) at
   * tlabs.CDR_Out_Report.main(CDR_Out_Report.java:380) at
   * sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
   * sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at
   * sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at
   * java.lang.reflect.Method.invoke(Method.java:622) at
   * org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:383)
   */
  @Test
  public void testPlanWithExternalClass() throws CompilerException, ProgramInvocationException {
    final Boolean callme[] = {
      false
    }; // create a final object reference, to be able to change its val later
    try {
      String[] parameters = {
        getTestJarPath(), "-c", TEST_JAR_CLASSLOADERTEST_CLASS, "some", "program"
      };
      CommandLine line =
          new PosixParser()
              .parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false);

      CliFrontend frontend = new CliFrontend();
      Object result = frontend.buildProgram(line);
      assertTrue(result instanceof PackagedProgram);

      PackagedProgram prog = spy((PackagedProgram) result);

      ClassLoader testClassLoader =
          new ClassLoader(prog.getUserCodeClassLoader()) {
            @Override
            public Class<?> loadClass(String name) throws ClassNotFoundException {
              assertTrue(name.equals("org.apache.hadoop.hive.ql.io.RCFileInputFormat"));
              callme[0] = true;
              return String.class; // Intentionally return the wrong class.
            }
          };
      when(prog.getUserCodeClassLoader()).thenReturn(testClassLoader);

      Assert.assertArrayEquals(new String[] {"some", "program"}, prog.getArguments());
      Assert.assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, prog.getMainClassName());

      Configuration c = new Configuration();
      c.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "devil");
      Client cli = new Client(c, getClass().getClassLoader());

      cli.getOptimizedPlanAsJson(prog, 666);
    } catch (ProgramInvocationException pie) {
      assertTrue("Classloader was not called", callme[0]);
      // class not found exception is expected as some point
      if (!(pie.getCause() instanceof ClassNotFoundException)) {
        System.err.println(pie.getMessage());
        pie.printStackTrace();
        fail("Program caused an exception: " + pie.getMessage());
      }
    } catch (Exception e) {
      assertTrue("Classloader was not called", callme[0]);
      System.err.println(e.getMessage());
      e.printStackTrace();
      fail("Program caused an exception: " + e.getMessage());
    }
  }
  @Override
  public void open() {
    out = new ByteArrayOutputStream();
    flinkConf = new org.apache.flink.configuration.Configuration();
    Properties intpProperty = getProperty();
    for (Object k : intpProperty.keySet()) {
      String key = (String) k;
      String val = toString(intpProperty.get(key));
      flinkConf.setString(key, val);
    }

    if (localMode()) {
      startFlinkMiniCluster();
    }

    flinkIloop =
        new FlinkILoop(
            getHost(), getPort(), flinkConf, (BufferedReader) null, new PrintWriter(out));

    flinkIloop.settings_$eq(createSettings());
    flinkIloop.createInterpreter();

    imain = flinkIloop.intp();

    org.apache.flink.api.scala.ExecutionEnvironment benv = flinkIloop.scalaBenv();
    // new ExecutionEnvironment(remoteBenv)
    org.apache.flink.streaming.api.scala.StreamExecutionEnvironment senv = flinkIloop.scalaSenv();

    senv.getConfig().disableSysoutLogging();
    benv.getConfig().disableSysoutLogging();

    // prepare bindings
    imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
    Map<String, Object> binder = (Map<String, Object>) getLastObject();

    // import libraries
    imain.interpret("import scala.tools.nsc.io._");
    imain.interpret("import Properties.userHome");
    imain.interpret("import scala.compat.Platform.EOL");

    imain.interpret("import org.apache.flink.api.scala._");
    imain.interpret("import org.apache.flink.api.common.functions._");

    binder.put("benv", benv);
    imain.interpret(
        "val benv = _binder.get(\"benv\").asInstanceOf[" + benv.getClass().getName() + "]");

    binder.put("senv", senv);
    imain.interpret(
        "val senv = _binder.get(\"senv\").asInstanceOf[" + senv.getClass().getName() + "]");
  }
  private Plan getTestPlanRightStatic(String strategy) {

    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);

    DataSet<Tuple3<Long, Long, Long>> bigInput =
        env.readCsvFile("file://bigFile").types(Long.class, Long.class, Long.class).name("bigFile");

    DataSet<Tuple3<Long, Long, Long>> smallInput =
        env.readCsvFile("file://smallFile")
            .types(Long.class, Long.class, Long.class)
            .name("smallFile");

    IterativeDataSet<Tuple3<Long, Long, Long>> iteration = bigInput.iterate(10);

    Configuration joinStrategy = new Configuration();
    joinStrategy.setString(
        Optimizer.HINT_SHIP_STRATEGY, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH);

    if (!strategy.equals("")) {
      joinStrategy.setString(Optimizer.HINT_LOCAL_STRATEGY, strategy);
    }

    DataSet<Tuple3<Long, Long, Long>> inner =
        iteration
            .join(smallInput)
            .where(0)
            .equalTo(0)
            .with(new DummyJoiner())
            .name("DummyJoiner")
            .withParameters(joinStrategy);

    DataSet<Tuple3<Long, Long, Long>> output = iteration.closeWith(inner);

    output.output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());

    return env.createProgramPlan();
  }
Пример #18
0
  /**
   * Package internal method to get a Flink {@link JobID} from a Storm topology name.
   *
   * @param id The Storm topology name.
   * @return Flink's internally used {@link JobID}.
   */
  JobID getTopologyJobId(final String id) {
    final Configuration configuration = GlobalConfiguration.getConfiguration();
    if (this.timeout != null) {
      configuration.setString(ConfigConstants.AKKA_ASK_TIMEOUT, this.timeout);
    }

    try {
      final ActorRef jobManager = this.getJobManager();

      final FiniteDuration askTimeout = this.getTimeout();
      final Future<Object> response =
          Patterns.ask(
              jobManager,
              JobManagerMessages.getRequestRunningJobsStatus(),
              new Timeout(askTimeout));

      Object result;
      try {
        result = Await.result(response, askTimeout);
      } catch (final Exception e) {
        throw new RuntimeException("Could not retrieve running jobs from the JobManager", e);
      }

      if (result instanceof RunningJobsStatus) {
        final List<JobStatusMessage> jobs = ((RunningJobsStatus) result).getStatusMessages();

        for (final JobStatusMessage status : jobs) {
          if (status.getJobName().equals(id)) {
            return status.getJobId();
          }
        }
      } else {
        throw new RuntimeException(
            "ReqeustRunningJobs requires a response of type "
                + "RunningJobs. Instead the response is of type "
                + result.getClass()
                + ".");
      }
    } catch (final IOException e) {
      throw new RuntimeException(
          "Could not connect to Flink JobManager with address "
              + this.jobManagerHost
              + ":"
              + this.jobManagerPort,
          e);
    }

    return null;
  }
Пример #19
0
 private static JMXReporter startJmxReporter(Configuration config) {
   JMXReporter reporter = null;
   try {
     Configuration reporterConfig = new Configuration();
     String portRange = config.getString(KEY_METRICS_JMX_PORT, null);
     if (portRange != null) {
       reporterConfig.setString(KEY_METRICS_JMX_PORT, portRange);
     }
     reporter = new JMXReporter();
     reporter.open(reporterConfig);
   } catch (Exception e) {
     LOG.error("Failed to instantiate JMX reporter.", e);
   } finally {
     return reporter;
   }
 }
Пример #20
0
  @BeforeClass
  public static void startCluster() {
    try {
      Configuration config = new Configuration();
      config.setInteger(
          ConfigConstants.LOCAL_INSTANCE_MANAGER_NUMBER_TASK_MANAGER, NUM_TASK_MANAGERS);
      config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_TASK_SLOTS);
      config.setString(ConfigConstants.DEFAULT_EXECUTION_RETRY_DELAY_KEY, "0 ms");
      config.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 12);

      cluster = new ForkableFlinkMiniCluster(config, false);
    } catch (Exception e) {
      e.printStackTrace();
      fail("Failed to start test cluster: " + e.getMessage());
    }
  }
  private static StreamTask<?, ?> createMockTask() {
    Configuration configuration = new Configuration();
    configuration.setString(ConfigConstants.STATE_BACKEND, "jobmanager");

    StreamTask<?, ?> task = mock(StreamTask.class);
    when(task.getAccumulatorMap()).thenReturn(new HashMap<String, Accumulator<?, ?>>());
    when(task.getName()).thenReturn("Test task name");
    when(task.getExecutionConfig()).thenReturn(new ExecutionConfig());

    final TaskManagerRuntimeInfo mockTaskManagerRuntimeInfo = mock(TaskManagerRuntimeInfo.class);
    when(mockTaskManagerRuntimeInfo.getConfiguration()).thenReturn(configuration);

    final Environment env = mock(Environment.class);
    when(env.getTaskInfo()).thenReturn(new TaskInfo("Test task name", 1, 0, 1, 0));
    when(env.getUserClassLoader())
        .thenReturn(AggregatingAlignedProcessingTimeWindowOperatorTest.class.getClassLoader());
    when(env.getMetricGroup()).thenReturn(new UnregisteredTaskMetricsGroup());

    when(task.getEnvironment()).thenReturn(env);
    return task;
  }
  private Plan getTestPlanLeftStatic(String strategy) {

    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);

    @SuppressWarnings("unchecked")
    DataSet<Tuple3<Long, Long, Long>> bigInput =
        env.fromElements(
                new Tuple3<Long, Long, Long>(1L, 2L, 3L),
                new Tuple3<Long, Long, Long>(1L, 2L, 3L),
                new Tuple3<Long, Long, Long>(1L, 2L, 3L))
            .name("Big");

    @SuppressWarnings("unchecked")
    DataSet<Tuple3<Long, Long, Long>> smallInput =
        env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Small");

    IterativeDataSet<Tuple3<Long, Long, Long>> iteration = bigInput.iterate(10);

    Configuration joinStrategy = new Configuration();
    joinStrategy.setString(Optimizer.HINT_LOCAL_STRATEGY, strategy);

    DataSet<Tuple3<Long, Long, Long>> inner =
        smallInput
            .join(iteration)
            .where(0)
            .equalTo(0)
            .with(new DummyJoiner())
            .name("DummyJoiner")
            .withParameters(joinStrategy);

    DataSet<Tuple3<Long, Long, Long>> output = iteration.closeWith(inner);

    output.output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());

    return env.createProgramPlan();
  }
  @Test
  public void testStartupWhenNetworkStackFailsToInitialize() {

    ServerSocket blocker = null;

    try {
      blocker = new ServerSocket(0, 50, InetAddress.getByName("localhost"));

      final Configuration cfg = new Configuration();
      cfg.setString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, "localhost");
      cfg.setInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, blocker.getLocalPort());
      cfg.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 1);

      new JavaTestKit(actorSystem) {
        {
          ActorRef taskManager = null;
          ActorRef jobManager = null;
          ActorRef resourceManager = null;

          try {
            // a simple JobManager
            jobManager = startJobManager(config);

            resourceManager = startResourceManager(config, jobManager);

            // start a task manager with a configuration that provides a blocked port
            taskManager =
                TaskManager.startTaskManagerComponentsAndActor(
                    cfg,
                    ResourceID.generate(),
                    actorSystem,
                    "localhost",
                    NONE_STRING, // no actor name -> random
                    new Some<LeaderRetrievalService>(
                        new StandaloneLeaderRetrievalService(jobManager.path().toString())),
                    false, // init network stack !!!
                    TaskManager.class);

            watch(taskManager);

            expectTerminated(timeout, taskManager);
          } catch (Exception e) {
            e.printStackTrace();
            fail(e.getMessage());
          } finally {
            stopActor(taskManager);
            stopActor(jobManager);
          }
        }
      };
    } catch (Exception e) {
      // does not work, skip test
      e.printStackTrace();
      fail(e.getMessage());
    } finally {
      if (blocker != null) {
        try {
          blocker.close();
        } catch (IOException e) {
          // ignore, best effort
        }
      }
    }
  }
  protected void internalNonBlockingStart() throws IOException {

    Plan plan = env.createProgramPlan();

    Optimizer optimizer =
        new Optimizer(new DataStatistics(), new org.apache.flink.configuration.Configuration());
    OptimizedPlan optimizedPlan = optimizer.compile(plan);

    final JobGraph jobGraph = new JobGraphGenerator().compileJobGraph(optimizedPlan);
    for (String jarPath : classPath) {
      jobGraph.addJar(new Path(jarPath));
    }

    jobID = jobGraph.getJobID();
    accumulatorCache.setJobID(jobID);

    if (isLocalExecution()) {

      flowStep.logInfo("Executing in local mode.");

      startLocalCluster();

      org.apache.flink.configuration.Configuration config =
          new org.apache.flink.configuration.Configuration();
      config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, localCluster.hostname());

      client = new Client(config);
      client.setPrintStatusDuringExecution(env.getConfig().isSysoutLoggingEnabled());

    } else {

      flowStep.logInfo("Executing in cluster mode.");

      try {
        String path =
            this.getClass().getProtectionDomain().getCodeSource().getLocation().toURI().getPath();
        jobGraph.addJar(new Path(path));
        classPath.add(path);
      } catch (URISyntaxException e) {
        throw new IOException("Could not add the submission JAR as a dependency.");
      }

      client = ((ContextEnvironment) env).getClient();
    }

    List<URL> fileList = new ArrayList<URL>(classPath.size());
    for (String path : classPath) {
      URL url;
      try {
        url = new URL(path);
      } catch (MalformedURLException e) {
        url = new URL("file://" + path);
      }
      fileList.add(url);
    }

    final ClassLoader loader =
        JobWithJars.buildUserCodeClassLoader(
            fileList, Collections.<URL>emptyList(), getClass().getClassLoader());

    accumulatorCache.setClient(client);

    final Callable<JobSubmissionResult> callable =
        new Callable<JobSubmissionResult>() {
          @Override
          public JobSubmissionResult call() throws Exception {
            return client.runBlocking(jobGraph, loader);
          }
        };

    jobSubmission = executorService.submit(callable);

    flowStep.logInfo("submitted Flink job: " + jobID);
  }
Пример #25
0
  /**
   * Entry point for the program.
   *
   * @param args arguments from the command line
   * @throws IOException
   */
  @SuppressWarnings("static-access")
  public static void main(String[] args) throws IOException {
    Option configDirOpt =
        OptionBuilder.withArgName("config directory")
            .hasArg()
            .withDescription("Specify configuration directory.")
            .create("configDir");
    // tempDir option is used by the YARN client.
    Option tempDir =
        OptionBuilder.withArgName("temporary directory (overwrites configured option)")
            .hasArg()
            .withDescription("Specify temporary directory.")
            .create(ARG_CONF_DIR);
    configDirOpt.setRequired(true);
    tempDir.setRequired(false);
    Options options = new Options();
    options.addOption(configDirOpt);
    options.addOption(tempDir);

    CommandLineParser parser = new GnuParser();
    CommandLine line = null;
    try {
      line = parser.parse(options, args);
    } catch (ParseException e) {
      System.err.println("CLI Parsing failed. Reason: " + e.getMessage());
      System.exit(FAILURE_RETURN_CODE);
    }

    String configDir = line.getOptionValue(configDirOpt.getOpt(), null);
    String tempDirVal = line.getOptionValue(tempDir.getOpt(), null);

    // First, try to load global configuration
    GlobalConfiguration.loadConfiguration(configDir);
    if (tempDirVal != null // the YARN TM runner has set a value for the temp dir
        // the configuration does not contain a temp direcory
        && GlobalConfiguration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, null) == null) {
      Configuration c = GlobalConfiguration.getConfiguration();
      c.setString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, tempDirVal);
      LOG.info("Setting temporary directory to " + tempDirVal);
      GlobalConfiguration.includeConfiguration(c);
    }

    // print some startup environment info, like user, code revision, etc
    EnvironmentInformation.logEnvironmentInfo(LOG, "TaskManager");

    // Create a new task manager object
    try {
      new TaskManager(ExecutionMode.CLUSTER);
    } catch (Exception e) {
      LOG.fatal("Taskmanager startup failed: " + e.getMessage(), e);
      System.exit(FAILURE_RETURN_CODE);
    }

    // park the main thread to keep the JVM alive (all other threads may be daemon threads)
    Object mon = new Object();
    synchronized (mon) {
      try {
        mon.wait();
      } catch (InterruptedException ex) {
      }
    }
  }