@BeforeClass public static void startActorSystem() { config = new Configuration(); config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "5 s"); config.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_INTERVAL, "200 ms"); config.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "2 s"); config.setDouble(ConfigConstants.AKKA_WATCH_THRESHOLD, 2.0); actorSystem = AkkaUtils.createLocalActorSystem(config); }
@Test public void testSavepointBackendFileSystemButCheckpointBackendJobManager() throws Exception { Configuration config = new Configuration(); // This combination does not make sense, because the checkpoints will be // lost after the job manager shuts down. config.setString(ConfigConstants.STATE_BACKEND, "jobmanager"); config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "filesystem"); SavepointStore store = SavepointStoreFactory.createFromConfig(config); assertTrue(store.getStateStore() instanceof HeapStateStore); }
static Configuration createReporterConfig(Configuration config, TimeUnit timeunit, long period) { Configuration reporterConfig = new Configuration(); reporterConfig.setLong("period", period); reporterConfig.setString("timeunit", timeunit.name()); String[] arguments = config.getString(KEY_METRICS_REPORTER_ARGUMENTS, "").split(" "); if (arguments.length > 1) { for (int x = 0; x < arguments.length; x += 2) { reporterConfig.setString(arguments[x].replace("--", ""), arguments[x + 1]); } } return reporterConfig; }
@Test public void testSavepointBackendFileSystem() throws Exception { Configuration config = new Configuration(); String rootPath = System.getProperty("java.io.tmpdir"); config.setString(ConfigConstants.STATE_BACKEND, "filesystem"); config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "filesystem"); config.setString(SavepointStoreFactory.SAVEPOINT_DIRECTORY_KEY, rootPath); SavepointStore store = SavepointStoreFactory.createFromConfig(config); assertTrue(store.getStateStore() instanceof FileSystemStateStore); FileSystemStateStore<CompletedCheckpoint> stateStore = (FileSystemStateStore<CompletedCheckpoint>) store.getStateStore(); assertEquals(new Path(rootPath), stateStore.getRootPath()); }
@Override public void open() { out = new ByteArrayOutputStream(); flinkConf = new org.apache.flink.configuration.Configuration(); Properties intpProperty = getProperty(); for (Object k : intpProperty.keySet()) { String key = (String) k; String val = toString(intpProperty.get(key)); flinkConf.setString(key, val); } if (localMode()) { startFlinkMiniCluster(); } flinkIloop = new FlinkILoop(getHost(), getPort(), (BufferedReader) null, new PrintWriter(out)); flinkIloop.settings_$eq(createSettings()); flinkIloop.createInterpreter(); imain = flinkIloop.intp(); // prepare bindings imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); binder = (Map<String, Object>) getValue("_binder"); // import libraries imain.interpret("import scala.tools.nsc.io._"); imain.interpret("import Properties.userHome"); imain.interpret("import scala.compat.Platform.EOL"); imain.interpret("import org.apache.flink.api.scala._"); imain.interpret("import org.apache.flink.api.common.functions._"); imain.bindValue("env", flinkIloop.scalaEnv()); }
/** * Checks if the local instance manager reads the default correctly from the configuration file. */ @Test public void testInstanceTypeFromConfiguration() { try { Configuration cfg = new Configuration(); cfg.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "127.0.0.1"); cfg.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 6123); cfg.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 1); cfg.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1); GlobalConfiguration.includeConfiguration(cfg); // start JobManager ExecutionMode executionMode = ExecutionMode.LOCAL; JobManager jm = new JobManager(executionMode); final TestInstanceListener testInstanceListener = new TestInstanceListener(); InstanceManager im = jm.getInstanceManager(); try { im.setInstanceListener(testInstanceListener); } catch (Exception e) { e.printStackTrace(); Assert.fail("Instantiation of LocalInstanceManager failed: " + e.getMessage()); } finally { jm.shutdown(); } } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail("Test caused an error: " + e.getMessage()); } }
@Test public void testSavepointBackendFileSystemButNoDirectory() throws Exception { Configuration config = new Configuration(); config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "filesystem"); SavepointStore store = SavepointStoreFactory.createFromConfig(config); assertTrue(store.getStateStore() instanceof HeapStateStore); }
@BeforeClass public static void setupJobManager() { Configuration config = new Configuration(); int port = NetUtils.getAvailablePort(); config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost"); config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, port); scala.Option<Tuple2<String, Object>> listeningAddress = scala.Option.apply(new Tuple2<String, Object>("localhost", port)); jobManagerSystem = AkkaUtils.createActorSystem(config, listeningAddress); ActorRef jobManagerActorRef = JobManager.startJobManagerActors( config, jobManagerSystem, StreamingMode.BATCH_ONLY, JobManager.class, MemoryArchivist.class) ._1(); try { LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(config); jmGateway = LeaderRetrievalUtils.retrieveLeaderGateway(lrs, jobManagerSystem, timeout); } catch (Exception e) { fail("Could not retrieve the JobManager gateway. " + e.getMessage()); } }
@Test public void testUnexpectedSavepointBackend() throws Exception { Configuration config = new Configuration(); config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "unexpected"); SavepointStore store = SavepointStoreFactory.createFromConfig(config); assertTrue(store.getStateStore() instanceof HeapStateStore); }
public void killTopologyWithOpts(final String name, final KillOptions options) throws NotAliveException { final JobID jobId = this.getTopologyJobId(name); if (jobId == null) { throw new NotAliveException("Storm topology with name " + name + " not found."); } if (options != null) { try { Thread.sleep(1000 * options.get_wait_secs()); } catch (final InterruptedException e) { throw new RuntimeException(e); } } final Configuration configuration = GlobalConfiguration.getConfiguration(); configuration.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, this.jobManagerHost); configuration.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, this.jobManagerPort); final Client client; try { client = new Client(configuration); } catch (final IOException e) { throw new RuntimeException("Could not establish a connection to the job manager", e); } try { client.cancel(jobId); } catch (final Exception e) { throw new RuntimeException("Cannot stop job.", e); } }
private FiniteDuration getTimeout() { final Configuration configuration = GlobalConfiguration.getConfiguration(); if (this.timeout != null) { configuration.setString(ConfigConstants.AKKA_ASK_TIMEOUT, this.timeout); } return AkkaUtils.getClientTimeout(configuration); }
@BeforeClass public static void setup() throws Exception { Configuration config = new Configuration(); config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, numTaskManagers); config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, slotsPerTaskManager); final File checkpointDir = temporaryFolder.newFolder(); final File savepointDir = temporaryFolder.newFolder(); config.setString(ConfigConstants.STATE_BACKEND, "filesystem"); config.setString( FsStateBackendFactory.CHECKPOINT_DIRECTORY_URI_CONF_KEY, checkpointDir.toURI().toString()); config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "filesystem"); config.setString( SavepointStoreFactory.SAVEPOINT_DIRECTORY_KEY, savepointDir.toURI().toString()); cluster = new TestingCluster(config); cluster.start(); }
@BeforeClass public static void startTestCluster() { Configuration config = new Configuration(); config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2); config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM / 2); config.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 48); config.setString(ConfigConstants.EXECUTION_RETRY_DELAY_KEY, "0 ms"); cluster = new ForkableFlinkMiniCluster(config, false); cluster.start(); }
/** * Parameter {@code uploadedJarLocation} is actually used to point to the local jar, because Flink * does not support uploading a jar file before hand. Jar files are always uploaded directly when * a program is submitted. */ public void submitTopologyWithOpts( final String name, final String uploadedJarLocation, final FlinkTopology topology) throws AlreadyAliveException, InvalidTopologyException { if (this.getTopologyJobId(name) != null) { throw new AlreadyAliveException(); } final URI uploadedJarUri; final URL uploadedJarUrl; try { uploadedJarUri = new File(uploadedJarLocation).getAbsoluteFile().toURI(); uploadedJarUrl = uploadedJarUri.toURL(); JobWithJars.checkJarFile(uploadedJarUrl); } catch (final IOException e) { throw new RuntimeException("Problem with jar file " + uploadedJarLocation, e); } try { FlinkClient.addStormConfigToTopology(topology, conf); } catch (ClassNotFoundException e) { LOG.error("Could not register class for Kryo serialization.", e); throw new InvalidTopologyException("Could not register class for Kryo serialization."); } final StreamGraph streamGraph = topology.getExecutionEnvironment().getStreamGraph(); streamGraph.setJobName(name); final JobGraph jobGraph = streamGraph.getJobGraph(); jobGraph.addJar(new Path(uploadedJarUri)); final Configuration configuration = jobGraph.getJobConfiguration(); configuration.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, jobManagerHost); configuration.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, jobManagerPort); final Client client; try { client = new Client(configuration); } catch (IOException e) { throw new RuntimeException("Could not establish a connection to the job manager", e); } try { ClassLoader classLoader = JobWithJars.buildUserCodeClassLoader( Lists.newArrayList(uploadedJarUrl), Collections.<URL>emptyList(), this.getClass().getClassLoader()); client.runDetached(jobGraph, classLoader); } catch (final ProgramInvocationException e) { throw new RuntimeException("Cannot execute job due to ProgramInvocationException", e); } }
/** * Ensure that we will never have the following error. * * <p>The test works as follows: - Use the CliFrontend to invoke a jar file that loads a class * which is only available in the jarfile itself (via a custom classloader) - Change the Usercode * classloader of the PackagedProgram to a special classloader for this test - the classloader * will accept the special class (and return a String.class) * * <p>org.apache.flink.client.program.ProgramInvocationException: The main method caused an error. * at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:398) at * org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:301) * at org.apache.flink.client.program.Client.getOptimizedPlan(Client.java:140) at * org.apache.flink.client.program.Client.getOptimizedPlanAsJson(Client.java:125) at * org.apache.flink.client.CliFrontend.info(CliFrontend.java:439) at * org.apache.flink.client.CliFrontend.parseParameters(CliFrontend.java:931) at * org.apache.flink.client.CliFrontend.main(CliFrontend.java:951) Caused by: java.io.IOException: * java.lang.RuntimeException: java.lang.ClassNotFoundException: * org.apache.hadoop.hive.ql.io.RCFileInputFormat at * org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:102) at * org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:54) at * tlabs.CDR_In_Report.createHCatInputFormat(CDR_In_Report.java:322) at * tlabs.CDR_Out_Report.main(CDR_Out_Report.java:380) at * sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at * sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at * sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at * java.lang.reflect.Method.invoke(Method.java:622) at * org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:383) */ @Test public void testPlanWithExternalClass() throws CompilerException, ProgramInvocationException { final Boolean callme[] = { false }; // create a final object reference, to be able to change its val later try { String[] parameters = { getTestJarPath(), "-c", TEST_JAR_CLASSLOADERTEST_CLASS, "some", "program" }; CommandLine line = new PosixParser() .parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false); CliFrontend frontend = new CliFrontend(); Object result = frontend.buildProgram(line); assertTrue(result instanceof PackagedProgram); PackagedProgram prog = spy((PackagedProgram) result); ClassLoader testClassLoader = new ClassLoader(prog.getUserCodeClassLoader()) { @Override public Class<?> loadClass(String name) throws ClassNotFoundException { assertTrue(name.equals("org.apache.hadoop.hive.ql.io.RCFileInputFormat")); callme[0] = true; return String.class; // Intentionally return the wrong class. } }; when(prog.getUserCodeClassLoader()).thenReturn(testClassLoader); Assert.assertArrayEquals(new String[] {"some", "program"}, prog.getArguments()); Assert.assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, prog.getMainClassName()); Configuration c = new Configuration(); c.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "devil"); Client cli = new Client(c, getClass().getClassLoader()); cli.getOptimizedPlanAsJson(prog, 666); } catch (ProgramInvocationException pie) { assertTrue("Classloader was not called", callme[0]); // class not found exception is expected as some point if (!(pie.getCause() instanceof ClassNotFoundException)) { System.err.println(pie.getMessage()); pie.printStackTrace(); fail("Program caused an exception: " + pie.getMessage()); } } catch (Exception e) { assertTrue("Classloader was not called", callme[0]); System.err.println(e.getMessage()); e.printStackTrace(); fail("Program caused an exception: " + e.getMessage()); } }
@Override public void open() { out = new ByteArrayOutputStream(); flinkConf = new org.apache.flink.configuration.Configuration(); Properties intpProperty = getProperty(); for (Object k : intpProperty.keySet()) { String key = (String) k; String val = toString(intpProperty.get(key)); flinkConf.setString(key, val); } if (localMode()) { startFlinkMiniCluster(); } flinkIloop = new FlinkILoop( getHost(), getPort(), flinkConf, (BufferedReader) null, new PrintWriter(out)); flinkIloop.settings_$eq(createSettings()); flinkIloop.createInterpreter(); imain = flinkIloop.intp(); org.apache.flink.api.scala.ExecutionEnvironment benv = flinkIloop.scalaBenv(); // new ExecutionEnvironment(remoteBenv) org.apache.flink.streaming.api.scala.StreamExecutionEnvironment senv = flinkIloop.scalaSenv(); senv.getConfig().disableSysoutLogging(); benv.getConfig().disableSysoutLogging(); // prepare bindings imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); Map<String, Object> binder = (Map<String, Object>) getLastObject(); // import libraries imain.interpret("import scala.tools.nsc.io._"); imain.interpret("import Properties.userHome"); imain.interpret("import scala.compat.Platform.EOL"); imain.interpret("import org.apache.flink.api.scala._"); imain.interpret("import org.apache.flink.api.common.functions._"); binder.put("benv", benv); imain.interpret( "val benv = _binder.get(\"benv\").asInstanceOf[" + benv.getClass().getName() + "]"); binder.put("senv", senv); imain.interpret( "val senv = _binder.get(\"senv\").asInstanceOf[" + senv.getClass().getName() + "]"); }
private Plan getTestPlanRightStatic(String strategy) { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(DEFAULT_PARALLELISM); DataSet<Tuple3<Long, Long, Long>> bigInput = env.readCsvFile("file://bigFile").types(Long.class, Long.class, Long.class).name("bigFile"); DataSet<Tuple3<Long, Long, Long>> smallInput = env.readCsvFile("file://smallFile") .types(Long.class, Long.class, Long.class) .name("smallFile"); IterativeDataSet<Tuple3<Long, Long, Long>> iteration = bigInput.iterate(10); Configuration joinStrategy = new Configuration(); joinStrategy.setString( Optimizer.HINT_SHIP_STRATEGY, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH); if (!strategy.equals("")) { joinStrategy.setString(Optimizer.HINT_LOCAL_STRATEGY, strategy); } DataSet<Tuple3<Long, Long, Long>> inner = iteration .join(smallInput) .where(0) .equalTo(0) .with(new DummyJoiner()) .name("DummyJoiner") .withParameters(joinStrategy); DataSet<Tuple3<Long, Long, Long>> output = iteration.closeWith(inner); output.output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>()); return env.createProgramPlan(); }
/** * Package internal method to get a Flink {@link JobID} from a Storm topology name. * * @param id The Storm topology name. * @return Flink's internally used {@link JobID}. */ JobID getTopologyJobId(final String id) { final Configuration configuration = GlobalConfiguration.getConfiguration(); if (this.timeout != null) { configuration.setString(ConfigConstants.AKKA_ASK_TIMEOUT, this.timeout); } try { final ActorRef jobManager = this.getJobManager(); final FiniteDuration askTimeout = this.getTimeout(); final Future<Object> response = Patterns.ask( jobManager, JobManagerMessages.getRequestRunningJobsStatus(), new Timeout(askTimeout)); Object result; try { result = Await.result(response, askTimeout); } catch (final Exception e) { throw new RuntimeException("Could not retrieve running jobs from the JobManager", e); } if (result instanceof RunningJobsStatus) { final List<JobStatusMessage> jobs = ((RunningJobsStatus) result).getStatusMessages(); for (final JobStatusMessage status : jobs) { if (status.getJobName().equals(id)) { return status.getJobId(); } } } else { throw new RuntimeException( "ReqeustRunningJobs requires a response of type " + "RunningJobs. Instead the response is of type " + result.getClass() + "."); } } catch (final IOException e) { throw new RuntimeException( "Could not connect to Flink JobManager with address " + this.jobManagerHost + ":" + this.jobManagerPort, e); } return null; }
private static JMXReporter startJmxReporter(Configuration config) { JMXReporter reporter = null; try { Configuration reporterConfig = new Configuration(); String portRange = config.getString(KEY_METRICS_JMX_PORT, null); if (portRange != null) { reporterConfig.setString(KEY_METRICS_JMX_PORT, portRange); } reporter = new JMXReporter(); reporter.open(reporterConfig); } catch (Exception e) { LOG.error("Failed to instantiate JMX reporter.", e); } finally { return reporter; } }
@BeforeClass public static void startCluster() { try { Configuration config = new Configuration(); config.setInteger( ConfigConstants.LOCAL_INSTANCE_MANAGER_NUMBER_TASK_MANAGER, NUM_TASK_MANAGERS); config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_TASK_SLOTS); config.setString(ConfigConstants.DEFAULT_EXECUTION_RETRY_DELAY_KEY, "0 ms"); config.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 12); cluster = new ForkableFlinkMiniCluster(config, false); } catch (Exception e) { e.printStackTrace(); fail("Failed to start test cluster: " + e.getMessage()); } }
private static StreamTask<?, ?> createMockTask() { Configuration configuration = new Configuration(); configuration.setString(ConfigConstants.STATE_BACKEND, "jobmanager"); StreamTask<?, ?> task = mock(StreamTask.class); when(task.getAccumulatorMap()).thenReturn(new HashMap<String, Accumulator<?, ?>>()); when(task.getName()).thenReturn("Test task name"); when(task.getExecutionConfig()).thenReturn(new ExecutionConfig()); final TaskManagerRuntimeInfo mockTaskManagerRuntimeInfo = mock(TaskManagerRuntimeInfo.class); when(mockTaskManagerRuntimeInfo.getConfiguration()).thenReturn(configuration); final Environment env = mock(Environment.class); when(env.getTaskInfo()).thenReturn(new TaskInfo("Test task name", 1, 0, 1, 0)); when(env.getUserClassLoader()) .thenReturn(AggregatingAlignedProcessingTimeWindowOperatorTest.class.getClassLoader()); when(env.getMetricGroup()).thenReturn(new UnregisteredTaskMetricsGroup()); when(task.getEnvironment()).thenReturn(env); return task; }
private Plan getTestPlanLeftStatic(String strategy) { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(DEFAULT_PARALLELISM); @SuppressWarnings("unchecked") DataSet<Tuple3<Long, Long, Long>> bigInput = env.fromElements( new Tuple3<Long, Long, Long>(1L, 2L, 3L), new Tuple3<Long, Long, Long>(1L, 2L, 3L), new Tuple3<Long, Long, Long>(1L, 2L, 3L)) .name("Big"); @SuppressWarnings("unchecked") DataSet<Tuple3<Long, Long, Long>> smallInput = env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Small"); IterativeDataSet<Tuple3<Long, Long, Long>> iteration = bigInput.iterate(10); Configuration joinStrategy = new Configuration(); joinStrategy.setString(Optimizer.HINT_LOCAL_STRATEGY, strategy); DataSet<Tuple3<Long, Long, Long>> inner = smallInput .join(iteration) .where(0) .equalTo(0) .with(new DummyJoiner()) .name("DummyJoiner") .withParameters(joinStrategy); DataSet<Tuple3<Long, Long, Long>> output = iteration.closeWith(inner); output.output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>()); return env.createProgramPlan(); }
@Test public void testStartupWhenNetworkStackFailsToInitialize() { ServerSocket blocker = null; try { blocker = new ServerSocket(0, 50, InetAddress.getByName("localhost")); final Configuration cfg = new Configuration(); cfg.setString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, "localhost"); cfg.setInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, blocker.getLocalPort()); cfg.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 1); new JavaTestKit(actorSystem) { { ActorRef taskManager = null; ActorRef jobManager = null; ActorRef resourceManager = null; try { // a simple JobManager jobManager = startJobManager(config); resourceManager = startResourceManager(config, jobManager); // start a task manager with a configuration that provides a blocked port taskManager = TaskManager.startTaskManagerComponentsAndActor( cfg, ResourceID.generate(), actorSystem, "localhost", NONE_STRING, // no actor name -> random new Some<LeaderRetrievalService>( new StandaloneLeaderRetrievalService(jobManager.path().toString())), false, // init network stack !!! TaskManager.class); watch(taskManager); expectTerminated(timeout, taskManager); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } finally { stopActor(taskManager); stopActor(jobManager); } } }; } catch (Exception e) { // does not work, skip test e.printStackTrace(); fail(e.getMessage()); } finally { if (blocker != null) { try { blocker.close(); } catch (IOException e) { // ignore, best effort } } } }
protected void internalNonBlockingStart() throws IOException { Plan plan = env.createProgramPlan(); Optimizer optimizer = new Optimizer(new DataStatistics(), new org.apache.flink.configuration.Configuration()); OptimizedPlan optimizedPlan = optimizer.compile(plan); final JobGraph jobGraph = new JobGraphGenerator().compileJobGraph(optimizedPlan); for (String jarPath : classPath) { jobGraph.addJar(new Path(jarPath)); } jobID = jobGraph.getJobID(); accumulatorCache.setJobID(jobID); if (isLocalExecution()) { flowStep.logInfo("Executing in local mode."); startLocalCluster(); org.apache.flink.configuration.Configuration config = new org.apache.flink.configuration.Configuration(); config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, localCluster.hostname()); client = new Client(config); client.setPrintStatusDuringExecution(env.getConfig().isSysoutLoggingEnabled()); } else { flowStep.logInfo("Executing in cluster mode."); try { String path = this.getClass().getProtectionDomain().getCodeSource().getLocation().toURI().getPath(); jobGraph.addJar(new Path(path)); classPath.add(path); } catch (URISyntaxException e) { throw new IOException("Could not add the submission JAR as a dependency."); } client = ((ContextEnvironment) env).getClient(); } List<URL> fileList = new ArrayList<URL>(classPath.size()); for (String path : classPath) { URL url; try { url = new URL(path); } catch (MalformedURLException e) { url = new URL("file://" + path); } fileList.add(url); } final ClassLoader loader = JobWithJars.buildUserCodeClassLoader( fileList, Collections.<URL>emptyList(), getClass().getClassLoader()); accumulatorCache.setClient(client); final Callable<JobSubmissionResult> callable = new Callable<JobSubmissionResult>() { @Override public JobSubmissionResult call() throws Exception { return client.runBlocking(jobGraph, loader); } }; jobSubmission = executorService.submit(callable); flowStep.logInfo("submitted Flink job: " + jobID); }
/** * Entry point for the program. * * @param args arguments from the command line * @throws IOException */ @SuppressWarnings("static-access") public static void main(String[] args) throws IOException { Option configDirOpt = OptionBuilder.withArgName("config directory") .hasArg() .withDescription("Specify configuration directory.") .create("configDir"); // tempDir option is used by the YARN client. Option tempDir = OptionBuilder.withArgName("temporary directory (overwrites configured option)") .hasArg() .withDescription("Specify temporary directory.") .create(ARG_CONF_DIR); configDirOpt.setRequired(true); tempDir.setRequired(false); Options options = new Options(); options.addOption(configDirOpt); options.addOption(tempDir); CommandLineParser parser = new GnuParser(); CommandLine line = null; try { line = parser.parse(options, args); } catch (ParseException e) { System.err.println("CLI Parsing failed. Reason: " + e.getMessage()); System.exit(FAILURE_RETURN_CODE); } String configDir = line.getOptionValue(configDirOpt.getOpt(), null); String tempDirVal = line.getOptionValue(tempDir.getOpt(), null); // First, try to load global configuration GlobalConfiguration.loadConfiguration(configDir); if (tempDirVal != null // the YARN TM runner has set a value for the temp dir // the configuration does not contain a temp direcory && GlobalConfiguration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, null) == null) { Configuration c = GlobalConfiguration.getConfiguration(); c.setString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, tempDirVal); LOG.info("Setting temporary directory to " + tempDirVal); GlobalConfiguration.includeConfiguration(c); } // print some startup environment info, like user, code revision, etc EnvironmentInformation.logEnvironmentInfo(LOG, "TaskManager"); // Create a new task manager object try { new TaskManager(ExecutionMode.CLUSTER); } catch (Exception e) { LOG.fatal("Taskmanager startup failed: " + e.getMessage(), e); System.exit(FAILURE_RETURN_CODE); } // park the main thread to keep the JVM alive (all other threads may be daemon threads) Object mon = new Object(); synchronized (mon) { try { mon.wait(); } catch (InterruptedException ex) { } } }