@BeforeClass
public static void startActorSystem() {
config = new Configuration();
config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "5 s");
config.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_INTERVAL, "200 ms");
config.setString(ConfigConstants.AKKA_WATCH_HEARTBEAT_PAUSE, "2 s");
config.setDouble(ConfigConstants.AKKA_WATCH_THRESHOLD, 2.0);
actorSystem = AkkaUtils.createLocalActorSystem(config);
}
@Test
public void testSavepointBackendFileSystemButCheckpointBackendJobManager() throws Exception {
Configuration config = new Configuration();
// This combination does not make sense, because the checkpoints will be
// lost after the job manager shuts down.
config.setString(ConfigConstants.STATE_BACKEND, "jobmanager");
config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "filesystem");
SavepointStore store = SavepointStoreFactory.createFromConfig(config);
assertTrue(store.getStateStore() instanceof HeapStateStore);
}
static Configuration createReporterConfig(Configuration config, TimeUnit timeunit, long period) {
Configuration reporterConfig = new Configuration();
reporterConfig.setLong("period", period);
reporterConfig.setString("timeunit", timeunit.name());
String[] arguments = config.getString(KEY_METRICS_REPORTER_ARGUMENTS, "").split(" ");
if (arguments.length > 1) {
for (int x = 0; x < arguments.length; x += 2) {
reporterConfig.setString(arguments[x].replace("--", ""), arguments[x + 1]);
}
}
return reporterConfig;
}
@Test
public void testSavepointBackendFileSystem() throws Exception {
Configuration config = new Configuration();
String rootPath = System.getProperty("java.io.tmpdir");
config.setString(ConfigConstants.STATE_BACKEND, "filesystem");
config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "filesystem");
config.setString(SavepointStoreFactory.SAVEPOINT_DIRECTORY_KEY, rootPath);
SavepointStore store = SavepointStoreFactory.createFromConfig(config);
assertTrue(store.getStateStore() instanceof FileSystemStateStore);
FileSystemStateStore<CompletedCheckpoint> stateStore =
(FileSystemStateStore<CompletedCheckpoint>) store.getStateStore();
assertEquals(new Path(rootPath), stateStore.getRootPath());
}
@Override
public void open() {
out = new ByteArrayOutputStream();
flinkConf = new org.apache.flink.configuration.Configuration();
Properties intpProperty = getProperty();
for (Object k : intpProperty.keySet()) {
String key = (String) k;
String val = toString(intpProperty.get(key));
flinkConf.setString(key, val);
}
if (localMode()) {
startFlinkMiniCluster();
}
flinkIloop = new FlinkILoop(getHost(), getPort(), (BufferedReader) null, new PrintWriter(out));
flinkIloop.settings_$eq(createSettings());
flinkIloop.createInterpreter();
imain = flinkIloop.intp();
// prepare bindings
imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
binder = (Map<String, Object>) getValue("_binder");
// import libraries
imain.interpret("import scala.tools.nsc.io._");
imain.interpret("import Properties.userHome");
imain.interpret("import scala.compat.Platform.EOL");
imain.interpret("import org.apache.flink.api.scala._");
imain.interpret("import org.apache.flink.api.common.functions._");
imain.bindValue("env", flinkIloop.scalaEnv());
}
/**
* Checks if the local instance manager reads the default correctly from the configuration file.
*/
@Test
public void testInstanceTypeFromConfiguration() {
try {
Configuration cfg = new Configuration();
cfg.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "127.0.0.1");
cfg.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, 6123);
cfg.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 1);
cfg.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
GlobalConfiguration.includeConfiguration(cfg);
// start JobManager
ExecutionMode executionMode = ExecutionMode.LOCAL;
JobManager jm = new JobManager(executionMode);
final TestInstanceListener testInstanceListener = new TestInstanceListener();
InstanceManager im = jm.getInstanceManager();
try {
im.setInstanceListener(testInstanceListener);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Instantiation of LocalInstanceManager failed: " + e.getMessage());
} finally {
jm.shutdown();
}
} catch (Exception e) {
System.err.println(e.getMessage());
e.printStackTrace();
Assert.fail("Test caused an error: " + e.getMessage());
}
}
@Test
public void testSavepointBackendFileSystemButNoDirectory() throws Exception {
Configuration config = new Configuration();
config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "filesystem");
SavepointStore store = SavepointStoreFactory.createFromConfig(config);
assertTrue(store.getStateStore() instanceof HeapStateStore);
}
@BeforeClass
public static void setupJobManager() {
Configuration config = new Configuration();
int port = NetUtils.getAvailablePort();
config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost");
config.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, port);
scala.Option<Tuple2<String, Object>> listeningAddress =
scala.Option.apply(new Tuple2<String, Object>("localhost", port));
jobManagerSystem = AkkaUtils.createActorSystem(config, listeningAddress);
ActorRef jobManagerActorRef =
JobManager.startJobManagerActors(
config,
jobManagerSystem,
StreamingMode.BATCH_ONLY,
JobManager.class,
MemoryArchivist.class)
._1();
try {
LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(config);
jmGateway = LeaderRetrievalUtils.retrieveLeaderGateway(lrs, jobManagerSystem, timeout);
} catch (Exception e) {
fail("Could not retrieve the JobManager gateway. " + e.getMessage());
}
}
@Test
public void testUnexpectedSavepointBackend() throws Exception {
Configuration config = new Configuration();
config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "unexpected");
SavepointStore store = SavepointStoreFactory.createFromConfig(config);
assertTrue(store.getStateStore() instanceof HeapStateStore);
}
public void killTopologyWithOpts(final String name, final KillOptions options)
throws NotAliveException {
final JobID jobId = this.getTopologyJobId(name);
if (jobId == null) {
throw new NotAliveException("Storm topology with name " + name + " not found.");
}
if (options != null) {
try {
Thread.sleep(1000 * options.get_wait_secs());
} catch (final InterruptedException e) {
throw new RuntimeException(e);
}
}
final Configuration configuration = GlobalConfiguration.getConfiguration();
configuration.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, this.jobManagerHost);
configuration.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, this.jobManagerPort);
final Client client;
try {
client = new Client(configuration);
} catch (final IOException e) {
throw new RuntimeException("Could not establish a connection to the job manager", e);
}
try {
client.cancel(jobId);
} catch (final Exception e) {
throw new RuntimeException("Cannot stop job.", e);
}
}
private FiniteDuration getTimeout() {
final Configuration configuration = GlobalConfiguration.getConfiguration();
if (this.timeout != null) {
configuration.setString(ConfigConstants.AKKA_ASK_TIMEOUT, this.timeout);
}
return AkkaUtils.getClientTimeout(configuration);
}
@BeforeClass
public static void setup() throws Exception {
Configuration config = new Configuration();
config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, numTaskManagers);
config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, slotsPerTaskManager);
final File checkpointDir = temporaryFolder.newFolder();
final File savepointDir = temporaryFolder.newFolder();
config.setString(ConfigConstants.STATE_BACKEND, "filesystem");
config.setString(
FsStateBackendFactory.CHECKPOINT_DIRECTORY_URI_CONF_KEY, checkpointDir.toURI().toString());
config.setString(SavepointStoreFactory.SAVEPOINT_BACKEND_KEY, "filesystem");
config.setString(
SavepointStoreFactory.SAVEPOINT_DIRECTORY_KEY, savepointDir.toURI().toString());
cluster = new TestingCluster(config);
cluster.start();
}
@BeforeClass
public static void startTestCluster() {
Configuration config = new Configuration();
config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM / 2);
config.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 48);
config.setString(ConfigConstants.EXECUTION_RETRY_DELAY_KEY, "0 ms");
cluster = new ForkableFlinkMiniCluster(config, false);
cluster.start();
}
/**
* Parameter {@code uploadedJarLocation} is actually used to point to the local jar, because Flink
* does not support uploading a jar file before hand. Jar files are always uploaded directly when
* a program is submitted.
*/
public void submitTopologyWithOpts(
final String name, final String uploadedJarLocation, final FlinkTopology topology)
throws AlreadyAliveException, InvalidTopologyException {
if (this.getTopologyJobId(name) != null) {
throw new AlreadyAliveException();
}
final URI uploadedJarUri;
final URL uploadedJarUrl;
try {
uploadedJarUri = new File(uploadedJarLocation).getAbsoluteFile().toURI();
uploadedJarUrl = uploadedJarUri.toURL();
JobWithJars.checkJarFile(uploadedJarUrl);
} catch (final IOException e) {
throw new RuntimeException("Problem with jar file " + uploadedJarLocation, e);
}
try {
FlinkClient.addStormConfigToTopology(topology, conf);
} catch (ClassNotFoundException e) {
LOG.error("Could not register class for Kryo serialization.", e);
throw new InvalidTopologyException("Could not register class for Kryo serialization.");
}
final StreamGraph streamGraph = topology.getExecutionEnvironment().getStreamGraph();
streamGraph.setJobName(name);
final JobGraph jobGraph = streamGraph.getJobGraph();
jobGraph.addJar(new Path(uploadedJarUri));
final Configuration configuration = jobGraph.getJobConfiguration();
configuration.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, jobManagerHost);
configuration.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, jobManagerPort);
final Client client;
try {
client = new Client(configuration);
} catch (IOException e) {
throw new RuntimeException("Could not establish a connection to the job manager", e);
}
try {
ClassLoader classLoader =
JobWithJars.buildUserCodeClassLoader(
Lists.newArrayList(uploadedJarUrl),
Collections.<URL>emptyList(),
this.getClass().getClassLoader());
client.runDetached(jobGraph, classLoader);
} catch (final ProgramInvocationException e) {
throw new RuntimeException("Cannot execute job due to ProgramInvocationException", e);
}
}
/**
* Ensure that we will never have the following error.
*
* <p>The test works as follows: - Use the CliFrontend to invoke a jar file that loads a class
* which is only available in the jarfile itself (via a custom classloader) - Change the Usercode
* classloader of the PackagedProgram to a special classloader for this test - the classloader
* will accept the special class (and return a String.class)
*
* <p>org.apache.flink.client.program.ProgramInvocationException: The main method caused an error.
* at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:398) at
* org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:301)
* at org.apache.flink.client.program.Client.getOptimizedPlan(Client.java:140) at
* org.apache.flink.client.program.Client.getOptimizedPlanAsJson(Client.java:125) at
* org.apache.flink.client.CliFrontend.info(CliFrontend.java:439) at
* org.apache.flink.client.CliFrontend.parseParameters(CliFrontend.java:931) at
* org.apache.flink.client.CliFrontend.main(CliFrontend.java:951) Caused by: java.io.IOException:
* java.lang.RuntimeException: java.lang.ClassNotFoundException:
* org.apache.hadoop.hive.ql.io.RCFileInputFormat at
* org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:102) at
* org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:54) at
* tlabs.CDR_In_Report.createHCatInputFormat(CDR_In_Report.java:322) at
* tlabs.CDR_Out_Report.main(CDR_Out_Report.java:380) at
* sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
* sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at
* sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at
* java.lang.reflect.Method.invoke(Method.java:622) at
* org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:383)
*/
@Test
public void testPlanWithExternalClass() throws CompilerException, ProgramInvocationException {
final Boolean callme[] = {
false
}; // create a final object reference, to be able to change its val later
try {
String[] parameters = {
getTestJarPath(), "-c", TEST_JAR_CLASSLOADERTEST_CLASS, "some", "program"
};
CommandLine line =
new PosixParser()
.parse(CliFrontend.getProgramSpecificOptions(new Options()), parameters, false);
CliFrontend frontend = new CliFrontend();
Object result = frontend.buildProgram(line);
assertTrue(result instanceof PackagedProgram);
PackagedProgram prog = spy((PackagedProgram) result);
ClassLoader testClassLoader =
new ClassLoader(prog.getUserCodeClassLoader()) {
@Override
public Class<?> loadClass(String name) throws ClassNotFoundException {
assertTrue(name.equals("org.apache.hadoop.hive.ql.io.RCFileInputFormat"));
callme[0] = true;
return String.class; // Intentionally return the wrong class.
}
};
when(prog.getUserCodeClassLoader()).thenReturn(testClassLoader);
Assert.assertArrayEquals(new String[] {"some", "program"}, prog.getArguments());
Assert.assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, prog.getMainClassName());
Configuration c = new Configuration();
c.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "devil");
Client cli = new Client(c, getClass().getClassLoader());
cli.getOptimizedPlanAsJson(prog, 666);
} catch (ProgramInvocationException pie) {
assertTrue("Classloader was not called", callme[0]);
// class not found exception is expected as some point
if (!(pie.getCause() instanceof ClassNotFoundException)) {
System.err.println(pie.getMessage());
pie.printStackTrace();
fail("Program caused an exception: " + pie.getMessage());
}
} catch (Exception e) {
assertTrue("Classloader was not called", callme[0]);
System.err.println(e.getMessage());
e.printStackTrace();
fail("Program caused an exception: " + e.getMessage());
}
}
@Override
public void open() {
out = new ByteArrayOutputStream();
flinkConf = new org.apache.flink.configuration.Configuration();
Properties intpProperty = getProperty();
for (Object k : intpProperty.keySet()) {
String key = (String) k;
String val = toString(intpProperty.get(key));
flinkConf.setString(key, val);
}
if (localMode()) {
startFlinkMiniCluster();
}
flinkIloop =
new FlinkILoop(
getHost(), getPort(), flinkConf, (BufferedReader) null, new PrintWriter(out));
flinkIloop.settings_$eq(createSettings());
flinkIloop.createInterpreter();
imain = flinkIloop.intp();
org.apache.flink.api.scala.ExecutionEnvironment benv = flinkIloop.scalaBenv();
// new ExecutionEnvironment(remoteBenv)
org.apache.flink.streaming.api.scala.StreamExecutionEnvironment senv = flinkIloop.scalaSenv();
senv.getConfig().disableSysoutLogging();
benv.getConfig().disableSysoutLogging();
// prepare bindings
imain.interpret("@transient var _binder = new java.util.HashMap[String, Object]()");
Map<String, Object> binder = (Map<String, Object>) getLastObject();
// import libraries
imain.interpret("import scala.tools.nsc.io._");
imain.interpret("import Properties.userHome");
imain.interpret("import scala.compat.Platform.EOL");
imain.interpret("import org.apache.flink.api.scala._");
imain.interpret("import org.apache.flink.api.common.functions._");
binder.put("benv", benv);
imain.interpret(
"val benv = _binder.get(\"benv\").asInstanceOf[" + benv.getClass().getName() + "]");
binder.put("senv", senv);
imain.interpret(
"val senv = _binder.get(\"senv\").asInstanceOf[" + senv.getClass().getName() + "]");
}
private Plan getTestPlanRightStatic(String strategy) {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(DEFAULT_PARALLELISM);
DataSet<Tuple3<Long, Long, Long>> bigInput =
env.readCsvFile("file://bigFile").types(Long.class, Long.class, Long.class).name("bigFile");
DataSet<Tuple3<Long, Long, Long>> smallInput =
env.readCsvFile("file://smallFile")
.types(Long.class, Long.class, Long.class)
.name("smallFile");
IterativeDataSet<Tuple3<Long, Long, Long>> iteration = bigInput.iterate(10);
Configuration joinStrategy = new Configuration();
joinStrategy.setString(
Optimizer.HINT_SHIP_STRATEGY, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH);
if (!strategy.equals("")) {
joinStrategy.setString(Optimizer.HINT_LOCAL_STRATEGY, strategy);
}
DataSet<Tuple3<Long, Long, Long>> inner =
iteration
.join(smallInput)
.where(0)
.equalTo(0)
.with(new DummyJoiner())
.name("DummyJoiner")
.withParameters(joinStrategy);
DataSet<Tuple3<Long, Long, Long>> output = iteration.closeWith(inner);
output.output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
return env.createProgramPlan();
}
/**
* Package internal method to get a Flink {@link JobID} from a Storm topology name.
*
* @param id The Storm topology name.
* @return Flink's internally used {@link JobID}.
*/
JobID getTopologyJobId(final String id) {
final Configuration configuration = GlobalConfiguration.getConfiguration();
if (this.timeout != null) {
configuration.setString(ConfigConstants.AKKA_ASK_TIMEOUT, this.timeout);
}
try {
final ActorRef jobManager = this.getJobManager();
final FiniteDuration askTimeout = this.getTimeout();
final Future<Object> response =
Patterns.ask(
jobManager,
JobManagerMessages.getRequestRunningJobsStatus(),
new Timeout(askTimeout));
Object result;
try {
result = Await.result(response, askTimeout);
} catch (final Exception e) {
throw new RuntimeException("Could not retrieve running jobs from the JobManager", e);
}
if (result instanceof RunningJobsStatus) {
final List<JobStatusMessage> jobs = ((RunningJobsStatus) result).getStatusMessages();
for (final JobStatusMessage status : jobs) {
if (status.getJobName().equals(id)) {
return status.getJobId();
}
}
} else {
throw new RuntimeException(
"ReqeustRunningJobs requires a response of type "
+ "RunningJobs. Instead the response is of type "
+ result.getClass()
+ ".");
}
} catch (final IOException e) {
throw new RuntimeException(
"Could not connect to Flink JobManager with address "
+ this.jobManagerHost
+ ":"
+ this.jobManagerPort,
e);
}
return null;
}
private static JMXReporter startJmxReporter(Configuration config) {
JMXReporter reporter = null;
try {
Configuration reporterConfig = new Configuration();
String portRange = config.getString(KEY_METRICS_JMX_PORT, null);
if (portRange != null) {
reporterConfig.setString(KEY_METRICS_JMX_PORT, portRange);
}
reporter = new JMXReporter();
reporter.open(reporterConfig);
} catch (Exception e) {
LOG.error("Failed to instantiate JMX reporter.", e);
} finally {
return reporter;
}
}
@BeforeClass
public static void startCluster() {
try {
Configuration config = new Configuration();
config.setInteger(
ConfigConstants.LOCAL_INSTANCE_MANAGER_NUMBER_TASK_MANAGER, NUM_TASK_MANAGERS);
config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, NUM_TASK_SLOTS);
config.setString(ConfigConstants.DEFAULT_EXECUTION_RETRY_DELAY_KEY, "0 ms");
config.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 12);
cluster = new ForkableFlinkMiniCluster(config, false);
} catch (Exception e) {
e.printStackTrace();
fail("Failed to start test cluster: " + e.getMessage());
}
}
private static StreamTask<?, ?> createMockTask() {
Configuration configuration = new Configuration();
configuration.setString(ConfigConstants.STATE_BACKEND, "jobmanager");
StreamTask<?, ?> task = mock(StreamTask.class);
when(task.getAccumulatorMap()).thenReturn(new HashMap<String, Accumulator<?, ?>>());
when(task.getName()).thenReturn("Test task name");
when(task.getExecutionConfig()).thenReturn(new ExecutionConfig());
final TaskManagerRuntimeInfo mockTaskManagerRuntimeInfo = mock(TaskManagerRuntimeInfo.class);
when(mockTaskManagerRuntimeInfo.getConfiguration()).thenReturn(configuration);
final Environment env = mock(Environment.class);
when(env.getTaskInfo()).thenReturn(new TaskInfo("Test task name", 1, 0, 1, 0));
when(env.getUserClassLoader())
.thenReturn(AggregatingAlignedProcessingTimeWindowOperatorTest.class.getClassLoader());
when(env.getMetricGroup()).thenReturn(new UnregisteredTaskMetricsGroup());
when(task.getEnvironment()).thenReturn(env);
return task;
}
private Plan getTestPlanLeftStatic(String strategy) {
ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(DEFAULT_PARALLELISM);
@SuppressWarnings("unchecked")
DataSet<Tuple3<Long, Long, Long>> bigInput =
env.fromElements(
new Tuple3<Long, Long, Long>(1L, 2L, 3L),
new Tuple3<Long, Long, Long>(1L, 2L, 3L),
new Tuple3<Long, Long, Long>(1L, 2L, 3L))
.name("Big");
@SuppressWarnings("unchecked")
DataSet<Tuple3<Long, Long, Long>> smallInput =
env.fromElements(new Tuple3<Long, Long, Long>(1L, 2L, 3L)).name("Small");
IterativeDataSet<Tuple3<Long, Long, Long>> iteration = bigInput.iterate(10);
Configuration joinStrategy = new Configuration();
joinStrategy.setString(Optimizer.HINT_LOCAL_STRATEGY, strategy);
DataSet<Tuple3<Long, Long, Long>> inner =
smallInput
.join(iteration)
.where(0)
.equalTo(0)
.with(new DummyJoiner())
.name("DummyJoiner")
.withParameters(joinStrategy);
DataSet<Tuple3<Long, Long, Long>> output = iteration.closeWith(inner);
output.output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
return env.createProgramPlan();
}
@Test
public void testStartupWhenNetworkStackFailsToInitialize() {
ServerSocket blocker = null;
try {
blocker = new ServerSocket(0, 50, InetAddress.getByName("localhost"));
final Configuration cfg = new Configuration();
cfg.setString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, "localhost");
cfg.setInteger(ConfigConstants.TASK_MANAGER_DATA_PORT_KEY, blocker.getLocalPort());
cfg.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 1);
new JavaTestKit(actorSystem) {
{
ActorRef taskManager = null;
ActorRef jobManager = null;
ActorRef resourceManager = null;
try {
// a simple JobManager
jobManager = startJobManager(config);
resourceManager = startResourceManager(config, jobManager);
// start a task manager with a configuration that provides a blocked port
taskManager =
TaskManager.startTaskManagerComponentsAndActor(
cfg,
ResourceID.generate(),
actorSystem,
"localhost",
NONE_STRING, // no actor name -> random
new Some<LeaderRetrievalService>(
new StandaloneLeaderRetrievalService(jobManager.path().toString())),
false, // init network stack !!!
TaskManager.class);
watch(taskManager);
expectTerminated(timeout, taskManager);
} catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
} finally {
stopActor(taskManager);
stopActor(jobManager);
}
}
};
} catch (Exception e) {
// does not work, skip test
e.printStackTrace();
fail(e.getMessage());
} finally {
if (blocker != null) {
try {
blocker.close();
} catch (IOException e) {
// ignore, best effort
}
}
}
}
protected void internalNonBlockingStart() throws IOException {
Plan plan = env.createProgramPlan();
Optimizer optimizer =
new Optimizer(new DataStatistics(), new org.apache.flink.configuration.Configuration());
OptimizedPlan optimizedPlan = optimizer.compile(plan);
final JobGraph jobGraph = new JobGraphGenerator().compileJobGraph(optimizedPlan);
for (String jarPath : classPath) {
jobGraph.addJar(new Path(jarPath));
}
jobID = jobGraph.getJobID();
accumulatorCache.setJobID(jobID);
if (isLocalExecution()) {
flowStep.logInfo("Executing in local mode.");
startLocalCluster();
org.apache.flink.configuration.Configuration config =
new org.apache.flink.configuration.Configuration();
config.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, localCluster.hostname());
client = new Client(config);
client.setPrintStatusDuringExecution(env.getConfig().isSysoutLoggingEnabled());
} else {
flowStep.logInfo("Executing in cluster mode.");
try {
String path =
this.getClass().getProtectionDomain().getCodeSource().getLocation().toURI().getPath();
jobGraph.addJar(new Path(path));
classPath.add(path);
} catch (URISyntaxException e) {
throw new IOException("Could not add the submission JAR as a dependency.");
}
client = ((ContextEnvironment) env).getClient();
}
List<URL> fileList = new ArrayList<URL>(classPath.size());
for (String path : classPath) {
URL url;
try {
url = new URL(path);
} catch (MalformedURLException e) {
url = new URL("file://" + path);
}
fileList.add(url);
}
final ClassLoader loader =
JobWithJars.buildUserCodeClassLoader(
fileList, Collections.<URL>emptyList(), getClass().getClassLoader());
accumulatorCache.setClient(client);
final Callable<JobSubmissionResult> callable =
new Callable<JobSubmissionResult>() {
@Override
public JobSubmissionResult call() throws Exception {
return client.runBlocking(jobGraph, loader);
}
};
jobSubmission = executorService.submit(callable);
flowStep.logInfo("submitted Flink job: " + jobID);
}
/**
* Entry point for the program.
*
* @param args arguments from the command line
* @throws IOException
*/
@SuppressWarnings("static-access")
public static void main(String[] args) throws IOException {
Option configDirOpt =
OptionBuilder.withArgName("config directory")
.hasArg()
.withDescription("Specify configuration directory.")
.create("configDir");
// tempDir option is used by the YARN client.
Option tempDir =
OptionBuilder.withArgName("temporary directory (overwrites configured option)")
.hasArg()
.withDescription("Specify temporary directory.")
.create(ARG_CONF_DIR);
configDirOpt.setRequired(true);
tempDir.setRequired(false);
Options options = new Options();
options.addOption(configDirOpt);
options.addOption(tempDir);
CommandLineParser parser = new GnuParser();
CommandLine line = null;
try {
line = parser.parse(options, args);
} catch (ParseException e) {
System.err.println("CLI Parsing failed. Reason: " + e.getMessage());
System.exit(FAILURE_RETURN_CODE);
}
String configDir = line.getOptionValue(configDirOpt.getOpt(), null);
String tempDirVal = line.getOptionValue(tempDir.getOpt(), null);
// First, try to load global configuration
GlobalConfiguration.loadConfiguration(configDir);
if (tempDirVal != null // the YARN TM runner has set a value for the temp dir
// the configuration does not contain a temp direcory
&& GlobalConfiguration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, null) == null) {
Configuration c = GlobalConfiguration.getConfiguration();
c.setString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY, tempDirVal);
LOG.info("Setting temporary directory to " + tempDirVal);
GlobalConfiguration.includeConfiguration(c);
}
// print some startup environment info, like user, code revision, etc
EnvironmentInformation.logEnvironmentInfo(LOG, "TaskManager");
// Create a new task manager object
try {
new TaskManager(ExecutionMode.CLUSTER);
} catch (Exception e) {
LOG.fatal("Taskmanager startup failed: " + e.getMessage(), e);
System.exit(FAILURE_RETURN_CODE);
}
// park the main thread to keep the JVM alive (all other threads may be daemon threads)
Object mon = new Object();
synchronized (mon) {
try {
mon.wait();
} catch (InterruptedException ex) {
}
}
}
