/** * Load the topology config * * @param topologyPackage, tar ball containing user submitted jar/tar, defn and config * @param topologyBinaryFile, name of the user submitted topology jar/tar/pex file * @param topology, proto in memory version of topology definition * @return config, the topology config */ protected static Config topologyConfigs( String topologyPackage, String topologyBinaryFile, String topologyDefnFile, TopologyAPI.Topology topology) { PackageType packageType = PackageType.getPackageType(topologyBinaryFile); Config config = Config.newBuilder() .put(Keys.topologyId(), topology.getId()) .put(Keys.topologyName(), topology.getName()) .put(Keys.topologyDefinitionFile(), topologyDefnFile) .put(Keys.topologyPackageFile(), topologyPackage) .put(Keys.topologyBinaryFile(), topologyBinaryFile) .put(Keys.topologyPackageType(), packageType) .build(); return config; }
public ExecutionEnvironment.ExecutionState createExecutionState() { String releaseUsername = Context.buildUser(config); // TODO(mfu): Currently we leave release tag empty String releaseTag = ""; String releaseVersion = Context.buildVersion(config); TopologyAPI.Topology topology = Runtime.topology(runtime); ExecutionEnvironment.ExecutionState.Builder builder = ExecutionEnvironment.ExecutionState.newBuilder(); // set the topology name, id, submitting user and time builder .setTopologyName(topology.getName()) .setTopologyId(topology.getId()) .setSubmissionTime(System.currentTimeMillis() / 1000) .setSubmissionUser(System.getProperty("user.name")) .setCluster(Context.cluster(config)) .setRole(Context.role(config)) .setEnviron(Context.environ(config)); // build the heron release state ExecutionEnvironment.HeronReleaseState.Builder releaseBuilder = ExecutionEnvironment.HeronReleaseState.newBuilder(); releaseBuilder.setReleaseUsername(releaseUsername); releaseBuilder.setReleaseTag(releaseTag); releaseBuilder.setReleaseVersion(releaseVersion); builder.setReleaseState(releaseBuilder); if (builder.isInitialized()) { return builder.build(); } else { throw new RuntimeException("Failed to create execution state"); } }
/** * Trim the topology definition for storing into state manager. This is because the user generated * spouts and bolts might be huge. * * @return trimmed topology */ public TopologyAPI.Topology trimTopology(TopologyAPI.Topology topology) { // create a copy of the topology physical plan TopologyAPI.Topology.Builder builder = TopologyAPI.Topology.newBuilder().mergeFrom(topology); // clear the state of user spout java objects - which can be potentially huge for (TopologyAPI.Spout.Builder spout : builder.getSpoutsBuilderList()) { spout.getCompBuilder().clearJavaObject(); } // clear the state of user spout java objects - which can be potentially huge for (TopologyAPI.Bolt.Builder bolt : builder.getBoltsBuilderList()) { bolt.getCompBuilder().clearJavaObject(); } return builder.build(); }
/** * Submit a topology 1. Instantiate necessary resources 2. Valid whether it is legal to submit a * topology 3. Call LauncherRunner */ public void submitTopology() throws TopologySubmissionException { // 1. Do prepare work // create an instance of state manager String statemgrClass = Context.stateManagerClass(config); IStateManager statemgr; // Create an instance of the launcher class String launcherClass = Context.launcherClass(config); ILauncher launcher; // create an instance of the uploader class String uploaderClass = Context.uploaderClass(config); IUploader uploader; // create an instance of state manager try { statemgr = ReflectionUtils.newInstance(statemgrClass); } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { throw new TopologySubmissionException( String.format("Failed to instantiate state manager class '%s'", statemgrClass), e); } // create an instance of launcher try { launcher = ReflectionUtils.newInstance(launcherClass); } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { throw new LauncherException( String.format("Failed to instantiate launcher class '%s'", launcherClass), e); } // create an instance of uploader try { uploader = ReflectionUtils.newInstance(uploaderClass); } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { throw new UploaderException( String.format("Failed to instantiate uploader class '%s'", uploaderClass), e); } // Put it in a try block so that we can always clean resources try { // initialize the state manager statemgr.initialize(config); // TODO(mfu): timeout should read from config SchedulerStateManagerAdaptor adaptor = new SchedulerStateManagerAdaptor(statemgr, 5000); validateSubmit(adaptor, topology.getName()); // 2. Try to submit topology if valid // invoke method to submit the topology LOG.log(Level.FINE, "Topology {0} to be submitted", topology.getName()); // Firstly, try to upload necessary packages URI packageURI = uploadPackage(uploader); // Secondly, try to submit a topology // build the runtime config Config runtime = Config.newBuilder() .putAll(LauncherUtils.getInstance().getPrimaryRuntime(topology, adaptor)) .put(Keys.topologyPackageUri(), packageURI) .put(Keys.launcherClassInstance(), launcher) .build(); callLauncherRunner(runtime); } catch (LauncherException | PackingException e) { // we undo uploading of topology package only if launcher fails to // launch topology, which will throw LauncherException or PackingException uploader.undo(); throw e; } finally { SysUtils.closeIgnoringExceptions(uploader); SysUtils.closeIgnoringExceptions(launcher); SysUtils.closeIgnoringExceptions(statemgr); } }
public static void main(String[] args) throws Exception { Options options = constructOptions(); Options helpOptions = constructHelpOptions(); CommandLineParser parser = new DefaultParser(); // parse the help options first. CommandLine cmd = parser.parse(helpOptions, args, true); if (cmd.hasOption("h")) { usage(options); return; } try { // Now parse the required options cmd = parser.parse(options, args); } catch (ParseException e) { usage(options); throw new RuntimeException("Error parsing command line options: ", e); } Boolean verbose = false; Level logLevel = Level.INFO; if (cmd.hasOption("v")) { logLevel = Level.ALL; verbose = true; } // init log LoggingHelper.loggerInit(logLevel, false); String cluster = cmd.getOptionValue("cluster"); String role = cmd.getOptionValue("role"); String environ = cmd.getOptionValue("environment"); String heronHome = cmd.getOptionValue("heron_home"); String configPath = cmd.getOptionValue("config_path"); String overrideConfigFile = cmd.getOptionValue("override_config_file"); String releaseFile = cmd.getOptionValue("release_file"); String topologyPackage = cmd.getOptionValue("topology_package"); String topologyDefnFile = cmd.getOptionValue("topology_defn"); String topologyBinaryFile = cmd.getOptionValue("topology_bin"); // load the topology definition into topology proto TopologyAPI.Topology topology = TopologyUtils.getTopology(topologyDefnFile); // first load the defaults, then the config from files to override it // next add config parameters from the command line // load the topology configs // build the final config by expanding all the variables Config config = Config.expand( Config.newBuilder() .putAll(defaultConfigs(heronHome, configPath, releaseFile)) .putAll(overrideConfigs(overrideConfigFile)) .putAll(commandLineConfigs(cluster, role, environ, verbose)) .putAll( topologyConfigs( topologyPackage, topologyBinaryFile, topologyDefnFile, topology)) .build()); LOG.fine("Static config loaded successfully"); LOG.fine(config.toString()); SubmitterMain submitterMain = new SubmitterMain(config, topology); try { submitterMain.submitTopology(); // SUPPRESS CHECKSTYLE IllegalCatch } catch (Exception e) { /* Since only stderr is used (by logging), we use stdout here to propagate error message back to Python's executor.py (invoke site). */ LOG.log(Level.FINE, "Exception when submitting topology", e); System.out.println(e.getMessage()); /* Meaning of exit status code: - status code = 0: program exits without error - 0 < status code < 100: program fails to execute before program execution. For example, JVM cannot find or load main class - status code >= 100: program fails to launch after program execution. For example, topology definition file fails to be loaded */ // Exit with status code 100 to indicate that error has happened on user-land // SUPPRESS CHECKSTYLE RegexpSinglelineJava System.exit(100); } LOG.log(Level.FINE, "Topology {0} submitted successfully", topology.getName()); }