private backtype.storm.Config getStormConfig(com.typesafe.config.Config config) {
backtype.storm.Config conf = new backtype.storm.Config();
conf.put(RichSpoutBatchExecutor.MAX_BATCH_SIZE_CONF, Int.box(64 * 1024));
conf.put(backtype.storm.Config.TOPOLOGY_RECEIVER_BUFFER_SIZE, Int.box(8));
conf.put(backtype.storm.Config.TOPOLOGY_TRANSFER_BUFFER_SIZE, Int.box(32));
conf.put(backtype.storm.Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, Int.box(16384));
conf.put(backtype.storm.Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, Int.box(16384));
conf.put(backtype.storm.Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE, Int.box(20480000));
String nimbusHost = STORM_NIMBUS_HOST_DEFAULT;
if (environment.config().hasPath(STORM_NIMBUS_HOST_CONF_PATH)) {
nimbusHost = environment.config().getString(STORM_NIMBUS_HOST_CONF_PATH);
LOG.info("Overriding {} = {}", STORM_NIMBUS_HOST_CONF_PATH, nimbusHost);
} else {
LOG.info("Using default {} = {}", STORM_NIMBUS_HOST_CONF_PATH, STORM_NIMBUS_HOST_DEFAULT);
}
Integer nimbusThriftPort = STORM_NIMBUS_THRIFT_DEFAULT;
if (environment.config().hasPath(STORM_NIMBUS_THRIFT_CONF_PATH)) {
nimbusThriftPort = environment.config().getInt(STORM_NIMBUS_THRIFT_CONF_PATH);
LOG.info("Overriding {} = {}", STORM_NIMBUS_THRIFT_CONF_PATH, nimbusThriftPort);
} else {
LOG.info("Using default {} = {}", STORM_NIMBUS_THRIFT_CONF_PATH, STORM_NIMBUS_THRIFT_DEFAULT);
}
conf.put(backtype.storm.Config.NIMBUS_HOST, nimbusHost);
conf.put(backtype.storm.Config.NIMBUS_THRIFT_PORT, nimbusThriftPort);
conf.put(
Config.STORM_THRIFT_TRANSPORT_PLUGIN, "backtype.storm.security.auth.SimpleTransportPlugin");
if (config.hasPath(WORKERS)) {
conf.setNumWorkers(config.getInt(WORKERS));
}
if (config.hasPath(TOPOLOGY_MESSAGE_TIMEOUT_SECS)) {
conf.put(TOPOLOGY_MESSAGE_TIMEOUT_SECS, config.getInt(TOPOLOGY_MESSAGE_TIMEOUT_SECS));
}
return conf;
}
@Override
public ApplicationEntity.Status status(
Application<StormEnvironment, StormTopology> executor, com.typesafe.config.Config config) {
String appId = config.getString("appId");
LOG.info("Fetching {} status", appId);
List<TopologySummary> topologySummaries;
ApplicationEntity.Status status = null;
try {
if (Objects.equals(config.getString("mode"), ApplicationEntity.Mode.CLUSTER.name())) {
Nimbus.Client stormClient =
NimbusClient.getConfiguredClient(getStormConfig(config)).getClient();
topologySummaries = stormClient.getClusterInfo().get_topologies();
} else {
topologySummaries = getLocalCluster().getClusterInfo().get_topologies();
}
for (TopologySummary topologySummary : topologySummaries) {
if (topologySummary.get_name().equalsIgnoreCase(appId)) {
if (topologySummary.get_status().equalsIgnoreCase("ACTIVE")) {
status = ApplicationEntity.Status.RUNNING;
} else if (topologySummary.get_status().equalsIgnoreCase("INACTIVE")) {
status = ApplicationEntity.Status.STOPPED;
} else if (topologySummary.get_status().equalsIgnoreCase("KILLED")) {
status = ApplicationEntity.Status.STOPPED;
} else {
LOG.error(
"Unknown storm topology ({}) status: {}",
topologySummary.get_status(),
topologySummary.get_status());
}
}
}
// If not exist, return removed
if (status == null) {
status = ApplicationEntity.Status.REMOVED;
}
} catch (TException e) {
LOG.error("Got error to fetch status of {}", appId, e);
status = ApplicationEntity.Status.UNKNOWN;
}
LOG.info("{} status is {}", appId, status);
return status;
}
@Override
public void stop(
Application<StormEnvironment, StormTopology> executor, com.typesafe.config.Config config) {
String appId = config.getString("appId");
LOG.info("Stopping topology {} ...", appId);
if (Objects.equals(config.getString("mode"), ApplicationEntity.Mode.CLUSTER.name())) {
Nimbus.Client stormClient =
NimbusClient.getConfiguredClient(getStormConfig(config)).getClient();
try {
stormClient.killTopologyWithOpts(appId, this.killOptions);
} catch (NotAliveException | TException e) {
LOG.error(
"Failed to kill topology named {}, due to: {}", appId, e.getMessage(), e.getCause());
throw new RuntimeException(e.getMessage(), e);
}
} else {
getLocalCluster().killTopologyWithOpts(appId, this.killOptions);
}
LOG.info("Stopped topology {}", appId);
}
@Override
public void start(
Application<StormEnvironment, StormTopology> executor, com.typesafe.config.Config config) {
String topologyName = config.getString("appId");
Preconditions.checkNotNull(
topologyName, "[appId] is required by null for " + executor.getClass().getCanonicalName());
StormTopology topology = executor.execute(config, environment);
LOG.info(
"Starting {} ({}), mode: {}",
topologyName,
executor.getClass().getCanonicalName(),
config.getString("mode"));
Config conf = getStormConfig(config);
if (ApplicationEntity.Mode.CLUSTER.name().equalsIgnoreCase(config.getString("mode"))) {
String jarFile = config.hasPath("jarPath") ? config.getString("jarPath") : null;
if (jarFile == null) {
jarFile = DynamicJarPathFinder.findPath(executor.getClass());
}
synchronized (StormExecutionRuntime.class) {
System.setProperty("storm.jar", jarFile);
LOG.info("Submitting as cluster mode ...");
try {
StormSubmitter.submitTopologyWithProgressBar(topologyName, conf, topology);
} catch (AlreadyAliveException | InvalidTopologyException e) {
LOG.error(e.getMessage(), e);
throw new RuntimeException(e.getMessage(), e);
} finally {
System.clearProperty("storm.jar");
}
}
} else {
LOG.info("Submitting as local mode ...");
getLocalCluster().submitTopology(topologyName, conf, topology);
LOG.info("Submitted");
}
LOG.info("Started {} ({})", topologyName, executor.getClass().getCanonicalName());
}