public TaskHeartbeatTrigger(
Map conf,
String name,
DisruptorQueue queue,
BlockingQueue<Object> controlQueue,
int taskId,
String componentId,
TopologyContext sysTopologyCtx,
ITaskReportErr reportError) {
this.name = name;
this.queue = queue;
this.controlQueue = controlQueue;
this.opCode = TimerConstants.TASK_HEARTBEAT;
this.taskId = taskId;
this.componentId = componentId;
this.sysTopologyCtx = sysTopologyCtx;
this.frequence = JStormUtils.parseInt(conf.get(Config.TASK_HEARTBEAT_FREQUENCY_SECS), 10);
this.firstTime = frequence;
this.executeThreadHbTime = TimeUtils.current_time_secs();
this.taskHbTimeout = JStormUtils.parseInt(conf.get(Config.NIMBUS_TASK_TIMEOUT_SECS), 180);
this.intervalCheck = new IntervalCheck();
this.intervalCheck.setInterval(taskHbTimeout);
this.intervalCheck.start();
this.reportError = reportError;
this.uptime = new UptimeComputer();
}
private int computeWorkerNum() {
Integer settingNum = JStormUtils.parseInt(stormConf.get(Config.TOPOLOGY_WORKERS));
int hintSum = 0;
Map<String, Object> components = ThriftTopologyUtils.getComponents(sysTopology);
for (Entry<String, Object> entry : components.entrySet()) {
String componentName = entry.getKey();
Object component = entry.getValue();
ComponentCommon common = null;
if (component instanceof Bolt) {
common = ((Bolt) component).get_common();
}
if (component instanceof SpoutSpec) {
common = ((SpoutSpec) component).get_common();
}
if (component instanceof StateSpoutSpec) {
common = ((StateSpoutSpec) component).get_common();
}
int hint = common.get_parallelism_hint();
hintSum += hint;
}
if (settingNum == null) {
return hintSum;
} else {
return Math.min(settingNum, hintSum);
}
}
@SuppressWarnings({"rawtypes", "unchecked"})
public Integer mkTaskMaker(
Map<Object, Object> stormConf,
Map<String, ?> cidSpec,
Map<Integer, String> rtn,
Integer cnt) {
if (cidSpec == null) {
LOG.warn("Component map is empty");
return cnt;
}
Set<?> entrySet = cidSpec.entrySet();
for (Iterator<?> it = entrySet.iterator(); it.hasNext(); ) {
Entry entry = (Entry) it.next();
Object obj = entry.getValue();
ComponentCommon common = null;
if (obj instanceof Bolt) {
common = ((Bolt) obj).get_common();
} else if (obj instanceof SpoutSpec) {
common = ((SpoutSpec) obj).get_common();
} else if (obj instanceof StateSpoutSpec) {
common = ((StateSpoutSpec) obj).get_common();
}
if (common == null) {
throw new RuntimeException("No ComponentCommon of " + entry.getKey());
}
int declared = Thrift.parallelismHint(common);
Integer parallelism = declared;
// Map tmp = (Map) Utils_clj.from_json(common.get_json_conf());
Map newStormConf = new HashMap(stormConf);
// newStormConf.putAll(tmp);
Integer maxParallelism =
JStormUtils.parseInt(newStormConf.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM));
if (maxParallelism != null) {
parallelism = Math.min(maxParallelism, declared);
}
for (int i = 0; i < parallelism; i++) {
cnt++;
rtn.put(cnt, (String) entry.getKey());
}
}
return cnt;
}
public BoltCollector(
int message_timeout_secs,
ITaskReportErr report_error,
TaskSendTargets _send_fn,
Map _storm_conf,
TaskTransfer _transfer_fn,
TopologyContext _topology_context,
Integer task_id,
RotatingMap<Tuple, Long> tuple_start_times,
CommonStatsRolling _task_stats) {
this.rotateTime = 1000L * message_timeout_secs / (Acker.TIMEOUT_BUCKET_NUM - 1);
this.reportError = report_error;
this.sendTargets = _send_fn;
this.storm_conf = _storm_conf;
this.taskTransfer = _transfer_fn;
this.topologyContext = _topology_context;
this.task_id = task_id;
this.task_stats = _task_stats;
this.pending_acks = new RotatingMap<Tuple, Long>(Acker.TIMEOUT_BUCKET_NUM);
// this.pending_acks = new TimeCacheMap<Tuple,
// Long>(message_timeout_secs,
// Acker.TIMEOUT_BUCKET_NUM);
this.tuple_start_times = tuple_start_times;
this.ackerNum = JStormUtils.parseInt(storm_conf.get(Config.TOPOLOGY_ACKER_EXECUTORS));
String componentId = topologyContext.getThisComponentId();
timer =
Metrics.registerTimer(
JStormServerUtils.getName(componentId, task_id),
MetricDef.EMIT_TIME,
String.valueOf(task_id),
Metrics.MetricType.TASK);
random = new Random();
random.setSeed(System.currentTimeMillis());
}
public SpoutCollector(
Integer task_id,
backtype.storm.spout.ISpout spout,
CommonStatsRolling task_stats,
TaskSendTargets sendTargets,
Map _storm_conf,
TaskTransfer _transfer_fn,
TimeOutMap<Long, TupleInfo> pending,
TopologyContext topology_context,
DisruptorQueue disruptorAckerQueue,
ITaskReportErr _report_error) {
this.sendTargets = sendTargets;
this.storm_conf = _storm_conf;
this.transfer_fn = _transfer_fn;
this.pending = pending;
this.topology_context = topology_context;
this.disruptorAckerQueue = disruptorAckerQueue;
this.task_stats = task_stats;
this.spout = spout;
this.task_id = task_id;
this.report_error = _report_error;
ackerNum = JStormUtils.parseInt(storm_conf.get(Config.TOPOLOGY_ACKER_EXECUTORS));
isDebug = JStormUtils.parseBoolean(storm_conf.get(Config.TOPOLOGY_DEBUG), false);
random = new Random();
random.setSeed(System.currentTimeMillis());
String componentId = topology_context.getThisComponentId();
emitTotalTimer =
Metrics.registerTimer(
JStormServerUtils.getName(componentId, task_id),
MetricDef.EMIT_TIME,
String.valueOf(task_id),
Metrics.MetricType.TASK);
}
@Override
public <T> Object execute(T... args) {
boolean isSetTaskInfo = false;
try {
Boolean reassign = (Boolean) args[1];
Map<Object, Object> conf = (Map<Object, Object>) args[2]; // args[0]:
// delay,
// args[1]:
// reassign_flag,
// args[2]:
// conf
if (conf != null) {
boolean isConfUpdate = false;
Map stormConf = data.getConf();
// Update topology code
Map topoConf = StormConfig.read_nimbus_topology_conf(stormConf, topologyid);
StormTopology rawOldTopology = StormConfig.read_nimbus_topology_code(stormConf, topologyid);
StormTopology rawNewTopology = NimbusUtils.normalizeTopology(conf, rawOldTopology, true);
StormTopology sysOldTopology = rawOldTopology.deepCopy();
StormTopology sysNewTopology = rawNewTopology.deepCopy();
if (conf.get(Config.TOPOLOGY_ACKER_EXECUTORS) != null) {
Common.add_acker(topoConf, sysOldTopology);
Common.add_acker(conf, sysNewTopology);
int ackerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS));
int oldAckerNum = JStormUtils.parseInt(topoConf.get(Config.TOPOLOGY_ACKER_EXECUTORS));
LOG.info("Update acker from oldAckerNum=" + oldAckerNum + " to ackerNum=" + ackerNum);
topoConf.put(Config.TOPOLOGY_ACKER_EXECUTORS, ackerNum);
isConfUpdate = true;
}
// If scale-out, setup task info for new added tasks
setTaskInfo(sysOldTopology, sysNewTopology);
isSetTaskInfo = true;
// If everything is OK, write topology code into disk
StormConfig.write_nimbus_topology_code(
stormConf, topologyid, Utils.serialize(rawNewTopology));
// Update topology conf if worker num has been updated
Set<Object> keys = conf.keySet();
Integer workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS));
if (workerNum != null) {
Integer oldWorkerNum = JStormUtils.parseInt(topoConf.get(Config.TOPOLOGY_WORKERS));
topoConf.put(Config.TOPOLOGY_WORKERS, workerNum);
isConfUpdate = true;
LOG.info("Update worker num from " + oldWorkerNum + " to " + workerNum);
}
if (keys.contains(Config.ISOLATION_SCHEDULER_MACHINES)) {
topoConf.put(
Config.ISOLATION_SCHEDULER_MACHINES, conf.get(Config.ISOLATION_SCHEDULER_MACHINES));
}
if (isConfUpdate) {
StormConfig.write_nimbus_topology_conf(stormConf, topologyid, topoConf);
}
}
TopologyAssignEvent event = new TopologyAssignEvent();
event.setTopologyId(topologyid);
event.setScratch(true);
event.setOldStatus(oldStatus);
event.setReassign(reassign);
if (conf != null) event.setScaleTopology(true);
TopologyAssign.push(event);
event.waitFinish();
} catch (Exception e) {
LOG.error("do-rebalance error!", e);
// Rollback the changes on ZK
if (isSetTaskInfo) {
try {
StormClusterState clusterState = data.getStormClusterState();
clusterState.remove_task(topologyid, newTasks);
} catch (Exception e1) {
LOG.error("Failed to rollback the changes on ZK for task-" + newTasks, e);
}
}
}
DelayStatusTransitionCallback delayCallback =
new DelayStatusTransitionCallback(
data, topologyid, oldStatus, StatusType.rebalancing, StatusType.done_rebalance);
return delayCallback.execute();
}
public BoltExecutors(
IBolt _bolt,
TaskTransfer _transfer_fn,
Map<Integer, DisruptorQueue> innerTaskTransfer,
Map storm_conf,
DisruptorQueue deserializeQueue,
TaskSendTargets _send_fn,
TaskStatus taskStatus,
TopologyContext sysTopologyCxt,
TopologyContext userTopologyCxt,
CommonStatsRolling _task_stats,
ITaskReportErr _report_error) {
super(
_transfer_fn,
storm_conf,
deserializeQueue,
innerTaskTransfer,
sysTopologyCxt,
userTopologyCxt,
_task_stats,
taskStatus,
_report_error);
this.bolt = _bolt;
// create TimeCacheMap
this.tuple_start_times = new RotatingMap<Tuple, Long>(Acker.TIMEOUT_BUCKET_NUM);
this.ackerNum = JStormUtils.parseInt(storm_conf.get(Config.TOPOLOGY_ACKER_EXECUTORS));
// don't use TimeoutQueue for recv_tuple_queue,
// then other place should check the queue size
// TimeCacheQueue.DefaultExpiredCallback<Tuple> logExpireCb = new
// TimeCacheQueue.DefaultExpiredCallback<Tuple>(
// idStr);
// this.recv_tuple_queue = new
// TimeCacheQueue<Tuple>(message_timeout_secs,
// TimeCacheQueue.DEFAULT_NUM_BUCKETS, logExpireCb);
// create BoltCollector
IOutputCollector output_collector =
new BoltCollector(
message_timeout_secs,
_report_error,
_send_fn,
storm_conf,
_transfer_fn,
sysTopologyCxt,
taskId,
tuple_start_times,
_task_stats);
outputCollector = new OutputCollector(output_collector);
boltExeTimer =
Metrics.registerTimer(
idStr, MetricDef.EXECUTE_TIME, String.valueOf(taskId), Metrics.MetricType.TASK);
Object tickFrequence = storm_conf.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
if (tickFrequence != null) {
Integer frequence = JStormUtils.parseInt(tickFrequence);
TickTupleTrigger tickTupleTrigger =
new TickTupleTrigger(
sysTopologyCxt, frequence, idStr + Constants.SYSTEM_TICK_STREAM_ID, exeQueue);
tickTupleTrigger.register();
}
try {
// do prepare
WorkerClassLoader.switchThreadContext();
// Method method = IBolt.class.getMethod("prepare", new Class[] {Map.class,
// TopologyContext.class,
// OutputCollector.class});
// method.invoke(bolt, new Object[] {storm_conf, userTopologyCxt, outputCollector});
bolt.prepare(storm_conf, userTopologyCtx, outputCollector);
} catch (Throwable e) {
error = e;
LOG.error("bolt prepare error ", e);
report_error.report(e);
} finally {
WorkerClassLoader.restoreThreadContext();
}
LOG.info("Successfully create BoltExecutors " + idStr);
}
public static int getCpuSlotPerWorker(Map conf) {
int slot = JStormUtils.parseInt(conf.get(CPU_SLOT_PER_WORKER), 1);
return slot > 0 ? slot : 1;
}
/**
* create and start one supervisor
*
* @param conf : configurationdefault.yaml storm.yaml
* @param sharedContext : null (right now)
* @return SupervisorManger: which is used to shutdown all workers and supervisor
*/
@SuppressWarnings("rawtypes")
public SupervisorManger mkSupervisor(Map conf, IContext sharedContext) throws Exception {
LOG.info("Starting Supervisor with conf " + conf);
active = new AtomicBoolean(true);
/** Step 1: cleanup all files in /storm-local-dir/supervisor/tmp */
String path = StormConfig.supervisorTmpDir(conf);
FileUtils.cleanDirectory(new File(path));
/*
* Step 2: create ZK operation instance StromClusterState
*/
StormClusterState stormClusterState = Cluster.mk_storm_cluster_state(conf);
/*
* Step 3, create LocalStat LocalStat is one KV database 4.1 create
* LocalState instance 4.2 get supervisorId, if no supervisorId, create
* one
*/
LocalState localState = StormConfig.supervisorState(conf);
String supervisorId = (String) localState.get(Common.LS_ID);
if (supervisorId == null) {
supervisorId = UUID.randomUUID().toString();
localState.put(Common.LS_ID, supervisorId);
}
Vector<SmartThread> threads = new Vector<SmartThread>();
// Step 5 create HeartBeat
// every supervisor.heartbeat.frequency.secs, write SupervisorInfo to ZK
String myHostName = null;
myHostName = ConfigExtension.getSupervisorHost(conf);
if (myHostName == null) {
myHostName = NetWorkUtils.hostname();
}
Heartbeat hb = new Heartbeat(conf, stormClusterState, supervisorId, myHostName, active);
hb.update();
AsyncLoopThread heartbeat = new AsyncLoopThread(hb, false, null, Thread.MIN_PRIORITY, true);
threads.add(heartbeat);
// Step 6 create and start sync Supervisor thread
// every supervisor.monitor.frequency.secs second run SyncSupervisor
EventManager processEventManager = new EventManagerImp(false);
ConcurrentHashMap<String, String> workerThreadPids = new ConcurrentHashMap<String, String>();
SyncProcessEvent syncProcessEvent =
new SyncProcessEvent(supervisorId, conf, localState, workerThreadPids, sharedContext);
EventManager syncSupEventManager = new EventManagerImp(false);
SyncSupervisorEvent syncSupervisorEvent =
new SyncSupervisorEvent(
supervisorId,
conf,
processEventManager,
syncSupEventManager,
stormClusterState,
localState,
syncProcessEvent);
int syncFrequence = JStormUtils.parseInt(conf.get(Config.SUPERVISOR_MONITOR_FREQUENCY_SECS));
EventManagerPusher syncSupervisorPusher =
new EventManagerPusher(syncSupEventManager, syncSupervisorEvent, active, syncFrequence);
AsyncLoopThread syncSupervisorThread = new AsyncLoopThread(syncSupervisorPusher);
threads.add(syncSupervisorThread);
// Step 7 start sync process thread
// every supervisor.monitor.frequency.secs run SyncProcesses
// skip thread to do syncProcess, due to nimbus will check whether
// worker is dead or not, if dead, it will reassign a new worker
//
// int syncProcessFrequence = syncFrequence/2;
// EventManagerPusher syncProcessPusher = new EventManagerPusher(
// processEventManager, syncProcessEvent, active,
// syncProcessFrequence);
// AsyncLoopThread syncProcessThread = new
// AsyncLoopThread(syncProcessPusher);
// threads.add(syncProcessThread);
// Step 7 start httpserver
Httpserver httpserver = new Httpserver(conf);
httpserver.start();
LOG.info("Starting supervisor with id " + supervisorId + " at host " + myHostName);
// SupervisorManger which can shutdown all supervisor and workers
return new SupervisorManger(
conf,
supervisorId,
active,
threads,
syncSupEventManager,
processEventManager,
httpserver,
stormClusterState,
workerThreadPids);
}
public static int getMemSlotPerTask(Map conf) {
return JStormUtils.parseInt(conf.get(MEM_SLOTS_PER_TASK), 1);
}
public static int getSpoutDelayRunSeconds(Map conf) {
return JStormUtils.parseInt(conf.get(SPOUT_DELAY_RUN), 30);
}
public static int getTopologyPortWeight(Map conf) {
return JStormUtils.parseInt(conf.get(TOPOLOGY_PORT_WEIGHT), DEFAULT_PORT_WEIGHT);
}
public static int getTopologyDiskWeight(Map conf) {
return JStormUtils.parseInt(conf.get(TOPOLOGY_DISK_WEIGHT), DEFAULT_DISK_WEIGHT);
}
public static int getCpuSlotsPerTask(Map conf) {
return JStormUtils.parseInt(conf.get(CPU_SLOTS_PER_TASK), 1);
}