public void shutdown() {
for (DataQueue q : topology.getAllDataQueue()) {
for (DataBolt bolt : q.bolts) {
bolt.shutdown();
}
}
workPool.shutdownNow();
}
/** 等待所有的Spout结束，用在分析单个文件的时候。等待所有日志处理完成。 */
public void waitAllSpoutDone() {
while (fileSpout != null && !fileSpout.isClosed()) {
synchronized (fileSpout) {
try {
fileSpout.wait(1000);
} catch (InterruptedException e) {
}
}
}
for (DataQueue q : topology.getAllDataQueue()) {
while (q.status == DataQueue.RUNNING) {
synchronized (q) {
try {
q.wait(1000);
} catch (InterruptedException e) {
}
}
}
}
}
/** 服务开始启动， 0. 初始化Redis数据库连接。 1. 加载所有blot，初始化状态数据。 2. 加载spout 开始处理数据 */
public void start() {
int coreWrokerSize = Settings.getInt(Settings.CORE_WORKER_SIZE, 10);
int queueSize = Settings.getInt(Settings.WRITE_LOG_QUEUE_SIZE, 1024);
taskQueue = new LinkedBlockingDeque<Runnable>(queueSize);
log.debug(
"start message process thread pool, core size:"
+ coreWrokerSize
+ ", queue size:"
+ queueSize);
workPool =
new ThreadPoolExecutor(coreWrokerSize, coreWrokerSize * 2, 10, TimeUnit.SECONDS, taskQueue);
String appKey = Settings.getString(Settings.TAODIAN_APPID, null); // System.getProperty("");
String appSecret = Settings.getString(Settings.TAODIAN_APPSECRET, null);
String appRoute = Settings.getString(Settings.TAODIAN_APPROUTE, "http://api.zaol.cn/api/route");
boolean inSAE = Settings.getString("in_sae", "n").equals("y");
if (appKey != null && appSecret != null) {
api = new TaodianApi(appKey, appSecret, appRoute, inSAE ? "simple" : "apache");
} else {
log.info("The taodian.api_id and taodian.api_secret Java properties are required.");
System.exit(255);
}
ds = new DataService();
if (!ds.start(workPool, api)) {
log.error("Data service start failed");
System.exit(255);
}
topology = new DefaultSimpleTopology(workPool);
TopologyBuilder builder = null; // new SimpleTopologyBuilder();
String topolgyName = Settings.getString(Settings.TOPOLOGY, "default_topology.cfg");
File f = new File(topolgyName);
InputStream ins = null;
if (f.isFile()) {
log.info("Load topology from file, " + f.getAbsolutePath());
try {
ins = new FileInputStream(f);
} catch (FileNotFoundException e) {
log.error(e, e);
}
} else {
ins = this.getClass().getClassLoader().getResourceAsStream(topolgyName);
}
if (ins != null) {
builder = new SimpleTopologyBuilder(ins);
}
builder.buildToplogy(topology, new ObjectFactory());
if (fileSpout != null) {
topology.setSpout(Settings.INPUT_SPOUT, fileSpout);
} else {
String clickGate = Settings.getString(Settings.CLICK_GATE_LOG_URL, "");
HTTPURLSpout spout = new HTTPURLSpout();
if (spout.connect(clickGate)) {
topology.setSpout(Settings.INPUT_SPOUT, spout);
}
}
topology.start();
}