Exemplo n.º 1
0
  public void shutdown() {
    for (DataQueue q : topology.getAllDataQueue()) {
      for (DataBolt bolt : q.bolts) {
        bolt.shutdown();
      }
    }

    workPool.shutdownNow();
  }
Exemplo n.º 2
0
 /** 等待所有的Spout结束,用在分析单个文件的时候。等待所有日志处理完成。 */
 public void waitAllSpoutDone() {
   while (fileSpout != null && !fileSpout.isClosed()) {
     synchronized (fileSpout) {
       try {
         fileSpout.wait(1000);
       } catch (InterruptedException e) {
       }
     }
   }
   for (DataQueue q : topology.getAllDataQueue()) {
     while (q.status == DataQueue.RUNNING) {
       synchronized (q) {
         try {
           q.wait(1000);
         } catch (InterruptedException e) {
         }
       }
     }
   }
 }
Exemplo n.º 3
0
  /** 服务开始启动, 0. 初始化Redis数据库连接。 1. 加载所有blot,初始化状态数据。 2. 加载spout 开始处理数据 */
  public void start() {
    int coreWrokerSize = Settings.getInt(Settings.CORE_WORKER_SIZE, 10);
    int queueSize = Settings.getInt(Settings.WRITE_LOG_QUEUE_SIZE, 1024);

    taskQueue = new LinkedBlockingDeque<Runnable>(queueSize);

    log.debug(
        "start message process thread pool, core size:"
            + coreWrokerSize
            + ", queue size:"
            + queueSize);
    workPool =
        new ThreadPoolExecutor(coreWrokerSize, coreWrokerSize * 2, 10, TimeUnit.SECONDS, taskQueue);

    String appKey = Settings.getString(Settings.TAODIAN_APPID, null); // System.getProperty("");
    String appSecret = Settings.getString(Settings.TAODIAN_APPSECRET, null);
    String appRoute = Settings.getString(Settings.TAODIAN_APPROUTE, "http://api.zaol.cn/api/route");
    boolean inSAE = Settings.getString("in_sae", "n").equals("y");

    if (appKey != null && appSecret != null) {
      api = new TaodianApi(appKey, appSecret, appRoute, inSAE ? "simple" : "apache");
    } else {
      log.info("The taodian.api_id and taodian.api_secret Java properties are required.");
      System.exit(255);
    }

    ds = new DataService();
    if (!ds.start(workPool, api)) {
      log.error("Data service start failed");
      System.exit(255);
    }

    topology = new DefaultSimpleTopology(workPool);
    TopologyBuilder builder = null; // new SimpleTopologyBuilder();
    String topolgyName = Settings.getString(Settings.TOPOLOGY, "default_topology.cfg");

    File f = new File(topolgyName);
    InputStream ins = null;
    if (f.isFile()) {
      log.info("Load topology from file, " + f.getAbsolutePath());
      try {
        ins = new FileInputStream(f);
      } catch (FileNotFoundException e) {
        log.error(e, e);
      }
    } else {
      ins = this.getClass().getClassLoader().getResourceAsStream(topolgyName);
    }
    if (ins != null) {
      builder = new SimpleTopologyBuilder(ins);
    }

    builder.buildToplogy(topology, new ObjectFactory());

    if (fileSpout != null) {
      topology.setSpout(Settings.INPUT_SPOUT, fileSpout);
    } else {
      String clickGate = Settings.getString(Settings.CLICK_GATE_LOG_URL, "");
      HTTPURLSpout spout = new HTTPURLSpout();
      if (spout.connect(clickGate)) {
        topology.setSpout(Settings.INPUT_SPOUT, spout);
      }
    }

    topology.start();
  }