Example #1
0
 @Override
 public int getCorePoolSize() {
   return Math.max(super.getCorePoolSize(), Config.instance().getVpsSize());
   //		return 1;
 }
Example #2
0
  void start0() {
    byte[] bytes = null;
    ParseShopTaskBean bean = null;
    Parser parser = null;
    try {
      bytes = TaskQueues.getParseShopTaskQueue().poll();
    } catch (Exception e) {
      log.error("error to poll queue:", e);
      return;
    }

    if (bytes == null) {
      return;
    }

    try {
      bean = ParseShopTaskBean.parse(bytes);
    } catch (Exception e) {
      log.error("Error to parse JSONString to ParseShopTaskBean: " + new String(bytes), e);
      return;
    }

    // 没有可用的 vps
    if (!this.getProvider().getIpPools().hasAvaliableIp(bean.getShopType())) {
      TaskQueues.getParseShopTaskQueue().add(bytes);
      return;
    }

    // 不做重复操作,防止把自个玩死
    if (!UrlCounters.add(
        bean.getReqFrom(),
        bean.getOuerUserId(),
        bean.getOuerShopId(),
        bean.getRequestUrl(),
        Config.instance().getShopUrlCacheTimeout())) {
      log.warn("url duplicateb in UrlCounters, url={}, now={}", bean.getShopUrl(), df.format(date));
      return;
    }

    switch (bean.getParserType()) {
      case TAOBAO_ITEMLIST:
        parser = new TaobaoItemListParser(this.getProvider(), bean);
        break;
      case TMALL_ITEMLIST:
        parser = new TmallItemListParser(this.getProvider(), bean);
        break;
      default:
        parser = null;
    }

    if (parser == null) {
      log.error("can not find ShopParser, ignore:" + bean.toString());
      return;
    }

    if (log.isDebugEnabled()) {
      log.debug("take from ShopTaskQueue:{}", bean);
    }

    try {
      parser.parse();
    } catch (Exception e) {
      log.error("Error to parse shop's items by " + bean.toString(), e);
      return;
    }
  }