Пример #1
0
  void start0() {
    byte[] bytes = null;
    ParseItemTaskBean bean = null;
    Parser parser = null;
    // while (true) {
    try {
      bytes = TaskQueues.getParseItemTaskQueue().poll();
    } catch (Exception e) {
      log.error("error to poll queue:", e);
      return;
    }

    if (bytes == null) {
      return;
    }

    try {
      bean = ParseItemTaskBean.parse(bytes);
    } catch (Exception e) {
      log.error("Error to parse JSONString to ParseItemTaskBean: " + new String(bytes), e);
      return;
    }

    if (StringUtils.isBlank(bean.getItemId())) {
      TaskCounters.decrementAndGet(bean.getTaskId());
      log.error("Can not find itemId, ignore: {}", bean);
      return;
    }

    // 没有可用的 vps
    if (!this.getProvider().getIpPools().hasAvaliableIp(bean.getShopType())) {
      if (!TaskQueues.getParseItemTaskQueue().add(bytes)) {
        TaskCounters.decrementAndGet(bean.getTaskId());
      }
      return;
    }

    // 不做重复操作
    if (!UrlCounters.add(
        bean.getReqFrom(), bean.getOuerUserId(), bean.getOuerShopId(), bean.getItemUrl())) {
      log.warn("url duplicateb in UrlCounters, url={} ", bean.getItemId());
      TaskCounters.decrementAndGet(bean.getTaskId());
      return;
    }
    if (!ItemCounters.add(
        bean.getReqFrom(),
        bean.getOuerUserId(),
        bean.getOuerShopId(),
        bean.getShopType(),
        bean.getItemId())) {
      log.warn("item duplicateb in ItemCounters, url={} ", bean.getItemId());
      TaskCounters.decrementAndGet(bean.getTaskId());
      return;
    }

    switch (bean.getShopType()) {
      case TAOBAO:
        parser = new TaobaoItemParser(this.getProvider(), bean, this.itemService);
        break;
      case TMALL:
        parser = new TmallItemParser(this.getProvider(), bean, this.itemService);
        break;
      default:
        parser = null;
    }

    if (parser == null) {
      TaskCounters.decrementAndGet(bean.getTaskId());
      log.error("can not find ItemParser, ignore:" + bean.toString());
      return;
    }

    if (log.isDebugEnabled()) {
      log.debug("take from ShopTaskQueue:{}", bean);
    }

    try {
      parser.parse();
    } catch (Exception e) {
      log.error("Error to parse item's info by " + bean.toString(), e);
      return;
    }
    // }
  }
Пример #2
0
  void start0() {
    byte[] bytes = null;
    ParseShopTaskBean bean = null;
    Parser parser = null;
    try {
      bytes = TaskQueues.getParseShopTaskQueue().poll();
    } catch (Exception e) {
      log.error("error to poll queue:", e);
      return;
    }

    if (bytes == null) {
      return;
    }

    try {
      bean = ParseShopTaskBean.parse(bytes);
    } catch (Exception e) {
      log.error("Error to parse JSONString to ParseShopTaskBean: " + new String(bytes), e);
      return;
    }

    // 没有可用的 vps
    if (!this.getProvider().getIpPools().hasAvaliableIp(bean.getShopType())) {
      TaskQueues.getParseShopTaskQueue().add(bytes);
      return;
    }

    // 不做重复操作,防止把自个玩死
    if (!UrlCounters.add(
        bean.getReqFrom(),
        bean.getOuerUserId(),
        bean.getOuerShopId(),
        bean.getRequestUrl(),
        Config.instance().getShopUrlCacheTimeout())) {
      log.warn("url duplicateb in UrlCounters, url={}, now={}", bean.getShopUrl(), df.format(date));
      return;
    }

    switch (bean.getParserType()) {
      case TAOBAO_ITEMLIST:
        parser = new TaobaoItemListParser(this.getProvider(), bean);
        break;
      case TMALL_ITEMLIST:
        parser = new TmallItemListParser(this.getProvider(), bean);
        break;
      default:
        parser = null;
    }

    if (parser == null) {
      log.error("can not find ShopParser, ignore:" + bean.toString());
      return;
    }

    if (log.isDebugEnabled()) {
      log.debug("take from ShopTaskQueue:{}", bean);
    }

    try {
      parser.parse();
    } catch (Exception e) {
      log.error("Error to parse shop's items by " + bean.toString(), e);
      return;
    }
  }