void start0() {
byte[] bytes = null;
ParseItemTaskBean bean = null;
Parser parser = null;
// while (true) {
try {
bytes = TaskQueues.getParseItemTaskQueue().poll();
} catch (Exception e) {
log.error("error to poll queue:", e);
return;
}
if (bytes == null) {
return;
}
try {
bean = ParseItemTaskBean.parse(bytes);
} catch (Exception e) {
log.error("Error to parse JSONString to ParseItemTaskBean: " + new String(bytes), e);
return;
}
if (StringUtils.isBlank(bean.getItemId())) {
TaskCounters.decrementAndGet(bean.getTaskId());
log.error("Can not find itemId, ignore: {}", bean);
return;
}
// 没有可用的 vps
if (!this.getProvider().getIpPools().hasAvaliableIp(bean.getShopType())) {
if (!TaskQueues.getParseItemTaskQueue().add(bytes)) {
TaskCounters.decrementAndGet(bean.getTaskId());
}
return;
}
// 不做重复操作
if (!UrlCounters.add(
bean.getReqFrom(), bean.getOuerUserId(), bean.getOuerShopId(), bean.getItemUrl())) {
log.warn("url duplicateb in UrlCounters, url={} ", bean.getItemId());
TaskCounters.decrementAndGet(bean.getTaskId());
return;
}
if (!ItemCounters.add(
bean.getReqFrom(),
bean.getOuerUserId(),
bean.getOuerShopId(),
bean.getShopType(),
bean.getItemId())) {
log.warn("item duplicateb in ItemCounters, url={} ", bean.getItemId());
TaskCounters.decrementAndGet(bean.getTaskId());
return;
}
switch (bean.getShopType()) {
case TAOBAO:
parser = new TaobaoItemParser(this.getProvider(), bean, this.itemService);
break;
case TMALL:
parser = new TmallItemParser(this.getProvider(), bean, this.itemService);
break;
default:
parser = null;
}
if (parser == null) {
TaskCounters.decrementAndGet(bean.getTaskId());
log.error("can not find ItemParser, ignore:" + bean.toString());
return;
}
if (log.isDebugEnabled()) {
log.debug("take from ShopTaskQueue:{}", bean);
}
try {
parser.parse();
} catch (Exception e) {
log.error("Error to parse item's info by " + bean.toString(), e);
return;
}
// }
}
void start0() {
byte[] bytes = null;
ParseShopTaskBean bean = null;
Parser parser = null;
try {
bytes = TaskQueues.getParseShopTaskQueue().poll();
} catch (Exception e) {
log.error("error to poll queue:", e);
return;
}
if (bytes == null) {
return;
}
try {
bean = ParseShopTaskBean.parse(bytes);
} catch (Exception e) {
log.error("Error to parse JSONString to ParseShopTaskBean: " + new String(bytes), e);
return;
}
// 没有可用的 vps
if (!this.getProvider().getIpPools().hasAvaliableIp(bean.getShopType())) {
TaskQueues.getParseShopTaskQueue().add(bytes);
return;
}
// 不做重复操作，防止把自个玩死
if (!UrlCounters.add(
bean.getReqFrom(),
bean.getOuerUserId(),
bean.getOuerShopId(),
bean.getRequestUrl(),
Config.instance().getShopUrlCacheTimeout())) {
log.warn("url duplicateb in UrlCounters, url={}, now={}", bean.getShopUrl(), df.format(date));
return;
}
switch (bean.getParserType()) {
case TAOBAO_ITEMLIST:
parser = new TaobaoItemListParser(this.getProvider(), bean);
break;
case TMALL_ITEMLIST:
parser = new TmallItemListParser(this.getProvider(), bean);
break;
default:
parser = null;
}
if (parser == null) {
log.error("can not find ShopParser, ignore:" + bean.toString());
return;
}
if (log.isDebugEnabled()) {
log.debug("take from ShopTaskQueue:{}", bean);
}
try {
parser.parse();
} catch (Exception e) {
log.error("Error to parse shop's items by " + bean.toString(), e);
return;
}
}