@Override
public int getCorePoolSize() {
return Math.max(super.getCorePoolSize(), Config.instance().getVpsSize());
// return 1;
}
void start0() {
byte[] bytes = null;
ParseShopTaskBean bean = null;
Parser parser = null;
try {
bytes = TaskQueues.getParseShopTaskQueue().poll();
} catch (Exception e) {
log.error("error to poll queue:", e);
return;
}
if (bytes == null) {
return;
}
try {
bean = ParseShopTaskBean.parse(bytes);
} catch (Exception e) {
log.error("Error to parse JSONString to ParseShopTaskBean: " + new String(bytes), e);
return;
}
// 没有可用的 vps
if (!this.getProvider().getIpPools().hasAvaliableIp(bean.getShopType())) {
TaskQueues.getParseShopTaskQueue().add(bytes);
return;
}
// 不做重复操作，防止把自个玩死
if (!UrlCounters.add(
bean.getReqFrom(),
bean.getOuerUserId(),
bean.getOuerShopId(),
bean.getRequestUrl(),
Config.instance().getShopUrlCacheTimeout())) {
log.warn("url duplicateb in UrlCounters, url={}, now={}", bean.getShopUrl(), df.format(date));
return;
}
switch (bean.getParserType()) {
case TAOBAO_ITEMLIST:
parser = new TaobaoItemListParser(this.getProvider(), bean);
break;
case TMALL_ITEMLIST:
parser = new TmallItemListParser(this.getProvider(), bean);
break;
default:
parser = null;
}
if (parser == null) {
log.error("can not find ShopParser, ignore:" + bean.toString());
return;
}
if (log.isDebugEnabled()) {
log.debug("take from ShopTaskQueue:{}", bean);
}
try {
parser.parse();
} catch (Exception e) {
log.error("Error to parse shop's items by " + bean.toString(), e);
return;
}
}