private List<TaobaoCommentBean> parseTaobaoComment(String result, Task task) {
JSONObject rateDetail = JSON.parseObject(result).getJSONObject("rateDetail");
JSONArray rateList = rateDetail.getJSONArray("rateList");
List<TaobaoCommentBean> beans = new ArrayList<TaobaoCommentBean>(rateList.size());
String itemId = HttpURLUtils.getUrlParams(task.getUrl(), "GBK").get("itemId");
TaobaoCommentBean bean = null;
for (int i = 0; i < rateList.size(); i++) {
bean = new TaobaoCommentBean();
bean.setType(task.getExtra());
bean.setItemId(itemId);
try {
JSONObject rate = rateList.getJSONObject(i);
bean.setId(rate.getString("id"));
bean.setConmment(rate.getString("rateContent"));
bean.setCommentTime(rate.getString("rateDate"));
bean.setReply(rate.getString("reply"));
String appendCommentJson = rate.getString("appendComment");
if (!StringUtils.isBlank(appendCommentJson)
&& StringUtils.startsWith(appendCommentJson, "{")) {
JSONObject appendComment = JSON.parseObject(appendCommentJson);
bean.setAppandConmment(appendComment.getString("content"));
} else {
bean.setAppandConmment(appendCommentJson);
}
bean.setServiceComment(rate.getString("serviceRateContent"));
// user
bean.setNick(rate.getString("displayUserNick"));
bean.setVip(rate.getString("tamllSweetLevel"));
beans.add(bean);
} catch (Exception e) {
}
}
return beans;
}
@Override
public void parse(String result, Task task) throws Exception {
List<ECBean> beans = new ArrayList<ECBean>();
JSONObject parseObject = JSON.parseObject(result);
Object object = parseObject.get("value");
Document doc = Jsoup.parse((String) object);
Elements eles = doc.select("div.mod_search_pro");
String categroy = "";
for (Element element : eles) {
ECBean bean = new ECBean("yhd");
Elements select = element.select("p.proName > a");
String url = select.first().attr("href");
String id = select.first().attr("pmid");
// 过滤没有pid数据
if (id.equals("0")) {
continue;
}
String name = select.first().attr("title");
// 抓取分类
if (StringUtils.isBlank(categroy)) {
Document document = GlobalComponents.fetcher.document(url);
Elements select2 = document.select("div.crumb > a");
StringBuilder sb = new StringBuilder();
for (int i = 1; i < select2.size() - 1; i++) {
sb.append(select2.get(i).text());
}
sb.deleteCharAt(sb.length() - 1);
categroy = sb.toString();
categroy = StringUtils.replaceChars(categroy, "", "/");
}
bean.setId(id);
bean.setUrl(url);
bean.setTitle(name);
bean.setCategory(categroy);
bean.setKeyword(task.getExtra());
beans.add(bean);
}
log.info("fetch list:" + beans.size());
if (!beans.isEmpty()) {
for (ECBean bean : beans) {
bean.saveOnNotExist();
}
}
beans.clear();
}
private List<TaobaoCommentBean> parseTmallComment(String result, Task task) {
String json = result.substring(1, result.length() - 1);
JSONObject parseObject = JSON.parseObject(json);
JSONArray jsonArray = parseObject.getJSONArray("comments");
List<TaobaoCommentBean> beans = new ArrayList<TaobaoCommentBean>(jsonArray.size());
JSONObject jsonObject = null;
TaobaoCommentBean bean = null;
for (int i = 0; i < jsonArray.size(); i++) {
bean = new TaobaoCommentBean();
bean.setType(task.getExtra());
try {
jsonObject = jsonArray.getJSONObject(i);
String aucNumId = jsonObject.getJSONObject("auction").getString("aucNumId");
bean.setItemId(aucNumId);
String rateId = jsonObject.getString("rateId");
bean.setId(rateId);
String content = jsonObject.getString("content");
bean.setConmment(content.trim());
String vip = jsonObject.getJSONObject("user").getString("vip");
bean.setVip(vip);
String nick = jsonObject.getJSONObject("user").getString("nick");
bean.setNick(nick);
String day = jsonObject.getString("date");
bean.setCommentTime(day);
JSONObject jsonTmp = jsonObject.getJSONObject("reply");
if (jsonTmp != null) {
String reply = jsonTmp.getString("content");
bean.setReply(reply.trim());
}
jsonTmp = jsonObject.getJSONObject("append");
if (jsonTmp != null) {
String append = jsonTmp.getString("content");
bean.setAppandConmment(append.trim());
}
beans.add(bean);
} catch (Exception e) {
}
}
return beans;
}
@Override
public void parse(String result, Task task) throws Exception {
if (StringUtils.isBlank(result)) {
return;
}
List<TaobaoCommentBean> beans = null;
if ("1".equals(task.getExtra())) {
beans = parseTmallComment(result, task);
} else {
beans = parseTaobaoComment(result, task);
}
if (beans != null && !beans.isEmpty()) {
for (TaobaoCommentBean commentBean : beans) {
try {
commentBean.persistOnNotExist();
} catch (Exception e) {
}
}
}
}
@Override
protected Task buildTask(String url) {
Task buildTask = super.buildTask(url);
buildTask.setExtra(keyword);
return buildTask;
}