private static void rollbackTx_() {
if (isJPAModel_()) {
JPAPlugin.closeTx(true);
}
}
private static void startTx_() {
if (isJPAModel_()) {
JPAPlugin.startTx(false);
}
}
private static void commitTx_() {
if (isJPAModel_()) {
JPAPlugin.closeTx(false);
}
}
@Override
@NoTransaction
public void doJob() {
int moreLinkCount = 0;
String url = "https://news.ycombinator.com";
parentPattern = Pattern.compile("<a href=\"item\\?id=\\d+\">parent</a>");
while (true) {
try {
String content = extractContent(url);
List<String> postList = extractPosts(content);
List<Item> newItemList = new ArrayList<Item>();
JPAPlugin.startTx(false);
for (String post : postList) {
try {
Item item = extractItem(post);
if (item != null) {
Item existing = Item.getByHnId(item.hnid);
if (existing != null) {
existing.update(item);
}
if (!JPA.em().getTransaction().isActive()) {
JPA.em().getTransaction().begin();
}
if (existing == null) {
item.save();
newItemList.add(item.clone());
} else {
existing.save();
newItemList.add(existing.clone());
}
JPA.em().getTransaction().commit();
}
} catch (Exception e) {
Logger.error(e, "Error while saving multiple post: %s", post);
}
}
JPAPlugin.closeTx(false);
ItemCache.getInstance().updateCache(newItemList);
url = extractMoreLink(content);
moreLinkCount++;
if (moreLinkCount == 6) {
url = "https://news.ycombinator.com";
} else if (moreLinkCount == 7) {
url = "https://news.ycombinator.com/best";
moreLinkCount = 0;
}
StatisticsMgr.instance().updateLastHnUpdateTime();
Logger.info("Last update : %s", Calendar.getInstance().getTime().toString());
} catch (Exception e) {
Logger.error(e, "Exception in forward crawler.");
} finally {
try {
Thread.sleep(((long) (MINUTE * Math.random())) * 10 + 5 * MINUTE);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}