Example #1
0
 private static void rollbackTx_() {
   if (isJPAModel_()) {
     JPAPlugin.closeTx(true);
   }
 }
Example #2
0
 private static void startTx_() {
   if (isJPAModel_()) {
     JPAPlugin.startTx(false);
   }
 }
Example #3
0
 private static void commitTx_() {
   if (isJPAModel_()) {
     JPAPlugin.closeTx(false);
   }
 }
Example #4
0
  @Override
  @NoTransaction
  public void doJob() {
    int moreLinkCount = 0;
    String url = "https://news.ycombinator.com";
    parentPattern = Pattern.compile("<a href=\"item\\?id=\\d+\">parent</a>");
    while (true) {
      try {

        String content = extractContent(url);
        List<String> postList = extractPosts(content);
        List<Item> newItemList = new ArrayList<Item>();
        JPAPlugin.startTx(false);
        for (String post : postList) {
          try {
            Item item = extractItem(post);
            if (item != null) {
              Item existing = Item.getByHnId(item.hnid);
              if (existing != null) {
                existing.update(item);
              }
              if (!JPA.em().getTransaction().isActive()) {
                JPA.em().getTransaction().begin();
              }

              if (existing == null) {
                item.save();
                newItemList.add(item.clone());
              } else {
                existing.save();
                newItemList.add(existing.clone());
              }
              JPA.em().getTransaction().commit();
            }
          } catch (Exception e) {
            Logger.error(e, "Error while saving multiple post: %s", post);
          }
        }
        JPAPlugin.closeTx(false);
        ItemCache.getInstance().updateCache(newItemList);

        url = extractMoreLink(content);
        moreLinkCount++;
        if (moreLinkCount == 6) {
          url = "https://news.ycombinator.com";
        } else if (moreLinkCount == 7) {
          url = "https://news.ycombinator.com/best";
          moreLinkCount = 0;
        }
        StatisticsMgr.instance().updateLastHnUpdateTime();
        Logger.info("Last update : %s", Calendar.getInstance().getTime().toString());

      } catch (Exception e) {
        Logger.error(e, "Exception in forward crawler.");
      } finally {
        try {
          Thread.sleep(((long) (MINUTE * Math.random())) * 10 + 5 * MINUTE);
        } catch (InterruptedException e) {
          e.printStackTrace();
        }
      }
    }
  }