コード例 #1
0
 public String getNextUidTillEnd(String currentUid) {
   int uidID = uidList.indexOf(currentUid);
   int nextUidID;
   nextUidID = uidID + 1;
   if (nextUidID < 0) {
     nextUidID = 0;
   }
   if (nextUidID >= this.uidList.size()) {
     return "finish";
   }
   if (uidID != nextUidID) {
     System.out.println("change uid.");
     ExpUtils.mylog(CrawlTool.splitFileNameByHour(this.logName), this.logSign + " Change uid.");
   }
   return uidList.get(nextUidID);
 }
コード例 #2
0
  private void process() {
    // check start crawl time
    if (!CrawlTool.checkStartCrawlTime(this.startCrawlTimeStamp)) { // wait
      // to
      // start
      Long nowTimeStamp = new Date().getTime();
      int waitTimeSecond =
          CrawlTool.safeLongToInt((this.startCrawlTimeStamp - nowTimeStamp) / 1000);
      System.out.println(
          this.logSign
              + " Sleep a while to start: "
              + waitTimeSecond
              + "s, or:"
              + waitTimeSecond / 60
              + "m.");
      CrawlTool.sleep(waitTimeSecond, this.logSign);
    }

    // the variables and flags
    String currentKey = "";
    int page = 0;
    String currentUid = "";
    boolean isResultEmpty = false;
    boolean isResultTooOld = false;
    boolean isError = false;

    while (true) {

      // check if it is time to stop
      boolean stopCrawl = CrawlTool.checkStopCrawlTime(this.stopCrawlTimeStamp);
      if (stopCrawl) {
        // report this round
        System.out.println("Time's up. Stop crawling now.");
        break;
      }

      // check if it is the max times to crawl
      if (!(this.requestedCrawlingTimes > 0 && this.requestedCrawlingTimes >= this.crawledTimes)) {
        return;
      }

      // prepare the key
      currentKey = CrawlTool.getNextKey(keyList, currentKey);

      // prepare the uid, page, count
      if (null == currentUid || "".equals(currentUid)) {
        currentUid = getNextUidTillEnd(currentUid);

        if (currentUid == "finish" || "finish".equals(currentUid)) { // all
          // uids
          // have
          // been
          // crawled
          String line = "All uids have been crawled, now finish! " + new Date();
          System.out.println(line);
          ExpUtils.mylog(CrawlTool.splitFileNameByHour(logName), line);
          break;
        }
        page = 0; // because later there will be page++
      }
      if (isResultEmpty || page > this.maxPageCount) {
        // empty result get or page > max page, goto next id, page = 1
        System.out.println("The result is empty.");
        currentUid = getNextUidTillEnd(currentUid);
        page = 0;

        if (currentUid == "finish" || "finish".equals(currentUid)) { // all
          // uids
          // have
          // been
          // crawled
          String line = "All uids have been crawled, now finish! " + new Date();
          System.out.println(line);
          ExpUtils.mylog(CrawlTool.splitFileNameByHour(logName), line);
          break;
        }
      }

      // if the posts got is older than the required date, then stop next
      // page, and change to the next userid
      // if (isResultTooOld) {
      // System.out.println("The last post is too old.");
      // currentUid = getNextUidTillEnd(currentUid);
      // page = 0;
      //
      // if (currentUid == "finish" || "finish".equals(currentUid)) {//
      // all
      // // uids
      // // have
      // // been
      // // crawled
      // String line = "All uids have been crawled, now finish! "
      // + new Date();
      // System.out.println(line);
      // ExpUtils.mylog(CrawlTool.splitFileNameByHour(logName), line);
      // break;
      // }
      //
      // }

      page++;

      // prepare the parameter map
      Map<String, String> map = getParaMap(currentUid, page, count);
      System.out.println(map.toString());
      this.crawledTimes++;
      System.out.println("Crawling for the times: " + this.crawledTimes);

      // sleep interval
      CrawlTool.sleep(this.interval, this.logSign);

      // try to crawl
      try {
        String line =
            "uid="
                + currentUid
                + ", page="
                + page
                + ", count="
                + count
                + ", Crawl times: "
                + this.crawledTimes;
        ExpUtils.mylog(CrawlTool.splitFileNameByHour(logName), line);

        // try to get the data
        String result = ExpUtils.crawlData(api, map, currentKey);
        // result = StringEscapeUtils.unescapeJava(result);

        // init and set the flags
        isResultEmpty = false;
        isError = false;

        if ("[]".equals(result)) {
          isResultEmpty = true;
        } else {
          isResultEmpty = false;
        }

        if (result.startsWith("{\"error\"")) {
          isError = true;
        } else {
          isError = false;
        }

        // if (isResultTooOld(result)) {
        // isResultTooOld = true;
        // } else {
        // isResultTooOld = false;
        // }

        if (!isResultEmpty && !isError) {
          ExpUtils.mylogJson(CrawlTool.splitFileNameByHour(JsonlogName), result);
        } else {
          // something wrong
          ExpUtils.mylog(CrawlTool.splitFileNameByHour(logName), result);
        }
      } catch (WeiboException e) {
        e.printStackTrace();
        ExpUtils.mylog(CrawlTool.splitFileNameByHour(logName), e.getError());
        continue;
      }
    }
  }