コード例 #1
0
 public String downloadImage(String imageUrl) throws IOException {
   String result = "";
   if (jcUser.isBackupThumbnail()) {
     // download thumbnail image
     byte[] thumbnailImage = NetUtil.readImage(imageUrl);
     File thumbnailDirectory = new File(path, "thumbnail");
     DataUtil.writeImage(
         thumbnailDirectory,
         imageUrl.substring(imageUrl.lastIndexOf("/"), imageUrl.length()),
         thumbnailImage);
     LOG.debug("saved thumbnail image:" + imageUrl);
     result = imageUrl.substring(imageUrl.lastIndexOf("thumbnail"), imageUrl.length());
     thumbnailCount++;
   }
   if (jcUser.isBackupLarge()) {
     // download large image
     String largeImageUrl = imageUrl.replaceFirst("thumbnail", "large");
     byte[] largeImage = NetUtil.readImage(largeImageUrl);
     File largeDirectory = new File(path, "large");
     DataUtil.writeImage(
         largeDirectory,
         largeImageUrl.substring(largeImageUrl.lastIndexOf("/"), largeImageUrl.length()),
         largeImage);
     result = imageUrl.substring(imageUrl.lastIndexOf("thumbnail"), imageUrl.length());
     LOG.debug("saved large image:" + largeImageUrl);
     largeCount++;
   }
   return result;
 }
コード例 #2
0
  public StatusWapper obtainWeibo(Timeline tm, JcUser jcUser, int page) {
    StatusWapper statusWapper = null;
    int i = 0;
    try {
      statusWapper = tm.getUserTimeline(COUNT_PER_PAGE, page, jcUser);
      LOG.debug(++callCount + "st call weibo api (now is userTimeline)");
    } catch (WeiboException e) {
      LOG.error("occured a exception when obtaining weibo use weibo api" + e);
      LOG.error(++callCount + "st call weibo api (now is comments) cause error");
      // outOfLimit = true;
      jcUser.setOutOfLimit(true);
      return null;
    }

    // download pictures and/or obtain comments
    List<Status> statuses = statusWapper.getStatuses();
    for (i = 0; i < statuses.size(); i++) {
      Status s = statuses.get(i);
      Place place = new Place();
      try {
        if (s.getLatitude() != -1 && s.getLongitude() != -1)
          s.setFormatedAddress(place.formattedAddress(s.getLatitude(), s.getLongitude()));
        else s.setFormatedAddress(null);
      } catch (Exception e) {
        LOG.error("occured a exception when format address " + e);
        LOG.error("try again");
        try {
          if (s.getLatitude() != -1 && s.getLongitude() != -1)
            s.setFormatedAddress(place.formattedAddress(s.getLatitude(), s.getLongitude()));
          else s.setFormatedAddress(null);
        } catch (Exception e2) {
          LOG.error("occured a exception when format address " + e2);
          s.setFormatedAddress(null);
        }
      }
      if (s.getCommentsCount() > 0) s.setComments(obtainComments(s));
      statuses.set(i, downloadImage(s));
      // if (callCount >= limit || outOfLimit == true)
      if (jcUser.isOutOfLimit()) break;
    }
    LOG.debug("this time get statues count is" + statuses.size());
    int index = 0;
    // if (callCount >= limit || outOfLimit == true)
    if (jcUser.isOutOfLimit()) index = i;
    else index = i + 1;
    int len = statuses.size();
    int times = len - index;
    for (int j = 0; j < times; j++) {
      Status s = statuses.remove(index);
      LOG.debug(
          "removing id:"
              + s.getId()
              + ", creatAt:"
              + StringUtil.formatDate(s.getCreatedAt(), "yyyy-MM-dd HH:mm"));
    }
    LOG.debug("after remove get statues count is" + statuses.size());
    LOG.debug("after remove get statusWapper.statues count is" + statusWapper.getStatuses().size());
    return statusWapper;
  }
コード例 #3
0
 @Override
 public FileInputStream downloadZip(JcUser jcUser) {
   FileInputStream in = null;
   try {
     in = new FileInputStream(jcUser.getZipPath());
   } catch (FileNotFoundException e) {
     LOG.error("File " + jcUser.getZipPath() + " is not found." + e);
   }
   return in;
 }
コード例 #4
0
 @Override
 public JcUser deleteWeibo(JcUser jcUser) {
   this.jcUser = jcUser;
   UserTask userTask = userTaskPersistenceService.getUserTaskByKey(jcUser.getAccessToken());
   if (userTask == null || !userTask.getTask().isAlive()) {
     if (userTask == null) userTask = new UserTask();
     else userTaskPersistenceService.delete(userTask);
     tm.client.setToken(jcUser.getAccessToken());
     List<Status> statuses = weiboPersistenceService.getAllTop(jcUser);
     if (statuses.size() == 0) {
       jcUser.setDeleteCount(-1);
       return jcUser;
     }
     DeleteTask deleteTask = new DeleteTask(jcUser.getAccessToken());
     deleteTask.setStatues(statuses);
     deleteTask.start();
     // jcUser.setDeleteTask(deleteTask);
     userTask.setKey(jcUser.getAccessToken());
     userTask.setTask(deleteTask);
     userTaskPersistenceService.save(userTask);
   }
   jcUser.setDeleteCount(destroyedIndex + 1);
   jcUser.setOutOfLimit(outOfLimit);
   return this.jcUser;
 }
コード例 #5
0
  /** call Weibo api to obtain weibos return count weibos have been saved */
  @Override
  public JcUser obtainWeibo(JcUser jcUser, ServletContext context) {
    int count = 0;
    int swoopCount = 0;
    int totalNumber = 0;
    int totalPage = 0;
    StatusWapper statusWapper = null;

    // get user's AccessToken by sessionId.
    // this.jcUser =
    // jcUserPersistenceService.getUserBySessionId(jcUser.getSession()); it
    // should be handled by security validation
    if (jcUser.getAccessToken() == null || jcUser.getAccessToken().length() == 0) return jcUser;
    this.jcUser = jcUser;
    preparePath(context);

    tm.client.setToken(jcUser.getAccessToken());
    c.client.setToken(jcUser.getAccessToken());

    // begin to obtain weibo from weibo api.
    statusWapper = obtainWeibo(tm, jcUser, 1);
    count += weiboPersistenceService.saveStatuses(statusWapper.getStatuses());
    swoopCount += statusWapper.getStatuses().size();

    totalNumber = (int) (statusWapper.getTotalNumber()); // total weibos the
    // user have
    totalPage = totalNumber % 100 == 0 ? totalNumber / 100 : totalNumber / 100 + 1;
    // because weibo api has a limit that only 100 weibos can be obtained
    // each request.
    for (int page = 2; page < totalPage; page++) {
      statusWapper = obtainWeibo(tm, jcUser, page);
      // if (callCount >= limit || outOfLimit == true)
      if (jcUser.isOutOfLimit()) break;
      count += weiboPersistenceService.saveStatuses(statusWapper.getStatuses());
      swoopCount += statusWapper.getStatuses().size();
    }
    LOG.debug("obtained " + swoopCount + " Weibos");
    LOG.debug("save " + count + " Weibos");
    this.jcUser.setWeiboCount(count);
    this.jcUser.setThumbnailCount(thumbnailCount);
    this.jcUser.setLargeCount(largeCount);
    this.jcUser.setCommentCount(commentCount);
    return this.jcUser;
  }
コード例 #6
0
 public List<Comment> obtainComments(Status s) {
   CommentWapper commentWapper = null;
   if (!jcUser.isBackupComment()) return null;
   try {
     commentWapper = c.getCommentById(s.getId());
     commentCount += commentWapper.getComments().size();
     LOG.debug(++callCount + "st call weibo api (now is comments)");
   } catch (WeiboException e) {
     LOG.error(
         "occured a exception when obtaining comments status'id="
             + s.getId()
             + ", Exception:"
             + e);
     LOG.error(++callCount + "st call weibo api (now is comments) cause error");
     // outOfLimit = true;
     jcUser.setOutOfLimit(true);
   }
   if (commentWapper != null) return commentWapper.getComments();
   else return null;
 }
コード例 #7
0
  @Override
  public JcUser packageZip(JcUser jcUser) {
    try {
      String weibo = compositeWeibo(jcUser);
      byte[] data = weibo.getBytes();
      BufferedOutputStream stream =
          new BufferedOutputStream(new FileOutputStream(new File(path, "index.html")));
      stream.write(data);
      stream.close();

      // package files in zip
      String zipPath = path + ".zip";
      File zip = new File(zipPath);
      FileUtils.zipFile(new File(path), zip);
      jcUser.setZipPath(zipPath);
      float fileSize = zip.length();
      jcUser.setFileSize(fileSize / (1024 * 1024));
    } catch (FileNotFoundException e) {
      LOG.debug("not found file' \n" + e);
    } catch (IOException e2) {
      LOG.debug("dowload zip function occured errer: \n" + e2);
    }
    return jcUser;
  }
コード例 #8
0
 public Status downloadImage(Status s) {
   if (s.getRetweetedStatus() != null) {
     s.setRetweetedStatus(downloadImage(s.getRetweetedStatus()));
   }
   if (s.getPicUrls() == null || s.getPicUrls().length == 0) return s;
   Map<Integer, String> failUrls = new HashMap<Integer, String>();
   // String[] imageUrls = new String[s.getPicUrls().length];
   String[] imageUrls = s.getPicUrls();
   // int index = 0;
   for (int i = 0; i < imageUrls.length; i++) {
     // for (String imageUrl : s.getPicUrls()) {
     try {
       String imageUrl = downloadImage(imageUrls[i]);
       imageUrls[i] = imageUrl;
     } catch (IOException e) {
       LOG.error("occured exception when download images:" + imageUrls[i] + e);
       LOG.error("keep it,then i will download it again.");
       failUrls.put(i, imageUrls[i]);
     }
   }
   if (failUrls.size() > 0) {
     for (int i : failUrls.keySet())
       try {
         LOG.debug("the second time to download image:" + failUrls.get(i));
         String imageUrl = downloadImage(failUrls.get(i));
         imageUrls[i] = imageUrl;
       } catch (IOException e) {
         LOG.error(
             "occured exception when the second time download images:"
                 + failUrls.get(i)
                 + " \n"
                 + e);
         imageUrls[i] = "images/fail.thumbnail.jpg";
       }
   }
   if (jcUser.isBackupThumbnail()) s.setPicUrls(imageUrls);
   // else //without any picture
   // s.setPicUrls(null);
   return s;
 }