public String downloadImage(String imageUrl) throws IOException {
String result = "";
if (jcUser.isBackupThumbnail()) {
// download thumbnail image
byte[] thumbnailImage = NetUtil.readImage(imageUrl);
File thumbnailDirectory = new File(path, "thumbnail");
DataUtil.writeImage(
thumbnailDirectory,
imageUrl.substring(imageUrl.lastIndexOf("/"), imageUrl.length()),
thumbnailImage);
LOG.debug("saved thumbnail image:" + imageUrl);
result = imageUrl.substring(imageUrl.lastIndexOf("thumbnail"), imageUrl.length());
thumbnailCount++;
}
if (jcUser.isBackupLarge()) {
// download large image
String largeImageUrl = imageUrl.replaceFirst("thumbnail", "large");
byte[] largeImage = NetUtil.readImage(largeImageUrl);
File largeDirectory = new File(path, "large");
DataUtil.writeImage(
largeDirectory,
largeImageUrl.substring(largeImageUrl.lastIndexOf("/"), largeImageUrl.length()),
largeImage);
result = imageUrl.substring(imageUrl.lastIndexOf("thumbnail"), imageUrl.length());
LOG.debug("saved large image:" + largeImageUrl);
largeCount++;
}
return result;
}
public StatusWapper obtainWeibo(Timeline tm, JcUser jcUser, int page) {
StatusWapper statusWapper = null;
int i = 0;
try {
statusWapper = tm.getUserTimeline(COUNT_PER_PAGE, page, jcUser);
LOG.debug(++callCount + "st call weibo api (now is userTimeline)");
} catch (WeiboException e) {
LOG.error("occured a exception when obtaining weibo use weibo api" + e);
LOG.error(++callCount + "st call weibo api (now is comments) cause error");
// outOfLimit = true;
jcUser.setOutOfLimit(true);
return null;
}
// download pictures and/or obtain comments
List<Status> statuses = statusWapper.getStatuses();
for (i = 0; i < statuses.size(); i++) {
Status s = statuses.get(i);
Place place = new Place();
try {
if (s.getLatitude() != -1 && s.getLongitude() != -1)
s.setFormatedAddress(place.formattedAddress(s.getLatitude(), s.getLongitude()));
else s.setFormatedAddress(null);
} catch (Exception e) {
LOG.error("occured a exception when format address " + e);
LOG.error("try again");
try {
if (s.getLatitude() != -1 && s.getLongitude() != -1)
s.setFormatedAddress(place.formattedAddress(s.getLatitude(), s.getLongitude()));
else s.setFormatedAddress(null);
} catch (Exception e2) {
LOG.error("occured a exception when format address " + e2);
s.setFormatedAddress(null);
}
}
if (s.getCommentsCount() > 0) s.setComments(obtainComments(s));
statuses.set(i, downloadImage(s));
// if (callCount >= limit || outOfLimit == true)
if (jcUser.isOutOfLimit()) break;
}
LOG.debug("this time get statues count is" + statuses.size());
int index = 0;
// if (callCount >= limit || outOfLimit == true)
if (jcUser.isOutOfLimit()) index = i;
else index = i + 1;
int len = statuses.size();
int times = len - index;
for (int j = 0; j < times; j++) {
Status s = statuses.remove(index);
LOG.debug(
"removing id:"
+ s.getId()
+ ", creatAt:"
+ StringUtil.formatDate(s.getCreatedAt(), "yyyy-MM-dd HH:mm"));
}
LOG.debug("after remove get statues count is" + statuses.size());
LOG.debug("after remove get statusWapper.statues count is" + statusWapper.getStatuses().size());
return statusWapper;
}
@Override
public FileInputStream downloadZip(JcUser jcUser) {
FileInputStream in = null;
try {
in = new FileInputStream(jcUser.getZipPath());
} catch (FileNotFoundException e) {
LOG.error("File " + jcUser.getZipPath() + " is not found." + e);
}
return in;
}
@Override
public JcUser deleteWeibo(JcUser jcUser) {
this.jcUser = jcUser;
UserTask userTask = userTaskPersistenceService.getUserTaskByKey(jcUser.getAccessToken());
if (userTask == null || !userTask.getTask().isAlive()) {
if (userTask == null) userTask = new UserTask();
else userTaskPersistenceService.delete(userTask);
tm.client.setToken(jcUser.getAccessToken());
List<Status> statuses = weiboPersistenceService.getAllTop(jcUser);
if (statuses.size() == 0) {
jcUser.setDeleteCount(-1);
return jcUser;
}
DeleteTask deleteTask = new DeleteTask(jcUser.getAccessToken());
deleteTask.setStatues(statuses);
deleteTask.start();
// jcUser.setDeleteTask(deleteTask);
userTask.setKey(jcUser.getAccessToken());
userTask.setTask(deleteTask);
userTaskPersistenceService.save(userTask);
}
jcUser.setDeleteCount(destroyedIndex + 1);
jcUser.setOutOfLimit(outOfLimit);
return this.jcUser;
}
/** call Weibo api to obtain weibos return count weibos have been saved */
@Override
public JcUser obtainWeibo(JcUser jcUser, ServletContext context) {
int count = 0;
int swoopCount = 0;
int totalNumber = 0;
int totalPage = 0;
StatusWapper statusWapper = null;
// get user's AccessToken by sessionId.
// this.jcUser =
// jcUserPersistenceService.getUserBySessionId(jcUser.getSession()); it
// should be handled by security validation
if (jcUser.getAccessToken() == null || jcUser.getAccessToken().length() == 0) return jcUser;
this.jcUser = jcUser;
preparePath(context);
tm.client.setToken(jcUser.getAccessToken());
c.client.setToken(jcUser.getAccessToken());
// begin to obtain weibo from weibo api.
statusWapper = obtainWeibo(tm, jcUser, 1);
count += weiboPersistenceService.saveStatuses(statusWapper.getStatuses());
swoopCount += statusWapper.getStatuses().size();
totalNumber = (int) (statusWapper.getTotalNumber()); // total weibos the
// user have
totalPage = totalNumber % 100 == 0 ? totalNumber / 100 : totalNumber / 100 + 1;
// because weibo api has a limit that only 100 weibos can be obtained
// each request.
for (int page = 2; page < totalPage; page++) {
statusWapper = obtainWeibo(tm, jcUser, page);
// if (callCount >= limit || outOfLimit == true)
if (jcUser.isOutOfLimit()) break;
count += weiboPersistenceService.saveStatuses(statusWapper.getStatuses());
swoopCount += statusWapper.getStatuses().size();
}
LOG.debug("obtained " + swoopCount + " Weibos");
LOG.debug("save " + count + " Weibos");
this.jcUser.setWeiboCount(count);
this.jcUser.setThumbnailCount(thumbnailCount);
this.jcUser.setLargeCount(largeCount);
this.jcUser.setCommentCount(commentCount);
return this.jcUser;
}
public List<Comment> obtainComments(Status s) {
CommentWapper commentWapper = null;
if (!jcUser.isBackupComment()) return null;
try {
commentWapper = c.getCommentById(s.getId());
commentCount += commentWapper.getComments().size();
LOG.debug(++callCount + "st call weibo api (now is comments)");
} catch (WeiboException e) {
LOG.error(
"occured a exception when obtaining comments status'id="
+ s.getId()
+ ", Exception:"
+ e);
LOG.error(++callCount + "st call weibo api (now is comments) cause error");
// outOfLimit = true;
jcUser.setOutOfLimit(true);
}
if (commentWapper != null) return commentWapper.getComments();
else return null;
}
@Override
public JcUser packageZip(JcUser jcUser) {
try {
String weibo = compositeWeibo(jcUser);
byte[] data = weibo.getBytes();
BufferedOutputStream stream =
new BufferedOutputStream(new FileOutputStream(new File(path, "index.html")));
stream.write(data);
stream.close();
// package files in zip
String zipPath = path + ".zip";
File zip = new File(zipPath);
FileUtils.zipFile(new File(path), zip);
jcUser.setZipPath(zipPath);
float fileSize = zip.length();
jcUser.setFileSize(fileSize / (1024 * 1024));
} catch (FileNotFoundException e) {
LOG.debug("not found file' \n" + e);
} catch (IOException e2) {
LOG.debug("dowload zip function occured errer: \n" + e2);
}
return jcUser;
}
public Status downloadImage(Status s) {
if (s.getRetweetedStatus() != null) {
s.setRetweetedStatus(downloadImage(s.getRetweetedStatus()));
}
if (s.getPicUrls() == null || s.getPicUrls().length == 0) return s;
Map<Integer, String> failUrls = new HashMap<Integer, String>();
// String[] imageUrls = new String[s.getPicUrls().length];
String[] imageUrls = s.getPicUrls();
// int index = 0;
for (int i = 0; i < imageUrls.length; i++) {
// for (String imageUrl : s.getPicUrls()) {
try {
String imageUrl = downloadImage(imageUrls[i]);
imageUrls[i] = imageUrl;
} catch (IOException e) {
LOG.error("occured exception when download images:" + imageUrls[i] + e);
LOG.error("keep it,then i will download it again.");
failUrls.put(i, imageUrls[i]);
}
}
if (failUrls.size() > 0) {
for (int i : failUrls.keySet())
try {
LOG.debug("the second time to download image:" + failUrls.get(i));
String imageUrl = downloadImage(failUrls.get(i));
imageUrls[i] = imageUrl;
} catch (IOException e) {
LOG.error(
"occured exception when the second time download images:"
+ failUrls.get(i)
+ " \n"
+ e);
imageUrls[i] = "images/fail.thumbnail.jpg";
}
}
if (jcUser.isBackupThumbnail()) s.setPicUrls(imageUrls);
// else //without any picture
// s.setPicUrls(null);
return s;
}
