private static void fetchLikesAndUpdateUsers(Post post, Set<User> users, Page page) { PagedList<Reference> likes = facebook.likeOperations().getLikes(post.getId()); int totalLikes = likes.size(); for (Reference userLike : likes) { User user = users .stream() .filter(f -> f.getId().equals(userLike.getId())) .findFirst() .orElse(new User(userLike.getId(), userLike.getName(), page)); user.addLike(post, page.getId()); users.add(user); } while (likes.getNextPage() != null) { likes = facebook.likeOperations().getLikes(post.getId(), likes.getNextPage()); totalLikes += likes.size(); for (Reference userLike : likes) { User user = users .stream() .filter(f -> f.getId().equals(userLike.getId())) .findFirst() .orElse(new User(userLike.getId(), userLike.getName(), page)); user.addLike(post, page.getId()); users.add(user); } } post.getExtraData().putIfAbsent("likesCount", totalLikes); }
public static void collect(CollectorInfo.Moment moment) { List<Page> pages = MongoService.getAllPages(); for (Page page : pages) { try { long lastMonth = DateTime.now().minusMonths(1).toDateTime().getMillis() / 1000; long now = DateTime.now().toDateTime().getMillis() / 1000; PagingParameters pagingParameters = new PagingParameters(25, null, lastMonth, now); PagedList<Post> posts; switch (moment) { case ALL: posts = facebook.feedOperations().getPosts(page.getId()); break; case RECENT: posts = facebook.feedOperations().getPosts(page.getId(), pagingParameters); break; default: posts = facebook.feedOperations().getPosts(page.getId()); break; } boolean firstTime = true; do { try { if (!firstTime) posts = facebook.feedOperations().getPosts(page.getId(), posts.getNextPage()); firstTime = false; for (Post post : posts) { Set<User> users = new HashSet<>(); Set<Comment> comments = new HashSet<>(); fetchCommentAndUpdateUsers(post, comments, users, page); fetchLikesAndUpdateUsers(post, users, page); MongoService.save(post); for (Comment comment : comments) { MongoService.save(comment); } // save or update users iterations MongoService.save(users); } } catch (Exception e) { Logger.debug("error on get more posts: " + e.getMessage()); } Logger.debug("update:" + page.getTitle() + " "); } while (posts.getNextPage() != null); Logger.debug("Finished page:" + page.getId()); } catch (Exception e) { Logger.debug("error on page:" + page.getId()); } } }
private static void processPage( PdfFile file, Document document, int i, int scaleInt, boolean isthumbnail) { float scale = scaleInt / 100.0f; float rotation = 0f; PDimension size = document.getPageDimension(i, rotation, scale); // double dpi = Math.sqrt((size.getWidth() * size.getWidth()) + // (size.getHeight() * size.getHeight())) // / Math.sqrt((8.5 * 8.5) + (11 * 11)); // if (dpi < (targetDPI - 0.1)) { // scale = (float) (targetDPI / dpi); size = document.getPageDimension(i, rotation, scale); // } int pageWidth = (int) (size.getWidth()); int pageHeight = (int) (size.getHeight()); BufferedImage image = new BufferedImage(pageWidth, pageHeight, BufferedImage.TYPE_3BYTE_BGR); Graphics2D g = image.createGraphics(); document.paintPage(i, g, GraphicsRenderingHints.SCREEN, Page.BOUNDARY_CROPBOX, rotation, scale); g.dispose(); if (!isthumbnail) { String pageImage = getPageImagePath(file, i, scaleInt, isthumbnail); ImageUtil.saveJPEG(image, pageImage); Logger.debug("Page File Path: %s", pageImage); } else { String pageThumbnail = getPageImagePath(file, i, scaleInt, isthumbnail); BufferedImage thumbnail = ImageUtil.thumbnail(image, 140, 160, false, false); ImageUtil.saveJPEG(thumbnail, pageThumbnail); Logger.debug("Page Thumbnail Path: %s", pageThumbnail); models.Page page = models.Page.find("file= ? and page = ?", file, i).first(); if (page == null) { page = new models.Page(); page.file = file; page.page = i; } page.content = document.getPageText(i).toString(); page.width = pageWidth; page.height = pageHeight; page.save(); } Logger.debug("process page %s", i); }
private static void fetchCommentAndUpdateUsers( Post post, Set<Comment> commentsToSave, Set<User> users, Page page) { PagedList<Comment> comments = facebook.commentOperations().getComments(post.getId()); int totalComments = comments.size(); for (Comment comment : comments) { User user = users .stream() .filter(f -> f.getId().equals(comment.getFrom().getId())) .findFirst() .orElse(new User(comment.getFrom().getId(), comment.getFrom().getName(), page)); user.addComment(comment, post, page.getId()); users.add(user); commentsToSave.add(comment); if (comment.getCommentCount() != null && comment.getCommentCount() > 0) { PagedList<Comment> commentsComments = facebook.commentOperations().getComments(comment.getId()); for (Comment commentsComment : commentsComments) { user = users .stream() .filter(f -> f.getId().equals(commentsComment.getFrom().getId())) .findFirst() .orElse(new User(comment.getFrom().getId(), comment.getFrom().getName(), page)); user.addComment(commentsComment, post, page.getId()); users.add(user); commentsToSave.add(comment); } while (commentsComments.getNextPage() != null) { commentsComments = facebook.commentOperations().getComments(post.getId(), comments.getNextPage()); for (Comment commentsComment : commentsComments) { user = users .stream() .filter(f -> f.getId().equals(commentsComment.getFrom().getId())) .findFirst() .orElse(new User(comment.getFrom().getId(), comment.getFrom().getName(), page)); user.addComment(commentsComment, post, page.getId()); users.add(user); commentsToSave.add(comment); } } } } while (comments.getNextPage() != null) { comments = facebook.commentOperations().getComments(post.getId(), comments.getNextPage()); totalComments += comments.size(); for (Comment comment : comments) { User user = users .stream() .filter(f -> f.getId().equals(comment.getFrom().getId())) .findFirst() .orElse(new User(comment.getFrom().getId(), comment.getFrom().getName(), page)); user.addComment(comment, post, page.getId()); users.add(user); commentsToSave.add(comment); if (comment.getCommentCount() != null && comment.getCommentCount() > 0) { PagedList<Comment> commentsComments = facebook.commentOperations().getComments(comment.getId()); for (Comment commentsComment : commentsComments) { user = users .stream() .filter(f -> f.getId().equals(commentsComment.getFrom().getId())) .findFirst() .orElse(new User(comment.getFrom().getId(), comment.getFrom().getName(), page)); user.addComment(commentsComment, post, page.getId()); users.add(user); commentsToSave.add(comment); } while (commentsComments.getNextPage() != null) { commentsComments = facebook.commentOperations().getComments(post.getId(), comments.getNextPage()); for (Comment commentsComment : commentsComments) { user = users .stream() .filter(f -> f.getId().equals(commentsComment.getFrom().getId())) .findFirst() .orElse( new User(comment.getFrom().getId(), comment.getFrom().getName(), page)); user.addComment(commentsComment, post, page.getId()); users.add(user); commentsToSave.add(comment); } } } } } post.getExtraData().putIfAbsent("commentsCount", totalComments); }