private void processPhotoSet(
final ArrayList<DownloadLink> decryptedLinks,
final Browser br,
final String puid,
final String fpname,
final String hashTagsStr)
throws Exception {
final String gc = getGoogleCarousel(br);
if (gc != null) {
FilePackage fp = null;
final String JSON =
new Regex(gc, "<script type=\"application/ld\\+json\">(.*?)</script>").getMatch(0);
final Map<String, Object> json = JavaScriptEngineFactory.jsonToJavaMap(JSON);
final String articleBody = (String) json.get("articleBody");
final String fpName =
articleBody != null ? articleBody.replaceAll("[\r\n]+", "").trim() : fpname;
if (fpName != null) {
fp = FilePackage.getInstance();
fp.setName(fpName);
}
// single entry objects are not in 'list'
ArrayList<Object> results = null;
try {
results = (ArrayList<Object>) JavaScriptEngineFactory.walkJson(json, "image/@list");
} catch (Throwable t) {
// single entry ?
final String[] a = new String[] {(String) json.get("image")};
results = new ArrayList<Object>(Arrays.asList(a));
}
if (results != null) {
int count = 1;
final DecimalFormat df = new DecimalFormat(results.size() < 100 ? "00" : "000");
for (final Object result : results) {
final String url = (String) result;
final DownloadLink dl = createDownloadlink("directhttp://" + url);
if (fp != null) {
fp.add(dl);
}
// cleanup...
final String filename =
setFileName(cleanupName(df.format(count) + " - " + fpName), puid)
+ getFileNameExtensionFromString(url);
if (!useOriginalFilename) {
dl.setFinalFileName(filename);
}
dl.setAvailable(true);
setMD5Hash(dl, url);
setImageLinkID(dl, url, puid);
if (hashTagsStr != null) {
dl.setProperty(PROPERTY_TAGS, hashTagsStr);
}
dl.setComment(hashTagsStr);
decryptedLinks.add(dl);
count++;
}
}
}
}
public ArrayList<DownloadLink> decryptIt(CryptedLink param, ProgressController progress)
throws Exception {
final ArrayList<DownloadLink> decryptedLinks = new ArrayList<DownloadLink>();
br.setFollowRedirects(true);
final String parameter = param.toString().replace("#", "");
if (parameter.matches(
"https?://(?:www\\.)?ge\\.tt/(?:developers|press|tools|notifications|blog|about|javascript|button|contact|terms|api|m).*?")) {
decryptedLinks.add(this.createOfflinelink(parameter));
return decryptedLinks;
}
br.setAllowedResponseCodes(410);
br.getPage(parameter);
if (this.br.getHttpConnection().getResponseCode() == 404
|| this.br.getHttpConnection().getResponseCode() == 410
|| br.containsHTML(
"Page not found|The page you were looking for was not found|Files removed|These files have been removed by the owner")) {
decryptedLinks.add(this.createOfflinelink(parameter));
return decryptedLinks;
}
// final String singleFile = new Regex(parameter, "/v/(\\d+)").getMatch(0);
final String folderid = new Regex(parameter, "ge\\.tt/([A-Za-z0-9]+)(/v/\\d+)?").getMatch(0);
br.getHeaders().put("Accept", "application/json, text/plain, */*");
br.getPage("//api.ge.tt/1/shares/" + folderid);
LinkedHashMap<String, Object> entries =
(LinkedHashMap<String, Object>) JavaScriptEngineFactory.jsonToJavaMap(br.toString());
final String error = (String) entries.get("error");
if (error != null || br.containsHTML(">404 Not Found<")) {
final DownloadLink dlink = this.createOfflinelink(parameter);
decryptedLinks.add(dlink);
return decryptedLinks;
}
final ArrayList<Object> files = (ArrayList<Object>) entries.get("files");
for (final Object fileo : files) {
entries = (LinkedHashMap<String, Object>) fileo;
final String filename = (String) entries.get("filename");
final String fileid = (String) entries.get("fileid");
final long filesize = JavaScriptEngineFactory.toLong(entries.get("size"), 0);
final DownloadLink dl =
createDownloadlink(
"http://proxy.ge.tt/1/files/" + folderid + "/" + fileid + "/blob?download");
dl.setContentUrl("http://ge.tt/" + folderid);
dl.setName(filename);
dl.setDownloadSize(filesize);
dl.setAvailable(true);
decryptedLinks.add(dl);
}
if (decryptedLinks == null || decryptedLinks.size() == 0) {
logger.info("ge.tt: Folder is empty! Link: " + parameter);
final DownloadLink dlink = this.createOfflinelink(parameter);
decryptedLinks.add(dlink);
return decryptedLinks;
}
return decryptedLinks;
}
private void decryptUser() throws Exception {
String nextPage = "";
int counter = 1;
boolean decryptSingle = parameter.matches("/page/\\d+");
br.getPage(parameter);
br.followRedirect(true);
if (br.containsHTML(GENERALOFFLINE)) {
logger.info("Link offline: " + parameter);
return;
}
handlePassword();
br.followRedirect(true);
final FilePackage fp = FilePackage.getInstance();
String fpName = new Regex(parameter, "//(.+?)\\.tumblr").getMatch(0);
fp.setName(fpName);
do {
if (this.isAbort()) {
logger.info("Decryption aborted by user");
return;
}
if (!"".equals(nextPage)) {
br.getPage(nextPage);
}
if (parameter.contains("/archive")) {
// archive we will need todo things differently!
final String[] posts =
br.getRegex("<a target=\"_blank\" class=\"hover\" title=\"[^\"]*\" href=\"(.*?)\"")
.getColumn(0);
if (posts != null) {
for (final String post : posts) {
final DownloadLink dl = createDownloadlink(post);
fp.add(dl);
distribute(dl);
decryptedLinks.add(dl);
}
}
} else {
// identify all posts on page then filter accordingly. best way todo this is via google
// carousel, due to different page
// layouts(templates) can be hard to find what you're looking for.
final String gc = getGoogleCarousel(br);
if (gc != null) {
final String JSON =
new Regex(gc, "<script type=\"application/ld\\+json\">(.*?)</script>").getMatch(0);
final Map<String, Object> json = JavaScriptEngineFactory.jsonToJavaMap(JSON);
final ArrayList<Object> results = (ArrayList<Object>) json.get("itemListElement");
for (final Object result : results) {
final LinkedHashMap<String, Object> j = (LinkedHashMap<String, Object>) result;
final String url = (String) j.get("url");
final DownloadLink dl = createDownloadlinkTumblr(url);
fp.add(dl);
distribute(dl);
decryptedLinks.add(dl);
}
}
}
if (decryptSingle) {
break;
}
nextPage =
parameter.contains("/archive")
? br.getRegex("\"(/archive(?:/[^\"]*)?\\?before_time=\\d+)\">Next Page").getMatch(0)
: br.getRegex("\"(/page/" + ++counter + ")\"").getMatch(0);
} while (nextPage != null);
logger.info("Decryption done - last 'nextPage' value was: " + nextPage);
}