private void processPhotoSet(
final ArrayList<DownloadLink> decryptedLinks,
final Browser br,
final String puid,
final String fpname,
final String hashTagsStr)
throws Exception {
final String gc = getGoogleCarousel(br);
if (gc != null) {
FilePackage fp = null;
final String JSON =
new Regex(gc, "<script type=\"application/ld\\+json\">(.*?)</script>").getMatch(0);
final Map<String, Object> json = JavaScriptEngineFactory.jsonToJavaMap(JSON);
final String articleBody = (String) json.get("articleBody");
final String fpName =
articleBody != null ? articleBody.replaceAll("[\r\n]+", "").trim() : fpname;
if (fpName != null) {
fp = FilePackage.getInstance();
fp.setName(fpName);
}
// single entry objects are not in 'list'
ArrayList<Object> results = null;
try {
results = (ArrayList<Object>) JavaScriptEngineFactory.walkJson(json, "image/@list");
} catch (Throwable t) {
// single entry ?
final String[] a = new String[] {(String) json.get("image")};
results = new ArrayList<Object>(Arrays.asList(a));
}
if (results != null) {
int count = 1;
final DecimalFormat df = new DecimalFormat(results.size() < 100 ? "00" : "000");
for (final Object result : results) {
final String url = (String) result;
final DownloadLink dl = createDownloadlink("directhttp://" + url);
if (fp != null) {
fp.add(dl);
}
// cleanup...
final String filename =
setFileName(cleanupName(df.format(count) + " - " + fpName), puid)
+ getFileNameExtensionFromString(url);
if (!useOriginalFilename) {
dl.setFinalFileName(filename);
}
dl.setAvailable(true);
setMD5Hash(dl, url);
setImageLinkID(dl, url, puid);
if (hashTagsStr != null) {
dl.setProperty(PROPERTY_TAGS, hashTagsStr);
}
dl.setComment(hashTagsStr);
decryptedLinks.add(dl);
count++;
}
}
}
}
public void setLinks(java.util.List<DownloadLinkStorable> links) {
if (links != null) {
this.links = links;
synchronized (filePackage) {
for (DownloadLinkStorable link : links) {
filePackage.add(link._getDownloadLink());
}
}
}
}
@Override
public ArrayList<DownloadLink> decryptIt(
final CryptedLink param, final ProgressController progress) throws Exception {
synchronized (LOCK) {
PROGRESS = progress;
final ArrayList<DownloadLink> decryptedLinks = new ArrayList<DownloadLink>();
final String parameter = correctCryptedLink(param.toString());
setBrowserExclusive();
br.setFollowRedirects(true);
br.getHeaders().put("User-Agent", UA);
/* Handle Captcha and/or password */
handleCaptchaAndPassword(parameter, param);
if (!br.getURL().contains("relink.us/")) {
try {
validateLastChallengeResponse();
} catch (final Throwable e) {
}
logger.info("Link offline: " + parameter);
return decryptedLinks;
}
if (br.containsHTML("<title>404</title>")) {
logger.info("Link offline: " + parameter);
return decryptedLinks;
}
if (ALLFORM != null && ALLFORM.getRegex("password").matches()) {
throw new DecrypterException(DecrypterException.PASSWORD);
}
if (ALLFORM != null && ALLFORM.getRegex("captcha").matches()) {
throw new DecrypterException(DecrypterException.CAPTCHA);
}
final String page = br.toString();
progress.setRange(0);
final String title =
br.getRegex("shrink\"><th>(Titel|Baslik|Title)</th><td>(.*?)</td></tr>").getMatch(1);
FilePackage fp = null;
if (title != null && title.trim().length() > 0) {
fp = FilePackage.getInstance();
fp.setName(title);
fp.setProperty("ALLOW_MERGE", true);
}
/* use cnl2 button if available */
String cnlUrl = "http://127\\.0\\.0\\.1:9666/flash/addcrypted2";
if (br.containsHTML(cnlUrl)) {
final Browser cnlbr = br.cloneBrowser();
Form cnlForm = null;
for (Form f : cnlbr.getForms()) {
if (f.containsHTML(cnlUrl)) {
cnlForm = f;
}
}
if (cnlForm != null) {
if (System.getProperty("jd.revision.jdownloaderrevision") != null) {
String jk =
cnlbr.getRegex("<input type=\"hidden\" name=\"jk\" value=\"([^\"]+)\"").getMatch(0);
HashMap<String, String> infos = new HashMap<String, String>();
infos.put(
"crypted", Encoding.urlDecode(cnlForm.getInputField("crypted").getValue(), false));
infos.put("jk", jk);
String source = cnlForm.getInputField("source").getValue();
if (StringUtils.isEmpty(source)) {
source = parameter.toString();
} else {
source = Encoding.urlDecode(source, true);
}
infos.put("source", source);
String json = JSonStorage.toString(infos);
final DownloadLink dl =
createDownloadlink(
"http://dummycnl.jdownloader.org/"
+ HexFormatter.byteArrayToHex(json.getBytes("UTF-8")));
if (fp != null) {
fp.add(dl);
}
try {
distribute(dl);
} catch (final Throwable e) {
/* does not exist in 09581 */
}
decryptedLinks.add(dl);
return decryptedLinks;
} else {
String jk =
cnlbr.getRegex("<input type=\"hidden\" name=\"jk\" value=\"([^\"]+)\"").getMatch(0);
cnlForm.remove("jk");
cnlForm.put("jk", (jk != null ? jk.replaceAll("\\+", "%2B") : "nothing"));
try {
cnlbr.submitForm(cnlForm);
if (cnlbr.containsHTML("success")) {
return decryptedLinks;
}
if (cnlbr.containsHTML("^failed")) {
logger.warning(
"relink.us: CNL2 Postrequest was failed! Please upload now a logfile, contact our support and add this loglink to your bugreport!");
logger.warning("relink.us: CNL2 Message: " + cnlbr.toString());
}
} catch (Throwable e) {
logger.info("relink.us: ExternInterface(CNL2) is disabled!");
}
}
}
}
if (!br.containsHTML("download.php\\?id=[a-f0-9]+") && !br.containsHTML("getFile\\(")) {
return null;
}
if (!decryptContainer(page, parameter, "dlc", decryptedLinks)) {
if (!decryptContainer(page, parameter, "ccf", decryptedLinks)) {
decryptContainer(page, parameter, "rsdf", decryptedLinks);
}
}
/* Webdecryption */
if (decryptedLinks.isEmpty()) {
decryptLinks(decryptedLinks, param);
final String more_links[] =
new Regex(
page,
Pattern.compile(
"<a href=\"(go\\.php\\?id=[a-zA-Z0-9]+\\&seite=\\d+)\">",
Pattern.CASE_INSENSITIVE))
.getColumn(0);
for (final String link : more_links) {
br.getPage("http://relink.us/" + link);
decryptLinks(decryptedLinks, param);
}
}
if (decryptedLinks.isEmpty() && br.containsHTML(cnlUrl)) {
throw new DecrypterException("CNL2 only, open this link in Browser");
}
try {
validateLastChallengeResponse();
} catch (final Throwable e) {
}
if (fp != null) {
fp.addLinks(decryptedLinks);
}
return decryptedLinks;
}
}
@SuppressWarnings({"unchecked"})
private void decryptUserLoggedIn() throws Exception {
final int limit = 10;
int offset = 0;
boolean decryptSingle = parameter.matches("/page/\\d+");
/* Access url for logged-out-users first because if we don't we cannot see whether a password is needed or not! */
final String url_for_logged_out_users = convertUserUrlToLoggedOutUser();
this.br.setFollowRedirects(false);
this.br.getPage(url_for_logged_out_users);
br.followRedirect(true);
if (this.handlePassword()) {
/* Bullshit - if a blog is password protected we can only display it in the "logged out" mode ... */
decryptUser();
return;
}
if (br.containsHTML(GENERALOFFLINE) || this.br.getHttpConnection().getResponseCode() == 404) {
logger.info("Link offline: " + parameter);
return;
}
handlePassword();
final FilePackage fp = FilePackage.getInstance();
final String username = getUsername(this.parameter);
String fpName = username;
fp.setName(fpName);
LinkedHashMap<String, Object> entries = null;
ArrayList<Object> ressourcelist = null;
do {
if (this.isAbort()) {
logger.info("Decryption aborted by user");
return;
}
br.getHeaders().put("Accept", "application/json, text/javascript, */*; q=0.01");
br.getHeaders().put("X-Requested-With", "XMLHttpRequest");
/* Not needed! */
// br.getHeaders().put("X-tumblr-form-key", "blaTest");
br.getPage(
"//www.tumblr.com/svc/indash_blog/posts?tumblelog_name_or_id="
+ username
+ "&post_id=&limit="
+ limit
+ "&offset="
+ offset);
entries =
(LinkedHashMap<String, Object>) JavaScriptEngineFactory.jsonToJavaObject(br.toString());
ressourcelist =
(ArrayList<Object>) JavaScriptEngineFactory.walkJson(entries, "response/posts");
for (final Object posto : ressourcelist) {
entries = (LinkedHashMap<String, Object>) posto;
final String type = (String) entries.get("type");
final String post_url = (String) entries.get("post_url");
String directlink = null;
String extension = null;
String extensionFallback = null;
if (type.equalsIgnoreCase("photo")) {
directlink =
(String) JavaScriptEngineFactory.walkJson(entries, "photos/{0}/original_size/url");
extensionFallback = ".jpg";
} else if (type.equalsIgnoreCase("video")) {
directlink = (String) entries.get("video_url");
final String url_hd = convertDirectVideoUrltoHD(directlink);
if (url_hd != null) {
directlink = url_hd;
}
extensionFallback = ".mp4";
} else {
/* There is type "text", "answer" and there might be type other types too! */
logger.info("Unsupported or un-downloadable tumblr-post-type: " + type);
}
if (directlink != null) {
extension = getFileNameExtensionFromURL(directlink);
if (extension == null) {
extension = extensionFallback;
}
String filename = getFileNameFromURL(new URL(directlink));
if (filename != null && !filename.endsWith(extension)) {
filename += extension;
}
final DownloadLink dl = this.createDownloadlink("directhttp://" + directlink);
if (this.passCode != null) {
dl.setDownloadPassword(this.passCode);
}
dl.setAvailable(true);
if (post_url != null) {
dl.setContentUrl(post_url);
}
if (filename != null && !useOriginalFilename) {
dl.setName(filename);
}
fp.add(dl);
decryptedLinks.add(dl);
distribute(dl);
}
offset++;
}
if (decryptSingle) {
break;
}
} while (ressourcelist.size() >= limit);
logger.info("Decryption done");
}
private void decryptUser() throws Exception {
String nextPage = "";
int counter = 1;
boolean decryptSingle = parameter.matches("/page/\\d+");
br.getPage(parameter);
br.followRedirect(true);
if (br.containsHTML(GENERALOFFLINE)) {
logger.info("Link offline: " + parameter);
return;
}
handlePassword();
br.followRedirect(true);
final FilePackage fp = FilePackage.getInstance();
String fpName = new Regex(parameter, "//(.+?)\\.tumblr").getMatch(0);
fp.setName(fpName);
do {
if (this.isAbort()) {
logger.info("Decryption aborted by user");
return;
}
if (!"".equals(nextPage)) {
br.getPage(nextPage);
}
if (parameter.contains("/archive")) {
// archive we will need todo things differently!
final String[] posts =
br.getRegex("<a target=\"_blank\" class=\"hover\" title=\"[^\"]*\" href=\"(.*?)\"")
.getColumn(0);
if (posts != null) {
for (final String post : posts) {
final DownloadLink dl = createDownloadlink(post);
fp.add(dl);
distribute(dl);
decryptedLinks.add(dl);
}
}
} else {
// identify all posts on page then filter accordingly. best way todo this is via google
// carousel, due to different page
// layouts(templates) can be hard to find what you're looking for.
final String gc = getGoogleCarousel(br);
if (gc != null) {
final String JSON =
new Regex(gc, "<script type=\"application/ld\\+json\">(.*?)</script>").getMatch(0);
final Map<String, Object> json = JavaScriptEngineFactory.jsonToJavaMap(JSON);
final ArrayList<Object> results = (ArrayList<Object>) json.get("itemListElement");
for (final Object result : results) {
final LinkedHashMap<String, Object> j = (LinkedHashMap<String, Object>) result;
final String url = (String) j.get("url");
final DownloadLink dl = createDownloadlinkTumblr(url);
fp.add(dl);
distribute(dl);
decryptedLinks.add(dl);
}
}
}
if (decryptSingle) {
break;
}
nextPage =
parameter.contains("/archive")
? br.getRegex("\"(/archive(?:/[^\"]*)?\\?before_time=\\d+)\">Next Page").getMatch(0)
: br.getRegex("\"(/page/" + ++counter + ")\"").getMatch(0);
} while (nextPage != null);
logger.info("Decryption done - last 'nextPage' value was: " + nextPage);
}