private void processPhotoSet(
final ArrayList<DownloadLink> decryptedLinks,
final Browser br,
final String puid,
final String fpname,
final String hashTagsStr)
throws Exception {
final String gc = getGoogleCarousel(br);
if (gc != null) {
FilePackage fp = null;
final String JSON =
new Regex(gc, "<script type=\"application/ld\\+json\">(.*?)</script>").getMatch(0);
final Map<String, Object> json = JavaScriptEngineFactory.jsonToJavaMap(JSON);
final String articleBody = (String) json.get("articleBody");
final String fpName =
articleBody != null ? articleBody.replaceAll("[\r\n]+", "").trim() : fpname;
if (fpName != null) {
fp = FilePackage.getInstance();
fp.setName(fpName);
}
// single entry objects are not in 'list'
ArrayList<Object> results = null;
try {
results = (ArrayList<Object>) JavaScriptEngineFactory.walkJson(json, "image/@list");
} catch (Throwable t) {
// single entry ?
final String[] a = new String[] {(String) json.get("image")};
results = new ArrayList<Object>(Arrays.asList(a));
}
if (results != null) {
int count = 1;
final DecimalFormat df = new DecimalFormat(results.size() < 100 ? "00" : "000");
for (final Object result : results) {
final String url = (String) result;
final DownloadLink dl = createDownloadlink("directhttp://" + url);
if (fp != null) {
fp.add(dl);
}
// cleanup...
final String filename =
setFileName(cleanupName(df.format(count) + " - " + fpName), puid)
+ getFileNameExtensionFromString(url);
if (!useOriginalFilename) {
dl.setFinalFileName(filename);
}
dl.setAvailable(true);
setMD5Hash(dl, url);
setImageLinkID(dl, url, puid);
if (hashTagsStr != null) {
dl.setProperty(PROPERTY_TAGS, hashTagsStr);
}
dl.setComment(hashTagsStr);
decryptedLinks.add(dl);
count++;
}
}
}
}
@SuppressWarnings({"unchecked", "deprecation"})
public ArrayList<DownloadLink> decryptIt(CryptedLink param, ProgressController progress)
throws Exception {
ArrayList<DownloadLink> decryptedLinks = new ArrayList<DownloadLink>();
final String parameter = param.toString();
jd.plugins.hoster.AmpyaCom.initializeSession(this.br);
br.getPage(jd.plugins.hoster.AmpyaCom.getApiUrl(parameter));
if (br.getHttpConnection().getResponseCode() == 404) {
decryptedLinks.add(this.createOfflinelink(parameter));
return decryptedLinks;
}
String fpName = null;
LinkedHashMap<String, Object> entries =
(LinkedHashMap<String, Object>) JavaScriptEngineFactory.jsonToJavaObject(br.toString());
final ArrayList<Object> ressourcelist = (ArrayList<Object>) entries.get("clips");
for (final Object clipo : ressourcelist) {
entries = (LinkedHashMap<String, Object>) clipo;
final String title = (String) entries.get("title");
final String artist = (String) entries.get("artist_name");
final String video_id =
Long.toString(JavaScriptEngineFactory.toLong(entries.get("video_id"), 0));
if (fpName == null) {
fpName = (String) entries.get("container_title");
}
if (title == null || artist == null || "0".equals(video_id)) {
continue;
}
final DownloadLink dl =
this.createDownloadlink(
"http://ampyadecrypted.com/"
+ System.currentTimeMillis()
+ new Random().nextInt(1000000000));
String filename = artist + " - " + title + ".mp4";
filename = encodeUnicode(filename);
dl.setFinalFileName(filename);
dl.setAvailable(true);
dl.setContentUrl(parameter);
dl.setProperty("mainlink", parameter);
dl.setProperty("videoid", video_id);
decryptedLinks.add(dl);
}
if (fpName != null) {
final FilePackage fp = FilePackage.getInstance();
fp.setName(Encoding.htmlDecode(fpName.trim()));
fp.addLinks(decryptedLinks);
}
return decryptedLinks;
}
@SuppressWarnings({"unchecked", "rawtypes"})
private void decryptMtvGermanyPlaylists() throws Exception {
br.getPage(parameter);
fpName = br.getRegex("<title>([^<>\"]*?)</title>").getMatch(0);
if (fpName == null) {
/* Fallback to url-packagename */
fpName = new Regex(this.parameter, "https?://[^/]+/(.+)").getMatch(0);
}
ArrayList<Object> ressourcelist = null;
LinkedHashMap<String, Object> entries = null;
try {
final String json = this.br.getRegex("window\\.pagePlaylist = (\\[\\{.*?\\}\\])").getMatch(0);
ressourcelist = (ArrayList) JavaScriptEngineFactory.jsonToJavaObject(json);
for (final Object object : ressourcelist) {
entries = (LinkedHashMap<String, Object>) object;
final String path = (String) entries.get("path");
final String url_mrss = (String) entries.get("mrss");
final String title = (String) entries.get("title");
final String subtitle = (String) entries.get("subtitle");
final String video_token = (String) entries.get("video_token");
final String mgid = jd.plugins.hoster.VivaTv.getMGIDOutOfURL(url_mrss);
if (url_mrss == null || title == null || video_token == null || mgid == null) {
throw new DecrypterException("Decrypter broken for link: " + parameter);
}
final String contenturl;
if (path != null) {
contenturl = "http://" + this.br.getHost() + path;
} else {
contenturl = this.parameter;
}
String temp_filename = title;
if (subtitle != null) {
temp_filename += " - " + subtitle;
}
temp_filename += jd.plugins.hoster.VivaTv.default_ext;
final DownloadLink dl = mgidSingleVideoGetDownloadLink(mgid);
dl.setLinkID(video_token);
dl.setName(temp_filename);
dl.setAvailable(true);
dl.setContentUrl(contenturl);
this.decryptedLinks.add(dl);
}
} catch (final Throwable e) {
return;
}
final FilePackage fp = FilePackage.getInstance();
fpName = Encoding.htmlDecode(fpName.trim());
fp.setName(fpName);
fp.addLinks(decryptedLinks);
}
private String md5hex(String s) {
Object result = new Object();
final ScriptEngineManager manager = JavaScriptEngineFactory.getScriptEngineManager(this);
final ScriptEngine engine = manager.getEngineByName("javascript");
final Invocable inv = (Invocable) engine;
try {
engine.eval(NONSTANDARDMD5);
result = inv.invokeFunction("hex_md5", s);
} catch (final Throwable e) {
return null;
}
return result != null ? result.toString() : "";
}
public ArrayList<DownloadLink> decryptIt(CryptedLink param, ProgressController progress)
throws Exception {
final ArrayList<DownloadLink> decryptedLinks = new ArrayList<DownloadLink>();
br.setFollowRedirects(true);
final String parameter = param.toString().replace("#", "");
if (parameter.matches(
"https?://(?:www\\.)?ge\\.tt/(?:developers|press|tools|notifications|blog|about|javascript|button|contact|terms|api|m).*?")) {
decryptedLinks.add(this.createOfflinelink(parameter));
return decryptedLinks;
}
br.setAllowedResponseCodes(410);
br.getPage(parameter);
if (this.br.getHttpConnection().getResponseCode() == 404
|| this.br.getHttpConnection().getResponseCode() == 410
|| br.containsHTML(
"Page not found|The page you were looking for was not found|Files removed|These files have been removed by the owner")) {
decryptedLinks.add(this.createOfflinelink(parameter));
return decryptedLinks;
}
// final String singleFile = new Regex(parameter, "/v/(\\d+)").getMatch(0);
final String folderid = new Regex(parameter, "ge\\.tt/([A-Za-z0-9]+)(/v/\\d+)?").getMatch(0);
br.getHeaders().put("Accept", "application/json, text/plain, */*");
br.getPage("//api.ge.tt/1/shares/" + folderid);
LinkedHashMap<String, Object> entries =
(LinkedHashMap<String, Object>) JavaScriptEngineFactory.jsonToJavaMap(br.toString());
final String error = (String) entries.get("error");
if (error != null || br.containsHTML(">404 Not Found<")) {
final DownloadLink dlink = this.createOfflinelink(parameter);
decryptedLinks.add(dlink);
return decryptedLinks;
}
final ArrayList<Object> files = (ArrayList<Object>) entries.get("files");
for (final Object fileo : files) {
entries = (LinkedHashMap<String, Object>) fileo;
final String filename = (String) entries.get("filename");
final String fileid = (String) entries.get("fileid");
final long filesize = JavaScriptEngineFactory.toLong(entries.get("size"), 0);
final DownloadLink dl =
createDownloadlink(
"http://proxy.ge.tt/1/files/" + folderid + "/" + fileid + "/blob?download");
dl.setContentUrl("http://ge.tt/" + folderid);
dl.setName(filename);
dl.setDownloadSize(filesize);
dl.setAvailable(true);
decryptedLinks.add(dl);
}
if (decryptedLinks == null || decryptedLinks.size() == 0) {
logger.info("ge.tt: Folder is empty! Link: " + parameter);
final DownloadLink dlink = this.createOfflinelink(parameter);
decryptedLinks.add(dlink);
return decryptedLinks;
}
return decryptedLinks;
}
@SuppressWarnings({"unchecked", "rawtypes"})
public ArrayList<DownloadLink> decryptIt(CryptedLink param, ProgressController progress)
throws Exception {
final ArrayList<DownloadLink> decryptedLinks = new ArrayList<DownloadLink>();
final String parameter = param.toString().replace("http://", "https://");
final String fid = new Regex(parameter, "/p/([A-Za-z0-9\\-_]+)").getMatch(0);
String subfolder = new Regex(parameter, "#dir=(.+)").getMatch(0);
if (subfolder == null) {
subfolder = "/";
} else {
subfolder = Encoding.htmlDecode(subfolder);
}
subfolder = Encoding.urlEncode(subfolder);
if (fid == null) {
decryptedLinks.add(createOfflinelink(parameter));
return decryptedLinks;
}
this.br.getPage(parameter);
final String pageJson = jd.plugins.hoster.JianguoyunCom.getWebsiteJson(this.br);
DownloadLink dl = null;
final String isdir =
new Regex(pageJson, "isdir[\t\n\r ]*?:[\t\n\r ]*?(true|false)").getMatch(0);
if ("false".equals(isdir)) {
/* Single file */
dl =
createDownloadlink(
"http://jianguoyundecrypted.com/"
+ System.currentTimeMillis()
+ new Random().nextInt(100000));
dl.setProperty("singlefile", true);
dl.setProperty("folderid", fid);
dl.setProperty("relPath", subfolder);
dl.setProperty("mainlink", parameter);
jd.plugins.hoster.JianguoyunCom.scanFileinfoFromWebsite(this.br, dl);
decryptedLinks.add(dl);
return decryptedLinks;
}
this.br.getPage(
"https://www.jianguoyun.com/d/ajax/dirops/pubDIRBrowse?hash="
+ fid
+ "&relPath="
+ subfolder
+ "&_="
+ System.currentTimeMillis());
LinkedHashMap<String, Object> entries =
(LinkedHashMap<String, Object>) JavaScriptEngineFactory.jsonToJavaObject(br.toString());
final ArrayList<Object> ressourcelist = (ArrayList) entries.get("objects");
for (final Object foldero : ressourcelist) {
entries = (LinkedHashMap<String, Object>) foldero;
final String type = (String) entries.get("type");
final String relPath = (String) entries.get("relPath");
final long filesize = JavaScriptEngineFactory.toLong(entries.get("size"), 0);
if (type == null || relPath == null) {
continue;
}
if (type.equals("directory")) {
dl =
createDownloadlink(
"https://www.jianguoyun.com/p/" + fid + "#dir=" + Encoding.urlEncode(relPath));
} else {
dl =
createDownloadlink(
"http://jianguoyundecrypted.com/"
+ System.currentTimeMillis()
+ new Random().nextInt(100000));
final Regex pathregex = new Regex(relPath, "^(/.*?)([^/]+)$");
final String contenturl =
"https://www.jianguoyun.com/p/"
+ fid
+ "#dir="
+ Encoding.urlEncode(pathregex.getMatch(0));
final String filename = pathregex.getMatch(1);
if (filesize > 0) {
dl.setDownloadSize(filesize);
dl.setAvailable(true);
}
if (filename != null) {
dl.setName(filename);
}
dl.setProperty("folderid", fid);
dl.setProperty("relPath", relPath);
dl.setProperty("mainlink", parameter);
dl.setContentUrl(contenturl);
dl.setLinkID(fid + relPath);
dl.setProperty(DownloadLink.RELATIVE_DOWNLOAD_FOLDER_PATH, relPath);
}
decryptedLinks.add(dl);
}
return decryptedLinks;
}
/* Old channel config url (see also rev 35204): http://www.rai.tv/dl/RaiTV/iphone/android/smartphone/advertising_config.html */
private void decryptWholeDay() throws Exception {
final String mainlink_urlpart = new Regex(parameter, "\\?(.+)").getMatch(0);
final String[] dates = new Regex(parameter, "(\\d{4}\\-\\d{2}\\-\\d{2})").getColumn(0);
final String[] channels = new Regex(parameter, "ch=(\\d+)").getColumn(0);
final String[] videoids = new Regex(parameter, "v=(\\d+)").getColumn(0);
final String date_user_input = dates[dates.length - 1];
final String date_user_input_underscore = date_user_input.replace("-", "_");
final String date_user_input_in_json_format = convertInputDateToJsonDateFormat(date_user_input);
final DownloadLink offline = this.createOfflinelink(parameter);
final String filename_offline;
if (mainlink_urlpart != null) {
filename_offline = date_user_input + "_" + mainlink_urlpart + ".mp4";
} else {
filename_offline = date_user_input + ".mp4";
}
offline.setFinalFileName(filename_offline);
String id_of_single_video_which_user_wants_to_have_only = null;
String chnumber_str = null;
if (channels != null && channels.length > 0) {
chnumber_str = channels[channels.length - 1];
}
if (chnumber_str == null) {
/* Small fallback */
chnumber_str = "1";
}
if (videoids != null && videoids.length > 0) {
id_of_single_video_which_user_wants_to_have_only = videoids[videoids.length - 1];
}
final FilePackage fp = FilePackage.getInstance();
fp.setName(date_user_input_underscore);
LinkedHashMap<String, Object> entries = null;
final String channel_name = "Rai" + chnumber_str;
final String channel_name_with_space = "Rai " + chnumber_str;
this.br.getHeaders().put("X-Requested-With", "XMLHttpRequest");
this.br.getPage(
"http://www.rai.it/dl/palinsesti/Page-e120a813-1b92-4057-a214-15943d95aa68-json.html?canale="
+ channel_name
+ "&giorno="
+ date_user_input);
if (br.getHttpConnection().getResponseCode() == 404) {
decryptedLinks.add(offline);
return;
}
entries =
(LinkedHashMap<String, Object>)
JavaScriptEngineFactory.jsonToJavaObject(this.br.toString());
final ArrayList<Object> daysList = (ArrayList<Object>) entries.get(channel_name_with_space);
/* Walk through all days. */
for (final Object dayO : daysList) {
entries = (LinkedHashMap<String, Object>) dayO;
final String date_of_this_item = (String) entries.get("giorno");
if (date_of_this_item == null || !date_of_this_item.equals(date_user_input_in_json_format)) {
/* Date is missing or not the date we want? Skip item! */
continue;
}
/* Get all items of the day. */
final ArrayList<Object> itemsOfThatDayList = (ArrayList<Object>) entries.get("palinsesto");
for (final Object itemsOfThatDayListO : itemsOfThatDayList) {
entries = (LinkedHashMap<String, Object>) itemsOfThatDayListO;
/* Get all programms of that day. */
final ArrayList<Object> programmsList = (ArrayList<Object>) entries.get("programmi");
/* Finally decrypt the programms. */
for (final Object programmO : programmsList) {
entries = (LinkedHashMap<String, Object>) programmO;
if (entries.isEmpty()) {
continue;
}
final boolean hasVideo = ((Boolean) entries.get("hasVideo")).booleanValue();
final String webLink = (String) entries.get("webLink");
if (!hasVideo || webLink == null || !webLink.startsWith("/")) {
continue;
}
final String url_for_user;
final String url_rai_replay = new Regex(webLink, "raiplay/(video/.+)").getMatch(0);
if (url_rai_replay != null) {
url_for_user = "******" + url_rai_replay;
} else {
url_for_user = "******" + webLink;
}
final DownloadLink dl = this.createDownloadlink(url_for_user);
decryptedLinks.add(dl);
}
}
}
}
private void decryptSingleVideo() throws DecrypterException, Exception {
String dllink = null;
String title = null;
String extension = ".mp4";
String date = null;
String date_formatted = null;
String description = null;
this.br.getPage(this.parameter);
final String jsredirect =
this.br.getRegex("document\\.location\\.replace\\(\\'(http[^<>\"]*?)\\'\\)").getMatch(0);
if (jsredirect != null) {
this.br.getPage(jsredirect.trim());
}
if (this.br.getHttpConnection().getResponseCode() == 404) {
decryptedLinks.add(this.createOfflinelink(this.parameter));
return;
}
/* Do NOT use value of "videoURL_MP4" here! */
/* E.g. http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-70996227-7fec-4be9-bc49-ba0a8104305a.html */
dllink =
this.br
.getRegex("var[\t\n\r ]*?videoURL[\t\n\r ]*?=[\t\n\r ]*?\"(http://[^<>\"]+)\"")
.getMatch(0);
String content_id_from_url = null;
if (this.parameter.matches(TYPE_CONTENTITEM)) {
content_id_from_url = new Regex(this.parameter, "(\\-[a-f0-9\\-]+)\\.html$").getMatch(0);
}
if (dllink == null) {
dllink = findRelinkerUrl();
}
title = this.br.getRegex("property=\"og:title\" content=\"([^<>\"]+)\"").getMatch(0);
date =
this.br
.getRegex(
"content=\"(\\d{4}\\-\\d{2}\\-\\d{2}) \\d{2}:\\d{2}:\\d{2}\" property=\"gen\\-date\"")
.getMatch(0);
final String contentset_id =
this.br
.getRegex(
"var[\t\n\r ]*?urlTop[\t\n\r ]*?=[\t\n\r ]*?\"[^<>\"]+/ContentSet([A-Za-z0-9\\-]+)\\.html")
.getMatch(0);
final String content_id_from_html =
this.br.getRegex("id=\"ContentItem(\\-[a-f0-9\\-]+)\"").getMatch(0);
if (br.getHttpConnection().getResponseCode() == 404
|| (contentset_id == null && content_id_from_html == null && dllink == null)) {
/* Probably not a video/offline */
decryptedLinks.add(this.createOfflinelink(this.parameter));
return;
}
if (dllink != null) {
if (title == null) {
/* Streamurls directly in html */
title = this.br.getRegex("id=\"idMedia\">([^<>]+)<").getMatch(0);
}
if (title == null) {
title = this.br.getRegex("var videoTitolo\\d*?=\\d*?\"([^<>\"]+)\";").getMatch(0);
}
if (date == null) {
date =
this.br
.getRegex("id=\"myGenDate\">(\\d{2}\\-\\d{2}\\-\\d{4} \\d{2}:\\d{2})<")
.getMatch(0);
}
if (date == null) {
/* 2017-01-06: New */
date =
this.br
.getRegex(
"property=\"gen\\-date\" content=\"(\\d{4}\\-\\d{2}\\-\\d{2} \\d{2}:\\d{2}:\\d{2})\"")
.getMatch(0);
}
if (date == null) {
date = this.br.getRegex("data\\-date=\"(\\d{2}/\\d{2}/\\d{4})\"").getMatch(0);
}
} else {
LinkedHashMap<String, Object> entries = null;
if (content_id_from_html != null) {
/* Easiest way to find videoinfo */
this.br.getPage(
"http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem"
+ content_id_from_html
+ ".html?json");
entries =
(LinkedHashMap<String, Object>) JavaScriptEngineFactory.jsonToJavaObject(br.toString());
}
if (entries == null) {
final ArrayList<Object> ressourcelist;
final String list_json_from_html =
this.br
.getRegex("\"list\"[\t\n\r ]*?:[\t\n\r ]*?(\\[.*?\\}[\t\n\r ]*?\\])")
.getMatch(0);
if (list_json_from_html != null) {
ressourcelist =
(ArrayList<Object>) JavaScriptEngineFactory.jsonToJavaObject(list_json_from_html);
} else {
br.getPage(
"http://www.rai.tv/dl/RaiTV/ondemand/ContentSet" + contentset_id + ".html?json");
if (br.getHttpConnection().getResponseCode() == 404) {
decryptedLinks.add(this.createOfflinelink(this.parameter));
return;
}
entries =
(LinkedHashMap<String, Object>)
JavaScriptEngineFactory.jsonToJavaObject(br.toString());
ressourcelist = (ArrayList<Object>) entries.get("list");
}
if (content_id_from_url == null) {
/* Hm probably not a video */
decryptedLinks.add(this.createOfflinelink(this.parameter));
return;
}
String content_id_temp = null;
boolean foundVideoInfo = false;
for (final Object videoo : ressourcelist) {
entries = (LinkedHashMap<String, Object>) videoo;
content_id_temp = (String) entries.get("itemId");
if (content_id_temp != null && content_id_temp.contains(content_id_from_url)) {
foundVideoInfo = true;
break;
}
}
if (!foundVideoInfo) {
/* Probably offline ... */
decryptedLinks.add(this.createOfflinelink(this.parameter));
return;
}
}
date = (String) entries.get("date");
title = (String) entries.get("name");
description = (String) entries.get("desc");
final String type = (String) entries.get("type");
if (type.equalsIgnoreCase("RaiTv Media Video Item")) {
} else {
/* TODO */
logger.warning("Unsupported media type!");
throw new DecrypterException(DecrypterException.PLUGIN_DEFECT);
}
extension = "mp4";
dllink = (String) entries.get("h264");
if (dllink == null || dllink.equals("")) {
dllink = (String) entries.get("m3u8");
extension = "mp4";
}
if (dllink == null || dllink.equals("")) {
dllink = (String) entries.get("wmv");
extension = "wmv";
}
if (dllink == null || dllink.equals("")) {
dllink = (String) entries.get("mediaUri");
extension = "mp4";
}
}
if (title == null) {
title = content_id_from_url;
}
date_formatted = jd.plugins.hoster.RaiTv.formatDate(date);
title = Encoding.htmlDecode(title);
title = date_formatted + "_raitv_" + title;
title = encodeUnicode(title);
final FilePackage fp = FilePackage.getInstance();
fp.setName(title);
decryptRelinker(dllink, title, extension, fp, description);
}
@SuppressWarnings({"deprecation", "unchecked"})
@Override
public boolean checkLinks(final DownloadLink[] urls) {
if (urls == null || urls.length == 0) {
return false;
}
try {
br.setCookiesExclusive(true);
final StringBuilder sb = new StringBuilder();
final ArrayList<DownloadLink> links = new ArrayList<DownloadLink>();
int index = 0;
int tempcounter = 0;
while (true) {
/* Reset tempcounter */
tempcounter = 0;
links.clear();
while (true) {
/* we test 10 links at once (max = 10) */
if (index == urls.length || links.size() > 9) {
break;
}
links.add(urls[index]);
index++;
}
sb.delete(0, sb.capacity());
sb.append("links=");
for (final DownloadLink dl : links) {
sb.append(Encoding.urlEncode(dl.getDownloadURL()));
if (tempcounter < links.size() - 1) {
sb.append(Encoding.urlEncode("\n"));
}
tempcounter++;
}
/* Reset tempcounter */
tempcounter = 0;
postPage(getAPIProtocol() + this.getHost() + "/download/json_check", sb.toString());
LinkedHashMap<String, Object> entries = null;
final ArrayList<Object> ressourcelist =
(ArrayList<Object>) JavaScriptEngineFactory.jsonToJavaObject(br.toString());
final String[] json_workaround_array = br.toString().split("\\},\\{");
for (final DownloadLink dl : links) {
final String state;
final String filename;
final long filesize;
if (tempcounter <= json_workaround_array.length - 1) {
entries = (LinkedHashMap<String, Object>) ressourcelist.get(tempcounter);
state = (String) entries.get("status");
filename = (String) entries.get("filename");
filesize = JavaScriptEngineFactory.toLong(entries.get("filesize"), 0);
} else {
state = null;
filename = null;
filesize = 0;
}
if ("offline".equals(state) || state == null) {
dl.setAvailable(false);
} else {
dl.setAvailable(true);
}
/* Trust API - offline urls can still have their filename- and size information available */
if (filename != null) {
dl.setFinalFileName(filename);
}
dl.setDownloadSize(filesize);
tempcounter++;
}
if (index == urls.length) {
break;
}
}
} catch (final Exception e) {
return false;
}
return true;
}
/** Finds mgids inside drupal json */
@SuppressWarnings({"unchecked", "rawtypes"})
private void crawlDrupal() {
final String playlist_id =
br.getRegex("name=\"vimn:entity_uuid\" content=\"([a-z0-9\\-:]*?)\"").getMatch(0);
final String js =
br.getRegex("jQuery\\.extend\\(Drupal\\.settings, (\\{.*?)\\);.*?</script>").getMatch(0);
if (js == null || playlist_id == null) {
return;
}
try {
LinkedHashMap<String, Object> entries =
(LinkedHashMap<String, Object>) JavaScriptEngineFactory.jsonToJavaObject(js);
final LinkedHashMap<String, Object> vimn_video =
(LinkedHashMap<String, Object>) entries.get("vimn_video");
ArrayList<Object> resources = null;
final Object playlist_o = vimn_video.get("playlists");
final Object embedded_videos_o = vimn_video.get("embedded_videos");
if (!(playlist_o instanceof ArrayList)) {
/* Playlist */
entries = (LinkedHashMap<String, Object>) playlist_o;
entries = (LinkedHashMap<String, Object>) entries.get(playlist_id);
resources = (ArrayList) entries.get("items");
final Object title_object = entries.get("title");
if (title_object instanceof String) {
fpName = (String) entries.get("title");
}
for (final Object pt : resources) {
final LinkedHashMap<String, Object> playlistentry = (LinkedHashMap<String, Object>) pt;
final String mgid = (String) playlistentry.get("guid");
final String partname = (String) playlistentry.get("title");
if (mgid == null || partname == null) {
continue;
}
if (fpName == null) {
fpName = doFilenameEncoding(partname);
}
final String final_filename = this.doFilenameEncoding(partname) + this.default_ext;
final DownloadLink fina = mgidSingleVideoGetDownloadLink(mgid);
fina.setFinalFileName(final_filename);
fina.setProperty("decryptedfilename", final_filename);
fina.setAvailable(true);
fina.setContentUrl(this.parameter);
decryptedLinks.add(fina);
}
}
if (embedded_videos_o instanceof ArrayList) {
resources = (ArrayList) embedded_videos_o;
for (final Object pt : resources) {
final LinkedHashMap<String, Object> playlistentry = (LinkedHashMap<String, Object>) pt;
final String mgid = (String) playlistentry.get("video_id");
final String partname = (String) playlistentry.get("video_title");
final String url = (String) playlistentry.get("video_url");
if (mgid == null || partname == null || url == null) {
continue;
}
final String final_filename = this.doFilenameEncoding(partname) + this.default_ext;
final DownloadLink fina = mgidSingleVideoGetDownloadLink(mgid);
fina.setFinalFileName(final_filename);
fina.setProperty("decryptedfilename", final_filename);
fina.setAvailable(true);
fina.setContentUrl("http://" + this.br.getHost() + url);
decryptedLinks.add(fina);
}
}
} catch (final Throwable e) {
}
}
@SuppressWarnings({"unchecked"})
private void decryptUserLoggedIn() throws Exception {
final int limit = 10;
int offset = 0;
boolean decryptSingle = parameter.matches("/page/\\d+");
/* Access url for logged-out-users first because if we don't we cannot see whether a password is needed or not! */
final String url_for_logged_out_users = convertUserUrlToLoggedOutUser();
this.br.setFollowRedirects(false);
this.br.getPage(url_for_logged_out_users);
br.followRedirect(true);
if (this.handlePassword()) {
/* Bullshit - if a blog is password protected we can only display it in the "logged out" mode ... */
decryptUser();
return;
}
if (br.containsHTML(GENERALOFFLINE) || this.br.getHttpConnection().getResponseCode() == 404) {
logger.info("Link offline: " + parameter);
return;
}
handlePassword();
final FilePackage fp = FilePackage.getInstance();
final String username = getUsername(this.parameter);
String fpName = username;
fp.setName(fpName);
LinkedHashMap<String, Object> entries = null;
ArrayList<Object> ressourcelist = null;
do {
if (this.isAbort()) {
logger.info("Decryption aborted by user");
return;
}
br.getHeaders().put("Accept", "application/json, text/javascript, */*; q=0.01");
br.getHeaders().put("X-Requested-With", "XMLHttpRequest");
/* Not needed! */
// br.getHeaders().put("X-tumblr-form-key", "blaTest");
br.getPage(
"//www.tumblr.com/svc/indash_blog/posts?tumblelog_name_or_id="
+ username
+ "&post_id=&limit="
+ limit
+ "&offset="
+ offset);
entries =
(LinkedHashMap<String, Object>) JavaScriptEngineFactory.jsonToJavaObject(br.toString());
ressourcelist =
(ArrayList<Object>) JavaScriptEngineFactory.walkJson(entries, "response/posts");
for (final Object posto : ressourcelist) {
entries = (LinkedHashMap<String, Object>) posto;
final String type = (String) entries.get("type");
final String post_url = (String) entries.get("post_url");
String directlink = null;
String extension = null;
String extensionFallback = null;
if (type.equalsIgnoreCase("photo")) {
directlink =
(String) JavaScriptEngineFactory.walkJson(entries, "photos/{0}/original_size/url");
extensionFallback = ".jpg";
} else if (type.equalsIgnoreCase("video")) {
directlink = (String) entries.get("video_url");
final String url_hd = convertDirectVideoUrltoHD(directlink);
if (url_hd != null) {
directlink = url_hd;
}
extensionFallback = ".mp4";
} else {
/* There is type "text", "answer" and there might be type other types too! */
logger.info("Unsupported or un-downloadable tumblr-post-type: " + type);
}
if (directlink != null) {
extension = getFileNameExtensionFromURL(directlink);
if (extension == null) {
extension = extensionFallback;
}
String filename = getFileNameFromURL(new URL(directlink));
if (filename != null && !filename.endsWith(extension)) {
filename += extension;
}
final DownloadLink dl = this.createDownloadlink("directhttp://" + directlink);
if (this.passCode != null) {
dl.setDownloadPassword(this.passCode);
}
dl.setAvailable(true);
if (post_url != null) {
dl.setContentUrl(post_url);
}
if (filename != null && !useOriginalFilename) {
dl.setName(filename);
}
fp.add(dl);
decryptedLinks.add(dl);
distribute(dl);
}
offset++;
}
if (decryptSingle) {
break;
}
} while (ressourcelist.size() >= limit);
logger.info("Decryption done");
}
private void decryptUser() throws Exception {
String nextPage = "";
int counter = 1;
boolean decryptSingle = parameter.matches("/page/\\d+");
br.getPage(parameter);
br.followRedirect(true);
if (br.containsHTML(GENERALOFFLINE)) {
logger.info("Link offline: " + parameter);
return;
}
handlePassword();
br.followRedirect(true);
final FilePackage fp = FilePackage.getInstance();
String fpName = new Regex(parameter, "//(.+?)\\.tumblr").getMatch(0);
fp.setName(fpName);
do {
if (this.isAbort()) {
logger.info("Decryption aborted by user");
return;
}
if (!"".equals(nextPage)) {
br.getPage(nextPage);
}
if (parameter.contains("/archive")) {
// archive we will need todo things differently!
final String[] posts =
br.getRegex("<a target=\"_blank\" class=\"hover\" title=\"[^\"]*\" href=\"(.*?)\"")
.getColumn(0);
if (posts != null) {
for (final String post : posts) {
final DownloadLink dl = createDownloadlink(post);
fp.add(dl);
distribute(dl);
decryptedLinks.add(dl);
}
}
} else {
// identify all posts on page then filter accordingly. best way todo this is via google
// carousel, due to different page
// layouts(templates) can be hard to find what you're looking for.
final String gc = getGoogleCarousel(br);
if (gc != null) {
final String JSON =
new Regex(gc, "<script type=\"application/ld\\+json\">(.*?)</script>").getMatch(0);
final Map<String, Object> json = JavaScriptEngineFactory.jsonToJavaMap(JSON);
final ArrayList<Object> results = (ArrayList<Object>) json.get("itemListElement");
for (final Object result : results) {
final LinkedHashMap<String, Object> j = (LinkedHashMap<String, Object>) result;
final String url = (String) j.get("url");
final DownloadLink dl = createDownloadlinkTumblr(url);
fp.add(dl);
distribute(dl);
decryptedLinks.add(dl);
}
}
}
if (decryptSingle) {
break;
}
nextPage =
parameter.contains("/archive")
? br.getRegex("\"(/archive(?:/[^\"]*)?\\?before_time=\\d+)\">Next Page").getMatch(0)
: br.getRegex("\"(/page/" + ++counter + ")\"").getMatch(0);
} while (nextPage != null);
logger.info("Decryption done - last 'nextPage' value was: " + nextPage);
}