Пример #1
0
 private void processPhotoSet(
     final ArrayList<DownloadLink> decryptedLinks,
     final Browser br,
     final String puid,
     final String fpname,
     final String hashTagsStr)
     throws Exception {
   final String gc = getGoogleCarousel(br);
   if (gc != null) {
     FilePackage fp = null;
     final String JSON =
         new Regex(gc, "<script type=\"application/ld\\+json\">(.*?)</script>").getMatch(0);
     final Map<String, Object> json = JavaScriptEngineFactory.jsonToJavaMap(JSON);
     final String articleBody = (String) json.get("articleBody");
     final String fpName =
         articleBody != null ? articleBody.replaceAll("[\r\n]+", "").trim() : fpname;
     if (fpName != null) {
       fp = FilePackage.getInstance();
       fp.setName(fpName);
     }
     // single entry objects are not in 'list'
     ArrayList<Object> results = null;
     try {
       results = (ArrayList<Object>) JavaScriptEngineFactory.walkJson(json, "image/@list");
     } catch (Throwable t) {
       // single entry ?
       final String[] a = new String[] {(String) json.get("image")};
       results = new ArrayList<Object>(Arrays.asList(a));
     }
     if (results != null) {
       int count = 1;
       final DecimalFormat df = new DecimalFormat(results.size() < 100 ? "00" : "000");
       for (final Object result : results) {
         final String url = (String) result;
         final DownloadLink dl = createDownloadlink("directhttp://" + url);
         if (fp != null) {
           fp.add(dl);
         }
         // cleanup...
         final String filename =
             setFileName(cleanupName(df.format(count) + " - " + fpName), puid)
                 + getFileNameExtensionFromString(url);
         if (!useOriginalFilename) {
           dl.setFinalFileName(filename);
         }
         dl.setAvailable(true);
         setMD5Hash(dl, url);
         setImageLinkID(dl, url, puid);
         if (hashTagsStr != null) {
           dl.setProperty(PROPERTY_TAGS, hashTagsStr);
         }
         dl.setComment(hashTagsStr);
         decryptedLinks.add(dl);
         count++;
       }
     }
   }
 }
Пример #2
0
  @SuppressWarnings({"unchecked", "deprecation"})
  public ArrayList<DownloadLink> decryptIt(CryptedLink param, ProgressController progress)
      throws Exception {
    ArrayList<DownloadLink> decryptedLinks = new ArrayList<DownloadLink>();
    final String parameter = param.toString();
    jd.plugins.hoster.AmpyaCom.initializeSession(this.br);

    br.getPage(jd.plugins.hoster.AmpyaCom.getApiUrl(parameter));
    if (br.getHttpConnection().getResponseCode() == 404) {
      decryptedLinks.add(this.createOfflinelink(parameter));
      return decryptedLinks;
    }

    String fpName = null;
    LinkedHashMap<String, Object> entries =
        (LinkedHashMap<String, Object>) JavaScriptEngineFactory.jsonToJavaObject(br.toString());
    final ArrayList<Object> ressourcelist = (ArrayList<Object>) entries.get("clips");
    for (final Object clipo : ressourcelist) {
      entries = (LinkedHashMap<String, Object>) clipo;
      final String title = (String) entries.get("title");
      final String artist = (String) entries.get("artist_name");
      final String video_id =
          Long.toString(JavaScriptEngineFactory.toLong(entries.get("video_id"), 0));
      if (fpName == null) {
        fpName = (String) entries.get("container_title");
      }
      if (title == null || artist == null || "0".equals(video_id)) {
        continue;
      }
      final DownloadLink dl =
          this.createDownloadlink(
              "http://ampyadecrypted.com/"
                  + System.currentTimeMillis()
                  + new Random().nextInt(1000000000));
      String filename = artist + " - " + title + ".mp4";
      filename = encodeUnicode(filename);
      dl.setFinalFileName(filename);
      dl.setAvailable(true);
      dl.setContentUrl(parameter);
      dl.setProperty("mainlink", parameter);
      dl.setProperty("videoid", video_id);
      decryptedLinks.add(dl);
    }

    if (fpName != null) {
      final FilePackage fp = FilePackage.getInstance();
      fp.setName(Encoding.htmlDecode(fpName.trim()));
      fp.addLinks(decryptedLinks);
    }

    return decryptedLinks;
  }
Пример #3
0
  @SuppressWarnings({"unchecked", "rawtypes"})
  private void decryptMtvGermanyPlaylists() throws Exception {
    br.getPage(parameter);
    fpName = br.getRegex("<title>([^<>\"]*?)</title>").getMatch(0);
    if (fpName == null) {
      /* Fallback to url-packagename */
      fpName = new Regex(this.parameter, "https?://[^/]+/(.+)").getMatch(0);
    }
    ArrayList<Object> ressourcelist = null;
    LinkedHashMap<String, Object> entries = null;
    try {
      final String json = this.br.getRegex("window\\.pagePlaylist = (\\[\\{.*?\\}\\])").getMatch(0);
      ressourcelist = (ArrayList) JavaScriptEngineFactory.jsonToJavaObject(json);

      for (final Object object : ressourcelist) {
        entries = (LinkedHashMap<String, Object>) object;
        final String path = (String) entries.get("path");
        final String url_mrss = (String) entries.get("mrss");
        final String title = (String) entries.get("title");
        final String subtitle = (String) entries.get("subtitle");
        final String video_token = (String) entries.get("video_token");
        final String mgid = jd.plugins.hoster.VivaTv.getMGIDOutOfURL(url_mrss);
        if (url_mrss == null || title == null || video_token == null || mgid == null) {
          throw new DecrypterException("Decrypter broken for link: " + parameter);
        }
        final String contenturl;
        if (path != null) {
          contenturl = "http://" + this.br.getHost() + path;
        } else {
          contenturl = this.parameter;
        }
        String temp_filename = title;
        if (subtitle != null) {
          temp_filename += " - " + subtitle;
        }
        temp_filename += jd.plugins.hoster.VivaTv.default_ext;

        final DownloadLink dl = mgidSingleVideoGetDownloadLink(mgid);
        dl.setLinkID(video_token);
        dl.setName(temp_filename);
        dl.setAvailable(true);
        dl.setContentUrl(contenturl);
        this.decryptedLinks.add(dl);
      }
    } catch (final Throwable e) {
      return;
    }
    final FilePackage fp = FilePackage.getInstance();
    fpName = Encoding.htmlDecode(fpName.trim());
    fp.setName(fpName);
    fp.addLinks(decryptedLinks);
  }
Пример #4
0
 private String md5hex(String s) {
   Object result = new Object();
   final ScriptEngineManager manager = JavaScriptEngineFactory.getScriptEngineManager(this);
   final ScriptEngine engine = manager.getEngineByName("javascript");
   final Invocable inv = (Invocable) engine;
   try {
     engine.eval(NONSTANDARDMD5);
     result = inv.invokeFunction("hex_md5", s);
   } catch (final Throwable e) {
     return null;
   }
   return result != null ? result.toString() : "";
 }
Пример #5
0
  public ArrayList<DownloadLink> decryptIt(CryptedLink param, ProgressController progress)
      throws Exception {
    final ArrayList<DownloadLink> decryptedLinks = new ArrayList<DownloadLink>();
    br.setFollowRedirects(true);
    final String parameter = param.toString().replace("#", "");

    if (parameter.matches(
        "https?://(?:www\\.)?ge\\.tt/(?:developers|press|tools|notifications|blog|about|javascript|button|contact|terms|api|m).*?")) {
      decryptedLinks.add(this.createOfflinelink(parameter));
      return decryptedLinks;
    }

    br.setAllowedResponseCodes(410);
    br.getPage(parameter);

    if (this.br.getHttpConnection().getResponseCode() == 404
        || this.br.getHttpConnection().getResponseCode() == 410
        || br.containsHTML(
            "Page not found|The page you were looking for was not found|Files removed|These files have been removed by the owner")) {
      decryptedLinks.add(this.createOfflinelink(parameter));
      return decryptedLinks;
    }

    // final String singleFile = new Regex(parameter, "/v/(\\d+)").getMatch(0);
    final String folderid = new Regex(parameter, "ge\\.tt/([A-Za-z0-9]+)(/v/\\d+)?").getMatch(0);
    br.getHeaders().put("Accept", "application/json, text/plain, */*");
    br.getPage("//api.ge.tt/1/shares/" + folderid);

    LinkedHashMap<String, Object> entries =
        (LinkedHashMap<String, Object>) JavaScriptEngineFactory.jsonToJavaMap(br.toString());
    final String error = (String) entries.get("error");

    if (error != null || br.containsHTML(">404 Not Found<")) {
      final DownloadLink dlink = this.createOfflinelink(parameter);
      decryptedLinks.add(dlink);
      return decryptedLinks;
    }

    final ArrayList<Object> files = (ArrayList<Object>) entries.get("files");

    for (final Object fileo : files) {
      entries = (LinkedHashMap<String, Object>) fileo;
      final String filename = (String) entries.get("filename");
      final String fileid = (String) entries.get("fileid");
      final long filesize = JavaScriptEngineFactory.toLong(entries.get("size"), 0);
      final DownloadLink dl =
          createDownloadlink(
              "http://proxy.ge.tt/1/files/" + folderid + "/" + fileid + "/blob?download");
      dl.setContentUrl("http://ge.tt/" + folderid);
      dl.setName(filename);
      dl.setDownloadSize(filesize);
      dl.setAvailable(true);
      decryptedLinks.add(dl);
    }

    if (decryptedLinks == null || decryptedLinks.size() == 0) {
      logger.info("ge.tt: Folder is empty! Link: " + parameter);
      final DownloadLink dlink = this.createOfflinelink(parameter);
      decryptedLinks.add(dlink);
      return decryptedLinks;
    }
    return decryptedLinks;
  }
Пример #6
0
  @SuppressWarnings({"unchecked", "rawtypes"})
  public ArrayList<DownloadLink> decryptIt(CryptedLink param, ProgressController progress)
      throws Exception {
    final ArrayList<DownloadLink> decryptedLinks = new ArrayList<DownloadLink>();
    final String parameter = param.toString().replace("http://", "https://");
    final String fid = new Regex(parameter, "/p/([A-Za-z0-9\\-_]+)").getMatch(0);
    String subfolder = new Regex(parameter, "#dir=(.+)").getMatch(0);
    if (subfolder == null) {
      subfolder = "/";
    } else {
      subfolder = Encoding.htmlDecode(subfolder);
    }
    subfolder = Encoding.urlEncode(subfolder);
    if (fid == null) {
      decryptedLinks.add(createOfflinelink(parameter));
      return decryptedLinks;
    }
    this.br.getPage(parameter);
    final String pageJson = jd.plugins.hoster.JianguoyunCom.getWebsiteJson(this.br);
    DownloadLink dl = null;
    final String isdir =
        new Regex(pageJson, "isdir[\t\n\r ]*?:[\t\n\r ]*?(true|false)").getMatch(0);
    if ("false".equals(isdir)) {
      /* Single file */
      dl =
          createDownloadlink(
              "http://jianguoyundecrypted.com/"
                  + System.currentTimeMillis()
                  + new Random().nextInt(100000));
      dl.setProperty("singlefile", true);
      dl.setProperty("folderid", fid);
      dl.setProperty("relPath", subfolder);
      dl.setProperty("mainlink", parameter);
      jd.plugins.hoster.JianguoyunCom.scanFileinfoFromWebsite(this.br, dl);
      decryptedLinks.add(dl);
      return decryptedLinks;
    }

    this.br.getPage(
        "https://www.jianguoyun.com/d/ajax/dirops/pubDIRBrowse?hash="
            + fid
            + "&relPath="
            + subfolder
            + "&_="
            + System.currentTimeMillis());
    LinkedHashMap<String, Object> entries =
        (LinkedHashMap<String, Object>) JavaScriptEngineFactory.jsonToJavaObject(br.toString());
    final ArrayList<Object> ressourcelist = (ArrayList) entries.get("objects");
    for (final Object foldero : ressourcelist) {
      entries = (LinkedHashMap<String, Object>) foldero;
      final String type = (String) entries.get("type");
      final String relPath = (String) entries.get("relPath");
      final long filesize = JavaScriptEngineFactory.toLong(entries.get("size"), 0);
      if (type == null || relPath == null) {
        continue;
      }
      if (type.equals("directory")) {
        dl =
            createDownloadlink(
                "https://www.jianguoyun.com/p/" + fid + "#dir=" + Encoding.urlEncode(relPath));
      } else {
        dl =
            createDownloadlink(
                "http://jianguoyundecrypted.com/"
                    + System.currentTimeMillis()
                    + new Random().nextInt(100000));
        final Regex pathregex = new Regex(relPath, "^(/.*?)([^/]+)$");
        final String contenturl =
            "https://www.jianguoyun.com/p/"
                + fid
                + "#dir="
                + Encoding.urlEncode(pathregex.getMatch(0));
        final String filename = pathregex.getMatch(1);
        if (filesize > 0) {
          dl.setDownloadSize(filesize);
          dl.setAvailable(true);
        }
        if (filename != null) {
          dl.setName(filename);
        }
        dl.setProperty("folderid", fid);
        dl.setProperty("relPath", relPath);
        dl.setProperty("mainlink", parameter);
        dl.setContentUrl(contenturl);
        dl.setLinkID(fid + relPath);
        dl.setProperty(DownloadLink.RELATIVE_DOWNLOAD_FOLDER_PATH, relPath);
      }
      decryptedLinks.add(dl);
    }

    return decryptedLinks;
  }
Пример #7
0
  /* Old channel config url (see also rev 35204): http://www.rai.tv/dl/RaiTV/iphone/android/smartphone/advertising_config.html */
  private void decryptWholeDay() throws Exception {
    final String mainlink_urlpart = new Regex(parameter, "\\?(.+)").getMatch(0);
    final String[] dates = new Regex(parameter, "(\\d{4}\\-\\d{2}\\-\\d{2})").getColumn(0);
    final String[] channels = new Regex(parameter, "ch=(\\d+)").getColumn(0);
    final String[] videoids = new Regex(parameter, "v=(\\d+)").getColumn(0);
    final String date_user_input = dates[dates.length - 1];
    final String date_user_input_underscore = date_user_input.replace("-", "_");
    final String date_user_input_in_json_format = convertInputDateToJsonDateFormat(date_user_input);

    final DownloadLink offline = this.createOfflinelink(parameter);
    final String filename_offline;
    if (mainlink_urlpart != null) {
      filename_offline = date_user_input + "_" + mainlink_urlpart + ".mp4";
    } else {
      filename_offline = date_user_input + ".mp4";
    }
    offline.setFinalFileName(filename_offline);

    String id_of_single_video_which_user_wants_to_have_only = null;
    String chnumber_str = null;
    if (channels != null && channels.length > 0) {
      chnumber_str = channels[channels.length - 1];
    }
    if (chnumber_str == null) {
      /* Small fallback */
      chnumber_str = "1";
    }
    if (videoids != null && videoids.length > 0) {
      id_of_single_video_which_user_wants_to_have_only = videoids[videoids.length - 1];
    }
    final FilePackage fp = FilePackage.getInstance();
    fp.setName(date_user_input_underscore);
    LinkedHashMap<String, Object> entries = null;
    final String channel_name = "Rai" + chnumber_str;
    final String channel_name_with_space = "Rai " + chnumber_str;

    this.br.getHeaders().put("X-Requested-With", "XMLHttpRequest");
    this.br.getPage(
        "http://www.rai.it/dl/palinsesti/Page-e120a813-1b92-4057-a214-15943d95aa68-json.html?canale="
            + channel_name
            + "&giorno="
            + date_user_input);
    if (br.getHttpConnection().getResponseCode() == 404) {
      decryptedLinks.add(offline);
      return;
    }
    entries =
        (LinkedHashMap<String, Object>)
            JavaScriptEngineFactory.jsonToJavaObject(this.br.toString());
    final ArrayList<Object> daysList = (ArrayList<Object>) entries.get(channel_name_with_space);

    /* Walk through all days. */
    for (final Object dayO : daysList) {
      entries = (LinkedHashMap<String, Object>) dayO;
      final String date_of_this_item = (String) entries.get("giorno");
      if (date_of_this_item == null || !date_of_this_item.equals(date_user_input_in_json_format)) {
        /* Date is missing or not the date we want? Skip item! */
        continue;
      }
      /* Get all items of the day. */
      final ArrayList<Object> itemsOfThatDayList = (ArrayList<Object>) entries.get("palinsesto");
      for (final Object itemsOfThatDayListO : itemsOfThatDayList) {
        entries = (LinkedHashMap<String, Object>) itemsOfThatDayListO;
        /* Get all programms of that day. */
        final ArrayList<Object> programmsList = (ArrayList<Object>) entries.get("programmi");
        /* Finally decrypt the programms. */
        for (final Object programmO : programmsList) {
          entries = (LinkedHashMap<String, Object>) programmO;
          if (entries.isEmpty()) {
            continue;
          }
          final boolean hasVideo = ((Boolean) entries.get("hasVideo")).booleanValue();
          final String webLink = (String) entries.get("webLink");
          if (!hasVideo || webLink == null || !webLink.startsWith("/")) {
            continue;
          }
          final String url_for_user;
          final String url_rai_replay = new Regex(webLink, "raiplay/(video/.+)").getMatch(0);
          if (url_rai_replay != null) {
            url_for_user = "******" + url_rai_replay;
          } else {
            url_for_user = "******" + webLink;
          }
          final DownloadLink dl = this.createDownloadlink(url_for_user);
          decryptedLinks.add(dl);
        }
      }
    }
  }
Пример #8
0
  private void decryptSingleVideo() throws DecrypterException, Exception {
    String dllink = null;
    String title = null;
    String extension = ".mp4";
    String date = null;
    String date_formatted = null;
    String description = null;

    this.br.getPage(this.parameter);
    final String jsredirect =
        this.br.getRegex("document\\.location\\.replace\\(\\'(http[^<>\"]*?)\\'\\)").getMatch(0);
    if (jsredirect != null) {
      this.br.getPage(jsredirect.trim());
    }
    if (this.br.getHttpConnection().getResponseCode() == 404) {
      decryptedLinks.add(this.createOfflinelink(this.parameter));
      return;
    }
    /* Do NOT use value of "videoURL_MP4" here! */
    /* E.g. http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem-70996227-7fec-4be9-bc49-ba0a8104305a.html */
    dllink =
        this.br
            .getRegex("var[\t\n\r ]*?videoURL[\t\n\r ]*?=[\t\n\r ]*?\"(http://[^<>\"]+)\"")
            .getMatch(0);
    String content_id_from_url = null;
    if (this.parameter.matches(TYPE_CONTENTITEM)) {
      content_id_from_url = new Regex(this.parameter, "(\\-[a-f0-9\\-]+)\\.html$").getMatch(0);
    }
    if (dllink == null) {
      dllink = findRelinkerUrl();
    }
    title = this.br.getRegex("property=\"og:title\" content=\"([^<>\"]+)\"").getMatch(0);
    date =
        this.br
            .getRegex(
                "content=\"(\\d{4}\\-\\d{2}\\-\\d{2}) \\d{2}:\\d{2}:\\d{2}\" property=\"gen\\-date\"")
            .getMatch(0);

    final String contentset_id =
        this.br
            .getRegex(
                "var[\t\n\r ]*?urlTop[\t\n\r ]*?=[\t\n\r ]*?\"[^<>\"]+/ContentSet([A-Za-z0-9\\-]+)\\.html")
            .getMatch(0);
    final String content_id_from_html =
        this.br.getRegex("id=\"ContentItem(\\-[a-f0-9\\-]+)\"").getMatch(0);
    if (br.getHttpConnection().getResponseCode() == 404
        || (contentset_id == null && content_id_from_html == null && dllink == null)) {
      /* Probably not a video/offline */
      decryptedLinks.add(this.createOfflinelink(this.parameter));
      return;
    }
    if (dllink != null) {
      if (title == null) {
        /* Streamurls directly in html */
        title = this.br.getRegex("id=\"idMedia\">([^<>]+)<").getMatch(0);
      }
      if (title == null) {
        title = this.br.getRegex("var videoTitolo\\d*?=\\d*?\"([^<>\"]+)\";").getMatch(0);
      }
      if (date == null) {
        date =
            this.br
                .getRegex("id=\"myGenDate\">(\\d{2}\\-\\d{2}\\-\\d{4} \\d{2}:\\d{2})<")
                .getMatch(0);
      }
      if (date == null) {
        /* 2017-01-06: New */
        date =
            this.br
                .getRegex(
                    "property=\"gen\\-date\" content=\"(\\d{4}\\-\\d{2}\\-\\d{2} \\d{2}:\\d{2}:\\d{2})\"")
                .getMatch(0);
      }
      if (date == null) {
        date = this.br.getRegex("data\\-date=\"(\\d{2}/\\d{2}/\\d{4})\"").getMatch(0);
      }
    } else {
      LinkedHashMap<String, Object> entries = null;
      if (content_id_from_html != null) {
        /* Easiest way to find videoinfo */
        this.br.getPage(
            "http://www.rai.tv/dl/RaiTV/programmi/media/ContentItem"
                + content_id_from_html
                + ".html?json");
        entries =
            (LinkedHashMap<String, Object>) JavaScriptEngineFactory.jsonToJavaObject(br.toString());
      }
      if (entries == null) {
        final ArrayList<Object> ressourcelist;
        final String list_json_from_html =
            this.br
                .getRegex("\"list\"[\t\n\r ]*?:[\t\n\r ]*?(\\[.*?\\}[\t\n\r ]*?\\])")
                .getMatch(0);
        if (list_json_from_html != null) {
          ressourcelist =
              (ArrayList<Object>) JavaScriptEngineFactory.jsonToJavaObject(list_json_from_html);
        } else {
          br.getPage(
              "http://www.rai.tv/dl/RaiTV/ondemand/ContentSet" + contentset_id + ".html?json");
          if (br.getHttpConnection().getResponseCode() == 404) {
            decryptedLinks.add(this.createOfflinelink(this.parameter));
            return;
          }
          entries =
              (LinkedHashMap<String, Object>)
                  JavaScriptEngineFactory.jsonToJavaObject(br.toString());
          ressourcelist = (ArrayList<Object>) entries.get("list");
        }

        if (content_id_from_url == null) {
          /* Hm probably not a video */
          decryptedLinks.add(this.createOfflinelink(this.parameter));
          return;
        }
        String content_id_temp = null;
        boolean foundVideoInfo = false;
        for (final Object videoo : ressourcelist) {
          entries = (LinkedHashMap<String, Object>) videoo;
          content_id_temp = (String) entries.get("itemId");
          if (content_id_temp != null && content_id_temp.contains(content_id_from_url)) {
            foundVideoInfo = true;
            break;
          }
        }
        if (!foundVideoInfo) {
          /* Probably offline ... */
          decryptedLinks.add(this.createOfflinelink(this.parameter));
          return;
        }
      }
      date = (String) entries.get("date");
      title = (String) entries.get("name");
      description = (String) entries.get("desc");
      final String type = (String) entries.get("type");
      if (type.equalsIgnoreCase("RaiTv Media Video Item")) {
      } else {
        /* TODO */
        logger.warning("Unsupported media type!");
        throw new DecrypterException(DecrypterException.PLUGIN_DEFECT);
      }
      extension = "mp4";
      dllink = (String) entries.get("h264");
      if (dllink == null || dllink.equals("")) {
        dllink = (String) entries.get("m3u8");
        extension = "mp4";
      }
      if (dllink == null || dllink.equals("")) {
        dllink = (String) entries.get("wmv");
        extension = "wmv";
      }
      if (dllink == null || dllink.equals("")) {
        dllink = (String) entries.get("mediaUri");
        extension = "mp4";
      }
    }
    if (title == null) {
      title = content_id_from_url;
    }
    date_formatted = jd.plugins.hoster.RaiTv.formatDate(date);
    title = Encoding.htmlDecode(title);
    title = date_formatted + "_raitv_" + title;
    title = encodeUnicode(title);

    final FilePackage fp = FilePackage.getInstance();
    fp.setName(title);

    decryptRelinker(dllink, title, extension, fp, description);
  }
Пример #9
0
  @SuppressWarnings({"deprecation", "unchecked"})
  @Override
  public boolean checkLinks(final DownloadLink[] urls) {
    if (urls == null || urls.length == 0) {
      return false;
    }
    try {
      br.setCookiesExclusive(true);
      final StringBuilder sb = new StringBuilder();
      final ArrayList<DownloadLink> links = new ArrayList<DownloadLink>();
      int index = 0;
      int tempcounter = 0;
      while (true) {
        /* Reset tempcounter */
        tempcounter = 0;
        links.clear();
        while (true) {
          /* we test 10 links at once (max = 10) */
          if (index == urls.length || links.size() > 9) {
            break;
          }
          links.add(urls[index]);
          index++;
        }
        sb.delete(0, sb.capacity());
        sb.append("links=");
        for (final DownloadLink dl : links) {
          sb.append(Encoding.urlEncode(dl.getDownloadURL()));
          if (tempcounter < links.size() - 1) {
            sb.append(Encoding.urlEncode("\n"));
          }
          tempcounter++;
        }
        /* Reset tempcounter */
        tempcounter = 0;

        postPage(getAPIProtocol() + this.getHost() + "/download/json_check", sb.toString());
        LinkedHashMap<String, Object> entries = null;
        final ArrayList<Object> ressourcelist =
            (ArrayList<Object>) JavaScriptEngineFactory.jsonToJavaObject(br.toString());
        final String[] json_workaround_array = br.toString().split("\\},\\{");
        for (final DownloadLink dl : links) {
          final String state;
          final String filename;
          final long filesize;
          if (tempcounter <= json_workaround_array.length - 1) {
            entries = (LinkedHashMap<String, Object>) ressourcelist.get(tempcounter);
            state = (String) entries.get("status");
            filename = (String) entries.get("filename");
            filesize = JavaScriptEngineFactory.toLong(entries.get("filesize"), 0);
          } else {
            state = null;
            filename = null;
            filesize = 0;
          }
          if ("offline".equals(state) || state == null) {
            dl.setAvailable(false);
          } else {
            dl.setAvailable(true);
          }
          /* Trust API - offline urls can still have their filename- and size information available */
          if (filename != null) {
            dl.setFinalFileName(filename);
          }
          dl.setDownloadSize(filesize);

          tempcounter++;
        }
        if (index == urls.length) {
          break;
        }
      }
    } catch (final Exception e) {
      return false;
    }
    return true;
  }
Пример #10
0
 /** Finds mgids inside drupal json */
 @SuppressWarnings({"unchecked", "rawtypes"})
 private void crawlDrupal() {
   final String playlist_id =
       br.getRegex("name=\"vimn:entity_uuid\" content=\"([a-z0-9\\-:]*?)\"").getMatch(0);
   final String js =
       br.getRegex("jQuery\\.extend\\(Drupal\\.settings, (\\{.*?)\\);.*?</script>").getMatch(0);
   if (js == null || playlist_id == null) {
     return;
   }
   try {
     LinkedHashMap<String, Object> entries =
         (LinkedHashMap<String, Object>) JavaScriptEngineFactory.jsonToJavaObject(js);
     final LinkedHashMap<String, Object> vimn_video =
         (LinkedHashMap<String, Object>) entries.get("vimn_video");
     ArrayList<Object> resources = null;
     final Object playlist_o = vimn_video.get("playlists");
     final Object embedded_videos_o = vimn_video.get("embedded_videos");
     if (!(playlist_o instanceof ArrayList)) {
       /* Playlist */
       entries = (LinkedHashMap<String, Object>) playlist_o;
       entries = (LinkedHashMap<String, Object>) entries.get(playlist_id);
       resources = (ArrayList) entries.get("items");
       final Object title_object = entries.get("title");
       if (title_object instanceof String) {
         fpName = (String) entries.get("title");
       }
       for (final Object pt : resources) {
         final LinkedHashMap<String, Object> playlistentry = (LinkedHashMap<String, Object>) pt;
         final String mgid = (String) playlistentry.get("guid");
         final String partname = (String) playlistentry.get("title");
         if (mgid == null || partname == null) {
           continue;
         }
         if (fpName == null) {
           fpName = doFilenameEncoding(partname);
         }
         final String final_filename = this.doFilenameEncoding(partname) + this.default_ext;
         final DownloadLink fina = mgidSingleVideoGetDownloadLink(mgid);
         fina.setFinalFileName(final_filename);
         fina.setProperty("decryptedfilename", final_filename);
         fina.setAvailable(true);
         fina.setContentUrl(this.parameter);
         decryptedLinks.add(fina);
       }
     }
     if (embedded_videos_o instanceof ArrayList) {
       resources = (ArrayList) embedded_videos_o;
       for (final Object pt : resources) {
         final LinkedHashMap<String, Object> playlistentry = (LinkedHashMap<String, Object>) pt;
         final String mgid = (String) playlistentry.get("video_id");
         final String partname = (String) playlistentry.get("video_title");
         final String url = (String) playlistentry.get("video_url");
         if (mgid == null || partname == null || url == null) {
           continue;
         }
         final String final_filename = this.doFilenameEncoding(partname) + this.default_ext;
         final DownloadLink fina = mgidSingleVideoGetDownloadLink(mgid);
         fina.setFinalFileName(final_filename);
         fina.setProperty("decryptedfilename", final_filename);
         fina.setAvailable(true);
         fina.setContentUrl("http://" + this.br.getHost() + url);
         decryptedLinks.add(fina);
       }
     }
   } catch (final Throwable e) {
   }
 }
Пример #11
0
  @SuppressWarnings({"unchecked"})
  private void decryptUserLoggedIn() throws Exception {
    final int limit = 10;
    int offset = 0;
    boolean decryptSingle = parameter.matches("/page/\\d+");
    /* Access url for logged-out-users first because if we don't we cannot see whether a password is needed or not! */
    final String url_for_logged_out_users = convertUserUrlToLoggedOutUser();
    this.br.setFollowRedirects(false);
    this.br.getPage(url_for_logged_out_users);
    br.followRedirect(true);
    if (this.handlePassword()) {
      /* Bullshit - if a blog is password protected we can only display it in the "logged out" mode ... */
      decryptUser();
      return;
    }
    if (br.containsHTML(GENERALOFFLINE) || this.br.getHttpConnection().getResponseCode() == 404) {
      logger.info("Link offline: " + parameter);
      return;
    }
    handlePassword();
    final FilePackage fp = FilePackage.getInstance();
    final String username = getUsername(this.parameter);
    String fpName = username;
    fp.setName(fpName);
    LinkedHashMap<String, Object> entries = null;
    ArrayList<Object> ressourcelist = null;
    do {
      if (this.isAbort()) {
        logger.info("Decryption aborted by user");
        return;
      }
      br.getHeaders().put("Accept", "application/json, text/javascript, */*; q=0.01");
      br.getHeaders().put("X-Requested-With", "XMLHttpRequest");
      /* Not needed! */
      // br.getHeaders().put("X-tumblr-form-key", "blaTest");
      br.getPage(
          "//www.tumblr.com/svc/indash_blog/posts?tumblelog_name_or_id="
              + username
              + "&post_id=&limit="
              + limit
              + "&offset="
              + offset);
      entries =
          (LinkedHashMap<String, Object>) JavaScriptEngineFactory.jsonToJavaObject(br.toString());
      ressourcelist =
          (ArrayList<Object>) JavaScriptEngineFactory.walkJson(entries, "response/posts");
      for (final Object posto : ressourcelist) {
        entries = (LinkedHashMap<String, Object>) posto;
        final String type = (String) entries.get("type");
        final String post_url = (String) entries.get("post_url");
        String directlink = null;
        String extension = null;
        String extensionFallback = null;
        if (type.equalsIgnoreCase("photo")) {
          directlink =
              (String) JavaScriptEngineFactory.walkJson(entries, "photos/{0}/original_size/url");
          extensionFallback = ".jpg";
        } else if (type.equalsIgnoreCase("video")) {
          directlink = (String) entries.get("video_url");
          final String url_hd = convertDirectVideoUrltoHD(directlink);
          if (url_hd != null) {
            directlink = url_hd;
          }
          extensionFallback = ".mp4";
        } else {
          /* There is type "text", "answer" and there might be type other types too! */
          logger.info("Unsupported or un-downloadable tumblr-post-type: " + type);
        }
        if (directlink != null) {
          extension = getFileNameExtensionFromURL(directlink);
          if (extension == null) {
            extension = extensionFallback;
          }
          String filename = getFileNameFromURL(new URL(directlink));
          if (filename != null && !filename.endsWith(extension)) {
            filename += extension;
          }
          final DownloadLink dl = this.createDownloadlink("directhttp://" + directlink);
          if (this.passCode != null) {
            dl.setDownloadPassword(this.passCode);
          }
          dl.setAvailable(true);
          if (post_url != null) {
            dl.setContentUrl(post_url);
          }
          if (filename != null && !useOriginalFilename) {
            dl.setName(filename);
          }
          fp.add(dl);
          decryptedLinks.add(dl);
          distribute(dl);
        }
        offset++;
      }

      if (decryptSingle) {
        break;
      }
    } while (ressourcelist.size() >= limit);
    logger.info("Decryption done");
  }
Пример #12
0
  private void decryptUser() throws Exception {
    String nextPage = "";
    int counter = 1;
    boolean decryptSingle = parameter.matches("/page/\\d+");
    br.getPage(parameter);
    br.followRedirect(true);
    if (br.containsHTML(GENERALOFFLINE)) {
      logger.info("Link offline: " + parameter);
      return;
    }
    handlePassword();
    br.followRedirect(true);
    final FilePackage fp = FilePackage.getInstance();
    String fpName = new Regex(parameter, "//(.+?)\\.tumblr").getMatch(0);
    fp.setName(fpName);
    do {
      if (this.isAbort()) {
        logger.info("Decryption aborted by user");
        return;
      }
      if (!"".equals(nextPage)) {
        br.getPage(nextPage);
      }
      if (parameter.contains("/archive")) {
        // archive we will need todo things differently!
        final String[] posts =
            br.getRegex("<a target=\"_blank\" class=\"hover\" title=\"[^\"]*\" href=\"(.*?)\"")
                .getColumn(0);
        if (posts != null) {
          for (final String post : posts) {
            final DownloadLink dl = createDownloadlink(post);
            fp.add(dl);
            distribute(dl);
            decryptedLinks.add(dl);
          }
        }
      } else {
        // identify all posts on page then filter accordingly. best way todo this is via google
        // carousel, due to different page
        // layouts(templates) can be hard to find what you're looking for.
        final String gc = getGoogleCarousel(br);
        if (gc != null) {
          final String JSON =
              new Regex(gc, "<script type=\"application/ld\\+json\">(.*?)</script>").getMatch(0);
          final Map<String, Object> json = JavaScriptEngineFactory.jsonToJavaMap(JSON);
          final ArrayList<Object> results = (ArrayList<Object>) json.get("itemListElement");
          for (final Object result : results) {
            final LinkedHashMap<String, Object> j = (LinkedHashMap<String, Object>) result;
            final String url = (String) j.get("url");
            final DownloadLink dl = createDownloadlinkTumblr(url);
            fp.add(dl);
            distribute(dl);
            decryptedLinks.add(dl);
          }
        }
      }

      if (decryptSingle) {
        break;
      }
      nextPage =
          parameter.contains("/archive")
              ? br.getRegex("\"(/archive(?:/[^\"]*)?\\?before_time=\\d+)\">Next Page").getMatch(0)
              : br.getRegex("\"(/page/" + ++counter + ")\"").getMatch(0);
    } while (nextPage != null);
    logger.info("Decryption done - last 'nextPage' value was: " + nextPage);
  }