Esempio n. 1
0
 protected <ReturnType> ReturnType handleCaptchaChallenge(Challenge<ReturnType> c)
     throws CaptchaException, InterruptedException, DecrypterException {
   if (c instanceof ImageCaptchaChallenge) {
     final File captchaFile = ((ImageCaptchaChallenge) c).getImageFile();
     cleanUpCaptchaFiles.add(captchaFile);
   }
   if (Thread.currentThread() instanceof SingleDownloadController) {
     logger.severe("PluginForDecrypt.getCaptchaCode inside SingleDownloadController!?");
   }
   c.setTimeout(getCaptchaTimeout());
   invalidateLastChallengeResponse();
   final BlacklistEntry blackListEntry = CaptchaBlackList.getInstance().matches(c);
   if (blackListEntry != null) {
     logger.warning("Cancel. Blacklist Matching");
     throw new CaptchaException(blackListEntry);
   }
   try {
     ChallengeResponseController.getInstance().handle(c);
   } catch (InterruptedException e) {
     LogSource.exception(logger, e);
     throw e;
   } catch (SkipException e) {
     LogSource.exception(logger, e);
     switch (e.getSkipRequest()) {
       case BLOCK_ALL_CAPTCHAS:
         CaptchaBlackList.getInstance().add(new BlockAllCrawlerCaptchasEntry(getCrawler()));
         break;
       case BLOCK_HOSTER:
         CaptchaBlackList.getInstance()
             .add(new BlockCrawlerCaptchasByHost(getCrawler(), getHost()));
         break;
       case BLOCK_PACKAGE:
         CaptchaBlackList.getInstance()
             .add(new BlockCrawlerCaptchasByPackage(getCrawler(), getCurrentLink()));
         break;
       case REFRESH:
         // refresh is not supported from the pluginsystem right now.
         return c.getRefreshTrigger();
       case STOP_CURRENT_ACTION:
         if (Thread.currentThread() instanceof LinkCrawlerThread) {
           LinkCollector.getInstance().abort();
           // Just to be sure
           CaptchaBlackList.getInstance().add(new BlockAllCrawlerCaptchasEntry(getCrawler()));
         }
         break;
       default:
         break;
     }
     throw new CaptchaException(e.getSkipRequest());
   }
   if (!c.isSolved()) {
     throw new DecrypterException(DecrypterException.CAPTCHA);
   }
   return c.getResult().getValue();
 }
Esempio n. 2
0
  private PollingResultAPIStorable getLinkGrabberState() {
    PollingResultAPIStorable prs = new PollingResultAPIStorable();
    prs.setEventName("linkGrabberState");

    LinkCollector lc = LinkCollector.getInstance();

    String status = "UNKNOWN";
    if (lc.getDefaultLinkChecker().isRunning()) {
      status = "RUNNING";
    } else {
      status = "IDLE";
    }

    org.jdownloader.myjdownloader.client.json.JsonMap eventData =
        new org.jdownloader.myjdownloader.client.json.JsonMap();
    eventData.put("data", status);
    prs.setEventData(eventData);

    return prs;
  }
Esempio n. 3
0
  @Override
  protected void setStringValue(final String value, final AbstractNode object) {
    if (StringUtils.isEmpty(value) || object == null) {
      return;
    }
    File oldPath = LinkTreeUtils.getDownloadDirectory(object);
    File newPath = LinkTreeUtils.getDownloadDirectory(value, null, object);
    if (oldPath.equals(newPath)) {
      /* both paths are same, so nothing to do */
      return;
    }
    if (object instanceof CrawledPackage) {
      LinkCollector.getInstance()
          .getQueue()
          .add(
              new QueueAction<Object, RuntimeException>(
                  org.appwork.utils.event.queue.Queue.QueuePriority.HIGH) {
                @Override
                protected Object run() {
                  ((CrawledPackage) object).setDownloadFolder(value);
                  return null;
                }
              });
      return;
    } else if (object instanceof CrawledLink) {
      final CrawledPackage p = ((CrawledLink) object).getParentNode();
      try {
        Dialog.getInstance()
            .showConfirmDialog(
                Dialog.LOGIC_DONOTSHOW_BASED_ON_TITLE_ONLY | Dialog.STYLE_SHOW_DO_NOT_DISPLAY_AGAIN,
                _JDT.T.SetDownloadFolderAction_actionPerformed_(p.getName()),
                _JDT.T.SetDownloadFolderAction_msg(p.getName(), 1),
                null,
                _JDT.T.SetDownloadFolderAction_yes(),
                _JDT.T.SetDownloadFolderAction_no());
        LinkCollector.getInstance()
            .getQueue()
            .add(
                new QueueAction<Object, RuntimeException>(
                    org.appwork.utils.event.queue.Queue.QueuePriority.HIGH) {
                  @Override
                  protected Object run() {
                    p.setDownloadFolder(value);
                    return null;
                  }
                });
        return;
      } catch (DialogClosedException e) {
        return;
      } catch (DialogCanceledException e) {
        /* user clicked no */
      }

      final CrawledPackage pkg = new CrawledPackage();
      pkg.setExpanded(CFG_LINKCOLLECTOR.CFG.isPackageAutoExpanded());
      if (TYPE.NORMAL != p.getType()) {
        final String pkgName;
        if (object instanceof AbstractPackageNode) {
          pkgName =
              LinknameCleaner.cleanFileName(
                  object.getName(),
                  false,
                  true,
                  LinknameCleaner.EXTENSION_SETTINGS.REMOVE_KNOWN,
                  true);
        } else {
          pkgName =
              LinknameCleaner.cleanFileName(
                  object.getName(),
                  false,
                  true,
                  LinknameCleaner.EXTENSION_SETTINGS.REMOVE_ALL,
                  true);
        }
        pkg.setName(pkgName);
      } else {
        pkg.setName(p.getName());
      }
      pkg.setComment(p.getComment());
      pkg.setDownloadFolder(value);

      final java.util.List<CrawledLink> links = new ArrayList<CrawledLink>();
      links.add((CrawledLink) object);
      LinkCollector.getInstance()
          .getQueue()
          .add(
              new QueueAction<Object, RuntimeException>(
                  org.appwork.utils.event.queue.Queue.QueuePriority.HIGH) {

                @Override
                protected Object run() {
                  LinkCollector.getInstance().moveOrAddAt(pkg, links, -1);
                  return null;
                }
              });
    }
  }
Esempio n. 4
0
  // @Override
  public ContainerStatus callDecryption(File lc) {
    ContainerStatus cs = new ContainerStatus(lc);

    // if (!getProperties().getBooleanProperty("USECCFRSDF", false)) {
    Vector<URL> services;
    try {
      services = new Vector<URL>();

      Collections.sort(
          services,
          new Comparator<Object>() {
            public int compare(Object a, Object b) {
              return (int) (Math.random() * 4.0 - 2.0);
            }
          });
      services.add(0, new URL("http://service.jdownloader.net/dlcrypt/getDLC.php"));
      Iterator<URL> it = services.iterator();

      String dlc = null;
      // Ua6LDQoSRm6XrkfussF7t1iM5D+WAT9y5QB0U3gwmkELcT+6+4
      // U0RUeKvv256VUPZdHttMCuf0/C
      // Ua6LDRJGbpeuR+6ywXu3WIzkj5YBP3LlAHRTeDCaQQtxkLr7hTRFR4q+/
      // bnpVQ9l0e20wK5/T8L1

      while (it.hasNext() && dlc == null) {
        URL service = it.next();
        try {
          // RequestInfo ri;
          // requestInfo = Plugin.postRequest(new
          // URL(ri.getLocation()), null, null, null, "src=ccf&data="
          // + data, true);
          Browser br = new Browser();
          br.setDebug(true);
          PostFormDataRequest r = (PostFormDataRequest) br.createPostFormDataRequest(service + "");
          r.addFormData(new FormData("upload", lc.getName(), lc));

          try {
            r.addFormData(
                new FormData(
                    "l",
                    getClass()
                            .forName(getClass().getPackage().getName() + ".Config")
                            .getField("CCF")
                            .get(null)
                        + ""));

          } catch (Throwable e) {

          }
          r.addFormData(new FormData("src", "ccf"));

          br.openRequestConnection(r);

          String ret = br.loadConnection(null) + "";

          if (ret != null && ret.contains("<dlc>")) {
            dlc = new Regex(ret, "<dlc>(.*)</dlc>").getMatch(0);
            if (dlc.length() < 90) {
              dlc = null;
            }
          }

        } catch (Exception e) {
          JDLogger.exception(e);
        }
      }
      if (dlc != null) {
        lc = new File(lc.getAbsolutePath().substring(0, lc.getAbsolutePath().length() - 3) + "dlc");
        JDIO.writeLocalFile(lc, dlc);
        LinkCollector.getInstance()
            .addCrawlerJob(new LinkCollectingJob("file://" + lc.getAbsolutePath()));
        cs.setStatus(ContainerStatus.STATUS_FINISHED);

        return cs;
      }
    } catch (MalformedURLException e) {

    }

    cs.setStatus(ContainerStatus.STATUS_FAILED);

    return cs;
  }
Esempio n. 5
0
  public void actionPerformed(ActionEvent e) {
    final SelectionInfo<CrawledPackage, CrawledLink> finalSelecction = getSelection();
    final String newName;
    final String newDownloadFolder;
    if (isMergePackages() && finalSelecction.getPackageViews().size() > 1) {
      if (isAskForNewDownloadFolderAndPackageName()) {
        try {
          final NewPackageDialog d =
              new NewPackageDialog(finalSelecction) {
                @Override
                public String getDontShowAgainKey() {
                  return "ABSTRACTDIALOG_DONT_SHOW_AGAIN_"
                      + SplitPackagesByHost.this.getClass().getSimpleName();
                }
              };

          Dialog.getInstance().showDialog(d);

          newName = d.getName();
          newDownloadFolder = d.getDownloadFolder();
          if (StringUtils.isEmpty(newName)) {
            return;
          }
        } catch (Throwable ignore) {
          return;
        }
      } else {
        newName = "";
        newDownloadFolder = finalSelecction.getFirstPackage().getRawDownloadFolder();
      }
    } else {
      newName = null;
      newDownloadFolder = null;
    }
    LinkCollector.getInstance()
        .getQueue()
        .add(
            new QueueAction<Void, RuntimeException>() {

              @Override
              protected Void run() throws RuntimeException {
                final HashMap<CrawledPackage, HashMap<String, ArrayList<CrawledLink>>> splitMap =
                    new HashMap<CrawledPackage, HashMap<String, ArrayList<CrawledLink>>>();
                int insertAt = -1;
                switch (getLocation()) {
                  case BEFORE_SELECTION:
                    insertAt = Integer.MAX_VALUE;
                }
                for (AbstractNode child : finalSelecction.getChildren()) {
                  if (child instanceof CrawledLink) {
                    final CrawledLink cL = (CrawledLink) child;
                    final CrawledPackage parent = isMergePackages() ? null : cL.getParentNode();

                    HashMap<String, ArrayList<CrawledLink>> parentMap = splitMap.get(parent);
                    if (parentMap == null) {
                      parentMap = new HashMap<String, ArrayList<CrawledLink>>();
                      splitMap.put(parent, parentMap);
                    }
                    final String host = cL.getDomainInfo().getTld();
                    ArrayList<CrawledLink> hostList = parentMap.get(host);
                    if (hostList == null) {
                      hostList = new ArrayList<CrawledLink>();
                      parentMap.put(host, hostList);
                    }
                    hostList.add(cL);
                    switch (getLocation()) {
                      case AFTER_SELECTION:
                        insertAt =
                            Math.max(
                                insertAt,
                                LinkCollector.getInstance()
                                        .indexOf(((CrawledLink) child).getParentNode())
                                    + 1);
                        break;
                      case BEFORE_SELECTION:
                        insertAt =
                            Math.min(
                                insertAt,
                                LinkCollector.getInstance()
                                    .indexOf(((CrawledLink) child).getParentNode()));
                        break;
                      case END_OF_LIST:
                        insertAt = -1;
                        break;
                      case TOP_OF_LIST:
                        insertAt = 0;
                        break;
                    }
                  }
                }
                if (insertAt == Integer.MAX_VALUE) {
                  insertAt = 0;
                }
                final String nameFactory =
                    JsonConfig.create(LinkgrabberSettings.class)
                        .getSplitPackageNameFactoryPattern();
                final boolean merge =
                    JsonConfig.create(LinkgrabberSettings.class).isSplitPackageMergeEnabled();
                final HashMap<String, CrawledPackage> mergedPackages =
                    new HashMap<String, CrawledPackage>();
                final Iterator<Entry<CrawledPackage, HashMap<String, ArrayList<CrawledLink>>>> it =
                    splitMap.entrySet().iterator();
                while (it.hasNext()) {
                  final Entry<CrawledPackage, HashMap<String, ArrayList<CrawledLink>>> next =
                      it.next();
                  // sourcePackage may be null
                  final CrawledPackage sourcePackage = next.getKey();
                  final HashMap<String, ArrayList<CrawledLink>> items = next.getValue();
                  final Iterator<Entry<String, ArrayList<CrawledLink>>> it2 =
                      items.entrySet().iterator();
                  while (it2.hasNext()) {
                    final Entry<String, ArrayList<CrawledLink>> next2 = it2.next();
                    final String host = next2.getKey();
                    final String newPackageName =
                        getNewPackageName(
                            nameFactory,
                            sourcePackage == null ? newName : sourcePackage.getName(),
                            host);
                    final CrawledPackage newPkg;
                    if (merge) {
                      CrawledPackage destPackage = mergedPackages.get(newPackageName);
                      if (destPackage == null) {
                        destPackage = new CrawledPackage();
                        destPackage.setExpanded(CFG_LINKCOLLECTOR.CFG.isPackageAutoExpanded());
                        if (sourcePackage != null) {
                          sourcePackage.copyPropertiesTo(destPackage);
                        } else {
                          destPackage.setDownloadFolder(newDownloadFolder);
                        }
                        destPackage.setName(newPackageName);
                        mergedPackages.put(newPackageName, destPackage);
                      }
                      newPkg = destPackage;
                    } else {
                      newPkg = new CrawledPackage();
                      newPkg.setExpanded(CFG_LINKCOLLECTOR.CFG.isPackageAutoExpanded());
                      if (sourcePackage != null) {
                        sourcePackage.copyPropertiesTo(newPkg);
                      } else {
                        newPkg.setDownloadFolder(newDownloadFolder);
                      }
                      newPkg.setName(newPackageName);
                    }
                    LinkCollector.getInstance().moveOrAddAt(newPkg, next2.getValue(), 0, insertAt);
                    insertAt++;
                  }
                }
                return null;
              }
            });
  }
Esempio n. 6
0
public class PollingAPIImpl implements PollingAPI {

  private DownloadWatchDog dwd = DownloadWatchDog.getInstance();
  private DownloadController dc = DownloadController.getInstance();
  private LinkCollector lc = LinkCollector.getInstance();
  private APIQuery queryParams;

  @Override
  public List<PollingResultAPIStorable> poll(APIQuery queryParams) {
    this.queryParams = queryParams;

    List<PollingResultAPIStorable> result = new ArrayList<PollingResultAPIStorable>();

    if (queryParams.containsKey("downloadProgress")) {
      result.add(getDownloadProgress());
    }
    if (queryParams.containsKey("jdState")) {
      result.add(getJDState());
    }
    if (queryParams.containsKey("linkGrabberState")) {
      result.add(getLinkGrabberState());
    }
    if (queryParams.containsKey("captchasWaiting")) {
      result.add(getCaptchasWaiting());
    }
    if (queryParams.containsKey("aggregatedNumbers")) {
      result.add(getAggregatedNumbers());
    }

    return result;
  }

  private PollingResultAPIStorable getAggregatedNumbers() {
    PollingResultAPIStorable prs = new PollingResultAPIStorable();
    prs.setEventName("aggregatedNumbers");
    org.jdownloader.myjdownloader.client.json.JsonMap eventData =
        new org.jdownloader.myjdownloader.client.json.JsonMap();
    eventData.put(
        "data",
        new AggregatedNumbersAPIStorable(
            new AggregatedNumbers(dc.getSelectionInfo()),
            new AggregatedCrawlerNumbers(lc.getSelectionInfo())));
    prs.setEventData(eventData);
    return prs;
  }

  @SuppressWarnings("rawtypes")
  private PollingResultAPIStorable getDownloadProgress() {

    // get packageUUIDs who should be filled with download progress of the containing links e.g
    // because they are expanded in the
    // view
    List<Long> expandedPackageUUIDs = new ArrayList<Long>();
    if (!queryParams._getQueryParam("downloadProgress", List.class, new ArrayList()).isEmpty()) {
      List uuidsFromQuery =
          queryParams._getQueryParam("downloadProgress", List.class, new ArrayList());
      for (Object o : uuidsFromQuery) {
        try {
          expandedPackageUUIDs.add((Long) o);
        } catch (ClassCastException e) {
          continue;
        }
      }
    }

    PollingResultAPIStorable prs = new PollingResultAPIStorable();
    prs.setEventName("downloadProgress");

    List<PollingAPIFilePackageStorable> fpas = new ArrayList<PollingAPIFilePackageStorable>();

    for (FilePackage fp : dwd.getRunningFilePackages()) {
      PollingAPIFilePackageStorable fps = new PollingAPIFilePackageStorable(fp);
      fps.setSpeed(dwd.getDownloadSpeedbyFilePackage(fp));

      // if packages is expanded in view, current state of all running links inside the package
      if (expandedPackageUUIDs.contains(fp.getUniqueID().getID())) {
        boolean readL = fp.getModifyLock().readLock();
        try {
          for (DownloadLink dl : fp.getChildren()) {
            if (dwd.getRunningDownloadLinks().contains(dl.getDownloadLinkController())) {
              PollingAPIDownloadLinkStorable pdls = new PollingAPIDownloadLinkStorable(dl);
              fps.getLinks().add(pdls);
            }
          }
        } finally {
          fp.getModifyLock().readUnlock(readL);
        }
      }
      fpas.add(fps);
    }

    org.jdownloader.myjdownloader.client.json.JsonMap eventData =
        new org.jdownloader.myjdownloader.client.json.JsonMap();
    eventData.put("data", fpas);
    prs.setEventData(eventData);

    return prs;
  }

  private PollingResultAPIStorable getJDState() {
    PollingResultAPIStorable prs = new PollingResultAPIStorable();
    prs.setEventName("jdState");

    org.jdownloader.myjdownloader.client.json.JsonMap eventData =
        new org.jdownloader.myjdownloader.client.json.JsonMap();
    eventData.put("data", dwd.getStateMachine().getState().getLabel());

    prs.setEventData(eventData);
    return prs;
  }

  private PollingResultAPIStorable getLinkGrabberState() {
    PollingResultAPIStorable prs = new PollingResultAPIStorable();
    prs.setEventName("linkGrabberState");

    LinkCollector lc = LinkCollector.getInstance();

    String status = "UNKNOWN";
    if (lc.getDefaultLinkChecker().isRunning()) {
      status = "RUNNING";
    } else {
      status = "IDLE";
    }

    org.jdownloader.myjdownloader.client.json.JsonMap eventData =
        new org.jdownloader.myjdownloader.client.json.JsonMap();
    eventData.put("data", status);
    prs.setEventData(eventData);

    return prs;
  }

  private CaptchaAPI captchaAPI = CaptchaAPISolver.getInstance();

  private PollingResultAPIStorable getCaptchasWaiting() {
    PollingResultAPIStorable prs = new PollingResultAPIStorable();
    prs.setEventName("captchasWaiting");

    org.jdownloader.myjdownloader.client.json.JsonMap eventData =
        new org.jdownloader.myjdownloader.client.json.JsonMap();
    eventData.put("data", captchaAPI.list());
    prs.setEventData(eventData);

    return prs;
  }
}