@Override public boolean matches(Challenge c) { final LinkCrawler lcrawler = getCrawler(); if (lcrawler != null && lcrawler.isRunning()) { final Plugin plugin = c.getPlugin(); if (plugin instanceof PluginForDecrypt) { final PluginForDecrypt decrypt = (PluginForDecrypt) plugin; return decrypt.getCrawler() == lcrawler && decrypt.getHost().equalsIgnoreCase(getHost()); } } return false; }
protected List<DownloadLink> loadContainerFile(File file) { final LinkCrawler lc = LinkCrawler.newInstance(); lc.crawl(file.toURI().toString()); lc.waitForCrawling(); final ArrayList<DownloadLink> ret = new ArrayList<DownloadLink>(lc.getCrawledLinks().size()); for (final CrawledLink link : lc.getCrawledLinks()) { DownloadLink dl = link.getDownloadLink(); if (dl == null) { final String url = link.getURL(); if (url != null) { dl = new DownloadLink(null, null, null, url, true); } } if (dl != null) { ret.add(dl); } } return ret; }
/** * Die Methode entschlüsselt einen einzelnen Link. Alle steps werden durchlaufen. Der letzte step * muss als parameter einen Vector <String> mit den decoded Links setzen * * @param cryptedLink Ein einzelner verschlüsselter Link * @return Ein Vector mit Klartext-links */ public ArrayList<DownloadLink> decryptLink(CrawledLink link) { if (link.getCryptedLink() == null) { return null; } ArrayList<DownloadLink> tmpLinks = null; Throwable throwable = null; boolean linkstatusOffline = false; boolean pwfailed = false; boolean captchafailed = false; try { challenges = null; setCurrentLink(link); /* * we now lets log into plugin specific loggers with all verbose/debug on */ br.setLogger(logger); br.setVerbose(true); br.setDebug(true); /* now we let the decrypter do its magic */ tmpLinks = decryptIt(link); validateLastChallengeResponse(); } catch (final Throwable e) { throwable = e; if (isAbort()) { throwable = null; } else if (processCaptchaException(e)) { /* User entered wrong captcha (too many times) */ throwable = null; captchafailed = true; } else if (DecrypterException.PLUGIN_DEFECT.equals(e.getMessage())) { // leave alone. } else if (DecrypterException.PASSWORD.equals(e.getMessage())) { /* User entered password captcha (too many times) */ throwable = null; pwfailed = true; } else if (DecrypterException.ACCOUNT.equals(e.getMessage())) { throwable = null; } else if (e instanceof DecrypterException || e.getCause() instanceof DecrypterException) { throwable = null; } else if (e instanceof PluginException) { // offline file linkstatus exception, this should not be treated as crawler error.. if (((PluginException) e).getLinkStatus() == 32) { throwable = null; linkstatusOffline = true; if (tmpLinks == null && LinkCrawler.getConfig().isAddDefectiveCrawlerTasksAsOfflineInLinkgrabber()) { tmpLinks = new ArrayList<DownloadLink>(); tmpLinks.add(createOfflinelink(link.getURL())); } } } if (throwable == null && logger instanceof LogSource) { if (logger instanceof LogSource) { /* make sure we use the right logger */ ((LogSource) logger).clear(); ((LogSource) logger).log(e); } else { LogSource.exception(logger, e); } } } finally { clean(); challenges = null; } if ((tmpLinks == null || throwable != null) && !isAbort() && !pwfailed && !captchafailed && !linkstatusOffline) { /* * null as return value? something must have happened, do not clear log */ errLog(throwable, br, link); logger.severe("CrawlerPlugin out of date: " + this + " :" + getVersion()); logger.severe("URL was: " + link.getURL()); /* * we can effectively create generic offline link here. For custom message/comments this must be done within the plugin. * -raztoki */ if (tmpLinks == null && LinkCrawler.getConfig().isAddDefectiveCrawlerTasksAsOfflineInLinkgrabber()) { tmpLinks = new ArrayList<DownloadLink>(); tmpLinks.add(createOfflinelink(link.getURL())); } /* lets forward the log */ if (logger instanceof LogSource) { /* make sure we use the right logger */ ((LogSource) logger).flush(); } } if (logger instanceof LogSource) { /* make sure we use the right logger */ ((LogSource) logger).clear(); } return tmpLinks; }
@Override public boolean canCleanUp() { final LinkCrawler lcrawler = getCrawler(); return lcrawler == null || !lcrawler.isRunning(); }