protected <ReturnType> ReturnType handleCaptchaChallenge(Challenge<ReturnType> c)
throws CaptchaException, InterruptedException, DecrypterException {
if (c instanceof ImageCaptchaChallenge) {
final File captchaFile = ((ImageCaptchaChallenge) c).getImageFile();
cleanUpCaptchaFiles.add(captchaFile);
}
if (Thread.currentThread() instanceof SingleDownloadController) {
logger.severe("PluginForDecrypt.getCaptchaCode inside SingleDownloadController!?");
}
c.setTimeout(getCaptchaTimeout());
invalidateLastChallengeResponse();
final BlacklistEntry blackListEntry = CaptchaBlackList.getInstance().matches(c);
if (blackListEntry != null) {
logger.warning("Cancel. Blacklist Matching");
throw new CaptchaException(blackListEntry);
}
try {
ChallengeResponseController.getInstance().handle(c);
} catch (InterruptedException e) {
LogSource.exception(logger, e);
throw e;
} catch (SkipException e) {
LogSource.exception(logger, e);
switch (e.getSkipRequest()) {
case BLOCK_ALL_CAPTCHAS:
CaptchaBlackList.getInstance().add(new BlockAllCrawlerCaptchasEntry(getCrawler()));
break;
case BLOCK_HOSTER:
CaptchaBlackList.getInstance()
.add(new BlockCrawlerCaptchasByHost(getCrawler(), getHost()));
break;
case BLOCK_PACKAGE:
CaptchaBlackList.getInstance()
.add(new BlockCrawlerCaptchasByPackage(getCrawler(), getCurrentLink()));
break;
case REFRESH:
// refresh is not supported from the pluginsystem right now.
return c.getRefreshTrigger();
case STOP_CURRENT_ACTION:
if (Thread.currentThread() instanceof LinkCrawlerThread) {
LinkCollector.getInstance().abort();
// Just to be sure
CaptchaBlackList.getInstance().add(new BlockAllCrawlerCaptchasEntry(getCrawler()));
}
break;
default:
break;
}
throw new CaptchaException(e.getSkipRequest());
}
if (!c.isSolved()) {
throw new DecrypterException(DecrypterException.CAPTCHA);
}
return c.getResult().getValue();
}
private PollingResultAPIStorable getLinkGrabberState() {
PollingResultAPIStorable prs = new PollingResultAPIStorable();
prs.setEventName("linkGrabberState");
LinkCollector lc = LinkCollector.getInstance();
String status = "UNKNOWN";
if (lc.getDefaultLinkChecker().isRunning()) {
status = "RUNNING";
} else {
status = "IDLE";
}
org.jdownloader.myjdownloader.client.json.JsonMap eventData =
new org.jdownloader.myjdownloader.client.json.JsonMap();
eventData.put("data", status);
prs.setEventData(eventData);
return prs;
}
private PollingResultAPIStorable getAggregatedNumbers() {
PollingResultAPIStorable prs = new PollingResultAPIStorable();
prs.setEventName("aggregatedNumbers");
org.jdownloader.myjdownloader.client.json.JsonMap eventData =
new org.jdownloader.myjdownloader.client.json.JsonMap();
eventData.put(
"data",
new AggregatedNumbersAPIStorable(
new AggregatedNumbers(dc.getSelectionInfo()),
new AggregatedCrawlerNumbers(lc.getSelectionInfo())));
prs.setEventData(eventData);
return prs;
}
@Override
protected void setStringValue(final String value, final AbstractNode object) {
if (StringUtils.isEmpty(value) || object == null) {
return;
}
File oldPath = LinkTreeUtils.getDownloadDirectory(object);
File newPath = LinkTreeUtils.getDownloadDirectory(value, null, object);
if (oldPath.equals(newPath)) {
/* both paths are same, so nothing to do */
return;
}
if (object instanceof CrawledPackage) {
LinkCollector.getInstance()
.getQueue()
.add(
new QueueAction<Object, RuntimeException>(
org.appwork.utils.event.queue.Queue.QueuePriority.HIGH) {
@Override
protected Object run() {
((CrawledPackage) object).setDownloadFolder(value);
return null;
}
});
return;
} else if (object instanceof CrawledLink) {
final CrawledPackage p = ((CrawledLink) object).getParentNode();
try {
Dialog.getInstance()
.showConfirmDialog(
Dialog.LOGIC_DONOTSHOW_BASED_ON_TITLE_ONLY | Dialog.STYLE_SHOW_DO_NOT_DISPLAY_AGAIN,
_JDT.T.SetDownloadFolderAction_actionPerformed_(p.getName()),
_JDT.T.SetDownloadFolderAction_msg(p.getName(), 1),
null,
_JDT.T.SetDownloadFolderAction_yes(),
_JDT.T.SetDownloadFolderAction_no());
LinkCollector.getInstance()
.getQueue()
.add(
new QueueAction<Object, RuntimeException>(
org.appwork.utils.event.queue.Queue.QueuePriority.HIGH) {
@Override
protected Object run() {
p.setDownloadFolder(value);
return null;
}
});
return;
} catch (DialogClosedException e) {
return;
} catch (DialogCanceledException e) {
/* user clicked no */
}
final CrawledPackage pkg = new CrawledPackage();
pkg.setExpanded(CFG_LINKCOLLECTOR.CFG.isPackageAutoExpanded());
if (TYPE.NORMAL != p.getType()) {
final String pkgName;
if (object instanceof AbstractPackageNode) {
pkgName =
LinknameCleaner.cleanFileName(
object.getName(),
false,
true,
LinknameCleaner.EXTENSION_SETTINGS.REMOVE_KNOWN,
true);
} else {
pkgName =
LinknameCleaner.cleanFileName(
object.getName(),
false,
true,
LinknameCleaner.EXTENSION_SETTINGS.REMOVE_ALL,
true);
}
pkg.setName(pkgName);
} else {
pkg.setName(p.getName());
}
pkg.setComment(p.getComment());
pkg.setDownloadFolder(value);
final java.util.List<CrawledLink> links = new ArrayList<CrawledLink>();
links.add((CrawledLink) object);
LinkCollector.getInstance()
.getQueue()
.add(
new QueueAction<Object, RuntimeException>(
org.appwork.utils.event.queue.Queue.QueuePriority.HIGH) {
@Override
protected Object run() {
LinkCollector.getInstance().moveOrAddAt(pkg, links, -1);
return null;
}
});
}
}
// @Override
public ContainerStatus callDecryption(File lc) {
ContainerStatus cs = new ContainerStatus(lc);
// if (!getProperties().getBooleanProperty("USECCFRSDF", false)) {
Vector<URL> services;
try {
services = new Vector<URL>();
Collections.sort(
services,
new Comparator<Object>() {
public int compare(Object a, Object b) {
return (int) (Math.random() * 4.0 - 2.0);
}
});
services.add(0, new URL("http://service.jdownloader.net/dlcrypt/getDLC.php"));
Iterator<URL> it = services.iterator();
String dlc = null;
// Ua6LDQoSRm6XrkfussF7t1iM5D+WAT9y5QB0U3gwmkELcT+6+4
// U0RUeKvv256VUPZdHttMCuf0/C
// Ua6LDRJGbpeuR+6ywXu3WIzkj5YBP3LlAHRTeDCaQQtxkLr7hTRFR4q+/
// bnpVQ9l0e20wK5/T8L1
while (it.hasNext() && dlc == null) {
URL service = it.next();
try {
// RequestInfo ri;
// requestInfo = Plugin.postRequest(new
// URL(ri.getLocation()), null, null, null, "src=ccf&data="
// + data, true);
Browser br = new Browser();
br.setDebug(true);
PostFormDataRequest r = (PostFormDataRequest) br.createPostFormDataRequest(service + "");
r.addFormData(new FormData("upload", lc.getName(), lc));
try {
r.addFormData(
new FormData(
"l",
getClass()
.forName(getClass().getPackage().getName() + ".Config")
.getField("CCF")
.get(null)
+ ""));
} catch (Throwable e) {
}
r.addFormData(new FormData("src", "ccf"));
br.openRequestConnection(r);
String ret = br.loadConnection(null) + "";
if (ret != null && ret.contains("<dlc>")) {
dlc = new Regex(ret, "<dlc>(.*)</dlc>").getMatch(0);
if (dlc.length() < 90) {
dlc = null;
}
}
} catch (Exception e) {
JDLogger.exception(e);
}
}
if (dlc != null) {
lc = new File(lc.getAbsolutePath().substring(0, lc.getAbsolutePath().length() - 3) + "dlc");
JDIO.writeLocalFile(lc, dlc);
LinkCollector.getInstance()
.addCrawlerJob(new LinkCollectingJob("file://" + lc.getAbsolutePath()));
cs.setStatus(ContainerStatus.STATUS_FINISHED);
return cs;
}
} catch (MalformedURLException e) {
}
cs.setStatus(ContainerStatus.STATUS_FAILED);
return cs;
}
public void actionPerformed(ActionEvent e) {
final SelectionInfo<CrawledPackage, CrawledLink> finalSelecction = getSelection();
final String newName;
final String newDownloadFolder;
if (isMergePackages() && finalSelecction.getPackageViews().size() > 1) {
if (isAskForNewDownloadFolderAndPackageName()) {
try {
final NewPackageDialog d =
new NewPackageDialog(finalSelecction) {
@Override
public String getDontShowAgainKey() {
return "ABSTRACTDIALOG_DONT_SHOW_AGAIN_"
+ SplitPackagesByHost.this.getClass().getSimpleName();
}
};
Dialog.getInstance().showDialog(d);
newName = d.getName();
newDownloadFolder = d.getDownloadFolder();
if (StringUtils.isEmpty(newName)) {
return;
}
} catch (Throwable ignore) {
return;
}
} else {
newName = "";
newDownloadFolder = finalSelecction.getFirstPackage().getRawDownloadFolder();
}
} else {
newName = null;
newDownloadFolder = null;
}
LinkCollector.getInstance()
.getQueue()
.add(
new QueueAction<Void, RuntimeException>() {
@Override
protected Void run() throws RuntimeException {
final HashMap<CrawledPackage, HashMap<String, ArrayList<CrawledLink>>> splitMap =
new HashMap<CrawledPackage, HashMap<String, ArrayList<CrawledLink>>>();
int insertAt = -1;
switch (getLocation()) {
case BEFORE_SELECTION:
insertAt = Integer.MAX_VALUE;
}
for (AbstractNode child : finalSelecction.getChildren()) {
if (child instanceof CrawledLink) {
final CrawledLink cL = (CrawledLink) child;
final CrawledPackage parent = isMergePackages() ? null : cL.getParentNode();
HashMap<String, ArrayList<CrawledLink>> parentMap = splitMap.get(parent);
if (parentMap == null) {
parentMap = new HashMap<String, ArrayList<CrawledLink>>();
splitMap.put(parent, parentMap);
}
final String host = cL.getDomainInfo().getTld();
ArrayList<CrawledLink> hostList = parentMap.get(host);
if (hostList == null) {
hostList = new ArrayList<CrawledLink>();
parentMap.put(host, hostList);
}
hostList.add(cL);
switch (getLocation()) {
case AFTER_SELECTION:
insertAt =
Math.max(
insertAt,
LinkCollector.getInstance()
.indexOf(((CrawledLink) child).getParentNode())
+ 1);
break;
case BEFORE_SELECTION:
insertAt =
Math.min(
insertAt,
LinkCollector.getInstance()
.indexOf(((CrawledLink) child).getParentNode()));
break;
case END_OF_LIST:
insertAt = -1;
break;
case TOP_OF_LIST:
insertAt = 0;
break;
}
}
}
if (insertAt == Integer.MAX_VALUE) {
insertAt = 0;
}
final String nameFactory =
JsonConfig.create(LinkgrabberSettings.class)
.getSplitPackageNameFactoryPattern();
final boolean merge =
JsonConfig.create(LinkgrabberSettings.class).isSplitPackageMergeEnabled();
final HashMap<String, CrawledPackage> mergedPackages =
new HashMap<String, CrawledPackage>();
final Iterator<Entry<CrawledPackage, HashMap<String, ArrayList<CrawledLink>>>> it =
splitMap.entrySet().iterator();
while (it.hasNext()) {
final Entry<CrawledPackage, HashMap<String, ArrayList<CrawledLink>>> next =
it.next();
// sourcePackage may be null
final CrawledPackage sourcePackage = next.getKey();
final HashMap<String, ArrayList<CrawledLink>> items = next.getValue();
final Iterator<Entry<String, ArrayList<CrawledLink>>> it2 =
items.entrySet().iterator();
while (it2.hasNext()) {
final Entry<String, ArrayList<CrawledLink>> next2 = it2.next();
final String host = next2.getKey();
final String newPackageName =
getNewPackageName(
nameFactory,
sourcePackage == null ? newName : sourcePackage.getName(),
host);
final CrawledPackage newPkg;
if (merge) {
CrawledPackage destPackage = mergedPackages.get(newPackageName);
if (destPackage == null) {
destPackage = new CrawledPackage();
destPackage.setExpanded(CFG_LINKCOLLECTOR.CFG.isPackageAutoExpanded());
if (sourcePackage != null) {
sourcePackage.copyPropertiesTo(destPackage);
} else {
destPackage.setDownloadFolder(newDownloadFolder);
}
destPackage.setName(newPackageName);
mergedPackages.put(newPackageName, destPackage);
}
newPkg = destPackage;
} else {
newPkg = new CrawledPackage();
newPkg.setExpanded(CFG_LINKCOLLECTOR.CFG.isPackageAutoExpanded());
if (sourcePackage != null) {
sourcePackage.copyPropertiesTo(newPkg);
} else {
newPkg.setDownloadFolder(newDownloadFolder);
}
newPkg.setName(newPackageName);
}
LinkCollector.getInstance().moveOrAddAt(newPkg, next2.getValue(), 0, insertAt);
insertAt++;
}
}
return null;
}
});
}
public class PollingAPIImpl implements PollingAPI {
private DownloadWatchDog dwd = DownloadWatchDog.getInstance();
private DownloadController dc = DownloadController.getInstance();
private LinkCollector lc = LinkCollector.getInstance();
private APIQuery queryParams;
@Override
public List<PollingResultAPIStorable> poll(APIQuery queryParams) {
this.queryParams = queryParams;
List<PollingResultAPIStorable> result = new ArrayList<PollingResultAPIStorable>();
if (queryParams.containsKey("downloadProgress")) {
result.add(getDownloadProgress());
}
if (queryParams.containsKey("jdState")) {
result.add(getJDState());
}
if (queryParams.containsKey("linkGrabberState")) {
result.add(getLinkGrabberState());
}
if (queryParams.containsKey("captchasWaiting")) {
result.add(getCaptchasWaiting());
}
if (queryParams.containsKey("aggregatedNumbers")) {
result.add(getAggregatedNumbers());
}
return result;
}
private PollingResultAPIStorable getAggregatedNumbers() {
PollingResultAPIStorable prs = new PollingResultAPIStorable();
prs.setEventName("aggregatedNumbers");
org.jdownloader.myjdownloader.client.json.JsonMap eventData =
new org.jdownloader.myjdownloader.client.json.JsonMap();
eventData.put(
"data",
new AggregatedNumbersAPIStorable(
new AggregatedNumbers(dc.getSelectionInfo()),
new AggregatedCrawlerNumbers(lc.getSelectionInfo())));
prs.setEventData(eventData);
return prs;
}
@SuppressWarnings("rawtypes")
private PollingResultAPIStorable getDownloadProgress() {
// get packageUUIDs who should be filled with download progress of the containing links e.g
// because they are expanded in the
// view
List<Long> expandedPackageUUIDs = new ArrayList<Long>();
if (!queryParams._getQueryParam("downloadProgress", List.class, new ArrayList()).isEmpty()) {
List uuidsFromQuery =
queryParams._getQueryParam("downloadProgress", List.class, new ArrayList());
for (Object o : uuidsFromQuery) {
try {
expandedPackageUUIDs.add((Long) o);
} catch (ClassCastException e) {
continue;
}
}
}
PollingResultAPIStorable prs = new PollingResultAPIStorable();
prs.setEventName("downloadProgress");
List<PollingAPIFilePackageStorable> fpas = new ArrayList<PollingAPIFilePackageStorable>();
for (FilePackage fp : dwd.getRunningFilePackages()) {
PollingAPIFilePackageStorable fps = new PollingAPIFilePackageStorable(fp);
fps.setSpeed(dwd.getDownloadSpeedbyFilePackage(fp));
// if packages is expanded in view, current state of all running links inside the package
if (expandedPackageUUIDs.contains(fp.getUniqueID().getID())) {
boolean readL = fp.getModifyLock().readLock();
try {
for (DownloadLink dl : fp.getChildren()) {
if (dwd.getRunningDownloadLinks().contains(dl.getDownloadLinkController())) {
PollingAPIDownloadLinkStorable pdls = new PollingAPIDownloadLinkStorable(dl);
fps.getLinks().add(pdls);
}
}
} finally {
fp.getModifyLock().readUnlock(readL);
}
}
fpas.add(fps);
}
org.jdownloader.myjdownloader.client.json.JsonMap eventData =
new org.jdownloader.myjdownloader.client.json.JsonMap();
eventData.put("data", fpas);
prs.setEventData(eventData);
return prs;
}
private PollingResultAPIStorable getJDState() {
PollingResultAPIStorable prs = new PollingResultAPIStorable();
prs.setEventName("jdState");
org.jdownloader.myjdownloader.client.json.JsonMap eventData =
new org.jdownloader.myjdownloader.client.json.JsonMap();
eventData.put("data", dwd.getStateMachine().getState().getLabel());
prs.setEventData(eventData);
return prs;
}
private PollingResultAPIStorable getLinkGrabberState() {
PollingResultAPIStorable prs = new PollingResultAPIStorable();
prs.setEventName("linkGrabberState");
LinkCollector lc = LinkCollector.getInstance();
String status = "UNKNOWN";
if (lc.getDefaultLinkChecker().isRunning()) {
status = "RUNNING";
} else {
status = "IDLE";
}
org.jdownloader.myjdownloader.client.json.JsonMap eventData =
new org.jdownloader.myjdownloader.client.json.JsonMap();
eventData.put("data", status);
prs.setEventData(eventData);
return prs;
}
private CaptchaAPI captchaAPI = CaptchaAPISolver.getInstance();
private PollingResultAPIStorable getCaptchasWaiting() {
PollingResultAPIStorable prs = new PollingResultAPIStorable();
prs.setEventName("captchasWaiting");
org.jdownloader.myjdownloader.client.json.JsonMap eventData =
new org.jdownloader.myjdownloader.client.json.JsonMap();
eventData.put("data", captchaAPI.list());
prs.setEventData(eventData);
return prs;
}
}
