public static serverObjects respond(
final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
if (post != null) {
if (post.containsKey("retrieve")) {
final String peerhash = post.get("peer", null);
final yacySeed seed = (peerhash == null) ? null : sb.peers.getConnected(peerhash);
final RSSFeed feed =
(seed == null) ? null : yacyClient.queryRemoteCrawlURLs(sb.peers, seed, 20, 60000);
if (feed != null) {
for (final Hit item : feed) {
// System.out.println("URL=" + item.getLink() + ", desc=" + item.getDescription() + ",
// pubDate=" + item.getPubDate());
// put url on remote crawl stack
DigestURI url;
try {
url = new DigestURI(item.getLink());
} catch (final MalformedURLException e) {
url = null;
}
Date loaddate;
loaddate = item.getPubDate();
final DigestURI referrer = null; // referrer needed!
final String urlRejectReason = sb.crawlStacker.urlInAcceptedDomain(url);
if (urlRejectReason == null) {
// stack url
if (sb.getLog().isFinest())
sb.getLog().logFinest("crawlOrder: stack: url='" + url + "'");
sb.crawlStacker.enqueueEntry(
new Request(
peerhash.getBytes(),
url,
(referrer == null) ? null : referrer.hash(),
"REMOTE-CRAWLING",
loaddate,
sb.crawler.defaultRemoteProfile.handle(),
0,
0,
0,
item.getSize()));
} else {
env.getLog()
.logWarning(
"crawlOrder: Rejected URL '" + urlToString(url) + "': " + urlRejectReason);
}
}
}
}
}
listHosts(sb, prop);
// return rewrite properties
return prop;
}
/**
* Generates a proxy-autoconfig-file (application/x-ns-proxy-autoconfig) See: <a
* href="http://wp.netscape.com/eng/mozilla/2.0/relnotes/demo/proxy-live.html">Proxy Auto-Config
* File Format</a>
*
* @param header the complete HTTP header of the request
* @param post any arguments for this servlet, the request carried with (GET as well as POST)
* @param env the serverSwitch object holding all runtime-data
* @return the rewrite-properties for the template
*/
public static serverObjects respond(
final RequestHeader header, final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
final boolean yacyonly = env.getConfigBool(SwitchboardConstants.PROXY_YACY_ONLY, false);
// get the http host header
final String hostSocket = header.get(HeaderFramework.CONNECTION_PROP_HOST);
String host = hostSocket;
int port = 80;
final int pos = hostSocket.indexOf(':', 0);
if (pos != -1) {
port = Integer.parseInt(hostSocket.substring(pos + 1));
host = hostSocket.substring(0, pos);
}
prop.put("yacy", yacyonly ? "0" : "1");
prop.put("yacy_host", host);
prop.put("yacy_port", port);
return prop;
}
public static serverObjects respond(
final RequestHeader header, final serverObjects post, final serverSwitch env) {
final serverObjects prop = new serverObjects();
// initialize the list manager
ListManager.switchboard = (Switchboard) env;
ListManager.listsPath =
new File(env.getDataPath(), env.getConfig("listManager.listsPath", "DATA/LISTS"));
String blacklistToUse = null;
// get the list of supported blacklist types
final String supportedBlacklistTypesStr = Blacklist.BLACKLIST_TYPES_STRING;
final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");
prop.put(DISABLED + "checked", "1");
if (post != null) {
final boolean allowRegex =
post.get("allowRegex", "off").equalsIgnoreCase("on") ? true : false;
prop.put(DISABLED + "checked", (allowRegex) ? "1" : "0");
if (post.containsKey("listNames")) {
blacklistToUse = post.get("listNames");
if (blacklistToUse.length() == 0
|| !ListManager.listSetContains("listManager.listsPath", blacklistToUse)) {
prop.put("results", "2");
}
}
putBlacklists(
prop,
FileUtils.getDirListing(ListManager.listsPath, BLACKLIST_FILENAME_FILTER),
blacklistToUse);
if (blacklistToUse != null) {
prop.put("results", "1");
if (post.containsKey("delete")) {
prop.put(RESULTS + "modified", "1");
prop.put(
RESULTS + "modified_delCount",
removeEntries(
blacklistToUse, supportedBlacklistTypes, getKeysByPrefix(post, "select", true)));
} else if (post.containsKey("alter")) {
prop.put(RESULTS + "modified", "2");
prop.put(
RESULTS + "modified_alterCount",
alterEntries(
blacklistToUse,
supportedBlacklistTypes,
getKeysByPrefix(post, "select", false),
getValuesByPrefix(post, "entry", false)));
}
// list illegal entries
final Map<String, BlacklistError> illegalEntries =
getIllegalEntries(blacklistToUse, Switchboard.urlBlacklist, allowRegex);
prop.put(RESULTS + "blList", blacklistToUse);
prop.put(RESULTS + "entries", illegalEntries.size());
prop.putHTML(RESULTS + "blEngine", Switchboard.urlBlacklist.getEngineInfo());
prop.put(RESULTS + "disabled", (illegalEntries.isEmpty()) ? "1" : "0");
if (!illegalEntries.isEmpty()) {
prop.put(RESULTS + DISABLED + "entries", illegalEntries.size());
int i = 0;
String key;
for (final Entry<String, BlacklistError> entry : illegalEntries.entrySet()) {
key = entry.getKey();
prop.put(RESULTS + DISABLED + ENTRIES + i + "_error", entry.getValue().getLong());
prop.putHTML(RESULTS + DISABLED + ENTRIES + i + "_entry", key);
i++;
}
}
}
} else {
prop.put("results", "0");
putBlacklists(
prop,
FileUtils.getDirListing(ListManager.listsPath, BLACKLIST_FILENAME_FILTER),
blacklistToUse);
}
return prop;
}
