public static serverObjects respond(
final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
if (post != null) {
if (post.containsKey("retrieve")) {
final String peerhash = post.get("peer", null);
final yacySeed seed = (peerhash == null) ? null : sb.peers.getConnected(peerhash);
final RSSFeed feed =
(seed == null) ? null : yacyClient.queryRemoteCrawlURLs(sb.peers, seed, 20, 60000);
if (feed != null) {
for (final Hit item : feed) {
// System.out.println("URL=" + item.getLink() + ", desc=" + item.getDescription() + ",
// pubDate=" + item.getPubDate());
// put url on remote crawl stack
DigestURI url;
try {
url = new DigestURI(item.getLink());
} catch (final MalformedURLException e) {
url = null;
}
Date loaddate;
loaddate = item.getPubDate();
final DigestURI referrer = null; // referrer needed!
final String urlRejectReason = sb.crawlStacker.urlInAcceptedDomain(url);
if (urlRejectReason == null) {
// stack url
if (sb.getLog().isFinest())
sb.getLog().logFinest("crawlOrder: stack: url='" + url + "'");
sb.crawlStacker.enqueueEntry(
new Request(
peerhash.getBytes(),
url,
(referrer == null) ? null : referrer.hash(),
"REMOTE-CRAWLING",
loaddate,
sb.crawler.defaultRemoteProfile.handle(),
0,
0,
0,
item.getSize()));
} else {
env.getLog()
.logWarning(
"crawlOrder: Rejected URL '" + urlToString(url) + "': " + urlRejectReason);
}
}
}
}
}
listHosts(sb, prop);
// return rewrite properties
return prop;
}
public static serverObjects respond(
final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
// define visible variables
String a = sb.peers.mySeed().getPublicAddress();
if (a == null) a = "localhost:" + sb.getConfig("port", "8090");
final boolean intranet = sb.getConfig(SwitchboardConstants.NETWORK_NAME, "").equals("intranet");
final String repository = "http://" + a + "/repository/";
prop.put("starturl", (intranet) ? repository : "http://");
prop.put("address", a);
// return rewrite properties
return prop;
}
/**
* Removes an element from a ListSet and updates the configuration file accordingly. If the
* element doesn't exist, then nothing will be changed.
*
* @param setName name of the ListSet.
* @param listName name of the element to remove from the ListSet.
*/
public static void removeFromListSet(final String setName, final String listName) {
final Set<String> listSet = getListSet(setName);
if (!listSet.isEmpty()) {
listSet.remove(listName);
switchboard.setConfig(setName, collection2string(listSet));
}
}
/** checks the resources and pauses crawls if necessary */
public void resourceObserverJob() {
MemoryControl.setDHTMbyte(getMinFreeMemory());
normalizedDiskFree = getNormalizedDiskFree();
normalizedMemoryFree = getNormalizedMemoryFree();
if (normalizedDiskFree.compareTo(Space.HIGH) < 0
|| normalizedMemoryFree.compareTo(Space.HIGH) < 0) {
if (normalizedDiskFree.compareTo(Space.HIGH) < 0) { // pause crawls
if (!sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {
log.logInfo("pausing local crawls");
sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
}
if (!sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) {
log.logInfo("pausing remote triggered crawls");
sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
}
}
if ((normalizedDiskFree == Space.LOW || normalizedMemoryFree.compareTo(Space.HIGH) < 0)
&& sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false)) {
log.logInfo("disabling index receive");
sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false);
sb.peers.mySeed().setFlagAcceptRemoteIndex(false);
sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, true);
}
} else {
if (sb.getConfigBool(
SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, false)) { // we were wrong!
log.logInfo("enabling index receive");
sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true);
sb.peers.mySeed().setFlagAcceptRemoteIndex(true);
sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, false);
}
log.logInfo("resources ok");
}
}
/** Load or reload all active Blacklists */
public static void reloadBlacklists() {
final String supportedBlacklistTypesStr = Blacklist.BLACKLIST_TYPES_STRING;
final String[] supportedBlacklistTypes = supportedBlacklistTypesStr.split(",");
final List<BlacklistFile> blacklistFiles =
new ArrayList<BlacklistFile>(supportedBlacklistTypes.length);
for (String supportedBlacklistType : supportedBlacklistTypes) {
final BlacklistFile blFile =
new BlacklistFile(
switchboard.getConfig(
supportedBlacklistType + ".BlackLists",
switchboard.getConfig("BlackLists.DefaultList", "url.default.black")),
supportedBlacklistType);
blacklistFiles.add(blFile);
}
Switchboard.urlBlacklist.clear();
Switchboard.urlBlacklist.loadList(
blacklistFiles.toArray(new BlacklistFile[blacklistFiles.size()]), "/");
SearchEventCache.cleanupEvents(true);
// switchboard.urlBlacklist.clear();
// if (f != "") switchboard.urlBlacklist.loadLists("black", f, "/");
}
public static serverObjects respond(
final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
final boolean authenticated = sb.adminAuthenticated(header) >= 2;
final int display = ((post == null) || (!authenticated)) ? 0 : post.getInt("display", 0);
prop.put("display", display);
final boolean showScore = ((post != null) && (post.containsKey("score")));
// access control
final boolean publicPage = sb.getConfigBool("publicSurftips", true);
final boolean authorizedAccess = sb.verifyAuthentication(header, false);
if ((publicPage) || (authorizedAccess)) {
// read voting
String hash;
if ((post != null) && ((hash = post.get("voteNegative", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
// make new news message with voting
if (!sb.isRobinsonMode()) {
final HashMap<String, String> map = new HashMap<String, String>();
map.put("urlhash", hash);
map.put("vote", "negative");
map.put("refid", post.get("refid", ""));
sb.peers.newsPool.publishMyNews(
sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map);
}
}
if ((post != null) && ((hash = post.get("votePositive", null)) != null)) {
if (!sb.verifyAuthentication(header, false)) {
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
// make new news message with voting
final HashMap<String, String> map = new HashMap<String, String>();
map.put("urlhash", hash);
map.put("url", crypt.simpleDecode(post.get("url", ""), null));
map.put("title", crypt.simpleDecode(post.get("title", ""), null));
map.put("description", crypt.simpleDecode(post.get("description", ""), null));
map.put("vote", "positive");
map.put("refid", post.get("refid", ""));
map.put("comment", post.get("comment", ""));
sb.peers.newsPool.publishMyNews(
sb.peers.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map);
}
// create Supporter
final HashMap<String, Integer> negativeHashes =
new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes)
final HashMap<String, Integer> positiveHashes =
new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes)
accumulateVotes(sb, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB);
// accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB);
// accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB);
final ScoreMap<String> ranking =
new ConcurrentScoreMap<String>(); // score cluster for url hashes
final Row rowdef =
new Row(
"String url-255, String title-120, String description-120, String refid-"
+ (GenericFormatter.PATTERN_SHORT_SECOND.length() + 12),
NaturalOrder.naturalOrder);
final HashMap<String, Entry> Supporter =
new HashMap<
String,
Entry>(); // a mapping from an url hash to a kelondroRow.Entry with display properties
accumulateSupporter(
sb, Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB);
// accumulateSupporter(Supporter, ranking, rowdef, negativeHashes, positiveHashes,
// yacyNewsPool.OUTGOING_DB);
// accumulateSupporter(Supporter, ranking, rowdef, negativeHashes, positiveHashes,
// yacyNewsPool.PUBLISHED_DB);
// read out surftipp array and create property entries
final Iterator<String> k = ranking.keys(false);
int i = 0;
Row.Entry row;
String url, urlhash, refid, title, description;
boolean voted;
while (k.hasNext()) {
urlhash = k.next();
if (urlhash == null) continue;
row = Supporter.get(urlhash);
if (row == null) continue;
url = row.getColString(0);
try {
if (Switchboard.urlBlacklist.isListed(
Blacklist.BLACKLIST_SURFTIPS, new DigestURI(url, urlhash.getBytes()))) continue;
} catch (final MalformedURLException e) {
continue;
}
title = row.getColString(1);
description = row.getColString(2);
if ((url == null) || (title == null) || (description == null)) continue;
refid = row.getColString(3);
voted =
(sb.peers.newsPool.getSpecific(
yacyNewsPool.OUTGOING_DB,
yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD,
"refid",
refid)
!= null)
|| (sb.peers.newsPool.getSpecific(
yacyNewsPool.PUBLISHED_DB,
yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD,
"refid",
refid)
!= null);
prop.put("supporter_results_" + i + "_authorized", authenticated ? "1" : "0");
prop.put("supporter_results_" + i + "_authorized_recommend", voted ? "0" : "1");
prop.put("supporter_results_" + i + "_authorized_recommend_urlhash", urlhash);
prop.put("supporter_results_" + i + "_authorized_recommend_refid", refid);
prop.put(
"supporter_results_" + i + "_authorized_recommend_url",
crypt.simpleEncode(url, null, 'b'));
prop.putHTML(
"supporter_results_" + i + "_authorized_recommend_title",
crypt.simpleEncode(title, null, 'b'));
prop.putHTML(
"supporter_results_" + i + "_authorized_recommend_description",
crypt.simpleEncode(description, null, 'b'));
prop.put("supporter_results_" + i + "_authorized_recommend_display", display);
prop.put(
"supporter_results_" + i + "_authorized_recommend_showScore", showScore ? "1" : "0");
prop.put("supporter_results_" + i + "_authorized_urlhash", urlhash);
prop.put("supporter_results_" + i + "_url", url);
prop.put("supporter_results_" + i + "_urlname", nxTools.shortenURLString(url, 60));
prop.put("supporter_results_" + i + "_urlhash", urlhash);
prop.putHTML(
"supporter_results_" + i + "_title",
(showScore) ? ("(" + ranking.get(urlhash) + ") " + title) : title);
prop.putHTML("supporter_results_" + i + "_description", description);
i++;
if (i >= 50) break;
}
prop.put("supporter_results", i);
prop.put("supporter", "1");
} else {
prop.put("supporter", "0");
}
return prop;
}
public static void initThread() {
final Switchboard sb = Switchboard.getSwitchboard();
sb.observer = new ResourceObserver(Switchboard.getSwitchboard());
sb.observer.resourceObserverJob();
}
public ResourceObserver(final Switchboard sb) {
this.sb = sb;
this.path = sb.getDataPath(SwitchboardConstants.INDEX_PRIMARY_PATH, "");
log.logInfo("path for disc space measurement: " + this.path);
}
/** @return amount of space (MiB) that should at least be free */
public long getMinFreeMemory() {
return sb.getConfigLong(SwitchboardConstants.MEMORY_ACCEPTDHT, 0);
}
/** @return amount of space (bytes) that should at least be kept free */
public long getMinFreeDiskSpace_hardlimit() {
return sb.getConfigLong(SwitchboardConstants.DISK_FREE_HARDLIMIT, 100) /* MiB */
* 1024L
* 1024L;
}
/** @return amount of space (bytes) that should be kept free */
public long getMinFreeDiskSpace() {
return sb.getConfigLong(SwitchboardConstants.DISK_FREE, 3000) /* MiB */ * 1024L * 1024L;
}
/**
* Adds an element to an existing ListSet. If the ListSet doesn't exist yet, a new one will be
* added. If the ListSet already contains an identical element, then nothing happens.
*
* <p>The new list will be written to the configuartion file.
*
* @param setName
* @param newListName
*/
public static void updateListSet(final String setName, final String newListName) {
final Set<String> listSet = getListSet(setName);
listSet.add(newListName);
switchboard.setConfig(setName, collection2string(listSet));
}
/**
* Get ListSet from configuration file and return it as a unified Set.
*
* <p><b>Meaning of ListSet</b>: There are various "lists" in YaCy which are actually disjunct
* (pairwise unequal) sets which themselves can be seperated into different subsets. E.g., there
* can be more than one blacklist of a type. A ListSet is the set of all those "lists" (subsets)
* of an equal type.
*
* @param setName name of the ListSet
* @return a ListSet from configuration file
*/
public static Set<String> getListSet(final String setName) {
return string2set(switchboard.getConfig(setName, ""));
}
