/**
* Loads crawl profiles from a DB file.
*
* @param file DB file
* @return crawl profile data
*/
private MapHeap loadFromDB(final File file) {
MapHeap ret;
try {
ret =
new MapHeap(file, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');
} catch (final IOException e) {
Log.logException(e);
Log.logException(e);
FileUtils.deletedelete(file);
try {
ret =
new MapHeap(
file, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');
} catch (final IOException e1) {
Log.logException(e1);
ret = null;
}
}
return ret;
}
private void resetProfiles() {
this.profilesActiveCrawlsCache.clear();
final File pdb = new File(this.queuesRoot, DBFILE_ACTIVE_CRAWL_PROFILES);
if (pdb.exists()) FileUtils.deletedelete(pdb);
try {
this.profilesActiveCrawls =
new MapHeap(pdb, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');
} catch (final IOException e1) {
Log.logException(e1);
this.profilesActiveCrawls = null;
}
initActiveCrawlProfiles();
}
public static serverObjects respond(
final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
final String EXT = header.get("EXT", "");
final boolean html = EXT.equals("html");
final boolean xml = EXT.equals("xml");
String table = (post == null) ? null : post.get("table");
if (post == null
|| (!post.containsKey("commitrow") && table != null && !sb.tables.hasHeap(table)))
table = null;
prop.put("showtable", 0);
prop.put("tablecount", sb.tables.size());
// apply deletion requests
if (table != null && post != null && post.containsKey("deletetable")) {
sb.tables.clear(table);
table = null;
}
if (table == null) {
// list all tables that we know
int c = 0;
for (final String name : sb.tables) {
try {
if (html) {
prop.putHTML("showtable_tables_" + c + "_table", name);
}
if (xml) {
prop.putXML("showtable_tables_" + c + "_table", name);
}
prop.put("showtable_tables_" + c + "_num", sb.tables.size(name));
c++;
} catch (final IOException e) {
}
}
prop.put("showtable_tables", c);
prop.put("tablecount", c);
return prop;
}
final boolean showpk = post.containsKey("pk");
final String selectKey = post.containsKey("selectKey") ? post.get("selectKey") : null;
final String selectValue =
(selectKey != null && post.containsKey("selectValue")) ? post.get("selectValue") : null;
final String counts = post.get("count", null);
int maxcount =
(counts == null || counts.equals("all")) ? Integer.MAX_VALUE : post.getInt("count", 10);
final String pattern = post.get("search", "");
final Pattern matcher =
(pattern.length() == 0 || pattern.equals(".*"))
? null
: Pattern.compile(".*" + pattern + ".*");
if (post.containsKey("deleterows")) {
for (final Map.Entry<String, String> entry : post.entrySet()) {
if (entry.getValue().startsWith("pk_"))
try {
sb.tables.delete(table, entry.getValue().substring(3).getBytes());
} catch (final IOException e) {
Log.logException(e);
}
}
}
if (post.containsKey("commitrow")) {
final String pk = post.get("pk");
final Map<String, byte[]> map = new HashMap<String, byte[]>();
for (final Map.Entry<String, String> entry : post.entrySet()) {
if (entry.getKey().startsWith("col_")) {
map.put(entry.getKey().substring(4), entry.getValue().getBytes());
}
}
try {
if (pk == null || pk.length() == 0) {
sb.tables.insert(table, map);
} else {
sb.tables.update(table, pk.getBytes(), map);
}
} catch (final IOException e) {
Log.logException(e);
} catch (final RowSpaceExceededException e) {
Log.logException(e);
}
}
// generate table
prop.put("showtable", 1);
prop.put("showtable_table", table);
// insert the columns
ArrayList<String> columns = null;
try {
columns = sb.tables.columns(table);
} catch (final IOException e) {
Log.logException(e);
columns = new ArrayList<String>();
}
// if a row attribute is given
// then order the columns according to the given order
final String[] row = post.get("row", "").split(",");
for (int i = 0; i < row.length; i++) {
if (columns.contains(row[i])) {
columns.remove(row[i]);
if (i < columns.size()) columns.add(i, row[i]);
}
}
prop.put("showtable_showpk", showpk ? 1 : 0);
for (int i = 0; i < columns.size(); i++) {
prop.putHTML("showtable_columns_" + i + "_header", columns.get(i));
}
prop.put("showtable_columns", columns.size());
// insert all rows
try {
maxcount = Math.min(maxcount, sb.tables.size(table));
} catch (final IOException e) {
Log.logException(e);
maxcount = 0;
}
int count = 0;
try {
final Iterator<Tables.Row> plainIterator = sb.tables.iterator(table, matcher);
final Iterator<Tables.Row> mapIterator =
sb.tables.orderByPK(plainIterator, maxcount).iterator();
Tables.Row trow;
boolean dark = true;
String cellName, cellValue;
rowloop:
while ((mapIterator.hasNext()) && (count < maxcount)) {
trow = mapIterator.next();
if (row == null) continue;
prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0));
dark = !dark;
prop.put("showtable_list_" + count + "_showpk", showpk ? 1 : 0);
prop.put("showtable_list_" + count + "_showpk_pk", UTF8.String(trow.getPK()));
prop.put("showtable_list_" + count + "_count", count);
for (int i = 0; i < columns.size(); i++) {
cellName = columns.get(i);
if (trow.containsKey(cellName)) {
cellValue = UTF8.String(trow.get(cellName));
if (selectKey != null && cellName.equals(selectKey) && !cellValue.matches(selectValue))
continue rowloop;
} else {
cellValue = "";
}
if (html) {
prop.putHTML("showtable_list_" + count + "_columns_" + i + "_column", cellName);
prop.putHTML("showtable_list_" + count + "_columns_" + i + "_cell", cellValue);
}
if (xml) {
prop.putXML("showtable_list_" + count + "_columns_" + i + "_column", cellName);
prop.putXML("showtable_list_" + count + "_columns_" + i + "_cell", cellValue);
}
}
prop.put("showtable_list_" + count + "_columns", columns.size());
count++;
}
} catch (final IOException e) {
Log.logException(e);
}
prop.put("showtable_list", count);
prop.put("showtable_num", count);
// return rewrite properties
return prop;
}
public static serverObjects respond(
final RequestHeader header, final serverObjects post, final serverSwitch env) {
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
final Switchboard sb = (Switchboard) env;
// set if this should be visible
if (yacyBuildProperties.isPkgManager()) {
prop.put("candeploy", "2");
return prop;
} else if (OS.canExecUnix || OS.isWindows) {
// we can deploy a new system with (i.e.)
// cd DATA/RELEASE;tar xfz $1;cp -Rf yacy/* ../../;rm -Rf yacy
prop.put("candeploy", "1");
} else {
prop.put("candeploy", "0");
}
prop.put("candeploy_configCommit", "0");
prop.put("candeploy_autoUpdate", "0");
prop.put("candeploy_downloadsAvailable", "0");
if (post != null) {
// check if update is supposed to be installed and a release is defined
if (post.containsKey("update") && !post.get("releaseinstall", "").isEmpty()) {
prop.put("forwardToSteering", "1");
prop.putHTML("forwardToSteering_release", post.get("releaseinstall", ""));
prop.put("deploys", "1");
prop.put("candeploy", "2"); // display nothing else
return prop;
}
if (post.containsKey("downloadRelease")) {
// download a release
final String release = post.get("releasedownload", "");
if (!release.isEmpty()) {
try {
yacyRelease versionToDownload = new yacyRelease(new DigestURI(release));
// replace this version with version which contains public key
final yacyRelease.DevAndMainVersions allReleases =
yacyRelease.allReleases(false, false);
final Set<yacyRelease> mostReleases =
versionToDownload.isMainRelease() ? allReleases.main : allReleases.dev;
for (final yacyRelease rel : mostReleases) {
if (rel.equals(versionToDownload)) {
versionToDownload = rel;
break;
}
}
versionToDownload.downloadRelease();
} catch (final IOException e) {
// TODO Auto-generated catch block
Log.logException(e);
}
}
}
if (post.containsKey("checkRelease")) {
yacyRelease.allReleases(true, false);
}
if (post.containsKey("deleteRelease")) {
final String release = post.get("releaseinstall", "");
if (!release.isEmpty()) {
try {
FileUtils.deletedelete(new File(sb.releasePath, release));
FileUtils.deletedelete(new File(sb.releasePath, release + ".sig"));
} catch (final NullPointerException e) {
sb.getLog()
.logSevere(
"AUTO-UPDATE: could not delete release " + release + ": " + e.getMessage());
}
}
}
if (post.containsKey("autoUpdate")) {
final yacyRelease updateVersion = yacyRelease.rulebasedUpdateInfo(true);
if (updateVersion == null) {
prop.put("candeploy_autoUpdate", "2"); // no more recent release found
} else {
// there is a version that is more recent. Load it and re-start with it
sb.getLog()
.logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.getUrl());
final File downloaded = updateVersion.downloadRelease();
prop.putHTML("candeploy_autoUpdate_downloadedRelease", updateVersion.getName());
final boolean devenvironment = new File(sb.getAppPath(), ".svn").exists();
if (devenvironment) {
sb.getLog()
.logInfo("AUTO-UPDATE: omitting update because this is a development environment");
prop.put("candeploy_autoUpdate", "3");
} else if ((downloaded == null) || (!downloaded.exists()) || (downloaded.length() == 0)) {
sb.getLog()
.logInfo(
"AUTO-UPDATE: omitting update because download failed (file cannot be found, is too small or signature was bad)");
prop.put("candeploy_autoUpdate", "4");
} else {
yacyRelease.deployRelease(downloaded);
sb.terminate(10, "manual release update to " + downloaded.getName());
sb.getLog().logInfo("AUTO-UPDATE: deploy and restart initiated");
prop.put("candeploy_autoUpdate", "1");
}
}
}
if (post.containsKey("configSubmit")) {
prop.put("candeploy_configCommit", "1");
sb.setConfig(
"update.process",
("manual".equals(post.get("updateMode", "manual"))) ? "manual" : "auto");
sb.setConfig("update.cycle", Math.max(12, post.getLong("cycle", 168)));
sb.setConfig("update.blacklist", post.get("blacklist", ""));
sb.setConfig(
"update.concept", ("any".equals(post.get("releaseType", "any"))) ? "any" : "main");
sb.setConfig(
"update.onlySignedFiles", (post.getBoolean("onlySignedFiles", false)) ? "1" : "0");
}
}
// version information
final String versionstring =
yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision();
prop.putHTML("candeploy_versionpp", versionstring);
final boolean devenvironment = new File(sb.getAppPath(), ".svn").exists();
float thisVersion = Float.parseFloat(yacyBuildProperties.getVersion());
// cut off the SVN Rev in the Version
try {
thisVersion = (float) (Math.round(thisVersion * 1000.0) / 1000.0);
} catch (final NumberFormatException e) {
}
// list downloaded releases
final File[] downloadedFiles = sb.releasePath.listFiles();
// list can be null if RELEASE directory has been deleted manually
final int downloadedFilesNum = (downloadedFiles == null) ? 0 : downloadedFiles.length;
prop.put(
"candeploy_deployenabled",
(downloadedFilesNum == 0)
? "0"
: ((devenvironment) ? "1" : "2")); // prevent that a developer-version is over-deployed
final NavigableSet<yacyRelease> downloadedReleases = new TreeSet<yacyRelease>();
for (final File downloaded : downloadedFiles) {
try {
final yacyRelease release = new yacyRelease(downloaded);
downloadedReleases.add(release);
} catch (final RuntimeException e) {
// not a valid release
// can be also a restart- or deploy-file
final File invalid = downloaded;
if (!(invalid.getName().endsWith(".bat")
|| invalid.getName().endsWith(".sh")
|| invalid
.getName()
.endsWith(".sig"))) { // Windows & Linux don't like deleted scripts while execution!
invalid.deleteOnExit();
}
}
}
// latest downloaded release
final yacyVersion dflt = (downloadedReleases.isEmpty()) ? null : downloadedReleases.last();
// check if there are any downloaded releases and if there are enable the update buttons
prop.put("candeploy_downloadsAvailable", (downloadedReleases.isEmpty()) ? "0" : "1");
prop.put(
"candeploy_deployenabled_buttonsActive",
(downloadedReleases.isEmpty() || devenvironment) ? "0" : "1");
int relcount = 0;
for (final yacyRelease release : downloadedReleases) {
prop.put(
"candeploy_downloadedreleases_" + relcount + "_name",
((release.isMainRelease()) ? "main" : "dev")
+ " "
+ release.getReleaseNr()
+ "/"
+ release.getSvn());
prop.put(
"candeploy_downloadedreleases_" + relcount + "_signature",
(release.getSignatureFile().exists() ? "1" : "0"));
prop.putHTML("candeploy_downloadedreleases_" + relcount + "_file", release.getName());
prop.put(
"candeploy_downloadedreleases_" + relcount + "_selected", (release == dflt) ? "1" : "0");
relcount++;
}
prop.put("candeploy_downloadedreleases", relcount);
// list remotely available releases
final yacyRelease.DevAndMainVersions releasess = yacyRelease.allReleases(false, false);
relcount = 0;
final ArrayList<yacyRelease> rlist = new ArrayList<yacyRelease>();
final Set<yacyRelease> remoteDevReleases = releasess.dev;
remoteDevReleases.removeAll(downloadedReleases);
for (final yacyRelease release : remoteDevReleases) {
rlist.add(release);
}
final Set<yacyRelease> remoteMainReleases = releasess.main;
remoteMainReleases.removeAll(downloadedReleases);
for (final yacyRelease release : remoteMainReleases) {
rlist.add(release);
}
yacyRelease release;
for (int i = rlist.size() - 1; i >= 0; i--) {
release = rlist.get(i);
prop.put(
"candeploy_availreleases_" + relcount + "_name",
((release.isMainRelease()) ? "main" : "dev")
+ " "
+ release.getReleaseNr()
+ "/"
+ release.getSvn());
prop.put("candeploy_availreleases_" + relcount + "_url", release.getUrl().toString());
prop.put(
"candeploy_availreleases_" + relcount + "_signatures",
(release.getPublicKey() != null ? "1" : "0"));
prop.put("candeploy_availreleases_" + relcount + "_selected", (relcount == 0) ? "1" : "0");
relcount++;
}
prop.put("candeploy_availreleases", relcount);
// properties for automated system update
prop.put(
"candeploy_manualUpdateChecked",
("manual".equals(sb.getConfig("update.process", "manual"))) ? "1" : "0");
prop.put(
"candeploy_autoUpdateChecked",
("auto".equals(sb.getConfig("update.process", "manual"))) ? "1" : "0");
prop.put("candeploy_cycle", sb.getConfigLong("update.cycle", 168));
prop.putHTML("candeploy_blacklist", sb.getConfig("update.blacklist", ""));
prop.put(
"candeploy_releaseTypeMainChecked",
("any".equals(sb.getConfig("update.concept", "any"))) ? "0" : "1");
prop.put(
"candeploy_releaseTypeAnyChecked",
("any".equals(sb.getConfig("update.concept", "any"))) ? "1" : "0");
prop.put("candeploy_lastlookup", (sb.getConfigLong("update.time.lookup", 0) == 0) ? "0" : "1");
prop.put(
"candeploy_lastlookup_time",
new Date(sb.getConfigLong("update.time.lookup", 0)).toString());
prop.put(
"candeploy_lastdownload", (sb.getConfigLong("update.time.download", 0) == 0) ? "0" : "1");
prop.put(
"candeploy_lastdownload_time",
new Date(sb.getConfigLong("update.time.download", 0)).toString());
prop.put("candeploy_lastdeploy", (sb.getConfigLong("update.time.deploy", 0) == 0) ? "0" : "1");
prop.put(
"candeploy_lastdeploy_time",
new Date(sb.getConfigLong("update.time.deploy", 0)).toString());
prop.put(
"candeploy_onlySignedFiles",
("1".equals(sb.getConfig("update.onlySignedFiles", "1"))) ? "1" : "0");
/*
if ((adminaccess) && (yacyVersion.latestRelease >= (thisVersion+0.01))) { // only new Versions(not new SVN)
if ((yacyVersion.latestMainRelease != null) ||
(yacyVersion.latestDevRelease != null)) {
prop.put("hintVersionDownload", 1);
} else if ((post != null) && (post.containsKey("aquirerelease"))) {
yacyVersion.aquireLatestReleaseInfo();
prop.put("hintVersionDownload", 1);
} else {
prop.put("hintVersionAvailable", 1);
}
}
prop.put("hintVersionAvailable", 1); // for testing
prop.putASIS("hintVersionDownload_versionResMain", (yacyVersion.latestMainRelease == null) ? "-" : yacyVersion.latestMainRelease.toAnchor());
prop.putASIS("hintVersionDownload_versionResDev", (yacyVersion.latestDevRelease == null) ? "-" : yacyVersion.latestDevRelease.toAnchor());
prop.put("hintVersionAvailable_latestVersion", Float.toString(yacyVersion.latestRelease));
*/
return prop;
}