/**
  * Changes existing entry in a blacklist.
  *
  * @param blacklistToUse The blacklist which contains the entry.
  * @param supportedBlacklistTypes Types of blacklists which the entry is to changed in.
  * @param oldEntry Entry to be changed.
  * @param newEntry Changed entry.
  * @return The length of the new entry.
  */
 private static int alterEntries(
     final String blacklistToUse,
     final String[] supportedBlacklistTypes,
     final String[] oldEntry,
     final String[] newEntry) {
   removeEntries(blacklistToUse, supportedBlacklistTypes, oldEntry);
   PrintWriter pw = null;
   try {
     pw = new PrintWriter(new FileWriter(new File(ListManager.listsPath, blacklistToUse), true));
     String host, path;
     for (final String n : newEntry) {
       int pos = n.indexOf('/');
       if (pos < 0) {
         host = n;
         path = ".*";
       } else {
         host = n.substring(0, pos);
         path = n.substring(pos + 1);
       }
       pw.println(host + "/" + path);
       for (final String s : supportedBlacklistTypes) {
         if (ListManager.listSetContains(s + ".BlackLists", blacklistToUse)) {
           Switchboard.urlBlacklist.add(s, host, path);
         }
       }
       SearchEventCache.cleanupEvents(true);
     }
     pw.close();
   } catch (final IOException e) {
     Log.logSevere("BLACKLIST-CLEANER", "error on writing altered entries to blacklist", e);
   }
   return newEntry.length;
 }
Exemple #2
0
  private static int importFromBookmarks(
      BookmarksDB db,
      final DigestURI baseURL,
      final InputStreamReader input,
      final String tag,
      final boolean importPublic) {

    int importCount = 0;

    Map<MultiProtocolURI, Properties> links = new HashMap<MultiProtocolURI, Properties>();
    String title;
    MultiProtocolURI url;
    Bookmark bm;
    final Set<String> tags = ListManager.string2set(tag); // this allow multiple default tags
    try {
      // load the links
      final ContentScraper scraper = new ContentScraper(baseURL);
      // OutputStream os = new htmlFilterOutputStream(null, scraper, null, false);
      final Writer writer = new TransformerWriter(null, null, scraper, null, false);
      FileUtils.copy(input, writer);
      writer.close();
      links = scraper.getAnchors();
    } catch (final IOException e) {
      Log.logWarning(
          "BOOKMARKS", "error during load of links: " + e.getClass() + " " + e.getMessage());
    }
    for (final Entry<MultiProtocolURI, Properties> link : links.entrySet()) {
      url = link.getKey();
      title = link.getValue().getProperty("name", "");
      Log.logInfo("BOOKMARKS", "links.get(url)");
      if ("".equals(title)) { // cannot be displayed
        title = url.toString();
      }
      bm = db.new Bookmark(url.toString());
      bm.setProperty(Bookmark.BOOKMARK_TITLE, title);
      bm.setTags(tags);
      bm.setPublic(importPublic);
      db.saveBookmark(bm);

      importCount++;
    }

    return importCount;
  }
Exemple #3
0
 /**
  * Loads crawl profiles from a DB file.
  *
  * @param file DB file
  * @return crawl profile data
  */
 private MapHeap loadFromDB(final File file) {
   MapHeap ret;
   try {
     ret =
         new MapHeap(file, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');
   } catch (final IOException e) {
     Log.logException(e);
     Log.logException(e);
     FileUtils.deletedelete(file);
     try {
       ret =
           new MapHeap(
               file, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');
     } catch (final IOException e1) {
       Log.logException(e1);
       ret = null;
     }
   }
   return ret;
 }
Exemple #4
0
 private void resetProfiles() {
   this.profilesActiveCrawlsCache.clear();
   final File pdb = new File(this.queuesRoot, DBFILE_ACTIVE_CRAWL_PROFILES);
   if (pdb.exists()) FileUtils.deletedelete(pdb);
   try {
     this.profilesActiveCrawls =
         new MapHeap(pdb, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');
   } catch (final IOException e1) {
     Log.logException(e1);
     this.profilesActiveCrawls = null;
   }
   initActiveCrawlProfiles();
 }
  /** checks the resources and pauses crawls if necessary */
  public void resourceObserverJob() {
    MemoryControl.setDHTMbyte(getMinFreeMemory());

    normalizedDiskFree = getNormalizedDiskFree();
    normalizedMemoryFree = getNormalizedMemoryFree();

    if (normalizedDiskFree.compareTo(Space.HIGH) < 0
        || normalizedMemoryFree.compareTo(Space.HIGH) < 0) {

      if (normalizedDiskFree.compareTo(Space.HIGH) < 0) { // pause crawls
        if (!sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL)) {
          log.logInfo("pausing local crawls");
          sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
        }
        if (!sb.crawlJobIsPaused(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL)) {
          log.logInfo("pausing remote triggered crawls");
          sb.pauseCrawlJob(SwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
        }
      }

      if ((normalizedDiskFree == Space.LOW || normalizedMemoryFree.compareTo(Space.HIGH) < 0)
          && sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false)) {
        log.logInfo("disabling index receive");
        sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, false);
        sb.peers.mySeed().setFlagAcceptRemoteIndex(false);
        sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, true);
      }
    } else {
      if (sb.getConfigBool(
          SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, false)) { // we were wrong!
        log.logInfo("enabling index receive");
        sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_ALLOW, true);
        sb.peers.mySeed().setFlagAcceptRemoteIndex(true);
        sb.setConfig(SwitchboardConstants.INDEX_RECEIVE_AUTODISABLED, false);
      }
      log.logInfo("resources ok");
    }
  }
  /**
   * Removes existing entries from a blacklist.
   *
   * @param blacklistToUse The blacklist which contains the
   * @param supportedBlacklistTypes Types of blacklists which the entry is to changed in.
   * @param entries Array of entries to be deleted.
   * @return Length of the list of entries to be removed.
   */
  private static int removeEntries(
      final String blacklistToUse, final String[] supportedBlacklistTypes, final String[] entries) {
    // load blacklist data from file
    final List<String> list =
        FileUtils.getListArray(new File(ListManager.listsPath, blacklistToUse));

    boolean listChanged = false;

    // delete the old entry from file
    for (final String entry : entries) {
      String s = entry;

      if (list != null) {

        // get rid of escape characters which make it impossible to
        // properly use contains()
        if (s.contains("\\\\")) {
          s = s.replaceAll(Pattern.quote("\\\\"), Matcher.quoteReplacement("\\"));
        }

        if (list.contains(s)) {
          listChanged = list.remove(s);
        }
      }

      // remove the entry from the running blacklist engine
      for (final String supportedBlacklistType : supportedBlacklistTypes) {
        if (ListManager.listSetContains(supportedBlacklistType + ".BlackLists", blacklistToUse)) {
          final String host = (s.indexOf('/') == -1) ? s : s.substring(0, s.indexOf('/'));
          final String path = (s.indexOf('/') == -1) ? ".*" : s.substring(s.indexOf('/') + 1);
          try {
            Switchboard.urlBlacklist.remove(supportedBlacklistType, host, path);
          } catch (final RuntimeException e) {
            Log.logSevere("BLACKLIST-CLEANER", e.getMessage() + ": " + host + "/" + path);
          }
        }
      }
      SearchEventCache.cleanupEvents(true);
    }
    if (listChanged) {
      FileUtils.writeList(
          new File(ListManager.listsPath, blacklistToUse), list.toArray(new String[list.size()]));
    }
    return entries.length;
  }
  /**
   * returns the amount of disk space available
   *
   * @return
   *     <ul>
   *       <li><code>HIGH</code> if disk space is available
   *       <li><code>MEDIUM</code> if low disk space is available
   *       <li><code>LOW</code> if lower than hardlimit disk space is available
   *     </ul>
   */
  private Space getNormalizedDiskFree() {
    final long currentSpace = getUsableSpace(this.path);
    if (currentSpace < 1L) return Space.HIGH;
    Space ret = Space.HIGH;

    if (currentSpace < getMinFreeDiskSpace()) {
      log.logWarning(
          "Volume "
              + this.path.toString()
              + ": free space ("
              + (currentSpace / 1024 / 1024)
              + " MB) is too low (< "
              + (getMinFreeDiskSpace() / 1024 / 1024)
              + " MB)");
      ret = Space.MEDIUM;
    }
    if (currentSpace < getMinFreeDiskSpace_hardlimit()) {
      ret = Space.LOW;
    }
    return ret;
  }
Exemple #8
0
  public static serverObjects respond(
      final RequestHeader header, final serverObjects post, final serverSwitch env) {
    final Switchboard sb = (Switchboard) env;
    final serverObjects prop = new serverObjects();

    final String EXT = header.get("EXT", "");
    final boolean html = EXT.equals("html");
    final boolean xml = EXT.equals("xml");

    String table = (post == null) ? null : post.get("table");
    if (post == null
        || (!post.containsKey("commitrow") && table != null && !sb.tables.hasHeap(table)))
      table = null;
    prop.put("showtable", 0);
    prop.put("tablecount", sb.tables.size());

    // apply deletion requests
    if (table != null && post != null && post.containsKey("deletetable")) {
      sb.tables.clear(table);
      table = null;
    }

    if (table == null) {
      // list all tables that we know
      int c = 0;
      for (final String name : sb.tables) {
        try {
          if (html) {
            prop.putHTML("showtable_tables_" + c + "_table", name);
          }
          if (xml) {
            prop.putXML("showtable_tables_" + c + "_table", name);
          }
          prop.put("showtable_tables_" + c + "_num", sb.tables.size(name));
          c++;
        } catch (final IOException e) {
        }
      }
      prop.put("showtable_tables", c);
      prop.put("tablecount", c);
      return prop;
    }

    final boolean showpk = post.containsKey("pk");

    final String selectKey = post.containsKey("selectKey") ? post.get("selectKey") : null;
    final String selectValue =
        (selectKey != null && post.containsKey("selectValue")) ? post.get("selectValue") : null;

    final String counts = post.get("count", null);
    int maxcount =
        (counts == null || counts.equals("all")) ? Integer.MAX_VALUE : post.getInt("count", 10);
    final String pattern = post.get("search", "");
    final Pattern matcher =
        (pattern.length() == 0 || pattern.equals(".*"))
            ? null
            : Pattern.compile(".*" + pattern + ".*");

    if (post.containsKey("deleterows")) {
      for (final Map.Entry<String, String> entry : post.entrySet()) {
        if (entry.getValue().startsWith("pk_"))
          try {
            sb.tables.delete(table, entry.getValue().substring(3).getBytes());
          } catch (final IOException e) {
            Log.logException(e);
          }
      }
    }

    if (post.containsKey("commitrow")) {
      final String pk = post.get("pk");
      final Map<String, byte[]> map = new HashMap<String, byte[]>();
      for (final Map.Entry<String, String> entry : post.entrySet()) {
        if (entry.getKey().startsWith("col_")) {
          map.put(entry.getKey().substring(4), entry.getValue().getBytes());
        }
      }
      try {
        if (pk == null || pk.length() == 0) {
          sb.tables.insert(table, map);
        } else {
          sb.tables.update(table, pk.getBytes(), map);
        }
      } catch (final IOException e) {
        Log.logException(e);
      } catch (final RowSpaceExceededException e) {
        Log.logException(e);
      }
    }

    // generate table
    prop.put("showtable", 1);
    prop.put("showtable_table", table);

    // insert the columns
    ArrayList<String> columns = null;
    try {
      columns = sb.tables.columns(table);
    } catch (final IOException e) {
      Log.logException(e);
      columns = new ArrayList<String>();
    }

    // if a row attribute is given
    // then order the columns according to the given order
    final String[] row = post.get("row", "").split(",");
    for (int i = 0; i < row.length; i++) {
      if (columns.contains(row[i])) {
        columns.remove(row[i]);
        if (i < columns.size()) columns.add(i, row[i]);
      }
    }
    prop.put("showtable_showpk", showpk ? 1 : 0);
    for (int i = 0; i < columns.size(); i++) {
      prop.putHTML("showtable_columns_" + i + "_header", columns.get(i));
    }
    prop.put("showtable_columns", columns.size());

    // insert all rows
    try {
      maxcount = Math.min(maxcount, sb.tables.size(table));
    } catch (final IOException e) {
      Log.logException(e);
      maxcount = 0;
    }
    int count = 0;
    try {
      final Iterator<Tables.Row> plainIterator = sb.tables.iterator(table, matcher);
      final Iterator<Tables.Row> mapIterator =
          sb.tables.orderByPK(plainIterator, maxcount).iterator();
      Tables.Row trow;
      boolean dark = true;
      String cellName, cellValue;
      rowloop:
      while ((mapIterator.hasNext()) && (count < maxcount)) {
        trow = mapIterator.next();
        if (row == null) continue;
        prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0));
        dark = !dark;
        prop.put("showtable_list_" + count + "_showpk", showpk ? 1 : 0);
        prop.put("showtable_list_" + count + "_showpk_pk", UTF8.String(trow.getPK()));
        prop.put("showtable_list_" + count + "_count", count);
        for (int i = 0; i < columns.size(); i++) {
          cellName = columns.get(i);
          if (trow.containsKey(cellName)) {
            cellValue = UTF8.String(trow.get(cellName));
            if (selectKey != null && cellName.equals(selectKey) && !cellValue.matches(selectValue))
              continue rowloop;
          } else {
            cellValue = "";
          }
          if (html) {
            prop.putHTML("showtable_list_" + count + "_columns_" + i + "_column", cellName);
            prop.putHTML("showtable_list_" + count + "_columns_" + i + "_cell", cellValue);
          }
          if (xml) {
            prop.putXML("showtable_list_" + count + "_columns_" + i + "_column", cellName);
            prop.putXML("showtable_list_" + count + "_columns_" + i + "_cell", cellValue);
          }
        }
        prop.put("showtable_list_" + count + "_columns", columns.size());

        count++;
      }
    } catch (final IOException e) {
      Log.logException(e);
    }
    prop.put("showtable_list", count);
    prop.put("showtable_num", count);

    // return rewrite properties
    return prop;
  }
 public ResourceObserver(final Switchboard sb) {
   this.sb = sb;
   this.path = sb.getDataPath(SwitchboardConstants.INDEX_PRIMARY_PATH, "");
   log.logInfo("path for disc space measurement: " + this.path);
 }
Exemple #10
0
  public static serverObjects respond(
      final RequestHeader header, final serverObjects post, final serverSwitch env) {
    // return variable that accumulates replacements
    final serverObjects prop = new serverObjects();
    final Switchboard sb = (Switchboard) env;

    // set if this should be visible
    if (yacyBuildProperties.isPkgManager()) {
      prop.put("candeploy", "2");
      return prop;
    } else if (OS.canExecUnix || OS.isWindows) {
      // we can deploy a new system with (i.e.)
      // cd DATA/RELEASE;tar xfz $1;cp -Rf yacy/* ../../;rm -Rf yacy
      prop.put("candeploy", "1");
    } else {
      prop.put("candeploy", "0");
    }

    prop.put("candeploy_configCommit", "0");
    prop.put("candeploy_autoUpdate", "0");
    prop.put("candeploy_downloadsAvailable", "0");

    if (post != null) {
      // check if update is supposed to be installed and a release is defined
      if (post.containsKey("update") && !post.get("releaseinstall", "").isEmpty()) {
        prop.put("forwardToSteering", "1");
        prop.putHTML("forwardToSteering_release", post.get("releaseinstall", ""));
        prop.put("deploys", "1");
        prop.put("candeploy", "2"); // display nothing else
        return prop;
      }

      if (post.containsKey("downloadRelease")) {
        // download a release
        final String release = post.get("releasedownload", "");
        if (!release.isEmpty()) {
          try {
            yacyRelease versionToDownload = new yacyRelease(new DigestURI(release));

            // replace this version with version which contains public key
            final yacyRelease.DevAndMainVersions allReleases =
                yacyRelease.allReleases(false, false);
            final Set<yacyRelease> mostReleases =
                versionToDownload.isMainRelease() ? allReleases.main : allReleases.dev;
            for (final yacyRelease rel : mostReleases) {
              if (rel.equals(versionToDownload)) {
                versionToDownload = rel;
                break;
              }
            }
            versionToDownload.downloadRelease();
          } catch (final IOException e) {
            // TODO Auto-generated catch block
            Log.logException(e);
          }
        }
      }

      if (post.containsKey("checkRelease")) {
        yacyRelease.allReleases(true, false);
      }

      if (post.containsKey("deleteRelease")) {
        final String release = post.get("releaseinstall", "");
        if (!release.isEmpty()) {
          try {
            FileUtils.deletedelete(new File(sb.releasePath, release));
            FileUtils.deletedelete(new File(sb.releasePath, release + ".sig"));
          } catch (final NullPointerException e) {
            sb.getLog()
                .logSevere(
                    "AUTO-UPDATE: could not delete release " + release + ": " + e.getMessage());
          }
        }
      }

      if (post.containsKey("autoUpdate")) {
        final yacyRelease updateVersion = yacyRelease.rulebasedUpdateInfo(true);
        if (updateVersion == null) {
          prop.put("candeploy_autoUpdate", "2"); // no more recent release found
        } else {
          // there is a version that is more recent. Load it and re-start with it
          sb.getLog()
              .logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.getUrl());
          final File downloaded = updateVersion.downloadRelease();
          prop.putHTML("candeploy_autoUpdate_downloadedRelease", updateVersion.getName());
          final boolean devenvironment = new File(sb.getAppPath(), ".svn").exists();
          if (devenvironment) {
            sb.getLog()
                .logInfo("AUTO-UPDATE: omitting update because this is a development environment");
            prop.put("candeploy_autoUpdate", "3");
          } else if ((downloaded == null) || (!downloaded.exists()) || (downloaded.length() == 0)) {
            sb.getLog()
                .logInfo(
                    "AUTO-UPDATE: omitting update because download failed (file cannot be found, is too small or signature was bad)");
            prop.put("candeploy_autoUpdate", "4");
          } else {
            yacyRelease.deployRelease(downloaded);
            sb.terminate(10, "manual release update to " + downloaded.getName());
            sb.getLog().logInfo("AUTO-UPDATE: deploy and restart initiated");
            prop.put("candeploy_autoUpdate", "1");
          }
        }
      }

      if (post.containsKey("configSubmit")) {
        prop.put("candeploy_configCommit", "1");
        sb.setConfig(
            "update.process",
            ("manual".equals(post.get("updateMode", "manual"))) ? "manual" : "auto");
        sb.setConfig("update.cycle", Math.max(12, post.getLong("cycle", 168)));
        sb.setConfig("update.blacklist", post.get("blacklist", ""));
        sb.setConfig(
            "update.concept", ("any".equals(post.get("releaseType", "any"))) ? "any" : "main");
        sb.setConfig(
            "update.onlySignedFiles", (post.getBoolean("onlySignedFiles", false)) ? "1" : "0");
      }
    }

    // version information
    final String versionstring =
        yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision();
    prop.putHTML("candeploy_versionpp", versionstring);
    final boolean devenvironment = new File(sb.getAppPath(), ".svn").exists();
    float thisVersion = Float.parseFloat(yacyBuildProperties.getVersion());
    // cut off the SVN Rev in the Version
    try {
      thisVersion = (float) (Math.round(thisVersion * 1000.0) / 1000.0);
    } catch (final NumberFormatException e) {
    }

    // list downloaded releases
    final File[] downloadedFiles = sb.releasePath.listFiles();
    // list can be null if RELEASE directory has been deleted manually
    final int downloadedFilesNum = (downloadedFiles == null) ? 0 : downloadedFiles.length;

    prop.put(
        "candeploy_deployenabled",
        (downloadedFilesNum == 0)
            ? "0"
            : ((devenvironment) ? "1" : "2")); // prevent that a developer-version is over-deployed

    final NavigableSet<yacyRelease> downloadedReleases = new TreeSet<yacyRelease>();
    for (final File downloaded : downloadedFiles) {
      try {
        final yacyRelease release = new yacyRelease(downloaded);
        downloadedReleases.add(release);
      } catch (final RuntimeException e) {
        // not a valid release
        // can be also a restart- or deploy-file
        final File invalid = downloaded;
        if (!(invalid.getName().endsWith(".bat")
            || invalid.getName().endsWith(".sh")
            || invalid
                .getName()
                .endsWith(".sig"))) { // Windows & Linux don't like deleted scripts while execution!
          invalid.deleteOnExit();
        }
      }
    }
    // latest downloaded release
    final yacyVersion dflt = (downloadedReleases.isEmpty()) ? null : downloadedReleases.last();
    // check if there are any downloaded releases and if there are enable the update buttons
    prop.put("candeploy_downloadsAvailable", (downloadedReleases.isEmpty()) ? "0" : "1");
    prop.put(
        "candeploy_deployenabled_buttonsActive",
        (downloadedReleases.isEmpty() || devenvironment) ? "0" : "1");

    int relcount = 0;
    for (final yacyRelease release : downloadedReleases) {
      prop.put(
          "candeploy_downloadedreleases_" + relcount + "_name",
          ((release.isMainRelease()) ? "main" : "dev")
              + " "
              + release.getReleaseNr()
              + "/"
              + release.getSvn());
      prop.put(
          "candeploy_downloadedreleases_" + relcount + "_signature",
          (release.getSignatureFile().exists() ? "1" : "0"));
      prop.putHTML("candeploy_downloadedreleases_" + relcount + "_file", release.getName());
      prop.put(
          "candeploy_downloadedreleases_" + relcount + "_selected", (release == dflt) ? "1" : "0");
      relcount++;
    }
    prop.put("candeploy_downloadedreleases", relcount);

    // list remotely available releases
    final yacyRelease.DevAndMainVersions releasess = yacyRelease.allReleases(false, false);
    relcount = 0;

    final ArrayList<yacyRelease> rlist = new ArrayList<yacyRelease>();
    final Set<yacyRelease> remoteDevReleases = releasess.dev;
    remoteDevReleases.removeAll(downloadedReleases);
    for (final yacyRelease release : remoteDevReleases) {
      rlist.add(release);
    }
    final Set<yacyRelease> remoteMainReleases = releasess.main;
    remoteMainReleases.removeAll(downloadedReleases);
    for (final yacyRelease release : remoteMainReleases) {
      rlist.add(release);
    }
    yacyRelease release;
    for (int i = rlist.size() - 1; i >= 0; i--) {
      release = rlist.get(i);
      prop.put(
          "candeploy_availreleases_" + relcount + "_name",
          ((release.isMainRelease()) ? "main" : "dev")
              + " "
              + release.getReleaseNr()
              + "/"
              + release.getSvn());
      prop.put("candeploy_availreleases_" + relcount + "_url", release.getUrl().toString());
      prop.put(
          "candeploy_availreleases_" + relcount + "_signatures",
          (release.getPublicKey() != null ? "1" : "0"));
      prop.put("candeploy_availreleases_" + relcount + "_selected", (relcount == 0) ? "1" : "0");
      relcount++;
    }

    prop.put("candeploy_availreleases", relcount);

    // properties for automated system update
    prop.put(
        "candeploy_manualUpdateChecked",
        ("manual".equals(sb.getConfig("update.process", "manual"))) ? "1" : "0");
    prop.put(
        "candeploy_autoUpdateChecked",
        ("auto".equals(sb.getConfig("update.process", "manual"))) ? "1" : "0");
    prop.put("candeploy_cycle", sb.getConfigLong("update.cycle", 168));
    prop.putHTML("candeploy_blacklist", sb.getConfig("update.blacklist", ""));
    prop.put(
        "candeploy_releaseTypeMainChecked",
        ("any".equals(sb.getConfig("update.concept", "any"))) ? "0" : "1");
    prop.put(
        "candeploy_releaseTypeAnyChecked",
        ("any".equals(sb.getConfig("update.concept", "any"))) ? "1" : "0");
    prop.put("candeploy_lastlookup", (sb.getConfigLong("update.time.lookup", 0) == 0) ? "0" : "1");
    prop.put(
        "candeploy_lastlookup_time",
        new Date(sb.getConfigLong("update.time.lookup", 0)).toString());
    prop.put(
        "candeploy_lastdownload", (sb.getConfigLong("update.time.download", 0) == 0) ? "0" : "1");
    prop.put(
        "candeploy_lastdownload_time",
        new Date(sb.getConfigLong("update.time.download", 0)).toString());
    prop.put("candeploy_lastdeploy", (sb.getConfigLong("update.time.deploy", 0) == 0) ? "0" : "1");
    prop.put(
        "candeploy_lastdeploy_time",
        new Date(sb.getConfigLong("update.time.deploy", 0)).toString());
    prop.put(
        "candeploy_onlySignedFiles",
        ("1".equals(sb.getConfig("update.onlySignedFiles", "1"))) ? "1" : "0");

    /*
    if ((adminaccess) && (yacyVersion.latestRelease >= (thisVersion+0.01))) { // only new Versions(not new SVN)
        if ((yacyVersion.latestMainRelease != null) ||
            (yacyVersion.latestDevRelease != null)) {
            prop.put("hintVersionDownload", 1);
        } else if ((post != null) && (post.containsKey("aquirerelease"))) {
            yacyVersion.aquireLatestReleaseInfo();
            prop.put("hintVersionDownload", 1);
        } else {
            prop.put("hintVersionAvailable", 1);
        }
    }
    prop.put("hintVersionAvailable", 1); // for testing

    prop.putASIS("hintVersionDownload_versionResMain", (yacyVersion.latestMainRelease == null) ? "-" : yacyVersion.latestMainRelease.toAnchor());
    prop.putASIS("hintVersionDownload_versionResDev", (yacyVersion.latestDevRelease == null) ? "-" : yacyVersion.latestDevRelease.toAnchor());
    prop.put("hintVersionAvailable_latestVersion", Float.toString(yacyVersion.latestRelease));
     */

    return prop;
  }
Exemple #11
0
  public CrawlSwitchboard(final String networkName, final Log log, final File queuesRoot) {

    log.logInfo("Initializing Word Index for the network '" + networkName + "'.");

    if (networkName == null || networkName.length() == 0) {
      log.logSevere("no network name given - shutting down");
      System.exit(0);
    }
    this.log = log;
    this.profilesActiveCrawlsCache =
        Collections.synchronizedMap(new TreeMap<byte[], CrawlProfile>(Base64Order.enhancedCoder));

    // make crawl profiles database and default profiles
    this.queuesRoot = queuesRoot;
    this.queuesRoot.mkdirs();
    this.log.logConfig("Initializing Crawl Profiles");

    final File profilesActiveFile = new File(queuesRoot, DBFILE_ACTIVE_CRAWL_PROFILES);
    this.profilesActiveCrawls = loadFromDB(profilesActiveFile);
    for (final byte[] handle : this.profilesActiveCrawls.keySet()) {
      CrawlProfile p;
      try {
        p = new CrawlProfile(this.profilesActiveCrawls.get(handle));
      } catch (final IOException e) {
        p = null;
      } catch (final RowSpaceExceededException e) {
        p = null;
      }
      if (p == null) continue;
      if (!RegexHelper.isValidRegex(p.get(CrawlProfile.FILTER_URL_MUSTMATCH))) {
        removeActive(handle);
        Log.logWarning(
            "CrawlProfiles",
            "removed Profile "
                + p.handle()
                + ": "
                + p.name()
                + " from active crawls since "
                + CrawlProfile.FILTER_URL_MUSTMATCH
                + " is no valid regular expression: "
                + p.get(CrawlProfile.FILTER_URL_MUSTMATCH));
      } else if (!RegexHelper.isValidRegex(p.get(CrawlProfile.FILTER_URL_MUSTNOTMATCH))) {
        removeActive(handle);
        Log.logWarning(
            "CrawlProfiles",
            "removed Profile "
                + p.handle()
                + ": "
                + p.name()
                + " from active crawls since "
                + CrawlProfile.FILTER_URL_MUSTNOTMATCH
                + " is no valid regular expression: "
                + p.get(CrawlProfile.FILTER_URL_MUSTNOTMATCH));
      } else {
        Log.logInfo("CrawlProfiles", "loaded Profile " + p.handle() + ": " + p.name());
      }
    }
    initActiveCrawlProfiles();
    log.logInfo(
        "Loaded active crawl profiles from file "
            + profilesActiveFile.getName()
            + ", "
            + this.profilesActiveCrawls.size()
            + " entries");

    final File profilesPassiveFile = new File(queuesRoot, DBFILE_PASSIVE_CRAWL_PROFILES);
    this.profilesPassiveCrawls = loadFromDB(profilesPassiveFile);
    for (final byte[] handle : this.profilesPassiveCrawls.keySet()) {
      CrawlProfile p;
      try {
        p = new CrawlProfile(this.profilesPassiveCrawls.get(handle));
        Log.logInfo("CrawlProfiles", "loaded Profile " + p.handle() + ": " + p.name());
      } catch (final IOException e) {
        continue;
      } catch (final RowSpaceExceededException e) {
        continue;
      }
    }
    log.logInfo(
        "Loaded passive crawl profiles from file "
            + profilesPassiveFile.getName()
            + ", "
            + this.profilesPassiveCrawls.size()
            + " entries"
            + ", "
            + profilesPassiveFile.length() / 1024);
  }