@Override
 public synchronized void close() {
   for (Map.Entry<Integer, Index> entry : this.depthStacks.entrySet()) {
     int size = entry.getValue().size();
     entry.getValue().close();
     if (size == 0) deletedelete(getFile(entry.getKey()));
   }
   this.depthStacks.clear();
   String[] l = this.hostPath.list();
   if ((l == null || l.length == 0) && this.hostPath != null) deletedelete(this.hostPath);
 }
 @Override
 public synchronized void clear() {
   for (Map.Entry<Integer, Index> entry : this.depthStacks.entrySet()) {
     entry.getValue().close();
     deletedelete(getFile(entry.getKey()));
   }
   this.depthStacks.clear();
   String[] l = this.hostPath.list();
   if (l != null)
     for (String s : l) {
       deletedelete(new File(this.hostPath, s));
     }
   deletedelete(this.hostPath);
 }
Esempio n. 3
0
 /**
  * @param from
  * @param to
  * @throws IOException
  */
 private static void forceMove(final File from, final File to) throws IOException {
   if (!(to.delete() && from.renameTo(to))) {
     // do it manually
     Files.copy(from, to);
     FileUtils.deletedelete(from);
   }
 }
 private int openAllStacks() {
   String[] l = this.hostPath.list();
   int c = 0;
   if (l != null)
     for (String s : l) {
       if (s.endsWith(indexSuffix))
         try {
           int depth = Integer.parseInt(s.substring(0, s.length() - indexSuffix.length()));
           File stackFile = new File(this.hostPath, s);
           Index depthStack = openStack(stackFile);
           if (depthStack != null) {
             int sz = depthStack.size();
             if (sz == 0) {
               depthStack.close();
               deletedelete(stackFile);
             } else {
               this.depthStacks.put(depth, depthStack);
               c += sz;
             }
           }
         } catch (NumberFormatException e) {
         }
     }
   return c;
 }
Esempio n. 5
0
 private void resetProfiles() {
   this.profilesActiveCrawlsCache.clear();
   final File pdb = new File(this.queuesRoot, DBFILE_ACTIVE_CRAWL_PROFILES);
   if (pdb.exists()) FileUtils.deletedelete(pdb);
   try {
     this.profilesActiveCrawls =
         new MapHeap(pdb, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');
   } catch (final IOException e1) {
     Log.logException(e1);
     this.profilesActiveCrawls = null;
   }
   initActiveCrawlProfiles();
 }
  /**
   * main - writes some data and checks the tables size (with time measureing)
   *
   * @param args
   */
  public static void main(final String[] args) {
    // open a file, add one entry and exit
    final File f = new File(args[0]);
    if (f.exists()) FileUtils.deletedelete(f);
    try {
      final Records t = new Records(f, 8);
      final byte[] b = new byte[8];
      t.add("01234567".getBytes(), 0);
      t.add("ABCDEFGH".getBytes(), 0);
      t.add("abcdefgh".getBytes(), 0);
      t.add("--------".getBytes(), 0);
      t.add("********".getBytes(), 0);
      for (int i = 0; i < 1000; i++) t.add("++++++++".getBytes(), 0);
      t.add("=======0".getBytes(), 0);
      t.add("=======1".getBytes(), 0);
      t.add("=======2".getBytes(), 0);
      t.cleanLast(b, 0);
      System.out.println(UTF8.String(b));
      t.cleanLast(b, 0);
      // t.clean(2, b, 0);
      System.out.println(UTF8.String(b));
      t.get(1, b, 0);
      System.out.println(UTF8.String(b));
      t.put(1, "AbCdEfGh".getBytes(), 0);
      t.get(1, b, 0);
      System.out.println(UTF8.String(b));
      t.get(3, b, 0);
      System.out.println(UTF8.String(b));
      t.get(4, b, 0);
      System.out.println(UTF8.String(b));
      System.out.println("size = " + t.size());
      // t.clean(t.size() - 2);
      t.cleanLast();
      final long start = System.currentTimeMillis();
      long c = 0;
      for (int i = 0; i < 100000; i++) {
        c = t.size();
      }
      System.out.println(
          "size() needs " + ((System.currentTimeMillis() - start) / 100) + " nanoseconds");
      System.out.println("size = " + c);

      t.close();
    } catch (final IOException e) {
      ConcurrentLog.logException(e);
    }
  }
 private Index getLowestStack() {
   while (this.depthStacks.size() > 0) {
     Map.Entry<Integer, Index> entry;
     synchronized (this) {
       entry = this.depthStacks.firstEntry();
     }
     if (entry == null) return null; // happens only if map is empty
     if (entry.getValue().size() == 0) {
       entry.getValue().close();
       deletedelete(getFile(entry.getKey()));
       this.depthStacks.remove(entry.getKey());
       continue;
     }
     return entry.getValue();
   }
   // this should not happen
   // assert false;
   return null;
 }
Esempio n. 8
0
 /**
  * Loads crawl profiles from a DB file.
  *
  * @param file DB file
  * @return crawl profile data
  */
 private MapHeap loadFromDB(final File file) {
   MapHeap ret;
   try {
     ret =
         new MapHeap(file, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');
   } catch (final IOException e) {
     Log.logException(e);
     Log.logException(e);
     FileUtils.deletedelete(file);
     try {
       ret =
           new MapHeap(
               file, Word.commonHashLength, NaturalOrder.naturalOrder, 1024 * 64, 500, ' ');
     } catch (final IOException e1) {
       Log.logException(e1);
       ret = null;
     }
   }
   return ret;
 }
Esempio n. 9
0
  public Document[] parse(
      final MultiProtocolURI location,
      final String mimeType,
      final String charset,
      final InputStream source)
      throws Parser.Failure, InterruptedException {

    File tempFile = null;
    Document[] docs = null;
    try {
      int read = 0;
      final byte[] data = new byte[1024];

      final GZIPInputStream zippedContent = new GZIPInputStream(source);

      tempFile = File.createTempFile("gunzip", "tmp");
      tempFile.deleteOnExit();

      // creating a temp file to store the uncompressed data
      final FileOutputStream out = new FileOutputStream(tempFile);

      // reading gzip file and store it uncompressed
      while ((read = zippedContent.read(data, 0, 1024)) != -1) {
        out.write(data, 0, read);
      }
      zippedContent.close();
      out.close();

      // creating a new parser class to parse the unzipped content
      docs = TextParser.parseSource(location, null, null, tempFile, false);
    } catch (final Exception e) {
      if (e instanceof InterruptedException) throw (InterruptedException) e;
      if (e instanceof Parser.Failure) throw (Parser.Failure) e;

      throw new Parser.Failure(
          "Unexpected error while parsing gzip file. " + e.getMessage(), location);
    } finally {
      if (tempFile != null) FileUtils.deletedelete(tempFile);
    }
    return docs;
  }
Esempio n. 10
0
  /**
   * delete files and directories if a directory is not empty, delete also everything inside because
   * deletion sometimes fails on windows, there is also a windows exec included
   *
   * @param path
   */
  public static void deletedelete(final File path) {
    if (path == null || !path.exists()) {
      return;
    }

    // empty the directory first
    if (path.isDirectory()) {
      final String[] list = path.list();
      if (list != null) {
        for (final String s : list) {
          deletedelete(new File(path, s));
        }
      }
    }

    if (path.exists()) path.delete();
    /*
    int c = 0;
    while ( c++ < 20 ) {
        if ( !path.exists() ) {
            break;
        }
        if ( path.delete() ) {
            break;
        }
        // some OS may be slow when giving up file pointer
        //System.runFinalization();
        //System.gc();
        try {
            Thread.sleep(200);
        } catch (final InterruptedException e ) {
            break;
        }
    }
    */
    if (path.exists()) {
      path.deleteOnExit();
      String p = "";
      try {
        p = path.getCanonicalPath();
      } catch (final IOException e1) {
        ConcurrentLog.logException(e1);
      }
      if (System.getProperties().getProperty("os.name", "").toLowerCase().startsWith("windows")) {
        // deleting files on windows sometimes does not work with java
        try {
          final String command = "cmd /C del /F /Q \"" + p + "\"";
          final Process r = Runtime.getRuntime().exec(command);
          if (r == null) {
            ConcurrentLog.severe("FileUtils", "cannot execute command: " + command);
          } else {
            final byte[] response = read(r.getInputStream());
            ConcurrentLog.info("FileUtils", "deletedelete: " + UTF8.String(response));
          }
        } catch (final IOException e) {
          ConcurrentLog.logException(e);
        }
      }
      if (path.exists()) {
        ConcurrentLog.severe("FileUtils", "cannot delete file " + p);
      }
    }
  }
Esempio n. 11
0
  public static serverObjects respond(
      final RequestHeader header, final serverObjects post, final serverSwitch env) {
    // return variable that accumulates replacements
    final serverObjects prop = new serverObjects();
    final Switchboard sb = (Switchboard) env;

    // set if this should be visible
    if (yacyBuildProperties.isPkgManager()) {
      prop.put("candeploy", "2");
      return prop;
    } else if (OS.canExecUnix || OS.isWindows) {
      // we can deploy a new system with (i.e.)
      // cd DATA/RELEASE;tar xfz $1;cp -Rf yacy/* ../../;rm -Rf yacy
      prop.put("candeploy", "1");
    } else {
      prop.put("candeploy", "0");
    }

    prop.put("candeploy_configCommit", "0");
    prop.put("candeploy_autoUpdate", "0");
    prop.put("candeploy_downloadsAvailable", "0");

    if (post != null) {
      // check if update is supposed to be installed and a release is defined
      if (post.containsKey("update") && !post.get("releaseinstall", "").isEmpty()) {
        prop.put("forwardToSteering", "1");
        prop.putHTML("forwardToSteering_release", post.get("releaseinstall", ""));
        prop.put("deploys", "1");
        prop.put("candeploy", "2"); // display nothing else
        return prop;
      }

      if (post.containsKey("downloadRelease")) {
        // download a release
        final String release = post.get("releasedownload", "");
        if (!release.isEmpty()) {
          try {
            yacyRelease versionToDownload = new yacyRelease(new DigestURI(release));

            // replace this version with version which contains public key
            final yacyRelease.DevAndMainVersions allReleases =
                yacyRelease.allReleases(false, false);
            final Set<yacyRelease> mostReleases =
                versionToDownload.isMainRelease() ? allReleases.main : allReleases.dev;
            for (final yacyRelease rel : mostReleases) {
              if (rel.equals(versionToDownload)) {
                versionToDownload = rel;
                break;
              }
            }
            versionToDownload.downloadRelease();
          } catch (final IOException e) {
            // TODO Auto-generated catch block
            Log.logException(e);
          }
        }
      }

      if (post.containsKey("checkRelease")) {
        yacyRelease.allReleases(true, false);
      }

      if (post.containsKey("deleteRelease")) {
        final String release = post.get("releaseinstall", "");
        if (!release.isEmpty()) {
          try {
            FileUtils.deletedelete(new File(sb.releasePath, release));
            FileUtils.deletedelete(new File(sb.releasePath, release + ".sig"));
          } catch (final NullPointerException e) {
            sb.getLog()
                .logSevere(
                    "AUTO-UPDATE: could not delete release " + release + ": " + e.getMessage());
          }
        }
      }

      if (post.containsKey("autoUpdate")) {
        final yacyRelease updateVersion = yacyRelease.rulebasedUpdateInfo(true);
        if (updateVersion == null) {
          prop.put("candeploy_autoUpdate", "2"); // no more recent release found
        } else {
          // there is a version that is more recent. Load it and re-start with it
          sb.getLog()
              .logInfo("AUTO-UPDATE: downloading more recent release " + updateVersion.getUrl());
          final File downloaded = updateVersion.downloadRelease();
          prop.putHTML("candeploy_autoUpdate_downloadedRelease", updateVersion.getName());
          final boolean devenvironment = new File(sb.getAppPath(), ".svn").exists();
          if (devenvironment) {
            sb.getLog()
                .logInfo("AUTO-UPDATE: omitting update because this is a development environment");
            prop.put("candeploy_autoUpdate", "3");
          } else if ((downloaded == null) || (!downloaded.exists()) || (downloaded.length() == 0)) {
            sb.getLog()
                .logInfo(
                    "AUTO-UPDATE: omitting update because download failed (file cannot be found, is too small or signature was bad)");
            prop.put("candeploy_autoUpdate", "4");
          } else {
            yacyRelease.deployRelease(downloaded);
            sb.terminate(10, "manual release update to " + downloaded.getName());
            sb.getLog().logInfo("AUTO-UPDATE: deploy and restart initiated");
            prop.put("candeploy_autoUpdate", "1");
          }
        }
      }

      if (post.containsKey("configSubmit")) {
        prop.put("candeploy_configCommit", "1");
        sb.setConfig(
            "update.process",
            ("manual".equals(post.get("updateMode", "manual"))) ? "manual" : "auto");
        sb.setConfig("update.cycle", Math.max(12, post.getLong("cycle", 168)));
        sb.setConfig("update.blacklist", post.get("blacklist", ""));
        sb.setConfig(
            "update.concept", ("any".equals(post.get("releaseType", "any"))) ? "any" : "main");
        sb.setConfig(
            "update.onlySignedFiles", (post.getBoolean("onlySignedFiles", false)) ? "1" : "0");
      }
    }

    // version information
    final String versionstring =
        yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision();
    prop.putHTML("candeploy_versionpp", versionstring);
    final boolean devenvironment = new File(sb.getAppPath(), ".svn").exists();
    float thisVersion = Float.parseFloat(yacyBuildProperties.getVersion());
    // cut off the SVN Rev in the Version
    try {
      thisVersion = (float) (Math.round(thisVersion * 1000.0) / 1000.0);
    } catch (final NumberFormatException e) {
    }

    // list downloaded releases
    final File[] downloadedFiles = sb.releasePath.listFiles();
    // list can be null if RELEASE directory has been deleted manually
    final int downloadedFilesNum = (downloadedFiles == null) ? 0 : downloadedFiles.length;

    prop.put(
        "candeploy_deployenabled",
        (downloadedFilesNum == 0)
            ? "0"
            : ((devenvironment) ? "1" : "2")); // prevent that a developer-version is over-deployed

    final NavigableSet<yacyRelease> downloadedReleases = new TreeSet<yacyRelease>();
    for (final File downloaded : downloadedFiles) {
      try {
        final yacyRelease release = new yacyRelease(downloaded);
        downloadedReleases.add(release);
      } catch (final RuntimeException e) {
        // not a valid release
        // can be also a restart- or deploy-file
        final File invalid = downloaded;
        if (!(invalid.getName().endsWith(".bat")
            || invalid.getName().endsWith(".sh")
            || invalid
                .getName()
                .endsWith(".sig"))) { // Windows & Linux don't like deleted scripts while execution!
          invalid.deleteOnExit();
        }
      }
    }
    // latest downloaded release
    final yacyVersion dflt = (downloadedReleases.isEmpty()) ? null : downloadedReleases.last();
    // check if there are any downloaded releases and if there are enable the update buttons
    prop.put("candeploy_downloadsAvailable", (downloadedReleases.isEmpty()) ? "0" : "1");
    prop.put(
        "candeploy_deployenabled_buttonsActive",
        (downloadedReleases.isEmpty() || devenvironment) ? "0" : "1");

    int relcount = 0;
    for (final yacyRelease release : downloadedReleases) {
      prop.put(
          "candeploy_downloadedreleases_" + relcount + "_name",
          ((release.isMainRelease()) ? "main" : "dev")
              + " "
              + release.getReleaseNr()
              + "/"
              + release.getSvn());
      prop.put(
          "candeploy_downloadedreleases_" + relcount + "_signature",
          (release.getSignatureFile().exists() ? "1" : "0"));
      prop.putHTML("candeploy_downloadedreleases_" + relcount + "_file", release.getName());
      prop.put(
          "candeploy_downloadedreleases_" + relcount + "_selected", (release == dflt) ? "1" : "0");
      relcount++;
    }
    prop.put("candeploy_downloadedreleases", relcount);

    // list remotely available releases
    final yacyRelease.DevAndMainVersions releasess = yacyRelease.allReleases(false, false);
    relcount = 0;

    final ArrayList<yacyRelease> rlist = new ArrayList<yacyRelease>();
    final Set<yacyRelease> remoteDevReleases = releasess.dev;
    remoteDevReleases.removeAll(downloadedReleases);
    for (final yacyRelease release : remoteDevReleases) {
      rlist.add(release);
    }
    final Set<yacyRelease> remoteMainReleases = releasess.main;
    remoteMainReleases.removeAll(downloadedReleases);
    for (final yacyRelease release : remoteMainReleases) {
      rlist.add(release);
    }
    yacyRelease release;
    for (int i = rlist.size() - 1; i >= 0; i--) {
      release = rlist.get(i);
      prop.put(
          "candeploy_availreleases_" + relcount + "_name",
          ((release.isMainRelease()) ? "main" : "dev")
              + " "
              + release.getReleaseNr()
              + "/"
              + release.getSvn());
      prop.put("candeploy_availreleases_" + relcount + "_url", release.getUrl().toString());
      prop.put(
          "candeploy_availreleases_" + relcount + "_signatures",
          (release.getPublicKey() != null ? "1" : "0"));
      prop.put("candeploy_availreleases_" + relcount + "_selected", (relcount == 0) ? "1" : "0");
      relcount++;
    }

    prop.put("candeploy_availreleases", relcount);

    // properties for automated system update
    prop.put(
        "candeploy_manualUpdateChecked",
        ("manual".equals(sb.getConfig("update.process", "manual"))) ? "1" : "0");
    prop.put(
        "candeploy_autoUpdateChecked",
        ("auto".equals(sb.getConfig("update.process", "manual"))) ? "1" : "0");
    prop.put("candeploy_cycle", sb.getConfigLong("update.cycle", 168));
    prop.putHTML("candeploy_blacklist", sb.getConfig("update.blacklist", ""));
    prop.put(
        "candeploy_releaseTypeMainChecked",
        ("any".equals(sb.getConfig("update.concept", "any"))) ? "0" : "1");
    prop.put(
        "candeploy_releaseTypeAnyChecked",
        ("any".equals(sb.getConfig("update.concept", "any"))) ? "1" : "0");
    prop.put("candeploy_lastlookup", (sb.getConfigLong("update.time.lookup", 0) == 0) ? "0" : "1");
    prop.put(
        "candeploy_lastlookup_time",
        new Date(sb.getConfigLong("update.time.lookup", 0)).toString());
    prop.put(
        "candeploy_lastdownload", (sb.getConfigLong("update.time.download", 0) == 0) ? "0" : "1");
    prop.put(
        "candeploy_lastdownload_time",
        new Date(sb.getConfigLong("update.time.download", 0)).toString());
    prop.put("candeploy_lastdeploy", (sb.getConfigLong("update.time.deploy", 0) == 0) ? "0" : "1");
    prop.put(
        "candeploy_lastdeploy_time",
        new Date(sb.getConfigLong("update.time.deploy", 0)).toString());
    prop.put(
        "candeploy_onlySignedFiles",
        ("1".equals(sb.getConfig("update.onlySignedFiles", "1"))) ? "1" : "0");

    /*
    if ((adminaccess) && (yacyVersion.latestRelease >= (thisVersion+0.01))) { // only new Versions(not new SVN)
        if ((yacyVersion.latestMainRelease != null) ||
            (yacyVersion.latestDevRelease != null)) {
            prop.put("hintVersionDownload", 1);
        } else if ((post != null) && (post.containsKey("aquirerelease"))) {
            yacyVersion.aquireLatestReleaseInfo();
            prop.put("hintVersionDownload", 1);
        } else {
            prop.put("hintVersionAvailable", 1);
        }
    }
    prop.put("hintVersionAvailable", 1); // for testing

    prop.putASIS("hintVersionDownload_versionResMain", (yacyVersion.latestMainRelease == null) ? "-" : yacyVersion.latestMainRelease.toAnchor());
    prop.putASIS("hintVersionDownload_versionResDev", (yacyVersion.latestDevRelease == null) ? "-" : yacyVersion.latestDevRelease.toAnchor());
    prop.put("hintVersionAvailable_latestVersion", Float.toString(yacyVersion.latestRelease));
     */

    return prop;
  }