Beispiel #1
0
 public boolean getContext(String filename, String path, List<Hit> hits) throws HistoryException {
   if (m == null) {
     return false;
   }
   File f = new File(filename);
   return getHistoryContext(HistoryGuru.getInstance().getHistory(f), path, null, hits, null);
 }
Beispiel #2
0
 public boolean getContext(String parent, String basename, String path, Writer out, String context)
     throws HistoryException {
   if (m == null) {
     return false;
   }
   History hist = HistoryGuru.getInstance().getHistory(new File(parent, basename));
   return getHistoryContext(hist, path, out, null, context);
 }
Beispiel #3
0
  /**
   * Check if I should accept this file into the index database
   *
   * @param file the file to check
   * @return true if the file should be included, false otherwise
   */
  private boolean accept(File file) {

    if (!includedNames.isEmpty()
        && // the filter should not affect directory names
        (!(file.isDirectory() || includedNames.match(file)))) {
      return false;
    }

    String absolutePath = file.getAbsolutePath();

    if (ignoredNames.ignore(file)) {
      LOGGER.log(Level.FINER, "ignoring {0}", absolutePath);
      return false;
    }

    if (!file.canRead()) {
      LOGGER.log(Level.WARNING, "Could not read {0}", absolutePath);
      return false;
    }

    try {
      String canonicalPath = file.getCanonicalPath();
      if (!absolutePath.equals(canonicalPath) && !acceptSymlink(absolutePath, canonicalPath)) {

        LOGGER.log(
            Level.FINE,
            "Skipped symlink ''{0}'' -> ''{1}''",
            new Object[] {absolutePath, canonicalPath});
        return false;
      }
      // below will only let go files and directories, anything else is considered special and is
      // not added
      if (!file.isFile() && !file.isDirectory()) {
        LOGGER.log(Level.WARNING, "Ignored special file {0}", absolutePath);
        return false;
      }
    } catch (IOException exp) {
      LOGGER.log(Level.WARNING, "Failed to resolve name: {0}", absolutePath);
      LOGGER.log(Level.FINE, "Stack Trace: ", exp);
    }

    if (file.isDirectory()) {
      // always accept directories so that their files can be examined
      return true;
    }

    if (HistoryGuru.getInstance().hasHistory(file)) {
      // versioned files should always be accepted
      return true;
    }

    // this is an unversioned file, check if it should be indexed
    return !RuntimeEnvironment.getInstance().isIndexVersionedFilesOnly();
  }
Beispiel #4
0
 /**
  * Get the annotation for the requested resource.
  *
  * @return {@code null} if not available or annotation was not requested, the cached annotation
  *     otherwise.
  */
 public Annotation getAnnotation() {
   if (isDir() || getResourcePath().equals("/") || !annotate()) {
     return null;
   }
   if (annotation != null) {
     return annotation;
   }
   getRequestedRevision();
   try {
     annotation =
         HistoryGuru.getInstance().annotate(resourceFile, rev.isEmpty() ? null : rev.substring(2));
   } catch (IOException e) {
     LOGGER.log(Level.WARNING, "Failed to get annotations: ", e);
     /* ignore */
   }
   return annotation;
 }
Beispiel #5
0
  /**
   * Populate a Lucene document with the required fields.
   *
   * @param doc The document to populate
   * @param file The file to index
   * @param path Where the file is located (from source root)
   * @param fa The analyzer to use on the file
   * @param xrefOut Where to write the xref (possibly {@code null})
   * @throws IOException If an exception occurs while collecting the data
   */
  public void populateDocument(
      Document doc, File file, String path, FileAnalyzer fa, Writer xrefOut) throws IOException {
    String date = DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND);
    doc.add(new Field(QueryBuilder.U, Util.path2uid(path, date), string_ft_stored_nanalyzed_norms));
    doc.add(
        new Field(
            QueryBuilder.FULLPATH, file.getAbsolutePath(), string_ft_nstored_nanalyzed_norms));
    doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH, new BytesRef(file.getAbsolutePath())));

    try {
      HistoryReader hr = HistoryGuru.getInstance().getHistoryReader(file);
      if (hr != null) {
        doc.add(new TextField(QueryBuilder.HIST, hr));
        // date = hr.getLastCommentDate() //RFE
      }
    } catch (HistoryException e) {
      LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e);
    }
    doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms));
    doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date)));
    if (path != null) {
      doc.add(new TextField(QueryBuilder.PATH, path, Store.YES));
      Project project = Project.getProject(path);
      if (project != null) {
        doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES));
      }
    }

    if (fa != null) {
      Genre g = fa.getGenre();
      if (g == Genre.PLAIN || g == Genre.XREFABLE || g == Genre.HTML) {
        doc.add(new Field(QueryBuilder.T, g.typeName(), string_ft_stored_nanalyzed_norms));
      }
      fa.analyze(doc, StreamSource.fromFile(file), xrefOut);

      String type = fa.getFileTypeName();
      doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES));
    }
  }
Beispiel #6
0
 /**
  * Dump the configuration as an HTML table.
  *
  * @param out destination for the HTML output
  * @throws IOException if an error happens while writing to {@code out}
  * @throws HistoryException if the history guru cannot be accesses
  */
 @SuppressWarnings("boxing")
 public static void dumpConfiguration(Appendable out) throws IOException, HistoryException {
   out.append("<table border=\"1\" width=\"100%\">");
   out.append("<tr><th>Variable</th><th>Value</th></tr>");
   RuntimeEnvironment env = RuntimeEnvironment.getInstance();
   printTableRow(out, "Source root", env.getSourceRootPath());
   printTableRow(out, "Data root", env.getDataRootPath());
   printTableRow(out, "CTags", env.getCtags());
   printTableRow(out, "Bug page", env.getBugPage());
   printTableRow(out, "Bug pattern", env.getBugPattern());
   printTableRow(out, "User page", env.getUserPage());
   printTableRow(out, "User page suffix", env.getUserPageSuffix());
   printTableRow(out, "Review page", env.getReviewPage());
   printTableRow(out, "Review pattern", env.getReviewPattern());
   printTableRow(out, "Using projects", env.hasProjects());
   out.append("<tr><td>Ignored files</td><td>");
   printUnorderedList(out, env.getIgnoredNames().getItems());
   out.append("</td></tr>");
   printTableRow(out, "Index word limit", env.getIndexWordLimit());
   printTableRow(out, "Allow leading wildcard in search", env.isAllowLeadingWildcard());
   printTableRow(out, "History cache", HistoryGuru.getInstance().getCacheInfo());
   out.append("</table>");
 }
Beispiel #7
0
 /**
  * Check, whether annotations are available for the related resource.
  *
  * @return {@code true} if annotations are available.
  */
 public boolean hasAnnotations() {
   if (hasAnnotation == null) {
     hasAnnotation = !isDir() && HistoryGuru.getInstance().hasHistory(getResourceFile());
   }
   return hasAnnotation;
 }
Beispiel #8
0
 /**
  * Check, whether the request related resource has history information.
  *
  * @return {@code true} if history is available.
  * @see HistoryGuru#hasHistory(File)
  */
 public boolean hasHistory() {
   if (hasHistory == null) {
     hasHistory = HistoryGuru.getInstance().hasHistory(getResourceFile());
   }
   return hasHistory;
 }
Beispiel #9
0
  /**
   * Get all data required to create a diff view wrt. to this request in one go.
   *
   * @return an instance with just enough information to render a sufficient view. If not all
   *     required parameters were given either they are supplemented with reasonable defaults if
   *     possible, otherwise the related field(s) are {@code null}. {@link DiffData#errorMsg} {@code
   *     != null} indicates, that an error occured and one should not try to render a view.
   */
  public DiffData getDiffData() {
    DiffData data = new DiffData();
    data.path = getPath().substring(0, path.lastIndexOf('/'));
    data.filename = Util.htmlize(getResourceFile().getName());

    String srcRoot = getSourceRootPath();
    String context = req.getContextPath();

    String[] filepath = new String[2];
    data.rev = new String[2];
    data.file = new String[2][];
    data.param = new String[2];

    /*
     * Basically the request URI looks like this:
     * http://$site/$webapp/diff/$resourceFile?r1=$fileA@$revA&r2=$fileB@$revB
     * The code below extracts file path and revision from the URI.
     */
    for (int i = 1; i <= 2; i++) {
      String[] tmp = null;
      String p = req.getParameter("r" + i);
      if (p != null) {
        tmp = p.split("@");
      }
      if (tmp != null && tmp.length == 2) {
        filepath[i - 1] = tmp[0];
        data.rev[i - 1] = tmp[1];
      }
    }
    if (data.rev[0] == null
        || data.rev[1] == null
        || data.rev[0].length() == 0
        || data.rev[1].length() == 0
        || data.rev[0].equals(data.rev[1])) {
      data.errorMsg =
          "Please pick two revisions to compare the changed "
              + "from the <a href=\""
              + context
              + Prefix.HIST_L
              + getUriEncodedPath()
              + "\">history</a>";
      return data;
    }
    data.genre = AnalyzerGuru.getGenre(getResourceFile().getName());

    if (data.genre == null || txtGenres.contains(data.genre)) {
      InputStream[] in = new InputStream[2];
      try {
        // Get input stream for both older and newer file.
        for (int i = 0; i < 2; i++) {
          File f = new File(srcRoot + filepath[i]);
          in[i] = HistoryGuru.getInstance().getRevision(f.getParent(), f.getName(), data.rev[i]);
          if (in[i] == null) {
            data.errorMsg =
                "Unable to get revision "
                    + Util.htmlize(data.rev[i])
                    + " for file: "
                    + Util.htmlize(getPath());
            return data;
          }
        }

        /*
         * If the genre of the older revision cannot be determined,
         * (this can happen if the file was empty), try with newer
         * version.
         */
        for (int i = 0; i < 2 && data.genre == null; i++) {
          try {
            data.genre = AnalyzerGuru.getGenre(in[i]);
          } catch (IOException e) {
            data.errorMsg = "Unable to determine the file type: " + Util.htmlize(e.getMessage());
          }
        }

        if (data.genre != Genre.PLAIN && data.genre != Genre.HTML) {
          return data;
        }

        ArrayList<String> lines = new ArrayList<>();
        Project p = getProject();
        for (int i = 0; i < 2; i++) {
          try (BufferedReader br =
              new BufferedReader(ExpandTabsReader.wrap(new InputStreamReader(in[i]), p))) {
            String line;
            while ((line = br.readLine()) != null) {
              lines.add(line);
            }
            data.file[i] = lines.toArray(new String[lines.size()]);
            lines.clear();
          }
          in[i] = null;
        }
      } catch (Exception e) {
        data.errorMsg = "Error reading revisions: " + Util.htmlize(e.getMessage());
      } finally {
        for (int i = 0; i < 2; i++) {
          IOUtils.close(in[i]);
        }
      }
      if (data.errorMsg != null) {
        return data;
      }
      try {
        data.revision = Diff.diff(data.file[0], data.file[1]);
      } catch (DifferentiationFailedException e) {
        data.errorMsg = "Unable to get diffs: " + Util.htmlize(e.getMessage());
      }
      for (int i = 0; i < 2; i++) {
        try {
          URI u = new URI(null, null, null, filepath[i] + "@" + data.rev[i], null);
          data.param[i] = u.getRawQuery();
        } catch (URISyntaxException e) {
          LOGGER.log(Level.WARNING, "Failed to create URI: ", e);
        }
      }
      data.full = fullDiff();
      data.type = getDiffType();
    }
    return data;
  }
Beispiel #10
0
  /**
   * Update the content of this index database
   *
   * @throws IOException if an error occurs
   * @throws HistoryException if an error occurs when accessing the history
   */
  public void update() throws IOException, HistoryException {
    synchronized (lock) {
      if (running) {
        throw new IOException("Indexer already running!");
      }
      running = true;
      interrupted = false;
    }

    String ctgs = RuntimeEnvironment.getInstance().getCtags();
    if (ctgs != null) {
      ctags = new Ctags();
      ctags.setBinary(ctgs);
    }
    if (ctags == null) {
      log.severe("Unable to run ctags! searching definitions will not work!");
    }

    if (ctags != null) {
      String filename = RuntimeEnvironment.getInstance().getCTagsExtraOptionsFile();
      if (filename != null) {
        ctags.setCTagsExtraOptionsFile(filename);
      }
    }

    try {
      Analyzer analyzer = AnalyzerGuru.getAnalyzer();
      IndexWriterConfig iwc = new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
      iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
      // iwc.setRAMBufferSizeMB(256.0);  //TODO check what is the sweet spot
      writer = new IndexWriter(indexDirectory, iwc);
      writer.commit(); // to make sure index exists on the disk
      // writer.setMaxFieldLength(RuntimeEnvironment.getInstance().getIndexWordLimit());

      if (directories.isEmpty()) {
        if (project == null) {
          directories.add("");
        } else {
          directories.add(project.getPath());
        }
      }

      for (String dir : directories) {
        File sourceRoot;
        if ("".equals(dir)) {
          sourceRoot = RuntimeEnvironment.getInstance().getSourceRootFile();
        } else {
          sourceRoot = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), dir);
        }

        HistoryGuru.getInstance().ensureHistoryCacheExists(sourceRoot);

        String startuid = Util.path2uid(dir, "");
        IndexReader reader = DirectoryReader.open(indexDirectory); // open existing index
        Terms terms = null;
        int numDocs = reader.numDocs();
        if (numDocs > 0) {
          Fields uFields = MultiFields.getFields(reader); // reader.getTermVectors(0);
          terms = uFields.terms(QueryBuilder.U);
        }

        try {
          if (numDocs > 0) {
            uidIter = terms.iterator(null);
            TermsEnum.SeekStatus stat = uidIter.seekCeil(new BytesRef(startuid), true); // init uid
            if (stat == TermsEnum.SeekStatus.END || stat == TermsEnum.SeekStatus.NOT_FOUND) {
              uidIter = null;
            }
          }
          // TODO below should be optional, since it traverses the tree once more to get total
          // count! :(
          int file_cnt = 0;
          if (RuntimeEnvironment.getInstance().isPrintProgress()) {
            log.log(Level.INFO, "Counting files in {0} ...", dir);
            file_cnt = indexDown(sourceRoot, dir, true, 0, 0);
            if (log.isLoggable(Level.INFO)) {
              log.log(
                  Level.INFO, "Need to process: {0} files for {1}", new Object[] {file_cnt, dir});
            }
          }

          indexDown(sourceRoot, dir, false, 0, file_cnt);

          while (uidIter != null
              && uidIter.term() != null
              && uidIter.term().utf8ToString().startsWith(startuid)) {
            removeFile();
            uidIter.next();
          }
        } finally {
          reader.close();
        }
      }
    } finally {
      if (writer != null) {
        try {
          writer.prepareCommit();
          writer.commit();
          writer.close();
        } catch (IOException e) {
          log.log(Level.WARNING, "An error occured while closing writer", e);
        }
      }

      if (ctags != null) {
        try {
          ctags.close();
        } catch (IOException e) {
          log.log(Level.WARNING, "An error occured while closing ctags process", e);
        }
      }

      synchronized (lock) {
        running = false;
      }
    }

    if (!isInterrupted() && isDirty()) {
      if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) {
        optimize();
      }
      createSpellingSuggestions();
      RuntimeEnvironment env = RuntimeEnvironment.getInstance();
      File timestamp = new File(env.getDataRootFile(), "timestamp");
      if (timestamp.exists()) {
        if (!timestamp.setLastModified(System.currentTimeMillis())) {
          log.log(
              Level.WARNING,
              "Failed to set last modified time on ''{0}'', used for timestamping the index database.",
              timestamp.getAbsolutePath());
        }
      } else {
        if (!timestamp.createNewFile()) {
          log.log(
              Level.WARNING,
              "Failed to create file ''{0}'', used for timestamping the index database.",
              timestamp.getAbsolutePath());
        }
      }
    }
  }