Esempio n. 1
1
  /**
   * Add a file to the Lucene index (and generate a xref file)
   *
   * @param file The file to add
   * @param path The path to the file (from source root)
   * @throws java.io.IOException if an error occurs
   */
  private void addFile(File file, String path) throws IOException {
    try (InputStream in = new BufferedInputStream(new FileInputStream(file))) {
      FileAnalyzer fa = AnalyzerGuru.getAnalyzer(in, path);
      for (IndexChangedListener listener : listeners) {
        listener.fileAdd(path, fa.getClass().getSimpleName());
      }
      fa.setCtags(ctags);
      fa.setProject(Project.getProject(path));

      Document d;
      try {
        d = analyzerGuru.getDocument(file, in, path, fa);
      } catch (Exception e) {
        log.log(
            Level.INFO,
            "Skipped file ''{0}'' because the analyzer didn''t " + "understand it.",
            path);
        StringBuilder stack = new StringBuilder();
        for (StackTraceElement ste : e.getStackTrace()) {
          stack.append(ste.toString()).append(System.lineSeparator());
        }
        StringBuilder sstack = new StringBuilder();
        for (Throwable t : e.getSuppressed()) {
          for (StackTraceElement ste : t.getStackTrace()) {
            sstack.append(ste.toString()).append(System.lineSeparator());
          }
        }
        log.log(
            Level.FINE,
            "Exception from analyzer {0}: {1} {2}{3}{4}{5}{6}",
            new String[] {
              fa.getClass().getName(),
              e.toString(),
              System.lineSeparator(),
              stack.toString(),
              System.lineSeparator(),
              sstack.toString()
            });
        return;
      }

      writer.addDocument(d, fa);
      Genre g = fa.getFactory().getGenre();
      if (xrefDir != null && (g == Genre.PLAIN || g == Genre.XREFABLE)) {
        File xrefFile = new File(xrefDir, path);
        // If mkdirs() returns false, the failure is most likely
        // because the file already exists. But to check for the
        // file first and only add it if it doesn't exists would
        // only increase the file IO...
        if (!xrefFile.getParentFile().mkdirs()) {
          assert xrefFile.getParentFile().exists();
        }
        fa.writeXref(xrefDir, path);
      }
      setDirty();
      for (IndexChangedListener listener : listeners) {
        listener.fileAdded(path, fa.getClass().getSimpleName());
      }
    }
  }
Esempio n. 2
0
 /**
  * Same as {@link #getRequestedProjects()}, but with a variable cookieName and parameter name.
  * This way it is trivial to implement a project filter ...
  *
  * @param paramName the name of the request parameter, which possibly contains the project list in
  *     question.
  * @param cookieName name of the cookie which possible contains project lists used as fallback
  * @return a possible empty set but never {@code null}.
  */
 protected SortedSet<String> getRequestedProjects(String paramName, String cookieName) {
   TreeSet<String> set = new TreeSet<>();
   List<Project> projects = getEnv().getProjects();
   if (projects == null) {
     return set;
   }
   if (projects.size() == 1 && authFramework.isAllowed(req, projects.get(0))) {
     set.add(projects.get(0).getDescription());
     return set;
   }
   List<String> vals = getParamVals(paramName);
   for (String s : vals) {
     Project x = Project.getByDescription(s);
     if (x != null && authFramework.isAllowed(req, x)) {
       set.add(s);
     }
   }
   if (set.isEmpty()) {
     List<String> cookies = getCookieVals(cookieName);
     for (String s : cookies) {
       Project x = Project.getByDescription(s);
       if (x != null && authFramework.isAllowed(req, x)) {
         set.add(s);
       }
     }
   }
   if (set.isEmpty()) {
     Project defaultProject = env.getDefaultProject();
     if (defaultProject != null && authFramework.isAllowed(req, defaultProject)) {
       set.add(defaultProject.getDescription());
     }
   }
   return set;
 }
Esempio n. 3
0
  static void listFrequentTokens(List<String> subFiles) throws IOException {
    final int limit = 4;

    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    if (env.hasProjects()) {
      if (subFiles == null || subFiles.isEmpty()) {
        for (Project project : env.getProjects()) {
          IndexDatabase db = new IndexDatabase(project);
          db.listTokens(4);
        }
      } else {
        for (String path : subFiles) {
          Project project = Project.getProject(path);
          if (project == null) {
            log.log(Level.WARNING, "Warning: Could not find a project for \"{0}\"", path);
          } else {
            IndexDatabase db = new IndexDatabase(project);
            db.listTokens(4);
          }
        }
      }
    } else {
      IndexDatabase db = new IndexDatabase();
      db.listTokens(limit);
    }
  }
Esempio n. 4
0
  @SuppressWarnings("PMD.CollapsibleIfStatements")
  private void initialize() throws IOException {
    synchronized (this) {
      RuntimeEnvironment env = RuntimeEnvironment.getInstance();
      File indexDir = new File(env.getDataRootFile(), INDEX_DIR);
      if (project != null) {
        indexDir = new File(indexDir, project.getPath());
      }

      if (!indexDir.exists() && !indexDir.mkdirs()) {
        // to avoid race conditions, just recheck..
        if (!indexDir.exists()) {
          throw new FileNotFoundException(
              "Failed to create root directory [" + indexDir.getAbsolutePath() + "]");
        }
      }

      if (!env.isUsingLuceneLocking()) {
        lockfact = NoLockFactory.INSTANCE;
      }
      indexDirectory = FSDirectory.open(indexDir.toPath(), lockfact);
      ignoredNames = env.getIgnoredNames();
      includedNames = env.getIncludedNames();
      analyzerGuru = new AnalyzerGuru();
      if (env.isGenerateHtml()) {
        xrefDir = new File(env.getDataRootFile(), "xref");
      }
      listeners = new ArrayList<>();
      dirtyFile = new File(indexDir, "dirty");
      dirty = dirtyFile.exists();
      directories = new ArrayList<>();
    }
  }
Esempio n. 5
0
  /**
   * Update the index database for a number of sub-directories
   *
   * @param executor An executor to run the job
   * @param listener where to signal the changes to the database
   * @param paths
   * @throws IOException if an error occurs
   */
  public static void update(
      ExecutorService executor, IndexChangedListener listener, List<String> paths)
      throws IOException {
    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();

    for (String path : paths) {
      Project project = Project.getProject(path);
      if (project == null && env.hasProjects()) {
        log.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
      } else {
        IndexDatabase db;

        try {
          if (project == null) {
            db = new IndexDatabase();
          } else {
            db = new IndexDatabase(project);
          }

          int idx = dbs.indexOf(db);
          if (idx != -1) {
            db = dbs.get(idx);
          }

          if (db.addDirectory(path)) {
            if (idx == -1) {
              dbs.add(db);
            }
          } else {
            log.log(Level.WARNING, "Directory does not exist \"{0}\"", path);
          }
        } catch (IOException e) {
          log.log(Level.WARNING, "An error occured while updating index", e);
        }
      }

      for (final IndexDatabase db : dbs) {
        db.addIndexChangedListener(listener);
        executor.submit(
            new Runnable() {
              @Override
              public void run() {
                try {
                  db.update();
                } catch (Throwable e) {
                  log.log(Level.SEVERE, "An error occured while updating index", e);
                }
              }
            });
      }
    }
  }
Esempio n. 6
0
  /**
   * Populate a Lucene document with the required fields.
   *
   * @param doc The document to populate
   * @param file The file to index
   * @param path Where the file is located (from source root)
   * @param fa The analyzer to use on the file
   * @param xrefOut Where to write the xref (possibly {@code null})
   * @throws IOException If an exception occurs while collecting the data
   */
  public void populateDocument(
      Document doc, File file, String path, FileAnalyzer fa, Writer xrefOut) throws IOException {
    String date = DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND);
    doc.add(new Field(QueryBuilder.U, Util.path2uid(path, date), string_ft_stored_nanalyzed_norms));
    doc.add(
        new Field(
            QueryBuilder.FULLPATH, file.getAbsolutePath(), string_ft_nstored_nanalyzed_norms));
    doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH, new BytesRef(file.getAbsolutePath())));

    try {
      HistoryReader hr = HistoryGuru.getInstance().getHistoryReader(file);
      if (hr != null) {
        doc.add(new TextField(QueryBuilder.HIST, hr));
        // date = hr.getLastCommentDate() //RFE
      }
    } catch (HistoryException e) {
      LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e);
    }
    doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms));
    doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date)));
    if (path != null) {
      doc.add(new TextField(QueryBuilder.PATH, path, Store.YES));
      Project project = Project.getProject(path);
      if (project != null) {
        doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES));
      }
    }

    if (fa != null) {
      Genre g = fa.getGenre();
      if (g == Genre.PLAIN || g == Genre.XREFABLE || g == Genre.HTML) {
        doc.add(new Field(QueryBuilder.T, g.typeName(), string_ft_stored_nanalyzed_norms));
      }
      fa.analyze(doc, StreamSource.fromFile(file), xrefOut);

      String type = fa.getFileTypeName();
      doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES));
    }
  }
Esempio n. 7
0
  /**
   * Check if a file is local to the current project. If we don't have projects, check if the file
   * is in the source root.
   *
   * @param path the path to a file
   * @return true if the file is local to the current repository
   */
  private boolean isLocal(String path) {
    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    String srcRoot = env.getSourceRootPath();

    boolean local = false;

    if (path.startsWith(srcRoot)) {
      if (env.hasProjects()) {
        String relPath = path.substring(srcRoot.length());
        if (project.equals(Project.getProject(relPath))) {
          // File is under the current project, so it's local.
          local = true;
        }
      } else {
        // File is under source root, and we don't have projects, so
        // consider it local.
        local = true;
      }
    }

    return local;
  }
Esempio n. 8
0
  /**
   * Get an indexReader for the Index database where a given file
   *
   * @param path the file to get the database for
   * @return The index database where the file should be located or null if it cannot be located.
   */
  public static IndexReader getIndexReader(String path) {
    IndexReader ret = null;

    RuntimeEnvironment env = RuntimeEnvironment.getInstance();
    File indexDir = new File(env.getDataRootFile(), "index");

    if (env.hasProjects()) {
      Project p = Project.getProject(path);
      if (p == null) {
        return null;
      }
      indexDir = new File(indexDir, p.getPath());
    }
    try {
      FSDirectory fdir = FSDirectory.open(indexDir, NoLockFactory.getNoLockFactory());
      if (indexDir.exists() && DirectoryReader.indexExists(fdir)) {
        ret = DirectoryReader.open(fdir);
      }
    } catch (Exception ex) {
      log.log(Level.SEVERE, "Failed to open index: {0}", indexDir.getAbsolutePath());
      log.log(Level.FINE, "Stack Trace: ", ex);
    }
    return ret;
  }
Esempio n. 9
0
  /**
   * Add a file to the Lucene index (and generate a xref file)
   *
   * @param file The file to add
   * @param path The path to the file (from source root)
   * @throws java.io.IOException if an error occurs
   */
  private void addFile(File file, String path) throws IOException {
    FileAnalyzer fa;
    try (InputStream in = new BufferedInputStream(new FileInputStream(file))) {
      fa = AnalyzerGuru.getAnalyzer(in, path);
    }

    for (IndexChangedListener listener : listeners) {
      listener.fileAdd(path, fa.getClass().getSimpleName());
    }
    fa.setCtags(ctags);
    fa.setProject(Project.getProject(path));
    fa.setScopesEnabled(RuntimeEnvironment.getInstance().isScopesEnabled());
    fa.setFoldingEnabled(RuntimeEnvironment.getInstance().isFoldingEnabled());

    Document doc = new Document();
    try (Writer xrefOut = getXrefWriter(fa, path)) {
      analyzerGuru.populateDocument(doc, file, path, fa, xrefOut);
    } catch (Exception e) {
      LOGGER.log(
          Level.INFO,
          "Skipped file ''{0}'' because the analyzer didn''t " + "understand it.",
          path);
      LOGGER.log(Level.FINE, "Exception from analyzer " + fa.getClass().getName(), e);
      cleanupResources(doc);
      return;
    }

    try {
      writer.addDocument(doc);
    } catch (Throwable t) {
      cleanupResources(doc);
      throw t;
    }

    setDirty();
    for (IndexChangedListener listener : listeners) {
      listener.fileAdded(path, fa.getClass().getSimpleName());
    }
  }
Esempio n. 10
0
 /**
  * List all files in some of the index databases
  *
  * @param subFiles Subdirectories for the various projects to list the files for (or null or an
  *     empty list to dump all projects)
  * @throws IOException if an error occurs
  */
 public static void listAllFiles(List<String> subFiles) throws IOException {
   RuntimeEnvironment env = RuntimeEnvironment.getInstance();
   if (env.hasProjects()) {
     if (subFiles == null || subFiles.isEmpty()) {
       for (Project project : env.getProjects()) {
         IndexDatabase db = new IndexDatabase(project);
         db.listFiles();
       }
     } else {
       for (String path : subFiles) {
         Project project = Project.getProject(path);
         if (project == null) {
           LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
         } else {
           IndexDatabase db = new IndexDatabase(project);
           db.listFiles();
         }
       }
     }
   } else {
     IndexDatabase db = new IndexDatabase();
     db.listFiles();
   }
 }
Esempio n. 11
0
 /**
  * Get the project {@link #getPath()} refers to.
  *
  * @return {@code null} if not available, the project otherwise.
  */
 public Project getProject() {
   return Project.getProject(getResourceFile());
 }
Esempio n. 12
0
  /**
   * Update the content of this index database
   *
   * @throws IOException if an error occurs
   * @throws HistoryException if an error occurs when accessing the history
   */
  public void update() throws IOException, HistoryException {
    synchronized (lock) {
      if (running) {
        throw new IOException("Indexer already running!");
      }
      running = true;
      interrupted = false;
    }

    String ctgs = RuntimeEnvironment.getInstance().getCtags();
    if (ctgs != null) {
      ctags = new Ctags();
      ctags.setBinary(ctgs);
    }
    if (ctags == null) {
      log.severe("Unable to run ctags! searching definitions will not work!");
    }

    if (ctags != null) {
      String filename = RuntimeEnvironment.getInstance().getCTagsExtraOptionsFile();
      if (filename != null) {
        ctags.setCTagsExtraOptionsFile(filename);
      }
    }

    try {
      Analyzer analyzer = AnalyzerGuru.getAnalyzer();
      IndexWriterConfig iwc = new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
      iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
      // iwc.setRAMBufferSizeMB(256.0);  //TODO check what is the sweet spot
      writer = new IndexWriter(indexDirectory, iwc);
      writer.commit(); // to make sure index exists on the disk
      // writer.setMaxFieldLength(RuntimeEnvironment.getInstance().getIndexWordLimit());

      if (directories.isEmpty()) {
        if (project == null) {
          directories.add("");
        } else {
          directories.add(project.getPath());
        }
      }

      for (String dir : directories) {
        File sourceRoot;
        if ("".equals(dir)) {
          sourceRoot = RuntimeEnvironment.getInstance().getSourceRootFile();
        } else {
          sourceRoot = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), dir);
        }

        HistoryGuru.getInstance().ensureHistoryCacheExists(sourceRoot);

        String startuid = Util.path2uid(dir, "");
        IndexReader reader = DirectoryReader.open(indexDirectory); // open existing index
        Terms terms = null;
        int numDocs = reader.numDocs();
        if (numDocs > 0) {
          Fields uFields = MultiFields.getFields(reader); // reader.getTermVectors(0);
          terms = uFields.terms(QueryBuilder.U);
        }

        try {
          if (numDocs > 0) {
            uidIter = terms.iterator(null);
            TermsEnum.SeekStatus stat = uidIter.seekCeil(new BytesRef(startuid), true); // init uid
            if (stat == TermsEnum.SeekStatus.END || stat == TermsEnum.SeekStatus.NOT_FOUND) {
              uidIter = null;
            }
          }
          // TODO below should be optional, since it traverses the tree once more to get total
          // count! :(
          int file_cnt = 0;
          if (RuntimeEnvironment.getInstance().isPrintProgress()) {
            log.log(Level.INFO, "Counting files in {0} ...", dir);
            file_cnt = indexDown(sourceRoot, dir, true, 0, 0);
            if (log.isLoggable(Level.INFO)) {
              log.log(
                  Level.INFO, "Need to process: {0} files for {1}", new Object[] {file_cnt, dir});
            }
          }

          indexDown(sourceRoot, dir, false, 0, file_cnt);

          while (uidIter != null
              && uidIter.term() != null
              && uidIter.term().utf8ToString().startsWith(startuid)) {
            removeFile();
            uidIter.next();
          }
        } finally {
          reader.close();
        }
      }
    } finally {
      if (writer != null) {
        try {
          writer.prepareCommit();
          writer.commit();
          writer.close();
        } catch (IOException e) {
          log.log(Level.WARNING, "An error occured while closing writer", e);
        }
      }

      if (ctags != null) {
        try {
          ctags.close();
        } catch (IOException e) {
          log.log(Level.WARNING, "An error occured while closing ctags process", e);
        }
      }

      synchronized (lock) {
        running = false;
      }
    }

    if (!isInterrupted() && isDirty()) {
      if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) {
        optimize();
      }
      createSpellingSuggestions();
      RuntimeEnvironment env = RuntimeEnvironment.getInstance();
      File timestamp = new File(env.getDataRootFile(), "timestamp");
      if (timestamp.exists()) {
        if (!timestamp.setLastModified(System.currentTimeMillis())) {
          log.log(
              Level.WARNING,
              "Failed to set last modified time on ''{0}'', used for timestamping the index database.",
              timestamp.getAbsolutePath());
        }
      } else {
        if (!timestamp.createNewFile()) {
          log.log(
              Level.WARNING,
              "Failed to create file ''{0}'', used for timestamping the index database.",
              timestamp.getAbsolutePath());
        }
      }
    }
  }
Esempio n. 13
0
 protected String getProjectPostfix(boolean encoded) {
   String amp = encoded ? "&amp;" : "&";
   return project == null ? "" : (amp + "project=" + project.getDescription());
 }
Esempio n. 14
0
 protected void appendProject() throws IOException {
   if (project != null) {
     out.write("&amp;project=");
     out.write(project.getDescription());
   }
 }