/** * Add a file to the Lucene index (and generate a xref file) * * @param file The file to add * @param path The path to the file (from source root) * @throws java.io.IOException if an error occurs */ private void addFile(File file, String path) throws IOException { try (InputStream in = new BufferedInputStream(new FileInputStream(file))) { FileAnalyzer fa = AnalyzerGuru.getAnalyzer(in, path); for (IndexChangedListener listener : listeners) { listener.fileAdd(path, fa.getClass().getSimpleName()); } fa.setCtags(ctags); fa.setProject(Project.getProject(path)); Document d; try { d = analyzerGuru.getDocument(file, in, path, fa); } catch (Exception e) { log.log( Level.INFO, "Skipped file ''{0}'' because the analyzer didn''t " + "understand it.", path); StringBuilder stack = new StringBuilder(); for (StackTraceElement ste : e.getStackTrace()) { stack.append(ste.toString()).append(System.lineSeparator()); } StringBuilder sstack = new StringBuilder(); for (Throwable t : e.getSuppressed()) { for (StackTraceElement ste : t.getStackTrace()) { sstack.append(ste.toString()).append(System.lineSeparator()); } } log.log( Level.FINE, "Exception from analyzer {0}: {1} {2}{3}{4}{5}{6}", new String[] { fa.getClass().getName(), e.toString(), System.lineSeparator(), stack.toString(), System.lineSeparator(), sstack.toString() }); return; } writer.addDocument(d, fa); Genre g = fa.getFactory().getGenre(); if (xrefDir != null && (g == Genre.PLAIN || g == Genre.XREFABLE)) { File xrefFile = new File(xrefDir, path); // If mkdirs() returns false, the failure is most likely // because the file already exists. But to check for the // file first and only add it if it doesn't exists would // only increase the file IO... if (!xrefFile.getParentFile().mkdirs()) { assert xrefFile.getParentFile().exists(); } fa.writeXref(xrefDir, path); } setDirty(); for (IndexChangedListener listener : listeners) { listener.fileAdded(path, fa.getClass().getSimpleName()); } } }
static void listFrequentTokens(List<String> subFiles) throws IOException { final int limit = 4; RuntimeEnvironment env = RuntimeEnvironment.getInstance(); if (env.hasProjects()) { if (subFiles == null || subFiles.isEmpty()) { for (Project project : env.getProjects()) { IndexDatabase db = new IndexDatabase(project); db.listTokens(4); } } else { for (String path : subFiles) { Project project = Project.getProject(path); if (project == null) { log.log(Level.WARNING, "Warning: Could not find a project for \"{0}\"", path); } else { IndexDatabase db = new IndexDatabase(project); db.listTokens(4); } } } } else { IndexDatabase db = new IndexDatabase(); db.listTokens(limit); } }
/** * Update the index database for a number of sub-directories * * @param executor An executor to run the job * @param listener where to signal the changes to the database * @param paths * @throws IOException if an error occurs */ public static void update( ExecutorService executor, IndexChangedListener listener, List<String> paths) throws IOException { RuntimeEnvironment env = RuntimeEnvironment.getInstance(); List<IndexDatabase> dbs = new ArrayList<IndexDatabase>(); for (String path : paths) { Project project = Project.getProject(path); if (project == null && env.hasProjects()) { log.log(Level.WARNING, "Could not find a project for \"{0}\"", path); } else { IndexDatabase db; try { if (project == null) { db = new IndexDatabase(); } else { db = new IndexDatabase(project); } int idx = dbs.indexOf(db); if (idx != -1) { db = dbs.get(idx); } if (db.addDirectory(path)) { if (idx == -1) { dbs.add(db); } } else { log.log(Level.WARNING, "Directory does not exist \"{0}\"", path); } } catch (IOException e) { log.log(Level.WARNING, "An error occured while updating index", e); } } for (final IndexDatabase db : dbs) { db.addIndexChangedListener(listener); executor.submit( new Runnable() { @Override public void run() { try { db.update(); } catch (Throwable e) { log.log(Level.SEVERE, "An error occured while updating index", e); } } }); } } }
/** * Populate a Lucene document with the required fields. * * @param doc The document to populate * @param file The file to index * @param path Where the file is located (from source root) * @param fa The analyzer to use on the file * @param xrefOut Where to write the xref (possibly {@code null}) * @throws IOException If an exception occurs while collecting the data */ public void populateDocument( Document doc, File file, String path, FileAnalyzer fa, Writer xrefOut) throws IOException { String date = DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND); doc.add(new Field(QueryBuilder.U, Util.path2uid(path, date), string_ft_stored_nanalyzed_norms)); doc.add( new Field( QueryBuilder.FULLPATH, file.getAbsolutePath(), string_ft_nstored_nanalyzed_norms)); doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH, new BytesRef(file.getAbsolutePath()))); try { HistoryReader hr = HistoryGuru.getInstance().getHistoryReader(file); if (hr != null) { doc.add(new TextField(QueryBuilder.HIST, hr)); // date = hr.getLastCommentDate() //RFE } } catch (HistoryException e) { LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e); } doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms)); doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date))); if (path != null) { doc.add(new TextField(QueryBuilder.PATH, path, Store.YES)); Project project = Project.getProject(path); if (project != null) { doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES)); } } if (fa != null) { Genre g = fa.getGenre(); if (g == Genre.PLAIN || g == Genre.XREFABLE || g == Genre.HTML) { doc.add(new Field(QueryBuilder.T, g.typeName(), string_ft_stored_nanalyzed_norms)); } fa.analyze(doc, StreamSource.fromFile(file), xrefOut); String type = fa.getFileTypeName(); doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES)); } }
/** * Check if a file is local to the current project. If we don't have projects, check if the file * is in the source root. * * @param path the path to a file * @return true if the file is local to the current repository */ private boolean isLocal(String path) { RuntimeEnvironment env = RuntimeEnvironment.getInstance(); String srcRoot = env.getSourceRootPath(); boolean local = false; if (path.startsWith(srcRoot)) { if (env.hasProjects()) { String relPath = path.substring(srcRoot.length()); if (project.equals(Project.getProject(relPath))) { // File is under the current project, so it's local. local = true; } } else { // File is under source root, and we don't have projects, so // consider it local. local = true; } } return local; }
/** * Add a file to the Lucene index (and generate a xref file) * * @param file The file to add * @param path The path to the file (from source root) * @throws java.io.IOException if an error occurs */ private void addFile(File file, String path) throws IOException { FileAnalyzer fa; try (InputStream in = new BufferedInputStream(new FileInputStream(file))) { fa = AnalyzerGuru.getAnalyzer(in, path); } for (IndexChangedListener listener : listeners) { listener.fileAdd(path, fa.getClass().getSimpleName()); } fa.setCtags(ctags); fa.setProject(Project.getProject(path)); fa.setScopesEnabled(RuntimeEnvironment.getInstance().isScopesEnabled()); fa.setFoldingEnabled(RuntimeEnvironment.getInstance().isFoldingEnabled()); Document doc = new Document(); try (Writer xrefOut = getXrefWriter(fa, path)) { analyzerGuru.populateDocument(doc, file, path, fa, xrefOut); } catch (Exception e) { LOGGER.log( Level.INFO, "Skipped file ''{0}'' because the analyzer didn''t " + "understand it.", path); LOGGER.log(Level.FINE, "Exception from analyzer " + fa.getClass().getName(), e); cleanupResources(doc); return; } try { writer.addDocument(doc); } catch (Throwable t) { cleanupResources(doc); throw t; } setDirty(); for (IndexChangedListener listener : listeners) { listener.fileAdded(path, fa.getClass().getSimpleName()); } }
/** * Get an indexReader for the Index database where a given file * * @param path the file to get the database for * @return The index database where the file should be located or null if it cannot be located. */ public static IndexReader getIndexReader(String path) { IndexReader ret = null; RuntimeEnvironment env = RuntimeEnvironment.getInstance(); File indexDir = new File(env.getDataRootFile(), "index"); if (env.hasProjects()) { Project p = Project.getProject(path); if (p == null) { return null; } indexDir = new File(indexDir, p.getPath()); } try { FSDirectory fdir = FSDirectory.open(indexDir, NoLockFactory.getNoLockFactory()); if (indexDir.exists() && DirectoryReader.indexExists(fdir)) { ret = DirectoryReader.open(fdir); } } catch (Exception ex) { log.log(Level.SEVERE, "Failed to open index: {0}", indexDir.getAbsolutePath()); log.log(Level.FINE, "Stack Trace: ", ex); } return ret; }
/** * List all files in some of the index databases * * @param subFiles Subdirectories for the various projects to list the files for (or null or an * empty list to dump all projects) * @throws IOException if an error occurs */ public static void listAllFiles(List<String> subFiles) throws IOException { RuntimeEnvironment env = RuntimeEnvironment.getInstance(); if (env.hasProjects()) { if (subFiles == null || subFiles.isEmpty()) { for (Project project : env.getProjects()) { IndexDatabase db = new IndexDatabase(project); db.listFiles(); } } else { for (String path : subFiles) { Project project = Project.getProject(path); if (project == null) { LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path); } else { IndexDatabase db = new IndexDatabase(project); db.listFiles(); } } } } else { IndexDatabase db = new IndexDatabase(); db.listFiles(); } }
/** * Get the project {@link #getPath()} refers to. * * @return {@code null} if not available, the project otherwise. */ public Project getProject() { return Project.getProject(getResourceFile()); }