/** * Add a file to the Lucene index (and generate a xref file) * * @param file The file to add * @param path The path to the file (from source root) * @throws java.io.IOException if an error occurs */ private void addFile(File file, String path) throws IOException { try (InputStream in = new BufferedInputStream(new FileInputStream(file))) { FileAnalyzer fa = AnalyzerGuru.getAnalyzer(in, path); for (IndexChangedListener listener : listeners) { listener.fileAdd(path, fa.getClass().getSimpleName()); } fa.setCtags(ctags); fa.setProject(Project.getProject(path)); Document d; try { d = analyzerGuru.getDocument(file, in, path, fa); } catch (Exception e) { log.log( Level.INFO, "Skipped file ''{0}'' because the analyzer didn''t " + "understand it.", path); StringBuilder stack = new StringBuilder(); for (StackTraceElement ste : e.getStackTrace()) { stack.append(ste.toString()).append(System.lineSeparator()); } StringBuilder sstack = new StringBuilder(); for (Throwable t : e.getSuppressed()) { for (StackTraceElement ste : t.getStackTrace()) { sstack.append(ste.toString()).append(System.lineSeparator()); } } log.log( Level.FINE, "Exception from analyzer {0}: {1} {2}{3}{4}{5}{6}", new String[] { fa.getClass().getName(), e.toString(), System.lineSeparator(), stack.toString(), System.lineSeparator(), sstack.toString() }); return; } writer.addDocument(d, fa); Genre g = fa.getFactory().getGenre(); if (xrefDir != null && (g == Genre.PLAIN || g == Genre.XREFABLE)) { File xrefFile = new File(xrefDir, path); // If mkdirs() returns false, the failure is most likely // because the file already exists. But to check for the // file first and only add it if it doesn't exists would // only increase the file IO... if (!xrefFile.getParentFile().mkdirs()) { assert xrefFile.getParentFile().exists(); } fa.writeXref(xrefDir, path); } setDirty(); for (IndexChangedListener listener : listeners) { listener.fileAdded(path, fa.getClass().getSimpleName()); } } }
/** * Same as {@link #getRequestedProjects()}, but with a variable cookieName and parameter name. * This way it is trivial to implement a project filter ... * * @param paramName the name of the request parameter, which possibly contains the project list in * question. * @param cookieName name of the cookie which possible contains project lists used as fallback * @return a possible empty set but never {@code null}. */ protected SortedSet<String> getRequestedProjects(String paramName, String cookieName) { TreeSet<String> set = new TreeSet<>(); List<Project> projects = getEnv().getProjects(); if (projects == null) { return set; } if (projects.size() == 1 && authFramework.isAllowed(req, projects.get(0))) { set.add(projects.get(0).getDescription()); return set; } List<String> vals = getParamVals(paramName); for (String s : vals) { Project x = Project.getByDescription(s); if (x != null && authFramework.isAllowed(req, x)) { set.add(s); } } if (set.isEmpty()) { List<String> cookies = getCookieVals(cookieName); for (String s : cookies) { Project x = Project.getByDescription(s); if (x != null && authFramework.isAllowed(req, x)) { set.add(s); } } } if (set.isEmpty()) { Project defaultProject = env.getDefaultProject(); if (defaultProject != null && authFramework.isAllowed(req, defaultProject)) { set.add(defaultProject.getDescription()); } } return set; }
static void listFrequentTokens(List<String> subFiles) throws IOException { final int limit = 4; RuntimeEnvironment env = RuntimeEnvironment.getInstance(); if (env.hasProjects()) { if (subFiles == null || subFiles.isEmpty()) { for (Project project : env.getProjects()) { IndexDatabase db = new IndexDatabase(project); db.listTokens(4); } } else { for (String path : subFiles) { Project project = Project.getProject(path); if (project == null) { log.log(Level.WARNING, "Warning: Could not find a project for \"{0}\"", path); } else { IndexDatabase db = new IndexDatabase(project); db.listTokens(4); } } } } else { IndexDatabase db = new IndexDatabase(); db.listTokens(limit); } }
@SuppressWarnings("PMD.CollapsibleIfStatements") private void initialize() throws IOException { synchronized (this) { RuntimeEnvironment env = RuntimeEnvironment.getInstance(); File indexDir = new File(env.getDataRootFile(), INDEX_DIR); if (project != null) { indexDir = new File(indexDir, project.getPath()); } if (!indexDir.exists() && !indexDir.mkdirs()) { // to avoid race conditions, just recheck.. if (!indexDir.exists()) { throw new FileNotFoundException( "Failed to create root directory [" + indexDir.getAbsolutePath() + "]"); } } if (!env.isUsingLuceneLocking()) { lockfact = NoLockFactory.INSTANCE; } indexDirectory = FSDirectory.open(indexDir.toPath(), lockfact); ignoredNames = env.getIgnoredNames(); includedNames = env.getIncludedNames(); analyzerGuru = new AnalyzerGuru(); if (env.isGenerateHtml()) { xrefDir = new File(env.getDataRootFile(), "xref"); } listeners = new ArrayList<>(); dirtyFile = new File(indexDir, "dirty"); dirty = dirtyFile.exists(); directories = new ArrayList<>(); } }
/** * Update the index database for a number of sub-directories * * @param executor An executor to run the job * @param listener where to signal the changes to the database * @param paths * @throws IOException if an error occurs */ public static void update( ExecutorService executor, IndexChangedListener listener, List<String> paths) throws IOException { RuntimeEnvironment env = RuntimeEnvironment.getInstance(); List<IndexDatabase> dbs = new ArrayList<IndexDatabase>(); for (String path : paths) { Project project = Project.getProject(path); if (project == null && env.hasProjects()) { log.log(Level.WARNING, "Could not find a project for \"{0}\"", path); } else { IndexDatabase db; try { if (project == null) { db = new IndexDatabase(); } else { db = new IndexDatabase(project); } int idx = dbs.indexOf(db); if (idx != -1) { db = dbs.get(idx); } if (db.addDirectory(path)) { if (idx == -1) { dbs.add(db); } } else { log.log(Level.WARNING, "Directory does not exist \"{0}\"", path); } } catch (IOException e) { log.log(Level.WARNING, "An error occured while updating index", e); } } for (final IndexDatabase db : dbs) { db.addIndexChangedListener(listener); executor.submit( new Runnable() { @Override public void run() { try { db.update(); } catch (Throwable e) { log.log(Level.SEVERE, "An error occured while updating index", e); } } }); } } }
/** * Populate a Lucene document with the required fields. * * @param doc The document to populate * @param file The file to index * @param path Where the file is located (from source root) * @param fa The analyzer to use on the file * @param xrefOut Where to write the xref (possibly {@code null}) * @throws IOException If an exception occurs while collecting the data */ public void populateDocument( Document doc, File file, String path, FileAnalyzer fa, Writer xrefOut) throws IOException { String date = DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND); doc.add(new Field(QueryBuilder.U, Util.path2uid(path, date), string_ft_stored_nanalyzed_norms)); doc.add( new Field( QueryBuilder.FULLPATH, file.getAbsolutePath(), string_ft_nstored_nanalyzed_norms)); doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH, new BytesRef(file.getAbsolutePath()))); try { HistoryReader hr = HistoryGuru.getInstance().getHistoryReader(file); if (hr != null) { doc.add(new TextField(QueryBuilder.HIST, hr)); // date = hr.getLastCommentDate() //RFE } } catch (HistoryException e) { LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e); } doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms)); doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date))); if (path != null) { doc.add(new TextField(QueryBuilder.PATH, path, Store.YES)); Project project = Project.getProject(path); if (project != null) { doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES)); } } if (fa != null) { Genre g = fa.getGenre(); if (g == Genre.PLAIN || g == Genre.XREFABLE || g == Genre.HTML) { doc.add(new Field(QueryBuilder.T, g.typeName(), string_ft_stored_nanalyzed_norms)); } fa.analyze(doc, StreamSource.fromFile(file), xrefOut); String type = fa.getFileTypeName(); doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES)); } }
/** * Check if a file is local to the current project. If we don't have projects, check if the file * is in the source root. * * @param path the path to a file * @return true if the file is local to the current repository */ private boolean isLocal(String path) { RuntimeEnvironment env = RuntimeEnvironment.getInstance(); String srcRoot = env.getSourceRootPath(); boolean local = false; if (path.startsWith(srcRoot)) { if (env.hasProjects()) { String relPath = path.substring(srcRoot.length()); if (project.equals(Project.getProject(relPath))) { // File is under the current project, so it's local. local = true; } } else { // File is under source root, and we don't have projects, so // consider it local. local = true; } } return local; }
/** * Get an indexReader for the Index database where a given file * * @param path the file to get the database for * @return The index database where the file should be located or null if it cannot be located. */ public static IndexReader getIndexReader(String path) { IndexReader ret = null; RuntimeEnvironment env = RuntimeEnvironment.getInstance(); File indexDir = new File(env.getDataRootFile(), "index"); if (env.hasProjects()) { Project p = Project.getProject(path); if (p == null) { return null; } indexDir = new File(indexDir, p.getPath()); } try { FSDirectory fdir = FSDirectory.open(indexDir, NoLockFactory.getNoLockFactory()); if (indexDir.exists() && DirectoryReader.indexExists(fdir)) { ret = DirectoryReader.open(fdir); } } catch (Exception ex) { log.log(Level.SEVERE, "Failed to open index: {0}", indexDir.getAbsolutePath()); log.log(Level.FINE, "Stack Trace: ", ex); } return ret; }
/** * Add a file to the Lucene index (and generate a xref file) * * @param file The file to add * @param path The path to the file (from source root) * @throws java.io.IOException if an error occurs */ private void addFile(File file, String path) throws IOException { FileAnalyzer fa; try (InputStream in = new BufferedInputStream(new FileInputStream(file))) { fa = AnalyzerGuru.getAnalyzer(in, path); } for (IndexChangedListener listener : listeners) { listener.fileAdd(path, fa.getClass().getSimpleName()); } fa.setCtags(ctags); fa.setProject(Project.getProject(path)); fa.setScopesEnabled(RuntimeEnvironment.getInstance().isScopesEnabled()); fa.setFoldingEnabled(RuntimeEnvironment.getInstance().isFoldingEnabled()); Document doc = new Document(); try (Writer xrefOut = getXrefWriter(fa, path)) { analyzerGuru.populateDocument(doc, file, path, fa, xrefOut); } catch (Exception e) { LOGGER.log( Level.INFO, "Skipped file ''{0}'' because the analyzer didn''t " + "understand it.", path); LOGGER.log(Level.FINE, "Exception from analyzer " + fa.getClass().getName(), e); cleanupResources(doc); return; } try { writer.addDocument(doc); } catch (Throwable t) { cleanupResources(doc); throw t; } setDirty(); for (IndexChangedListener listener : listeners) { listener.fileAdded(path, fa.getClass().getSimpleName()); } }
/** * List all files in some of the index databases * * @param subFiles Subdirectories for the various projects to list the files for (or null or an * empty list to dump all projects) * @throws IOException if an error occurs */ public static void listAllFiles(List<String> subFiles) throws IOException { RuntimeEnvironment env = RuntimeEnvironment.getInstance(); if (env.hasProjects()) { if (subFiles == null || subFiles.isEmpty()) { for (Project project : env.getProjects()) { IndexDatabase db = new IndexDatabase(project); db.listFiles(); } } else { for (String path : subFiles) { Project project = Project.getProject(path); if (project == null) { LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path); } else { IndexDatabase db = new IndexDatabase(project); db.listFiles(); } } } } else { IndexDatabase db = new IndexDatabase(); db.listFiles(); } }
/** * Get the project {@link #getPath()} refers to. * * @return {@code null} if not available, the project otherwise. */ public Project getProject() { return Project.getProject(getResourceFile()); }
/** * Update the content of this index database * * @throws IOException if an error occurs * @throws HistoryException if an error occurs when accessing the history */ public void update() throws IOException, HistoryException { synchronized (lock) { if (running) { throw new IOException("Indexer already running!"); } running = true; interrupted = false; } String ctgs = RuntimeEnvironment.getInstance().getCtags(); if (ctgs != null) { ctags = new Ctags(); ctags.setBinary(ctgs); } if (ctags == null) { log.severe("Unable to run ctags! searching definitions will not work!"); } if (ctags != null) { String filename = RuntimeEnvironment.getInstance().getCTagsExtraOptionsFile(); if (filename != null) { ctags.setCTagsExtraOptionsFile(filename); } } try { Analyzer analyzer = AnalyzerGuru.getAnalyzer(); IndexWriterConfig iwc = new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer); iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); // iwc.setRAMBufferSizeMB(256.0); //TODO check what is the sweet spot writer = new IndexWriter(indexDirectory, iwc); writer.commit(); // to make sure index exists on the disk // writer.setMaxFieldLength(RuntimeEnvironment.getInstance().getIndexWordLimit()); if (directories.isEmpty()) { if (project == null) { directories.add(""); } else { directories.add(project.getPath()); } } for (String dir : directories) { File sourceRoot; if ("".equals(dir)) { sourceRoot = RuntimeEnvironment.getInstance().getSourceRootFile(); } else { sourceRoot = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), dir); } HistoryGuru.getInstance().ensureHistoryCacheExists(sourceRoot); String startuid = Util.path2uid(dir, ""); IndexReader reader = DirectoryReader.open(indexDirectory); // open existing index Terms terms = null; int numDocs = reader.numDocs(); if (numDocs > 0) { Fields uFields = MultiFields.getFields(reader); // reader.getTermVectors(0); terms = uFields.terms(QueryBuilder.U); } try { if (numDocs > 0) { uidIter = terms.iterator(null); TermsEnum.SeekStatus stat = uidIter.seekCeil(new BytesRef(startuid), true); // init uid if (stat == TermsEnum.SeekStatus.END || stat == TermsEnum.SeekStatus.NOT_FOUND) { uidIter = null; } } // TODO below should be optional, since it traverses the tree once more to get total // count! :( int file_cnt = 0; if (RuntimeEnvironment.getInstance().isPrintProgress()) { log.log(Level.INFO, "Counting files in {0} ...", dir); file_cnt = indexDown(sourceRoot, dir, true, 0, 0); if (log.isLoggable(Level.INFO)) { log.log( Level.INFO, "Need to process: {0} files for {1}", new Object[] {file_cnt, dir}); } } indexDown(sourceRoot, dir, false, 0, file_cnt); while (uidIter != null && uidIter.term() != null && uidIter.term().utf8ToString().startsWith(startuid)) { removeFile(); uidIter.next(); } } finally { reader.close(); } } } finally { if (writer != null) { try { writer.prepareCommit(); writer.commit(); writer.close(); } catch (IOException e) { log.log(Level.WARNING, "An error occured while closing writer", e); } } if (ctags != null) { try { ctags.close(); } catch (IOException e) { log.log(Level.WARNING, "An error occured while closing ctags process", e); } } synchronized (lock) { running = false; } } if (!isInterrupted() && isDirty()) { if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) { optimize(); } createSpellingSuggestions(); RuntimeEnvironment env = RuntimeEnvironment.getInstance(); File timestamp = new File(env.getDataRootFile(), "timestamp"); if (timestamp.exists()) { if (!timestamp.setLastModified(System.currentTimeMillis())) { log.log( Level.WARNING, "Failed to set last modified time on ''{0}'', used for timestamping the index database.", timestamp.getAbsolutePath()); } } else { if (!timestamp.createNewFile()) { log.log( Level.WARNING, "Failed to create file ''{0}'', used for timestamping the index database.", timestamp.getAbsolutePath()); } } } }
protected String getProjectPostfix(boolean encoded) { String amp = encoded ? "&" : "&"; return project == null ? "" : (amp + "project=" + project.getDescription()); }
protected void appendProject() throws IOException { if (project != null) { out.write("&project="); out.write(project.getDescription()); } }