/**
* Add a file to the Lucene index (and generate a xref file)
*
* @param file The file to add
* @param path The path to the file (from source root)
* @throws java.io.IOException if an error occurs
*/
private void addFile(File file, String path) throws IOException {
try (InputStream in = new BufferedInputStream(new FileInputStream(file))) {
FileAnalyzer fa = AnalyzerGuru.getAnalyzer(in, path);
for (IndexChangedListener listener : listeners) {
listener.fileAdd(path, fa.getClass().getSimpleName());
}
fa.setCtags(ctags);
fa.setProject(Project.getProject(path));
Document d;
try {
d = analyzerGuru.getDocument(file, in, path, fa);
} catch (Exception e) {
log.log(
Level.INFO,
"Skipped file ''{0}'' because the analyzer didn''t " + "understand it.",
path);
StringBuilder stack = new StringBuilder();
for (StackTraceElement ste : e.getStackTrace()) {
stack.append(ste.toString()).append(System.lineSeparator());
}
StringBuilder sstack = new StringBuilder();
for (Throwable t : e.getSuppressed()) {
for (StackTraceElement ste : t.getStackTrace()) {
sstack.append(ste.toString()).append(System.lineSeparator());
}
}
log.log(
Level.FINE,
"Exception from analyzer {0}: {1} {2}{3}{4}{5}{6}",
new String[] {
fa.getClass().getName(),
e.toString(),
System.lineSeparator(),
stack.toString(),
System.lineSeparator(),
sstack.toString()
});
return;
}
writer.addDocument(d, fa);
Genre g = fa.getFactory().getGenre();
if (xrefDir != null && (g == Genre.PLAIN || g == Genre.XREFABLE)) {
File xrefFile = new File(xrefDir, path);
// If mkdirs() returns false, the failure is most likely
// because the file already exists. But to check for the
// file first and only add it if it doesn't exists would
// only increase the file IO...
if (!xrefFile.getParentFile().mkdirs()) {
assert xrefFile.getParentFile().exists();
}
fa.writeXref(xrefDir, path);
}
setDirty();
for (IndexChangedListener listener : listeners) {
listener.fileAdded(path, fa.getClass().getSimpleName());
}
}
}
/**
* Same as {@link #getRequestedProjects()}, but with a variable cookieName and parameter name.
* This way it is trivial to implement a project filter ...
*
* @param paramName the name of the request parameter, which possibly contains the project list in
* question.
* @param cookieName name of the cookie which possible contains project lists used as fallback
* @return a possible empty set but never {@code null}.
*/
protected SortedSet<String> getRequestedProjects(String paramName, String cookieName) {
TreeSet<String> set = new TreeSet<>();
List<Project> projects = getEnv().getProjects();
if (projects == null) {
return set;
}
if (projects.size() == 1 && authFramework.isAllowed(req, projects.get(0))) {
set.add(projects.get(0).getDescription());
return set;
}
List<String> vals = getParamVals(paramName);
for (String s : vals) {
Project x = Project.getByDescription(s);
if (x != null && authFramework.isAllowed(req, x)) {
set.add(s);
}
}
if (set.isEmpty()) {
List<String> cookies = getCookieVals(cookieName);
for (String s : cookies) {
Project x = Project.getByDescription(s);
if (x != null && authFramework.isAllowed(req, x)) {
set.add(s);
}
}
}
if (set.isEmpty()) {
Project defaultProject = env.getDefaultProject();
if (defaultProject != null && authFramework.isAllowed(req, defaultProject)) {
set.add(defaultProject.getDescription());
}
}
return set;
}
static void listFrequentTokens(List<String> subFiles) throws IOException {
final int limit = 4;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.hasProjects()) {
if (subFiles == null || subFiles.isEmpty()) {
for (Project project : env.getProjects()) {
IndexDatabase db = new IndexDatabase(project);
db.listTokens(4);
}
} else {
for (String path : subFiles) {
Project project = Project.getProject(path);
if (project == null) {
log.log(Level.WARNING, "Warning: Could not find a project for \"{0}\"", path);
} else {
IndexDatabase db = new IndexDatabase(project);
db.listTokens(4);
}
}
}
} else {
IndexDatabase db = new IndexDatabase();
db.listTokens(limit);
}
}
@SuppressWarnings("PMD.CollapsibleIfStatements")
private void initialize() throws IOException {
synchronized (this) {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File indexDir = new File(env.getDataRootFile(), INDEX_DIR);
if (project != null) {
indexDir = new File(indexDir, project.getPath());
}
if (!indexDir.exists() && !indexDir.mkdirs()) {
// to avoid race conditions, just recheck..
if (!indexDir.exists()) {
throw new FileNotFoundException(
"Failed to create root directory [" + indexDir.getAbsolutePath() + "]");
}
}
if (!env.isUsingLuceneLocking()) {
lockfact = NoLockFactory.INSTANCE;
}
indexDirectory = FSDirectory.open(indexDir.toPath(), lockfact);
ignoredNames = env.getIgnoredNames();
includedNames = env.getIncludedNames();
analyzerGuru = new AnalyzerGuru();
if (env.isGenerateHtml()) {
xrefDir = new File(env.getDataRootFile(), "xref");
}
listeners = new ArrayList<>();
dirtyFile = new File(indexDir, "dirty");
dirty = dirtyFile.exists();
directories = new ArrayList<>();
}
}
/**
* Update the index database for a number of sub-directories
*
* @param executor An executor to run the job
* @param listener where to signal the changes to the database
* @param paths
* @throws IOException if an error occurs
*/
public static void update(
ExecutorService executor, IndexChangedListener listener, List<String> paths)
throws IOException {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
for (String path : paths) {
Project project = Project.getProject(path);
if (project == null && env.hasProjects()) {
log.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
} else {
IndexDatabase db;
try {
if (project == null) {
db = new IndexDatabase();
} else {
db = new IndexDatabase(project);
}
int idx = dbs.indexOf(db);
if (idx != -1) {
db = dbs.get(idx);
}
if (db.addDirectory(path)) {
if (idx == -1) {
dbs.add(db);
}
} else {
log.log(Level.WARNING, "Directory does not exist \"{0}\"", path);
}
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while updating index", e);
}
}
for (final IndexDatabase db : dbs) {
db.addIndexChangedListener(listener);
executor.submit(
new Runnable() {
@Override
public void run() {
try {
db.update();
} catch (Throwable e) {
log.log(Level.SEVERE, "An error occured while updating index", e);
}
}
});
}
}
}
/**
* Populate a Lucene document with the required fields.
*
* @param doc The document to populate
* @param file The file to index
* @param path Where the file is located (from source root)
* @param fa The analyzer to use on the file
* @param xrefOut Where to write the xref (possibly {@code null})
* @throws IOException If an exception occurs while collecting the data
*/
public void populateDocument(
Document doc, File file, String path, FileAnalyzer fa, Writer xrefOut) throws IOException {
String date = DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND);
doc.add(new Field(QueryBuilder.U, Util.path2uid(path, date), string_ft_stored_nanalyzed_norms));
doc.add(
new Field(
QueryBuilder.FULLPATH, file.getAbsolutePath(), string_ft_nstored_nanalyzed_norms));
doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH, new BytesRef(file.getAbsolutePath())));
try {
HistoryReader hr = HistoryGuru.getInstance().getHistoryReader(file);
if (hr != null) {
doc.add(new TextField(QueryBuilder.HIST, hr));
// date = hr.getLastCommentDate() //RFE
}
} catch (HistoryException e) {
LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e);
}
doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms));
doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date)));
if (path != null) {
doc.add(new TextField(QueryBuilder.PATH, path, Store.YES));
Project project = Project.getProject(path);
if (project != null) {
doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES));
}
}
if (fa != null) {
Genre g = fa.getGenre();
if (g == Genre.PLAIN || g == Genre.XREFABLE || g == Genre.HTML) {
doc.add(new Field(QueryBuilder.T, g.typeName(), string_ft_stored_nanalyzed_norms));
}
fa.analyze(doc, StreamSource.fromFile(file), xrefOut);
String type = fa.getFileTypeName();
doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES));
}
}
/**
* Check if a file is local to the current project. If we don't have projects, check if the file
* is in the source root.
*
* @param path the path to a file
* @return true if the file is local to the current repository
*/
private boolean isLocal(String path) {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
String srcRoot = env.getSourceRootPath();
boolean local = false;
if (path.startsWith(srcRoot)) {
if (env.hasProjects()) {
String relPath = path.substring(srcRoot.length());
if (project.equals(Project.getProject(relPath))) {
// File is under the current project, so it's local.
local = true;
}
} else {
// File is under source root, and we don't have projects, so
// consider it local.
local = true;
}
}
return local;
}
/**
* Get an indexReader for the Index database where a given file
*
* @param path the file to get the database for
* @return The index database where the file should be located or null if it cannot be located.
*/
public static IndexReader getIndexReader(String path) {
IndexReader ret = null;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File indexDir = new File(env.getDataRootFile(), "index");
if (env.hasProjects()) {
Project p = Project.getProject(path);
if (p == null) {
return null;
}
indexDir = new File(indexDir, p.getPath());
}
try {
FSDirectory fdir = FSDirectory.open(indexDir, NoLockFactory.getNoLockFactory());
if (indexDir.exists() && DirectoryReader.indexExists(fdir)) {
ret = DirectoryReader.open(fdir);
}
} catch (Exception ex) {
log.log(Level.SEVERE, "Failed to open index: {0}", indexDir.getAbsolutePath());
log.log(Level.FINE, "Stack Trace: ", ex);
}
return ret;
}
/**
* Add a file to the Lucene index (and generate a xref file)
*
* @param file The file to add
* @param path The path to the file (from source root)
* @throws java.io.IOException if an error occurs
*/
private void addFile(File file, String path) throws IOException {
FileAnalyzer fa;
try (InputStream in = new BufferedInputStream(new FileInputStream(file))) {
fa = AnalyzerGuru.getAnalyzer(in, path);
}
for (IndexChangedListener listener : listeners) {
listener.fileAdd(path, fa.getClass().getSimpleName());
}
fa.setCtags(ctags);
fa.setProject(Project.getProject(path));
fa.setScopesEnabled(RuntimeEnvironment.getInstance().isScopesEnabled());
fa.setFoldingEnabled(RuntimeEnvironment.getInstance().isFoldingEnabled());
Document doc = new Document();
try (Writer xrefOut = getXrefWriter(fa, path)) {
analyzerGuru.populateDocument(doc, file, path, fa, xrefOut);
} catch (Exception e) {
LOGGER.log(
Level.INFO,
"Skipped file ''{0}'' because the analyzer didn''t " + "understand it.",
path);
LOGGER.log(Level.FINE, "Exception from analyzer " + fa.getClass().getName(), e);
cleanupResources(doc);
return;
}
try {
writer.addDocument(doc);
} catch (Throwable t) {
cleanupResources(doc);
throw t;
}
setDirty();
for (IndexChangedListener listener : listeners) {
listener.fileAdded(path, fa.getClass().getSimpleName());
}
}
/**
* List all files in some of the index databases
*
* @param subFiles Subdirectories for the various projects to list the files for (or null or an
* empty list to dump all projects)
* @throws IOException if an error occurs
*/
public static void listAllFiles(List<String> subFiles) throws IOException {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.hasProjects()) {
if (subFiles == null || subFiles.isEmpty()) {
for (Project project : env.getProjects()) {
IndexDatabase db = new IndexDatabase(project);
db.listFiles();
}
} else {
for (String path : subFiles) {
Project project = Project.getProject(path);
if (project == null) {
LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
} else {
IndexDatabase db = new IndexDatabase(project);
db.listFiles();
}
}
}
} else {
IndexDatabase db = new IndexDatabase();
db.listFiles();
}
}
/**
* Get the project {@link #getPath()} refers to.
*
* @return {@code null} if not available, the project otherwise.
*/
public Project getProject() {
return Project.getProject(getResourceFile());
}
/**
* Update the content of this index database
*
* @throws IOException if an error occurs
* @throws HistoryException if an error occurs when accessing the history
*/
public void update() throws IOException, HistoryException {
synchronized (lock) {
if (running) {
throw new IOException("Indexer already running!");
}
running = true;
interrupted = false;
}
String ctgs = RuntimeEnvironment.getInstance().getCtags();
if (ctgs != null) {
ctags = new Ctags();
ctags.setBinary(ctgs);
}
if (ctags == null) {
log.severe("Unable to run ctags! searching definitions will not work!");
}
if (ctags != null) {
String filename = RuntimeEnvironment.getInstance().getCTagsExtraOptionsFile();
if (filename != null) {
ctags.setCTagsExtraOptionsFile(filename);
}
}
try {
Analyzer analyzer = AnalyzerGuru.getAnalyzer();
IndexWriterConfig iwc = new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
// iwc.setRAMBufferSizeMB(256.0); //TODO check what is the sweet spot
writer = new IndexWriter(indexDirectory, iwc);
writer.commit(); // to make sure index exists on the disk
// writer.setMaxFieldLength(RuntimeEnvironment.getInstance().getIndexWordLimit());
if (directories.isEmpty()) {
if (project == null) {
directories.add("");
} else {
directories.add(project.getPath());
}
}
for (String dir : directories) {
File sourceRoot;
if ("".equals(dir)) {
sourceRoot = RuntimeEnvironment.getInstance().getSourceRootFile();
} else {
sourceRoot = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), dir);
}
HistoryGuru.getInstance().ensureHistoryCacheExists(sourceRoot);
String startuid = Util.path2uid(dir, "");
IndexReader reader = DirectoryReader.open(indexDirectory); // open existing index
Terms terms = null;
int numDocs = reader.numDocs();
if (numDocs > 0) {
Fields uFields = MultiFields.getFields(reader); // reader.getTermVectors(0);
terms = uFields.terms(QueryBuilder.U);
}
try {
if (numDocs > 0) {
uidIter = terms.iterator(null);
TermsEnum.SeekStatus stat = uidIter.seekCeil(new BytesRef(startuid), true); // init uid
if (stat == TermsEnum.SeekStatus.END || stat == TermsEnum.SeekStatus.NOT_FOUND) {
uidIter = null;
}
}
// TODO below should be optional, since it traverses the tree once more to get total
// count! :(
int file_cnt = 0;
if (RuntimeEnvironment.getInstance().isPrintProgress()) {
log.log(Level.INFO, "Counting files in {0} ...", dir);
file_cnt = indexDown(sourceRoot, dir, true, 0, 0);
if (log.isLoggable(Level.INFO)) {
log.log(
Level.INFO, "Need to process: {0} files for {1}", new Object[] {file_cnt, dir});
}
}
indexDown(sourceRoot, dir, false, 0, file_cnt);
while (uidIter != null
&& uidIter.term() != null
&& uidIter.term().utf8ToString().startsWith(startuid)) {
removeFile();
uidIter.next();
}
} finally {
reader.close();
}
}
} finally {
if (writer != null) {
try {
writer.prepareCommit();
writer.commit();
writer.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing writer", e);
}
}
if (ctags != null) {
try {
ctags.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing ctags process", e);
}
}
synchronized (lock) {
running = false;
}
}
if (!isInterrupted() && isDirty()) {
if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) {
optimize();
}
createSpellingSuggestions();
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File timestamp = new File(env.getDataRootFile(), "timestamp");
if (timestamp.exists()) {
if (!timestamp.setLastModified(System.currentTimeMillis())) {
log.log(
Level.WARNING,
"Failed to set last modified time on ''{0}'', used for timestamping the index database.",
timestamp.getAbsolutePath());
}
} else {
if (!timestamp.createNewFile()) {
log.log(
Level.WARNING,
"Failed to create file ''{0}'', used for timestamping the index database.",
timestamp.getAbsolutePath());
}
}
}
}
protected String getProjectPostfix(boolean encoded) {
String amp = encoded ? "&" : "&";
return project == null ? "" : (amp + "project=" + project.getDescription());
}
protected void appendProject() throws IOException {
if (project != null) {
out.write("&project=");
out.write(project.getDescription());
}
}