/**
* Add a file to the Lucene index (and generate a xref file)
*
* @param file The file to add
* @param path The path to the file (from source root)
* @throws java.io.IOException if an error occurs
*/
private void addFile(File file, String path) throws IOException {
try (InputStream in = new BufferedInputStream(new FileInputStream(file))) {
FileAnalyzer fa = AnalyzerGuru.getAnalyzer(in, path);
for (IndexChangedListener listener : listeners) {
listener.fileAdd(path, fa.getClass().getSimpleName());
}
fa.setCtags(ctags);
fa.setProject(Project.getProject(path));
Document d;
try {
d = analyzerGuru.getDocument(file, in, path, fa);
} catch (Exception e) {
log.log(
Level.INFO,
"Skipped file ''{0}'' because the analyzer didn''t " + "understand it.",
path);
StringBuilder stack = new StringBuilder();
for (StackTraceElement ste : e.getStackTrace()) {
stack.append(ste.toString()).append(System.lineSeparator());
}
StringBuilder sstack = new StringBuilder();
for (Throwable t : e.getSuppressed()) {
for (StackTraceElement ste : t.getStackTrace()) {
sstack.append(ste.toString()).append(System.lineSeparator());
}
}
log.log(
Level.FINE,
"Exception from analyzer {0}: {1} {2}{3}{4}{5}{6}",
new String[] {
fa.getClass().getName(),
e.toString(),
System.lineSeparator(),
stack.toString(),
System.lineSeparator(),
sstack.toString()
});
return;
}
writer.addDocument(d, fa);
Genre g = fa.getFactory().getGenre();
if (xrefDir != null && (g == Genre.PLAIN || g == Genre.XREFABLE)) {
File xrefFile = new File(xrefDir, path);
// If mkdirs() returns false, the failure is most likely
// because the file already exists. But to check for the
// file first and only add it if it doesn't exists would
// only increase the file IO...
if (!xrefFile.getParentFile().mkdirs()) {
assert xrefFile.getParentFile().exists();
}
fa.writeXref(xrefDir, path);
}
setDirty();
for (IndexChangedListener listener : listeners) {
listener.fileAdded(path, fa.getClass().getSimpleName());
}
}
}
static void listFrequentTokens(List<String> subFiles) throws IOException {
final int limit = 4;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.hasProjects()) {
if (subFiles == null || subFiles.isEmpty()) {
for (Project project : env.getProjects()) {
IndexDatabase db = new IndexDatabase(project);
db.listTokens(4);
}
} else {
for (String path : subFiles) {
Project project = Project.getProject(path);
if (project == null) {
log.log(Level.WARNING, "Warning: Could not find a project for \"{0}\"", path);
} else {
IndexDatabase db = new IndexDatabase(project);
db.listTokens(4);
}
}
}
} else {
IndexDatabase db = new IndexDatabase();
db.listTokens(limit);
}
}
/**
* Update the index database for a number of sub-directories
*
* @param executor An executor to run the job
* @param listener where to signal the changes to the database
* @param paths
* @throws IOException if an error occurs
*/
public static void update(
ExecutorService executor, IndexChangedListener listener, List<String> paths)
throws IOException {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
for (String path : paths) {
Project project = Project.getProject(path);
if (project == null && env.hasProjects()) {
log.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
} else {
IndexDatabase db;
try {
if (project == null) {
db = new IndexDatabase();
} else {
db = new IndexDatabase(project);
}
int idx = dbs.indexOf(db);
if (idx != -1) {
db = dbs.get(idx);
}
if (db.addDirectory(path)) {
if (idx == -1) {
dbs.add(db);
}
} else {
log.log(Level.WARNING, "Directory does not exist \"{0}\"", path);
}
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while updating index", e);
}
}
for (final IndexDatabase db : dbs) {
db.addIndexChangedListener(listener);
executor.submit(
new Runnable() {
@Override
public void run() {
try {
db.update();
} catch (Throwable e) {
log.log(Level.SEVERE, "An error occured while updating index", e);
}
}
});
}
}
}
/**
* Populate a Lucene document with the required fields.
*
* @param doc The document to populate
* @param file The file to index
* @param path Where the file is located (from source root)
* @param fa The analyzer to use on the file
* @param xrefOut Where to write the xref (possibly {@code null})
* @throws IOException If an exception occurs while collecting the data
*/
public void populateDocument(
Document doc, File file, String path, FileAnalyzer fa, Writer xrefOut) throws IOException {
String date = DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND);
doc.add(new Field(QueryBuilder.U, Util.path2uid(path, date), string_ft_stored_nanalyzed_norms));
doc.add(
new Field(
QueryBuilder.FULLPATH, file.getAbsolutePath(), string_ft_nstored_nanalyzed_norms));
doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH, new BytesRef(file.getAbsolutePath())));
try {
HistoryReader hr = HistoryGuru.getInstance().getHistoryReader(file);
if (hr != null) {
doc.add(new TextField(QueryBuilder.HIST, hr));
// date = hr.getLastCommentDate() //RFE
}
} catch (HistoryException e) {
LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e);
}
doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms));
doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date)));
if (path != null) {
doc.add(new TextField(QueryBuilder.PATH, path, Store.YES));
Project project = Project.getProject(path);
if (project != null) {
doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES));
}
}
if (fa != null) {
Genre g = fa.getGenre();
if (g == Genre.PLAIN || g == Genre.XREFABLE || g == Genre.HTML) {
doc.add(new Field(QueryBuilder.T, g.typeName(), string_ft_stored_nanalyzed_norms));
}
fa.analyze(doc, StreamSource.fromFile(file), xrefOut);
String type = fa.getFileTypeName();
doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES));
}
}
/**
* Check if a file is local to the current project. If we don't have projects, check if the file
* is in the source root.
*
* @param path the path to a file
* @return true if the file is local to the current repository
*/
private boolean isLocal(String path) {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
String srcRoot = env.getSourceRootPath();
boolean local = false;
if (path.startsWith(srcRoot)) {
if (env.hasProjects()) {
String relPath = path.substring(srcRoot.length());
if (project.equals(Project.getProject(relPath))) {
// File is under the current project, so it's local.
local = true;
}
} else {
// File is under source root, and we don't have projects, so
// consider it local.
local = true;
}
}
return local;
}
/**
* Add a file to the Lucene index (and generate a xref file)
*
* @param file The file to add
* @param path The path to the file (from source root)
* @throws java.io.IOException if an error occurs
*/
private void addFile(File file, String path) throws IOException {
FileAnalyzer fa;
try (InputStream in = new BufferedInputStream(new FileInputStream(file))) {
fa = AnalyzerGuru.getAnalyzer(in, path);
}
for (IndexChangedListener listener : listeners) {
listener.fileAdd(path, fa.getClass().getSimpleName());
}
fa.setCtags(ctags);
fa.setProject(Project.getProject(path));
fa.setScopesEnabled(RuntimeEnvironment.getInstance().isScopesEnabled());
fa.setFoldingEnabled(RuntimeEnvironment.getInstance().isFoldingEnabled());
Document doc = new Document();
try (Writer xrefOut = getXrefWriter(fa, path)) {
analyzerGuru.populateDocument(doc, file, path, fa, xrefOut);
} catch (Exception e) {
LOGGER.log(
Level.INFO,
"Skipped file ''{0}'' because the analyzer didn''t " + "understand it.",
path);
LOGGER.log(Level.FINE, "Exception from analyzer " + fa.getClass().getName(), e);
cleanupResources(doc);
return;
}
try {
writer.addDocument(doc);
} catch (Throwable t) {
cleanupResources(doc);
throw t;
}
setDirty();
for (IndexChangedListener listener : listeners) {
listener.fileAdded(path, fa.getClass().getSimpleName());
}
}
/**
* Get an indexReader for the Index database where a given file
*
* @param path the file to get the database for
* @return The index database where the file should be located or null if it cannot be located.
*/
public static IndexReader getIndexReader(String path) {
IndexReader ret = null;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File indexDir = new File(env.getDataRootFile(), "index");
if (env.hasProjects()) {
Project p = Project.getProject(path);
if (p == null) {
return null;
}
indexDir = new File(indexDir, p.getPath());
}
try {
FSDirectory fdir = FSDirectory.open(indexDir, NoLockFactory.getNoLockFactory());
if (indexDir.exists() && DirectoryReader.indexExists(fdir)) {
ret = DirectoryReader.open(fdir);
}
} catch (Exception ex) {
log.log(Level.SEVERE, "Failed to open index: {0}", indexDir.getAbsolutePath());
log.log(Level.FINE, "Stack Trace: ", ex);
}
return ret;
}
/**
* List all files in some of the index databases
*
* @param subFiles Subdirectories for the various projects to list the files for (or null or an
* empty list to dump all projects)
* @throws IOException if an error occurs
*/
public static void listAllFiles(List<String> subFiles) throws IOException {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.hasProjects()) {
if (subFiles == null || subFiles.isEmpty()) {
for (Project project : env.getProjects()) {
IndexDatabase db = new IndexDatabase(project);
db.listFiles();
}
} else {
for (String path : subFiles) {
Project project = Project.getProject(path);
if (project == null) {
LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
} else {
IndexDatabase db = new IndexDatabase(project);
db.listFiles();
}
}
}
} else {
IndexDatabase db = new IndexDatabase();
db.listFiles();
}
}
/**
* Get the project {@link #getPath()} refers to.
*
* @return {@code null} if not available, the project otherwise.
*/
public Project getProject() {
return Project.getProject(getResourceFile());
}
