public boolean getContext(String filename, String path, List<Hit> hits) throws HistoryException {
if (m == null) {
return false;
}
File f = new File(filename);
return getHistoryContext(HistoryGuru.getInstance().getHistory(f), path, null, hits, null);
}
public boolean getContext(String parent, String basename, String path, Writer out, String context)
throws HistoryException {
if (m == null) {
return false;
}
History hist = HistoryGuru.getInstance().getHistory(new File(parent, basename));
return getHistoryContext(hist, path, out, null, context);
}
/**
* Check if I should accept this file into the index database
*
* @param file the file to check
* @return true if the file should be included, false otherwise
*/
private boolean accept(File file) {
if (!includedNames.isEmpty()
&& // the filter should not affect directory names
(!(file.isDirectory() || includedNames.match(file)))) {
return false;
}
String absolutePath = file.getAbsolutePath();
if (ignoredNames.ignore(file)) {
LOGGER.log(Level.FINER, "ignoring {0}", absolutePath);
return false;
}
if (!file.canRead()) {
LOGGER.log(Level.WARNING, "Could not read {0}", absolutePath);
return false;
}
try {
String canonicalPath = file.getCanonicalPath();
if (!absolutePath.equals(canonicalPath) && !acceptSymlink(absolutePath, canonicalPath)) {
LOGGER.log(
Level.FINE,
"Skipped symlink ''{0}'' -> ''{1}''",
new Object[] {absolutePath, canonicalPath});
return false;
}
// below will only let go files and directories, anything else is considered special and is
// not added
if (!file.isFile() && !file.isDirectory()) {
LOGGER.log(Level.WARNING, "Ignored special file {0}", absolutePath);
return false;
}
} catch (IOException exp) {
LOGGER.log(Level.WARNING, "Failed to resolve name: {0}", absolutePath);
LOGGER.log(Level.FINE, "Stack Trace: ", exp);
}
if (file.isDirectory()) {
// always accept directories so that their files can be examined
return true;
}
if (HistoryGuru.getInstance().hasHistory(file)) {
// versioned files should always be accepted
return true;
}
// this is an unversioned file, check if it should be indexed
return !RuntimeEnvironment.getInstance().isIndexVersionedFilesOnly();
}
/**
* Get the annotation for the requested resource.
*
* @return {@code null} if not available or annotation was not requested, the cached annotation
* otherwise.
*/
public Annotation getAnnotation() {
if (isDir() || getResourcePath().equals("/") || !annotate()) {
return null;
}
if (annotation != null) {
return annotation;
}
getRequestedRevision();
try {
annotation =
HistoryGuru.getInstance().annotate(resourceFile, rev.isEmpty() ? null : rev.substring(2));
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Failed to get annotations: ", e);
/* ignore */
}
return annotation;
}
/**
* Populate a Lucene document with the required fields.
*
* @param doc The document to populate
* @param file The file to index
* @param path Where the file is located (from source root)
* @param fa The analyzer to use on the file
* @param xrefOut Where to write the xref (possibly {@code null})
* @throws IOException If an exception occurs while collecting the data
*/
public void populateDocument(
Document doc, File file, String path, FileAnalyzer fa, Writer xrefOut) throws IOException {
String date = DateTools.timeToString(file.lastModified(), DateTools.Resolution.MILLISECOND);
doc.add(new Field(QueryBuilder.U, Util.path2uid(path, date), string_ft_stored_nanalyzed_norms));
doc.add(
new Field(
QueryBuilder.FULLPATH, file.getAbsolutePath(), string_ft_nstored_nanalyzed_norms));
doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH, new BytesRef(file.getAbsolutePath())));
try {
HistoryReader hr = HistoryGuru.getInstance().getHistoryReader(file);
if (hr != null) {
doc.add(new TextField(QueryBuilder.HIST, hr));
// date = hr.getLastCommentDate() //RFE
}
} catch (HistoryException e) {
LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e);
}
doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms));
doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date)));
if (path != null) {
doc.add(new TextField(QueryBuilder.PATH, path, Store.YES));
Project project = Project.getProject(path);
if (project != null) {
doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES));
}
}
if (fa != null) {
Genre g = fa.getGenre();
if (g == Genre.PLAIN || g == Genre.XREFABLE || g == Genre.HTML) {
doc.add(new Field(QueryBuilder.T, g.typeName(), string_ft_stored_nanalyzed_norms));
}
fa.analyze(doc, StreamSource.fromFile(file), xrefOut);
String type = fa.getFileTypeName();
doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES));
}
}
/**
* Dump the configuration as an HTML table.
*
* @param out destination for the HTML output
* @throws IOException if an error happens while writing to {@code out}
* @throws HistoryException if the history guru cannot be accesses
*/
@SuppressWarnings("boxing")
public static void dumpConfiguration(Appendable out) throws IOException, HistoryException {
out.append("<table border=\"1\" width=\"100%\">");
out.append("<tr><th>Variable</th><th>Value</th></tr>");
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
printTableRow(out, "Source root", env.getSourceRootPath());
printTableRow(out, "Data root", env.getDataRootPath());
printTableRow(out, "CTags", env.getCtags());
printTableRow(out, "Bug page", env.getBugPage());
printTableRow(out, "Bug pattern", env.getBugPattern());
printTableRow(out, "User page", env.getUserPage());
printTableRow(out, "User page suffix", env.getUserPageSuffix());
printTableRow(out, "Review page", env.getReviewPage());
printTableRow(out, "Review pattern", env.getReviewPattern());
printTableRow(out, "Using projects", env.hasProjects());
out.append("<tr><td>Ignored files</td><td>");
printUnorderedList(out, env.getIgnoredNames().getItems());
out.append("</td></tr>");
printTableRow(out, "Index word limit", env.getIndexWordLimit());
printTableRow(out, "Allow leading wildcard in search", env.isAllowLeadingWildcard());
printTableRow(out, "History cache", HistoryGuru.getInstance().getCacheInfo());
out.append("</table>");
}
/**
* Check, whether annotations are available for the related resource.
*
* @return {@code true} if annotations are available.
*/
public boolean hasAnnotations() {
if (hasAnnotation == null) {
hasAnnotation = !isDir() && HistoryGuru.getInstance().hasHistory(getResourceFile());
}
return hasAnnotation;
}
/**
* Check, whether the request related resource has history information.
*
* @return {@code true} if history is available.
* @see HistoryGuru#hasHistory(File)
*/
public boolean hasHistory() {
if (hasHistory == null) {
hasHistory = HistoryGuru.getInstance().hasHistory(getResourceFile());
}
return hasHistory;
}
/**
* Get all data required to create a diff view wrt. to this request in one go.
*
* @return an instance with just enough information to render a sufficient view. If not all
* required parameters were given either they are supplemented with reasonable defaults if
* possible, otherwise the related field(s) are {@code null}. {@link DiffData#errorMsg} {@code
* != null} indicates, that an error occured and one should not try to render a view.
*/
public DiffData getDiffData() {
DiffData data = new DiffData();
data.path = getPath().substring(0, path.lastIndexOf('/'));
data.filename = Util.htmlize(getResourceFile().getName());
String srcRoot = getSourceRootPath();
String context = req.getContextPath();
String[] filepath = new String[2];
data.rev = new String[2];
data.file = new String[2][];
data.param = new String[2];
/*
* Basically the request URI looks like this:
* http://$site/$webapp/diff/$resourceFile?r1=$fileA@$revA&r2=$fileB@$revB
* The code below extracts file path and revision from the URI.
*/
for (int i = 1; i <= 2; i++) {
String[] tmp = null;
String p = req.getParameter("r" + i);
if (p != null) {
tmp = p.split("@");
}
if (tmp != null && tmp.length == 2) {
filepath[i - 1] = tmp[0];
data.rev[i - 1] = tmp[1];
}
}
if (data.rev[0] == null
|| data.rev[1] == null
|| data.rev[0].length() == 0
|| data.rev[1].length() == 0
|| data.rev[0].equals(data.rev[1])) {
data.errorMsg =
"Please pick two revisions to compare the changed "
+ "from the <a href=\""
+ context
+ Prefix.HIST_L
+ getUriEncodedPath()
+ "\">history</a>";
return data;
}
data.genre = AnalyzerGuru.getGenre(getResourceFile().getName());
if (data.genre == null || txtGenres.contains(data.genre)) {
InputStream[] in = new InputStream[2];
try {
// Get input stream for both older and newer file.
for (int i = 0; i < 2; i++) {
File f = new File(srcRoot + filepath[i]);
in[i] = HistoryGuru.getInstance().getRevision(f.getParent(), f.getName(), data.rev[i]);
if (in[i] == null) {
data.errorMsg =
"Unable to get revision "
+ Util.htmlize(data.rev[i])
+ " for file: "
+ Util.htmlize(getPath());
return data;
}
}
/*
* If the genre of the older revision cannot be determined,
* (this can happen if the file was empty), try with newer
* version.
*/
for (int i = 0; i < 2 && data.genre == null; i++) {
try {
data.genre = AnalyzerGuru.getGenre(in[i]);
} catch (IOException e) {
data.errorMsg = "Unable to determine the file type: " + Util.htmlize(e.getMessage());
}
}
if (data.genre != Genre.PLAIN && data.genre != Genre.HTML) {
return data;
}
ArrayList<String> lines = new ArrayList<>();
Project p = getProject();
for (int i = 0; i < 2; i++) {
try (BufferedReader br =
new BufferedReader(ExpandTabsReader.wrap(new InputStreamReader(in[i]), p))) {
String line;
while ((line = br.readLine()) != null) {
lines.add(line);
}
data.file[i] = lines.toArray(new String[lines.size()]);
lines.clear();
}
in[i] = null;
}
} catch (Exception e) {
data.errorMsg = "Error reading revisions: " + Util.htmlize(e.getMessage());
} finally {
for (int i = 0; i < 2; i++) {
IOUtils.close(in[i]);
}
}
if (data.errorMsg != null) {
return data;
}
try {
data.revision = Diff.diff(data.file[0], data.file[1]);
} catch (DifferentiationFailedException e) {
data.errorMsg = "Unable to get diffs: " + Util.htmlize(e.getMessage());
}
for (int i = 0; i < 2; i++) {
try {
URI u = new URI(null, null, null, filepath[i] + "@" + data.rev[i], null);
data.param[i] = u.getRawQuery();
} catch (URISyntaxException e) {
LOGGER.log(Level.WARNING, "Failed to create URI: ", e);
}
}
data.full = fullDiff();
data.type = getDiffType();
}
return data;
}
/**
* Update the content of this index database
*
* @throws IOException if an error occurs
* @throws HistoryException if an error occurs when accessing the history
*/
public void update() throws IOException, HistoryException {
synchronized (lock) {
if (running) {
throw new IOException("Indexer already running!");
}
running = true;
interrupted = false;
}
String ctgs = RuntimeEnvironment.getInstance().getCtags();
if (ctgs != null) {
ctags = new Ctags();
ctags.setBinary(ctgs);
}
if (ctags == null) {
log.severe("Unable to run ctags! searching definitions will not work!");
}
if (ctags != null) {
String filename = RuntimeEnvironment.getInstance().getCTagsExtraOptionsFile();
if (filename != null) {
ctags.setCTagsExtraOptionsFile(filename);
}
}
try {
Analyzer analyzer = AnalyzerGuru.getAnalyzer();
IndexWriterConfig iwc = new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
// iwc.setRAMBufferSizeMB(256.0); //TODO check what is the sweet spot
writer = new IndexWriter(indexDirectory, iwc);
writer.commit(); // to make sure index exists on the disk
// writer.setMaxFieldLength(RuntimeEnvironment.getInstance().getIndexWordLimit());
if (directories.isEmpty()) {
if (project == null) {
directories.add("");
} else {
directories.add(project.getPath());
}
}
for (String dir : directories) {
File sourceRoot;
if ("".equals(dir)) {
sourceRoot = RuntimeEnvironment.getInstance().getSourceRootFile();
} else {
sourceRoot = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), dir);
}
HistoryGuru.getInstance().ensureHistoryCacheExists(sourceRoot);
String startuid = Util.path2uid(dir, "");
IndexReader reader = DirectoryReader.open(indexDirectory); // open existing index
Terms terms = null;
int numDocs = reader.numDocs();
if (numDocs > 0) {
Fields uFields = MultiFields.getFields(reader); // reader.getTermVectors(0);
terms = uFields.terms(QueryBuilder.U);
}
try {
if (numDocs > 0) {
uidIter = terms.iterator(null);
TermsEnum.SeekStatus stat = uidIter.seekCeil(new BytesRef(startuid), true); // init uid
if (stat == TermsEnum.SeekStatus.END || stat == TermsEnum.SeekStatus.NOT_FOUND) {
uidIter = null;
}
}
// TODO below should be optional, since it traverses the tree once more to get total
// count! :(
int file_cnt = 0;
if (RuntimeEnvironment.getInstance().isPrintProgress()) {
log.log(Level.INFO, "Counting files in {0} ...", dir);
file_cnt = indexDown(sourceRoot, dir, true, 0, 0);
if (log.isLoggable(Level.INFO)) {
log.log(
Level.INFO, "Need to process: {0} files for {1}", new Object[] {file_cnt, dir});
}
}
indexDown(sourceRoot, dir, false, 0, file_cnt);
while (uidIter != null
&& uidIter.term() != null
&& uidIter.term().utf8ToString().startsWith(startuid)) {
removeFile();
uidIter.next();
}
} finally {
reader.close();
}
}
} finally {
if (writer != null) {
try {
writer.prepareCommit();
writer.commit();
writer.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing writer", e);
}
}
if (ctags != null) {
try {
ctags.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing ctags process", e);
}
}
synchronized (lock) {
running = false;
}
}
if (!isInterrupted() && isDirty()) {
if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) {
optimize();
}
createSpellingSuggestions();
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File timestamp = new File(env.getDataRootFile(), "timestamp");
if (timestamp.exists()) {
if (!timestamp.setLastModified(System.currentTimeMillis())) {
log.log(
Level.WARNING,
"Failed to set last modified time on ''{0}'', used for timestamping the index database.",
timestamp.getAbsolutePath());
}
} else {
if (!timestamp.createNewFile()) {
log.log(
Level.WARNING,
"Failed to create file ''{0}'', used for timestamping the index database.",
timestamp.getAbsolutePath());
}
}
}
}