protected JFlexXref() {
try {
// TODO when bug #16053 is fixed, we should add a getter to a file
// that's included from all the Xref classes so that we avoid the
// reflection.
Field f = getClass().getField("YYEOF");
yyeof = f.getInt(null);
userPageLink = RuntimeEnvironment.getInstance().getUserPage();
if (userPageLink != null && userPageLink.length() == 0) {
userPageLink = null;
}
userPageSuffix = RuntimeEnvironment.getInstance().getUserPageSuffix();
if (userPageSuffix != null && userPageSuffix.length() == 0) {
userPageSuffix = null;
}
} catch (NoSuchFieldException
| SecurityException
| IllegalArgumentException
| IllegalAccessException e) {
// The auto-generated constructors for the Xref classes don't
// expect a checked exception, so wrap it in an AssertionError.
// This should never happen, since all the Xref classes will get
// a public static YYEOF field from JFlex.
// NOPMD (stack trace is preserved by initCause(), but
// PMD thinks it's lost)
throw new AssertionError("Couldn't initialize yyeof", e);
}
}
/**
* Set the name of the external client command that should be used to access the repository wrt.
* the given parameters. Does nothing, if this repository's <var>RepoCommand</var> has already
* been set (i.e. has a non-{@code null} value).
*
* @param propertyKey property key to lookup the corresponding system property.
* @param fallbackCommand the command to use, if lookup fails.
* @return the command to use.
* @see #RepoCommand
*/
protected String ensureCommand(String propertyKey, String fallbackCommand) {
if (RepoCommand != null) {
return RepoCommand;
}
RepoCommand = RuntimeEnvironment.getInstance().getRepoCmd(this.getClass().getCanonicalName());
if (RepoCommand == null) {
RepoCommand = System.getProperty(propertyKey, fallbackCommand);
RuntimeEnvironment.getInstance().setRepoCmd(this.getClass().getCanonicalName(), RepoCommand);
}
return RepoCommand;
}
/**
* Update the index database for all of the projects
*
* @param executor An executor to run the job
* @param listener where to signal the changes to the database
* @throws IOException if an error occurs
*/
static void updateAll(ExecutorService executor, IndexChangedListener listener)
throws IOException {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
List<IndexDatabase> dbs = new ArrayList<>();
if (env.hasProjects()) {
for (Project project : env.getProjects()) {
dbs.add(new IndexDatabase(project));
}
} else {
dbs.add(new IndexDatabase());
}
for (IndexDatabase d : dbs) {
final IndexDatabase db = d;
if (listener != null) {
db.addIndexChangedListener(listener);
}
executor.submit(
new Runnable() {
@Override
public void run() {
try {
db.update();
} catch (Throwable e) {
LOGGER.log(Level.SEVERE, "Problem updating lucene index database: ", e);
}
}
});
}
}
/**
* Get a writer to which the xref can be written, or null if no xref should be produced for files
* of this type.
*/
private Writer getXrefWriter(FileAnalyzer fa, String path) throws IOException {
Genre g = fa.getFactory().getGenre();
if (xrefDir != null && (g == Genre.PLAIN || g == Genre.XREFABLE)) {
File xrefFile = new File(xrefDir, path);
// If mkdirs() returns false, the failure is most likely
// because the file already exists. But to check for the
// file first and only add it if it doesn't exists would
// only increase the file IO...
if (!xrefFile.getParentFile().mkdirs()) {
assert xrefFile.getParentFile().exists();
}
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
boolean compressed = env.isCompressXref();
File file = new File(xrefDir, path + (compressed ? ".gz" : ""));
return new BufferedWriter(
new OutputStreamWriter(
compressed
? new GZIPOutputStream(new FileOutputStream(file))
: new FileOutputStream(file)));
}
// no Xref for this analyzer
return null;
}
/**
* Write an e-mail address. The address will be obfuscated if {@link
* RuntimeEnvironment#isObfuscatingEMailAddresses()} returns {@code true}.
*
* @param address the address to write
* @throws IOException if an error occurs while writing to the stream
*/
protected void writeEMailAddress(String address) throws IOException {
if (RuntimeEnvironment.getInstance().isObfuscatingEMailAddresses()) {
out.write(address.replace("@", " (at) "));
} else {
out.write(address);
}
}
/**
* Optimize all index databases
*
* @param executor An executor to run the job
* @throws IOException if an error occurs
*/
static void optimizeAll(ExecutorService executor) throws IOException {
List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.hasProjects()) {
for (Project project : env.getProjects()) {
dbs.add(new IndexDatabase(project));
}
} else {
dbs.add(new IndexDatabase());
}
for (IndexDatabase d : dbs) {
final IndexDatabase db = d;
if (db.isDirty()) {
executor.submit(
new Runnable() {
@Override
public void run() {
try {
db.update();
} catch (Throwable e) {
log.log(Level.SEVERE, "Problem updating lucene index database: ", e);
}
}
});
}
}
}
static void listFrequentTokens(List<String> subFiles) throws IOException {
final int limit = 4;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.hasProjects()) {
if (subFiles == null || subFiles.isEmpty()) {
for (Project project : env.getProjects()) {
IndexDatabase db = new IndexDatabase(project);
db.listTokens(4);
}
} else {
for (String path : subFiles) {
Project project = Project.getProject(path);
if (project == null) {
log.log(Level.WARNING, "Warning: Could not find a project for \"{0}\"", path);
} else {
IndexDatabase db = new IndexDatabase(project);
db.listTokens(4);
}
}
}
} else {
IndexDatabase db = new IndexDatabase();
db.listTokens(limit);
}
}
@BeforeClass
public static void setUpClass() throws Exception {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
assertTrue("No ctags available", env.validateExuberantCtags());
repository = new TestRepository();
repository.create(IndexDatabase.class.getResourceAsStream("source.zip"));
env.setSourceRoot(repository.getSourceRoot());
env.setDataRoot(repository.getDataRoot());
Indexer indexer = Indexer.getInstance();
indexer.prepareIndexer(
env,
true,
true,
"/c",
null,
false,
false,
false,
null,
null,
new ArrayList<String>(),
false);
indexer.doIndexerExecution(true, 1, null, null);
}
/**
* Remove a stale file (uidIter.term().text()) from the index database (and the xref file)
*
* @throws java.io.IOException if an error occurs
*/
private void removeFile() throws IOException {
String path = Util.uid2url(uidIter.term().utf8ToString());
for (IndexChangedListener listener : listeners) {
listener.fileRemove(path);
}
writer.deleteDocuments(new Term(QueryBuilder.U, uidIter.term()));
writer.prepareCommit();
writer.commit();
File xrefFile;
if (RuntimeEnvironment.getInstance().isCompressXref()) {
xrefFile = new File(xrefDir, path + ".gz");
} else {
xrefFile = new File(xrefDir, path);
}
File parent = xrefFile.getParentFile();
if (!xrefFile.delete() && xrefFile.exists()) {
log.log(Level.INFO, "Failed to remove obsolete xref-file: {0}", xrefFile.getAbsolutePath());
}
// Remove the parent directory if it's empty
if (parent.delete()) {
log.log(Level.FINE, "Removed empty xref dir:{0}", parent.getAbsolutePath());
}
setDirty();
for (IndexChangedListener listener : listeners) {
listener.fileRemoved(path);
}
}
@SuppressWarnings("PMD.CollapsibleIfStatements")
private void initialize() throws IOException {
synchronized (this) {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File indexDir = new File(env.getDataRootFile(), INDEX_DIR);
if (project != null) {
indexDir = new File(indexDir, project.getPath());
}
if (!indexDir.exists() && !indexDir.mkdirs()) {
// to avoid race conditions, just recheck..
if (!indexDir.exists()) {
throw new FileNotFoundException(
"Failed to create root directory [" + indexDir.getAbsolutePath() + "]");
}
}
if (!env.isUsingLuceneLocking()) {
lockfact = NoLockFactory.INSTANCE;
}
indexDirectory = FSDirectory.open(indexDir.toPath(), lockfact);
ignoredNames = env.getIgnoredNames();
includedNames = env.getIncludedNames();
analyzerGuru = new AnalyzerGuru();
if (env.isGenerateHtml()) {
xrefDir = new File(env.getDataRootFile(), "xref");
}
listeners = new ArrayList<>();
dirtyFile = new File(indexDir, "dirty");
dirty = dirtyFile.exists();
directories = new ArrayList<>();
}
}
/**
* Create a history log cache for all files in this repository. {@code getHistory()} is used to
* fetch the history for the entire repository. If {@code hasHistoryForDirectories()} returns
* {@code false}, this method is a no-op.
*
* @param cache the cache instance in which to store the history log
* @param sinceRevision if non-null, incrementally update the cache with all revisions after the
* specified revision; otherwise, create the full history starting with the initial revision
* @throws HistoryException on error
*/
final void createCache(HistoryCache cache, String sinceRevision) throws HistoryException {
if (!isWorking()) {
return;
}
// If we don't have a directory parser, we can't create the cache
// this way. Just give up and return.
if (!hasHistoryForDirectories()) {
Logger.getLogger(getClass().getName())
.log(
Level.INFO,
"Skipping creation of history cache for {0}, since retrieval "
+ "of history for directories is not implemented for this "
+ "repository type.",
getDirectoryName());
return;
}
File directory = new File(getDirectoryName());
History history;
try {
history = getHistory(directory, sinceRevision);
} catch (HistoryException he) {
if (sinceRevision == null) {
// Failed to get full history, so fail.
throw he;
}
// Failed to get partial history. This may have been caused
// by changes in the revision numbers since the last update
// (bug #14724) so we'll try to regenerate the cache from
// scratch instead.
OpenGrokLogger.getLogger()
.log(
Level.INFO,
"Failed to get partial history. Attempting to "
+ "recreate the history cache from scratch.",
he);
history = null;
}
if (sinceRevision != null && history == null) {
// Failed to get partial history, now get full history instead.
history = getHistory(directory);
// Got full history successfully. Clear the history cache so that
// we can recreate it from scratch.
cache.clear(this);
}
// We need to refresh list of tags for incremental reindex.
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.isTagsEnabled() && this.hasFileBasedTags()) {
this.buildTagList(new File(this.directoryName));
}
if (history != null) {
cache.store(history, this);
}
}
/** Helper for {@link #get(File, Repository)}. */
private History getHistory(File file, Repository repository, boolean withFiles)
throws HistoryException, SQLException {
final String filePath = getSourceRootRelativePath(file);
final String reposPath = toUnixPath(repository.getDirectoryName());
final ArrayList<HistoryEntry> entries = new ArrayList<HistoryEntry>();
final ConnectionResource conn = connectionManager.getConnectionResource();
try {
final PreparedStatement ps;
if (file.isDirectory()) {
// Fetch history for all files under this directory.
ps = conn.getStatement(GET_DIR_HISTORY);
ps.setString(2, filePath);
} else {
// Fetch history for a single file only.
ps = conn.getStatement(GET_FILE_HISTORY);
ps.setString(2, getParentPath(filePath));
ps.setString(3, getBaseName(filePath));
}
ps.setString(1, reposPath);
final PreparedStatement filePS = withFiles ? conn.getStatement(GET_CS_FILES) : null;
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
// Get the information about a changeset
String revision = rs.getString(1);
String author = rs.getString(2);
Timestamp time = rs.getTimestamp(3);
String message = rs.getString(4);
HistoryEntry entry = new HistoryEntry(revision, time, author, null, message, true);
entries.add(entry);
// Fill the list of files touched by the changeset, if
// requested.
if (withFiles) {
int changeset = rs.getInt(5);
filePS.setInt(1, changeset);
try (ResultSet fileRS = filePS.executeQuery()) {
while (fileRS.next()) {
entry.addFile(fileRS.getString(1));
}
}
}
}
}
} finally {
connectionManager.releaseConnection(conn);
}
History history = new History();
history.setHistoryEntries(entries);
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.isTagsEnabled() && repository.hasFileBasedTags()) {
repository.assignTagsInHistory(history);
}
return history;
}
@Override
History getHistory(File file, String sinceRevision) throws HistoryException {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
History result = new BazaarHistoryParser(this).parse(file, sinceRevision);
// Assign tags to changesets they represent
// We don't need to check if this repository supports tags, because we know it:-)
if (env.isTagsEnabled()) {
assignTagsInHistory(result);
}
return result;
}
/**
* Check if I should accept this file into the index database
*
* @param file the file to check
* @return true if the file should be included, false otherwise
*/
private boolean accept(File file) {
if (!includedNames.isEmpty()
&& // the filter should not affect directory names
(!(file.isDirectory() || includedNames.match(file)))) {
return false;
}
String absolutePath = file.getAbsolutePath();
if (ignoredNames.ignore(file)) {
LOGGER.log(Level.FINER, "ignoring {0}", absolutePath);
return false;
}
if (!file.canRead()) {
LOGGER.log(Level.WARNING, "Could not read {0}", absolutePath);
return false;
}
try {
String canonicalPath = file.getCanonicalPath();
if (!absolutePath.equals(canonicalPath) && !acceptSymlink(absolutePath, canonicalPath)) {
LOGGER.log(
Level.FINE,
"Skipped symlink ''{0}'' -> ''{1}''",
new Object[] {absolutePath, canonicalPath});
return false;
}
// below will only let go files and directories, anything else is considered special and is
// not added
if (!file.isFile() && !file.isDirectory()) {
LOGGER.log(Level.WARNING, "Ignored special file {0}", absolutePath);
return false;
}
} catch (IOException exp) {
LOGGER.log(Level.WARNING, "Failed to resolve name: {0}", absolutePath);
LOGGER.log(Level.FINE, "Stack Trace: ", exp);
}
if (file.isDirectory()) {
// always accept directories so that their files can be examined
return true;
}
if (HistoryGuru.getInstance().hasHistory(file)) {
// versioned files should always be accepted
return true;
}
// this is an unversioned file, check if it should be indexed
return !RuntimeEnvironment.getInstance().isIndexVersionedFilesOnly();
}
/**
* Update the index database for a number of sub-directories
*
* @param executor An executor to run the job
* @param listener where to signal the changes to the database
* @param paths
* @throws IOException if an error occurs
*/
public static void update(
ExecutorService executor, IndexChangedListener listener, List<String> paths)
throws IOException {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
List<IndexDatabase> dbs = new ArrayList<IndexDatabase>();
for (String path : paths) {
Project project = Project.getProject(path);
if (project == null && env.hasProjects()) {
log.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
} else {
IndexDatabase db;
try {
if (project == null) {
db = new IndexDatabase();
} else {
db = new IndexDatabase(project);
}
int idx = dbs.indexOf(db);
if (idx != -1) {
db = dbs.get(idx);
}
if (db.addDirectory(path)) {
if (idx == -1) {
dbs.add(db);
}
} else {
log.log(Level.WARNING, "Directory does not exist \"{0}\"", path);
}
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while updating index", e);
}
}
for (final IndexDatabase db : dbs) {
db.addIndexChangedListener(listener);
executor.submit(
new Runnable() {
@Override
public void run() {
try {
db.update();
} catch (Throwable e) {
log.log(Level.SEVERE, "An error occured while updating index", e);
}
}
});
}
}
}
/**
* Add a file to the Lucene index (and generate a xref file)
*
* @param file The file to add
* @param path The path to the file (from source root)
* @throws java.io.IOException if an error occurs
*/
private void addFile(File file, String path) throws IOException {
FileAnalyzer fa;
try (InputStream in = new BufferedInputStream(new FileInputStream(file))) {
fa = AnalyzerGuru.getAnalyzer(in, path);
}
for (IndexChangedListener listener : listeners) {
listener.fileAdd(path, fa.getClass().getSimpleName());
}
fa.setCtags(ctags);
fa.setProject(Project.getProject(path));
fa.setScopesEnabled(RuntimeEnvironment.getInstance().isScopesEnabled());
fa.setFoldingEnabled(RuntimeEnvironment.getInstance().isFoldingEnabled());
Document doc = new Document();
try (Writer xrefOut = getXrefWriter(fa, path)) {
analyzerGuru.populateDocument(doc, file, path, fa, xrefOut);
} catch (Exception e) {
LOGGER.log(
Level.INFO,
"Skipped file ''{0}'' because the analyzer didn''t " + "understand it.",
path);
LOGGER.log(Level.FINE, "Exception from analyzer " + fa.getClass().getName(), e);
cleanupResources(doc);
return;
}
try {
writer.addDocument(doc);
} catch (Throwable t) {
cleanupResources(doc);
throw t;
}
setDirty();
for (IndexChangedListener listener : listeners) {
listener.fileAdded(path, fa.getClass().getSimpleName());
}
}
/**
* By default the indexer will traverse all directories in the project. If you add directories
* with this function update will just process the specified directories.
*
* @param dir The directory to scan
* @return <code>true</code> if the file is added, false otherwise
*/
@SuppressWarnings("PMD.UseStringBufferForStringAppends")
public boolean addDirectory(String dir) {
String directory = dir;
if (directory.startsWith("\\")) {
directory = directory.replace('\\', '/');
} else if (directory.charAt(0) != '/') {
directory = "/" + directory;
}
File file = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), directory);
if (file.exists()) {
directories.add(directory);
return true;
}
return false;
}
/**
* Check if I should accept the path containing a symlink
*
* @param absolutePath the path with a symlink to check
* @param canonicalPath the canonical path to the file
* @return true if the file should be accepted, false otherwise
*/
private boolean acceptSymlink(String absolutePath, String canonicalPath) throws IOException {
// Always accept local symlinks
if (isLocal(canonicalPath)) {
return true;
}
for (String allowedSymlink : RuntimeEnvironment.getInstance().getAllowedSymlinks()) {
if (absolutePath.startsWith(allowedSymlink)) {
String allowedTarget = new File(allowedSymlink).getCanonicalPath();
if (canonicalPath.startsWith(allowedTarget)
&& absolutePath
.substring(allowedSymlink.length())
.equals(canonicalPath.substring(allowedTarget.length()))) {
return true;
}
}
}
return false;
}
/**
* Get the latest definitions for a file from the index.
*
* @param file the file whose definitions to find
* @return definitions for the file, or {@code null} if they could not be found
* @throws IOException if an error happens when accessing the index
* @throws ParseException if an error happens when building the Lucene query
* @throws ClassNotFoundException if the class for the stored definitions instance cannot be found
*/
public static Definitions getDefinitions(File file)
throws IOException, ParseException, ClassNotFoundException {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
String path = env.getPathRelativeToSourceRoot(file, 0);
// sanitize windows path delimiters
// in order not to conflict with Lucene escape character
path = path.replace("\\", "/");
IndexReader ireader = getIndexReader(path);
if (ireader == null) {
// No index, no definitions...
return null;
}
try {
Query q = new QueryBuilder().setPath(path).build();
IndexSearcher searcher = new IndexSearcher(ireader);
TopDocs top = searcher.search(q, 1);
if (top.totalHits == 0) {
// No hits, no definitions...
return null;
}
Document doc = searcher.doc(top.scoreDocs[0].doc);
String foundPath = doc.get(QueryBuilder.PATH);
// Only use the definitions if we found an exact match.
if (path.equals(foundPath)) {
IndexableField tags = doc.getField(QueryBuilder.TAGS);
if (tags != null) {
return Definitions.deserialize(tags.binaryValue().bytes);
}
}
} finally {
ireader.close();
}
// Didn't find any definitions.
return null;
}
/**
* Check if a file is local to the current project. If we don't have projects, check if the file
* is in the source root.
*
* @param path the path to a file
* @return true if the file is local to the current repository
*/
private boolean isLocal(String path) {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
String srcRoot = env.getSourceRootPath();
boolean local = false;
if (path.startsWith(srcRoot)) {
if (env.hasProjects()) {
String relPath = path.substring(srcRoot.length());
if (project.equals(Project.getProject(relPath))) {
// File is under the current project, so it's local.
local = true;
}
} else {
// File is under source root, and we don't have projects, so
// consider it local.
local = true;
}
}
return local;
}
/**
* Dump the configuration as an HTML table.
*
* @param out destination for the HTML output
* @throws IOException if an error happens while writing to {@code out}
* @throws HistoryException if the history guru cannot be accesses
*/
@SuppressWarnings("boxing")
public static void dumpConfiguration(Appendable out) throws IOException, HistoryException {
out.append("<table border=\"1\" width=\"100%\">");
out.append("<tr><th>Variable</th><th>Value</th></tr>");
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
printTableRow(out, "Source root", env.getSourceRootPath());
printTableRow(out, "Data root", env.getDataRootPath());
printTableRow(out, "CTags", env.getCtags());
printTableRow(out, "Bug page", env.getBugPage());
printTableRow(out, "Bug pattern", env.getBugPattern());
printTableRow(out, "User page", env.getUserPage());
printTableRow(out, "User page suffix", env.getUserPageSuffix());
printTableRow(out, "Review page", env.getReviewPage());
printTableRow(out, "Review pattern", env.getReviewPattern());
printTableRow(out, "Using projects", env.hasProjects());
out.append("<tr><td>Ignored files</td><td>");
printUnorderedList(out, env.getIgnoredNames().getItems());
out.append("</td></tr>");
printTableRow(out, "Index word limit", env.getIndexWordLimit());
printTableRow(out, "Allow leading wildcard in search", env.isAllowLeadingWildcard());
printTableRow(out, "History cache", HistoryGuru.getInstance().getCacheInfo());
out.append("</table>");
}
/**
* List all files in some of the index databases
*
* @param subFiles Subdirectories for the various projects to list the files for (or null or an
* empty list to dump all projects)
* @throws IOException if an error occurs
*/
public static void listAllFiles(List<String> subFiles) throws IOException {
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
if (env.hasProjects()) {
if (subFiles == null || subFiles.isEmpty()) {
for (Project project : env.getProjects()) {
IndexDatabase db = new IndexDatabase(project);
db.listFiles();
}
} else {
for (String path : subFiles) {
Project project = Project.getProject(path);
if (project == null) {
LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path);
} else {
IndexDatabase db = new IndexDatabase(project);
db.listFiles();
}
}
}
} else {
IndexDatabase db = new IndexDatabase();
db.listFiles();
}
}
/**
* Get an indexReader for the Index database where a given file
*
* @param path the file to get the database for
* @return The index database where the file should be located or null if it cannot be located.
*/
public static IndexReader getIndexReader(String path) {
IndexReader ret = null;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File indexDir = new File(env.getDataRootFile(), "index");
if (env.hasProjects()) {
Project p = Project.getProject(path);
if (p == null) {
return null;
}
indexDir = new File(indexDir, p.getPath());
}
try {
FSDirectory fdir = FSDirectory.open(indexDir, NoLockFactory.getNoLockFactory());
if (indexDir.exists() && DirectoryReader.indexExists(fdir)) {
ret = DirectoryReader.open(fdir);
}
} catch (Exception ex) {
log.log(Level.SEVERE, "Failed to open index: {0}", indexDir.getAbsolutePath());
log.log(Level.FINE, "Stack Trace: ", ex);
}
return ret;
}
/**
* Get the path of a file relative to the source root.
*
* @param file the file to get the path for
* @return relative path for {@code file} with unix file separators
*/
private static String getSourceRootRelativePath(File file) throws HistoryException {
String filePath = toUnixPath(file);
String rootPath = RuntimeEnvironment.getInstance().getSourceRootPath();
return getRelativePath(filePath, rootPath);
}
/**
* Get the current runtime environment.
*
* @return the runtime env.
* @see RuntimeEnvironment#getInstance()
* @see RuntimeEnvironment#register()
*/
public RuntimeEnvironment getEnv() {
if (env == null) {
env = RuntimeEnvironment.getInstance().register();
}
return env;
}
/**
* Generate indexes recursively
*
* @param dir the root indexDirectory to generate indexes for
* @param path the path
* @param count_only if true will just traverse the source root and count files
* @param cur_count current count during the traversal of the tree
* @param est_total estimate total files to process
*/
private int indexDown(File dir, String parent, boolean count_only, int cur_count, int est_total)
throws IOException {
int lcur_count = cur_count;
if (isInterrupted()) {
return lcur_count;
}
if (!accept(dir)) {
return lcur_count;
}
File[] files = dir.listFiles();
if (files == null) {
log.log(Level.SEVERE, "Failed to get file listing for: {0}", dir.getAbsolutePath());
return lcur_count;
}
Arrays.sort(
files,
new Comparator<File>() {
@Override
public int compare(File p1, File p2) {
return p1.getName().compareTo(p2.getName());
}
});
for (File file : files) {
if (accept(dir, file)) {
String path = parent + '/' + file.getName();
if (file.isDirectory()) {
lcur_count = indexDown(file, path, count_only, lcur_count, est_total);
} else {
lcur_count++;
if (count_only) {
continue;
}
if (RuntimeEnvironment.getInstance().isPrintProgress()
&& est_total > 0
&& log.isLoggable(Level.INFO)) {
log.log(
Level.INFO,
"Progress: {0} ({1}%)",
new Object[] {lcur_count, (lcur_count * 100.0f / est_total)});
}
if (uidIter != null) {
String uid =
Util.path2uid(
path,
DateTools.timeToString(
file.lastModified(),
DateTools.Resolution.MILLISECOND)); // construct uid for doc
BytesRef buid = new BytesRef(uid);
while (uidIter.term() != null
&& uidIter.term().compareTo(emptyBR) != 0
&& uidIter.term().compareTo(buid) < 0) {
removeFile();
uidIter.next();
}
if (uidIter.term() != null && uidIter.term().bytesEquals(buid)) {
uidIter.next(); // keep matching docs
continue;
}
}
try {
addFile(file, path);
} catch (Exception e) {
log.log(Level.WARNING, "Failed to add file " + file.getAbsolutePath(), e);
}
}
}
}
return lcur_count;
}
/**
* Base class for Xref lexers.
*
* @author Lubos Kosco
*/
public abstract class JFlexXref {
public Writer out;
public String urlPrefix = RuntimeEnvironment.getInstance().getUrlPrefix();
public Annotation annotation;
public Project project;
protected Definitions defs;
private boolean scopesEnabled = false;
private boolean foldingEnabled = false;
private boolean scopeOpen = false;
protected Scopes scopes = new Scopes();
protected Scope scope;
private int scopeLevel = 0;
/** EOF value returned by yylex(). */
private final int yyeof;
/**
* See {@link RuntimeEnvironment#getUserPage()}. Per default initialized in the constructor and
* here to be consistent and avoid lot of unnecessary lookups.
*
* @see #startNewLine()
*/
protected String userPageLink;
/**
* See {@link RuntimeEnvironment#getUserPageSuffix()}. Per default initialized in the constructor
* and here to be consistent and avoid lot of unnecessary lookups.
*
* @see #startNewLine()
*/
protected String userPageSuffix;
protected Stack<Integer> stack = new Stack<>();
protected Stack<String> stackPopString = new Stack<>();
/** Description of the style to use for a type of definitions. */
private static class Style {
/** Name of the style definition as given by CTags. */
final String name;
/** Class name used by the style sheets when rendering the xref. */
final String ssClass;
/**
* The title of the section to which this type belongs, or {@code null} if this type should not
* be listed in the navigation panel.
*/
final String title;
/** Construct a style description. */
Style(String name, String ssClass, String title) {
this.name = name;
this.ssClass = ssClass;
this.title = title;
}
}
/** Description of styles to use for different types of definitions. */
private static final Style[] DEFINITION_STYLES = {
new Style("macro", "xm", "Macro"),
new Style("argument", "xa", null),
new Style("local", "xl", null),
new Style("variable", "xv", "Variable"),
new Style("class", "xc", "Class"),
new Style("package", "xp", "Package"),
new Style("interface", "xi", "Interface"),
new Style("namespace", "xn", "Namespace"),
new Style("enumerator", "xer", null),
new Style("enum", "xe", "Enum"),
new Style("struct", "xs", "Struct"),
new Style("typedefs", "xts", null),
new Style("typedef", "xt", "Typedef"),
new Style("union", "xu", null),
new Style("field", "xfld", null),
new Style("member", "xmb", null),
new Style("function", "xf", "Function"),
new Style("method", "xmt", "Method"),
new Style("subroutine", "xsr", "Subroutine"),
};
protected JFlexXref() {
try {
// TODO when bug #16053 is fixed, we should add a getter to a file
// that's included from all the Xref classes so that we avoid the
// reflection.
Field f = getClass().getField("YYEOF");
yyeof = f.getInt(null);
userPageLink = RuntimeEnvironment.getInstance().getUserPage();
if (userPageLink != null && userPageLink.length() == 0) {
userPageLink = null;
}
userPageSuffix = RuntimeEnvironment.getInstance().getUserPageSuffix();
if (userPageSuffix != null && userPageSuffix.length() == 0) {
userPageSuffix = null;
}
} catch (NoSuchFieldException
| SecurityException
| IllegalArgumentException
| IllegalAccessException e) {
// The auto-generated constructors for the Xref classes don't
// expect a checked exception, so wrap it in an AssertionError.
// This should never happen, since all the Xref classes will get
// a public static YYEOF field from JFlex.
// NOPMD (stack trace is preserved by initCause(), but
// PMD thinks it's lost)
throw new AssertionError("Couldn't initialize yyeof", e);
}
}
/**
* Reinitialize the xref with new contents.
*
* @param contents a char buffer with text to analyze
* @param length the number of characters to use from the char buffer
*/
public void reInit(char[] contents, int length) {
reInit(new CharArrayReader(contents, 0, length));
}
/**
* Reinitialize the lexer with new reader.
*
* @param reader new reader for this lexer
*/
public final void reInit(Reader reader) {
this.yyreset(reader);
annotation = null;
scopes = new Scopes();
scope = null;
scopeLevel = 0;
scopeOpen = false;
}
/**
* set definitions
*
* @param defs definitions
*/
public void setDefs(Definitions defs) {
this.defs = defs;
}
/**
* set scopes
*
* @param scopesEnabled if they should be enabled or disabled
*/
public void setScopesEnabled(boolean scopesEnabled) {
this.scopesEnabled = scopesEnabled;
}
/**
* set folding of code
*
* @param foldingEnabled whether to fold or not
*/
public void setFoldingEnabled(boolean foldingEnabled) {
this.foldingEnabled = foldingEnabled;
}
protected void appendProject() throws IOException {
if (project != null) {
out.write("&project=");
out.write(project.getDescription());
}
}
protected void appendLink(String url) throws IOException {
out.write("<a href=\"");
out.write(Util.formQuoteEscape(url));
out.write("\">");
Util.htmlize(url, out);
out.write("</a>");
}
protected String getProjectPostfix(boolean encoded) {
String amp = encoded ? "&" : "&";
return project == null ? "" : (amp + "project=" + project.getDescription());
}
protected void startScope() {
if (scopesEnabled && scope == null) {
int line = getLineNumber();
List<Tag> tags = defs.getTags(line);
if (tags != null) {
for (Tag tag : tags) {
if (tag.type.startsWith("function") || tag.type.startsWith("method")) {
scope = new Scope(tag.line, tag.line, tag.symbol, tag.namespace, tag.signature);
scopeLevel = 0;
break;
}
}
}
}
}
protected void incScope() {
if (scope != null) {
scopeLevel++;
}
}
protected void decScope() {
if (scope != null && scopeLevel > 0) {
scopeLevel--;
if (scopeLevel == 0) {
scope.setLineTo(getLineNumber());
scopes.addScope(scope);
scope = null;
}
}
}
protected void endScope() {
if (scope != null && scopeLevel == 0) {
scope.setLineTo(getLineNumber());
scopes.addScope(scope);
scope = null;
}
}
/**
* Get generated scopes.
*
* @return scopes for current line
*/
public Scopes getScopes() {
return scopes;
}
/**
* Get the next token from the scanner.
*
* @return state number (e.g. YYEOF)
* @throws java.io.IOException in case of any I/O prob
*/
public abstract int yylex() throws IOException;
/**
* Reset the scanner.
*
* @param reader new reader to reinit this
*/
public abstract void yyreset(Reader reader);
/**
* Get the value of {@code yyline}.
*
* @return line number
*/
protected abstract int getLineNumber();
/**
* Set the value of {@code yyline}.
*
* @param x line number
*/
protected abstract void setLineNumber(int x);
/**
* start new analysis
*
* @param newState state to begin from
*/
public abstract void yybegin(int newState);
/**
* returns current state of analysis
*
* @return id of state
*/
public abstract int yystate();
/**
* Write xref to the specified {@code Writer}.
*
* @param out xref destination
* @throws IOException on error when writing the xref
*/
public void write(Writer out) throws IOException {
this.out = out;
writeSymbolTable();
setLineNumber(0);
startNewLine();
while (yylex() != yyeof) { // NOPMD while statement intentionally empty
// nothing to do here, yylex() will do the work
}
// terminate scopes
if (scopeOpen) {
out.write("</div>");
scopeOpen = false;
}
while (!stack.empty()) {
yypop();
}
writeScopesFooter();
}
/** Write a JavaScript function that display scopes panel if scopes are available */
private void writeScopesFooter() throws IOException {
// TODO try to get rid of included js scripts generated from here (all js should ideally be in
// util)
if (scopesEnabled && scopes != null && scopes.size() > 0) {
out.append(
"<script type=\"text/javascript\">document.getElementById(\"scope\").style.display = \"block\";</script>");
}
}
/**
* Write a JavaScript function that returns an array with the definitions to list in the
* navigation panel. Each element of the array is itself an array containing the name of the
* definition type, the CSS class name for the type, and an array of (symbol, line) pairs for the
* definitions of that type.
*/
private void writeSymbolTable() throws IOException {
if (defs == null) {
// No definitions, no symbol table to write
return;
}
// We want the symbol table to be sorted
Comparator<Tag> cmp =
new Comparator<Tag>() {
@Override
public int compare(Tag tag1, Tag tag2) {
// Order by symbol name, and then by line number if multiple
// definitions use the same symbol name
int ret = tag1.symbol.compareTo(tag2.symbol);
if (ret == 0) {
ret = tag1.line - tag2.line;
}
return ret;
}
};
Map<String, SortedSet<Tag>> symbols = new HashMap<>();
for (Tag tag : defs.getTags()) {
Style style = getStyle(tag.type);
if (style != null && style.title != null) {
SortedSet<Tag> tags = symbols.get(style.name);
if (tags == null) {
tags = new TreeSet<>(cmp);
symbols.put(style.name, tags);
}
tags.add(tag);
}
}
// TODO try to get rid of included js scripts generated from here (all js should ideally be in
// util)
out.append("<script type=\"text/javascript\">/* <![CDATA[ */\n");
out.append("function get_sym_list(){return [");
boolean first = true;
for (Style style : DEFINITION_STYLES) {
SortedSet<Tag> tags = symbols.get(style.name);
if (tags != null) {
if (!first) {
out.append(',');
}
out.append("[\"");
out.append(style.title);
out.append("\",\"");
out.append(style.ssClass);
out.append("\",[");
boolean firstTag = true;
for (Tag tag : tags) {
if (!firstTag) {
out.append(',');
}
out.append('[');
out.append(Util.jsStringLiteral(tag.symbol));
out.append(',');
out.append(Integer.toString(tag.line));
out.append(']');
firstTag = false;
}
out.append("]]");
first = false;
}
}
/* no LF intentionally - xml is whitespace aware ... */
out.append("];} /* ]]> */</script>");
}
/**
* Get the style description for a definition type.
*
* @param type the definition type
* @return the style of a definition type, or {@code null} if no style is defined for the type
* @see #DEFINITION_STYLES
*/
private Style getStyle(String type) {
for (Style style : DEFINITION_STYLES) {
if (type.startsWith(style.name)) {
return style;
}
}
return null;
}
/**
* Generate span id for namespace based on line number, name, and signature (more functions with
* same name and signature can be defined in single file)
*
* @param scope Scope to generate id from
* @return generated span id
*/
private String generateId(Scope scope) {
String name = Integer.toString(scope.getLineFrom()) + scope.getName() + scope.getSignature();
int hash = name.hashCode();
return "scope_id_" + Integer.toHexString(hash);
}
/**
* Simple escape of html characters in raw string.
*
* @param raw Raw string
* @return String with escaped html characters
*/
protected String htmlize(String raw) {
return raw.replace("&", "&")
.replace("<", "<")
.replace(">", ">")
.replace("\"", """)
.replace("'", "'");
}
/**
* Terminate the current line and insert preamble for the next line. The line count will be
* incremented.
*
* @throws IOException on error when writing the xref
*/
protected void startNewLine() throws IOException {
String iconId = null;
int line = getLineNumber() + 1;
boolean skipNl = false;
setLineNumber(line);
if (scopesEnabled) {
startScope();
if (scopeOpen && scope == null) {
scopeOpen = false;
out.write("</span>");
skipNl = true;
} else if (scope != null) {
String scopeId = generateId(scope);
if (scope.getLineFrom() == line) {
out.write("<span id='");
out.write(scopeId);
out.write("' class='scope-head'><span class='scope-signature'>");
out.write(htmlize(scope.getName() + scope.getSignature()));
out.write("</span>");
iconId = scopeId + "_fold_icon";
skipNl = true;
} else if (scope.getLineFrom() == line - 1) {
if (scopeOpen) {
out.write("</span>");
}
out.write("<span id='");
out.write(scopeId);
out.write("_fold' class='scope-body'>");
skipNl = true;
}
scopeOpen = true;
}
}
Util.readableLine(
line, out, annotation, userPageLink, userPageSuffix, getProjectPostfix(true), skipNl);
if (foldingEnabled && scopesEnabled) {
if (iconId != null) {
out.write("<a href=\"#\" onclick='fold(this.parentNode.id)' id='");
out.write(iconId);
/* space inside span for IE support */
out.write("'><span class='fold-icon'> </span></a>");
} else {
out.write("<span class='fold-space'> </span>");
}
}
}
/**
* Write a symbol and generate links as appropriate.
*
* @param symbol the symbol to write
* @param keywords a set of keywords recognized by this analyzer (no links will be generated if
* the symbol is a keyword)
* @param line the line number on which the symbol appears
* @throws IOException if an error occurs while writing to the stream
*/
protected void writeSymbol(String symbol, Set<String> keywords, int line) throws IOException {
writeSymbol(symbol, keywords, line, true);
}
/**
* Write a symbol and generate links as appropriate.
*
* @param symbol the symbol to write
* @param keywords a set of keywords recognized by this analyzer (no links will be generated if
* the symbol is a keyword)
* @param line the line number on which the symbol appears
* @param caseSensitive Whether the keyword list is case sensitive
* @throws IOException if an error occurs while writing to the stream
*/
protected void writeSymbol(String symbol, Set<String> keywords, int line, boolean caseSensitive)
throws IOException {
String[] strs = new String[1];
strs[0] = "";
String jsEscapedSymbol = symbol.replace("'", "\\'");
String check = caseSensitive ? symbol : symbol.toLowerCase();
if (keywords != null && keywords.contains(check)) {
// This is a keyword, so we don't create a link.
out.append("<b>").append(symbol).append("</b>");
} else if (defs != null && defs.hasDefinitionAt(symbol, line, strs)) {
// This is the definition of the symbol.
String type = strs[0];
String style_class = "d";
Style style = getStyle(type);
if (style != null) {
style_class = style.ssClass;
}
// 1) Create an anchor for direct links. (Perhaps we should only
// do this when there's exactly one definition of the symbol in
// this file? Otherwise, we may end up with multiple anchors with
// the same name.)
//
// Note: In HTML 4, the name must start with a letter, and can
// only contain letters, digits, hyphens, underscores, colons,
// and periods. https://www.w3.org/TR/html4/types.html#type-name
// Skip the anchor if the symbol name is not a valid anchor
// name. This restriction is lifted in HTML 5.
if (symbol.matches("[a-zA-Z][a-zA-Z0-9_:.-]*")) {
out.append("<a class=\"");
out.append(style_class);
out.append("\" name=\"");
out.append(symbol);
out.append("\"/>");
}
// 2) Create a link that searches for all references to this symbol.
out.append("<a href=\"");
out.append(urlPrefix);
out.append("refs=");
out.append(symbol);
appendProject();
out.append("\" class=\"");
out.append(style_class);
out.append("\" onmouseover=\"onMouseOverSymbol('");
out.append(jsEscapedSymbol);
out.append("', 'def')");
out.append("\">");
out.append(symbol);
out.append("</a>");
} else if (defs != null && defs.occurrences(symbol) == 1) {
// This is a reference to a symbol defined exactly once in this file.
String style_class = "d";
// Generate a direct link to the symbol definition.
out.append("<a class=\"");
out.append(style_class);
out.append("\" href=\"#");
out.append(symbol);
out.append("\" onmouseover=\"onMouseOverSymbol('");
out.append(jsEscapedSymbol);
out.append("', 'defined-in-file')");
out.append("\">");
out.append(symbol);
out.append("</a>");
} else {
// This is a symbol that is not defined in this file, or a symbol
// that is defined more than once in this file. In either case, we
// can't generate a direct link to the definition, so generate a
// link to search for all definitions of that symbol instead.
out.append("<a href=\"");
out.append(urlPrefix);
out.append("defs=");
out.append(symbol);
appendProject();
out.append("\" onmouseover=\"onMouseOverSymbol('");
out.append(jsEscapedSymbol);
out.append("', 'undefined-in-file')");
out.append("\">");
out.append(symbol);
out.append("</a>");
}
}
/**
* Write HTML escape sequence for the specified Unicode character, unless it's an ISO control
* character, in which case it is ignored.
*
* @param c the character to write
* @throws IOException if an error occurs while writing to the stream
*/
protected void writeUnicodeChar(char c) throws IOException {
if (!Character.isISOControl(c)) {
out.append("&#").append(Integer.toString(c)).append(';');
}
}
/**
* Write an e-mail address. The address will be obfuscated if {@link
* RuntimeEnvironment#isObfuscatingEMailAddresses()} returns {@code true}.
*
* @param address the address to write
* @throws IOException if an error occurs while writing to the stream
*/
protected void writeEMailAddress(String address) throws IOException {
if (RuntimeEnvironment.getInstance().isObfuscatingEMailAddresses()) {
out.write(address.replace("@", " (at) "));
} else {
out.write(address);
}
}
/**
* save current yy state to stack
*
* @param newState state id
* @param popString string for the state
*/
public void yypush(int newState, String popString) {
this.stack.push(yystate());
this.stackPopString.push(popString);
yybegin(newState);
}
/**
* pop last state from stack
*
* @throws IOException in case of any I/O problem
*/
public void yypop() throws IOException {
yybegin(this.stack.pop());
String popString = this.stackPopString.pop();
if (popString != null) {
out.write(popString);
}
}
}
/** Create a new cache instance with the default JDBC driver and URL. */
JDBCHistoryCache() {
this(
RuntimeEnvironment.getInstance().getDatabaseDriver(),
RuntimeEnvironment.getInstance().getDatabaseUrl());
}
private void storeHistory(ConnectionResource conn, History history, Repository repository)
throws SQLException {
Integer reposId = null;
Map<String, Integer> authors = null;
Map<String, Integer> files = null;
Map<String, Integer> directories = null;
PreparedStatement addChangeset = null;
PreparedStatement addDirchange = null;
PreparedStatement addFilechange = null;
PreparedStatement addFilemove = null;
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
// return immediately when there is nothing to do
List<HistoryEntry> entries = history.getHistoryEntries();
if (entries.isEmpty()) {
return;
}
for (int i = 0; ; i++) {
try {
if (reposId == null) {
reposId = getRepositoryId(conn, repository);
conn.commit();
}
if (authors == null) {
authors = getAuthors(conn, history, reposId);
conn.commit();
}
if (directories == null || files == null) {
Map<String, Integer> dirs = new HashMap<String, Integer>();
Map<String, Integer> fls = new HashMap<String, Integer>();
getFilesAndDirectories(conn, history, reposId, dirs, fls);
conn.commit();
directories = dirs;
files = fls;
}
if (addChangeset == null) {
addChangeset = conn.getStatement(ADD_CHANGESET);
}
if (addDirchange == null) {
addDirchange = conn.getStatement(ADD_DIRCHANGE);
}
if (addFilechange == null) {
addFilechange = conn.getStatement(ADD_FILECHANGE);
}
if (addFilemove == null) {
addFilemove = conn.getStatement(ADD_FILEMOVE);
}
// Success! Break out of the loop.
break;
} catch (SQLException sqle) {
handleSQLException(sqle, i);
conn.rollback();
}
}
addChangeset.setInt(1, reposId);
// getHistoryEntries() returns the entries in reverse chronological
// order, but we want to insert them in chronological order so that
// their auto-generated identity column can be used as a chronological
// ordering column. Otherwise, incremental updates will make the
// identity column unusable for chronological ordering. So therefore
// we walk the list backwards.
for (ListIterator<HistoryEntry> it = entries.listIterator(entries.size()); it.hasPrevious(); ) {
HistoryEntry entry = it.previous();
retry:
for (int i = 0; ; i++) {
try {
addChangeset.setString(2, entry.getRevision());
addChangeset.setInt(3, authors.get(entry.getAuthor()));
addChangeset.setTimestamp(4, new Timestamp(entry.getDate().getTime()));
String msg = entry.getMessage();
// Truncate the message if it can't fit in a VARCHAR
// (bug #11663).
if (msg.length() > MAX_MESSAGE_LENGTH) {
msg = truncate(msg, MAX_MESSAGE_LENGTH);
}
addChangeset.setString(5, msg);
int changesetId = nextChangesetId.getAndIncrement();
addChangeset.setInt(6, changesetId);
addChangeset.executeUpdate();
// Add one row for each file in FILECHANGES, and one row
// for each path element of the directories in DIRCHANGES.
Set<String> addedDirs = new HashSet<String>();
addDirchange.setInt(1, changesetId);
addFilechange.setInt(1, changesetId);
for (String file : entry.getFiles()) {
// ignore ignored files
String repodir = "";
try {
repodir = env.getPathRelativeToSourceRoot(new File(repository.getDirectoryName()), 0);
} catch (IOException ex) {
Logger.getLogger(JDBCHistoryCache.class.getName()).log(Level.SEVERE, null, ex);
}
String fullPath = toUnixPath(file);
if (!history.isIgnored(file.substring(repodir.length() + 1))) {
int fileId = files.get(fullPath);
addFilechange.setInt(2, fileId);
addFilechange.executeUpdate();
}
String[] pathElts = splitPath(fullPath);
for (int j = 0; j < pathElts.length; j++) {
String dir = unsplitPath(pathElts, j);
// Only add to DIRCHANGES if we haven't already
// added this dir/changeset combination.
if (!addedDirs.contains(dir)) {
addDirchange.setInt(2, directories.get(dir));
addDirchange.executeUpdate();
addedDirs.add(dir);
}
}
}
conn.commit();
// Successfully added the entry. Break out of retry loop.
break retry;
} catch (SQLException sqle) {
handleSQLException(sqle, i);
conn.rollback();
}
}
}
/*
* Special handling for certain files - this is mainly for files which
* have been renamed in Mercurial repository.
* This ensures that their complete history (follow) will be saved.
*/
for (String filename : history.getIgnoredFiles()) {
String file_path = repository.getDirectoryName() + File.separatorChar + filename;
File file = new File(file_path);
String repo_path = file_path.substring(env.getSourceRootPath().length());
History hist;
try {
hist = repository.getHistory(file);
} catch (HistoryException ex) {
Logger.getLogger(JDBCHistoryCache.class.getName()).log(Level.SEVERE, null, ex);
continue;
}
int fileId = files.get(repo_path);
for (HistoryEntry entry : hist.getHistoryEntries()) {
retry:
for (int i = 0; ; i++) {
try {
int changesetId = getIdForRevision(entry.getRevision());
/*
* If the file exists in the changeset, store it in
* the table tracking moves of the file when it had
* one of its precedent names so it can be found
* when performing historyget on directory.
*/
if (entry.getFiles().contains(repo_path)) {
addFilechange.setInt(1, changesetId);
addFilechange.setInt(2, fileId);
addFilechange.executeUpdate();
} else {
addFilemove.setInt(1, changesetId);
addFilemove.setInt(2, fileId);
addFilemove.executeUpdate();
}
conn.commit();
break retry;
} catch (SQLException sqle) {
handleSQLException(sqle, i);
conn.rollback();
}
}
}
}
}
/**
* Update the content of this index database
*
* @throws IOException if an error occurs
* @throws HistoryException if an error occurs when accessing the history
*/
public void update() throws IOException, HistoryException {
synchronized (lock) {
if (running) {
throw new IOException("Indexer already running!");
}
running = true;
interrupted = false;
}
String ctgs = RuntimeEnvironment.getInstance().getCtags();
if (ctgs != null) {
ctags = new Ctags();
ctags.setBinary(ctgs);
}
if (ctags == null) {
log.severe("Unable to run ctags! searching definitions will not work!");
}
if (ctags != null) {
String filename = RuntimeEnvironment.getInstance().getCTagsExtraOptionsFile();
if (filename != null) {
ctags.setCTagsExtraOptionsFile(filename);
}
}
try {
Analyzer analyzer = AnalyzerGuru.getAnalyzer();
IndexWriterConfig iwc = new IndexWriterConfig(SearchEngine.LUCENE_VERSION, analyzer);
iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
// iwc.setRAMBufferSizeMB(256.0); //TODO check what is the sweet spot
writer = new IndexWriter(indexDirectory, iwc);
writer.commit(); // to make sure index exists on the disk
// writer.setMaxFieldLength(RuntimeEnvironment.getInstance().getIndexWordLimit());
if (directories.isEmpty()) {
if (project == null) {
directories.add("");
} else {
directories.add(project.getPath());
}
}
for (String dir : directories) {
File sourceRoot;
if ("".equals(dir)) {
sourceRoot = RuntimeEnvironment.getInstance().getSourceRootFile();
} else {
sourceRoot = new File(RuntimeEnvironment.getInstance().getSourceRootFile(), dir);
}
HistoryGuru.getInstance().ensureHistoryCacheExists(sourceRoot);
String startuid = Util.path2uid(dir, "");
IndexReader reader = DirectoryReader.open(indexDirectory); // open existing index
Terms terms = null;
int numDocs = reader.numDocs();
if (numDocs > 0) {
Fields uFields = MultiFields.getFields(reader); // reader.getTermVectors(0);
terms = uFields.terms(QueryBuilder.U);
}
try {
if (numDocs > 0) {
uidIter = terms.iterator(null);
TermsEnum.SeekStatus stat = uidIter.seekCeil(new BytesRef(startuid), true); // init uid
if (stat == TermsEnum.SeekStatus.END || stat == TermsEnum.SeekStatus.NOT_FOUND) {
uidIter = null;
}
}
// TODO below should be optional, since it traverses the tree once more to get total
// count! :(
int file_cnt = 0;
if (RuntimeEnvironment.getInstance().isPrintProgress()) {
log.log(Level.INFO, "Counting files in {0} ...", dir);
file_cnt = indexDown(sourceRoot, dir, true, 0, 0);
if (log.isLoggable(Level.INFO)) {
log.log(
Level.INFO, "Need to process: {0} files for {1}", new Object[] {file_cnt, dir});
}
}
indexDown(sourceRoot, dir, false, 0, file_cnt);
while (uidIter != null
&& uidIter.term() != null
&& uidIter.term().utf8ToString().startsWith(startuid)) {
removeFile();
uidIter.next();
}
} finally {
reader.close();
}
}
} finally {
if (writer != null) {
try {
writer.prepareCommit();
writer.commit();
writer.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing writer", e);
}
}
if (ctags != null) {
try {
ctags.close();
} catch (IOException e) {
log.log(Level.WARNING, "An error occured while closing ctags process", e);
}
}
synchronized (lock) {
running = false;
}
}
if (!isInterrupted() && isDirty()) {
if (RuntimeEnvironment.getInstance().isOptimizeDatabase()) {
optimize();
}
createSpellingSuggestions();
RuntimeEnvironment env = RuntimeEnvironment.getInstance();
File timestamp = new File(env.getDataRootFile(), "timestamp");
if (timestamp.exists()) {
if (!timestamp.setLastModified(System.currentTimeMillis())) {
log.log(
Level.WARNING,
"Failed to set last modified time on ''{0}'', used for timestamping the index database.",
timestamp.getAbsolutePath());
}
} else {
if (!timestamp.createNewFile()) {
log.log(
Level.WARNING,
"Failed to create file ''{0}'', used for timestamping the index database.",
timestamp.getAbsolutePath());
}
}
}
}
