private void enter_file(String filename, InputStream in) {
if (!fFileList.contains(filename)) {
fFileList.add(filename);
} else {
// System.out.println("File: " + filename + " already included");
}
// Add a marker noting that we're switching to a new file
int file_id = 0;
if (fFileMapper != null) {
file_id = fFileMapper.mapFilePathToId(filename, true);
}
SVFileBuffer in_b = new SVFileBuffer(in);
SVPreProc2InputData in_data = new SVPreProc2InputData(this, in_b, filename, file_id);
add_file_change_info(file_id, in_data.getLineNo());
in_data.setFileTree(new SVDBFileTree(new SVDBFile(filename)));
// Record the current state of the pre-processor
// in_data.fFileTree.fMacroEntryState.putAll(fMacroMap);
if (!fInputStack.empty()) {
SVPreProc2InputData p_data = fInputCurr;
in_data.getFileTree().setParent(p_data.getFileTree());
p_data.getFileTree().addIncludedFileTree(in_data.getFileTree());
}
fInputStack.push(in_data);
fInputCurr = in_data;
}
public SVDBMacroDef findMacro(String name, int lineno) {
SVDBMacroDef m = fMacroMap.get(name);
// SVPreProc2InputData in = fInputCurr;
// TODO: Add macro reference to current file data
fInputCurr.addRefMacro(name, m);
return m;
}
public void unget_ch(int ch) {
fInputCurr.unget_ch(ch);
/*
if (ch != -1 && fCaptureEnabled && fCaptureBuffer.length() > 0) {
fCaptureBuffer.deleteCharAt(fCaptureBuffer.length()-1);
}
*/
}
private void endComment() {
if (!fInComment) {
return;
}
fInComment = false;
String comment = fCommentBuffer.toString();
String title = fDocCommentParser.isDocComment(comment);
if (title != null) {
SVPreProc2InputData in = fInputCurr;
SVDBDocComment doc_comment = new SVDBDocComment(title, comment);
SVDBLocation loc =
new SVDBLocation(
fCommentStart.getFileId(), fCommentStart.getLineNo(), fCommentStart.getLinePos());
doc_comment.setLocation(loc);
if (in.getFileTree() != null) {
in.getFileTree().getSVDBFile().addChildItem(doc_comment);
}
}
}
private void leave_file() {
SVPreProc2InputData old_file = fInputCurr;
// Clean up (if needed) after the macro stack
// Only cleanup after leaving a file where file content (not expanded macro content)
// was present
if (fInputCurr.incPos()) {
cleanup_preproc_leftovers();
}
fInputCurr.leave_file();
if (fIndexStats != null) {
// Update stats
if (old_file.getInput() instanceof SVFileBuffer) {
fIndexStats.incLastIndexFileReadTime(((SVFileBuffer) old_file.getInput()).getReadTime());
}
fIndexStats.incNumLines(old_file.getLineCount());
fIndexStats.incNumProcessedFiles();
}
/*SVPreProc2InputData curr_in =*/ fInputStack.pop();
SVPreProc2InputData new_file = fInputStack.peek();
fInputCurr = new_file;
int file_idx = fFileList.indexOf(new_file.getFileName());
// We will not have registered ANONYMOUS files
if (file_idx != -1) {
int file_id = 0;
if (fFileMapper != null) {
file_id = fFileMapper.mapFilePathToId(new_file.getFileName(), true);
}
// Add a marker noting that we're switching to a new file
SVPreProcOutput.FileChangeInfo file_info =
new SVPreProcOutput.FileChangeInfo(fOutputLen, file_id, new_file.getLineNo());
fFileMap.add(file_info);
}
}
private void update_unprocessed_region(ScanLocation scan_loc, boolean enabled_pre) {
boolean enabled_post = ifdef_enabled();
// TODO: need to identify exclusions due to multiple includes?
// eg:
// `ifndef FILE_INCLUDED
// `define FILE_INCLUDED
//
// `endif
fInputCurr.update_unprocessed_region(scan_loc, enabled_pre, enabled_post);
}
public int get_ch() {
int ch = fInputCurr.get_ch();
if (ch == -1) {
while (fInputStack.size() > 0) {
SVPreProc2InputData in = fInputStack.peek();
ch = in.get_ch();
if (ch == -1 && fInputStack.size() > 1) {
leave_file();
continue;
} else {
// Always leave one on the stack
break;
}
}
}
return ch;
}
private void cleanup_preproc_leftovers() {
int file_id = fInputCurr.getFileId();
while (fPreProcLoc.size() > 0 && fPreProcLoc.peek().getFileId() == file_id) {
// Leftovers indicates unbalanced directives
ScanLocation loc = fPreProcLoc.pop();
fPreProcEn.pop();
SVDBLocation location = new SVDBLocation(loc.getFileId(), loc.getLineNo(), loc.getLinePos());
SVDBMarker m =
new SVDBMarker(
MarkerType.Error,
MarkerKind.UnbalancedDirective,
"Unbalanced pre-processor directive");
if (fDebugEn) {
fLog.debug("Cleanup pre-proc leftover @ " + loc.getLineNo());
}
m.setLocation(location);
if (fInputCurr.getFileTree() != null) {
fInputCurr.getFileTree().fMarkers.add(m);
}
}
}
private void add_macro_reference(String macro) {
SVPreProc2InputData in = fInputCurr;
SVDBMacroDef def = fMacroProvider.findMacro(macro, -1);
in.addReferencedMacro(macro, def);
}
private void handle_preproc_directive() {
int ch = -1;
while ((ch = get_ch()) != -1 && Character.isWhitespace(ch)) {}
ScanLocation scan_loc = getLocation();
String type;
if (ch == -1) {
type = "";
} else {
type = readPreProcIdentifier(ch);
if (type == null) {
type = "";
}
}
if (type.equals("ifdef") || type.equals("ifndef") || type.equals("elsif")) {
// TODO: line number tracking
ch = skipWhite(get_ch());
// TODO: evaluate the expression?
String remainder = readPreProcIdentifier(ch);
if (remainder != null) {
remainder = remainder.trim();
} else {
remainder = "";
}
// Add a new entry to the referenced macros
add_macro_reference(remainder);
if (type.equals("ifdef")) {
if (fDebugEn) {
fLog.debug("ifdef \"" + remainder + "\": " + fDefineProvider.isDefined(remainder, -1));
}
boolean active = fDefineProvider.isDefined(remainder, -1);
enter_ifdef(scan_loc, active);
} else if (type.equals("ifndef")) {
boolean active = !fDefineProvider.isDefined(remainder, -1);
if (fDebugEn) {
fLog.debug("ifndef \"" + remainder + "\": " + active);
}
enter_ifdef(scan_loc, active);
} else { // elsif
boolean active = fDefineProvider.isDefined(remainder, -1);
enter_elsif(scan_loc, active);
}
} else if (type.equals("else")) {
enter_else(scan_loc);
} else if (type.equals("endif")) {
leave_ifdef(scan_loc);
} else if (fIgnoredDirectives.contains(type)) {
// Skip entire line
readLine(get_ch());
} else if (type.equals("define")) {
SVDBLocation location =
new SVDBLocation(scan_loc.getFileId(), scan_loc.getLineNo(), scan_loc.getLinePos());
SVDBMacroDef m = new SVDBMacroDef();
// TODO: save file?
// TODO: line numbers
ch = skipWhite(get_ch());
m.setName(readIdentifier(ch));
m.setLocation(location);
fParamList.clear();
ch = get_ch();
if (ch == '(') {
// Has parameters
List<SVDBMacroDefParam> param_list = new ArrayList<SVDBMacroDefParam>();
do {
ch = skipWhite(get_ch());
if (!(Character.isJavaIdentifierPart(ch))) {
break;
} else {
String p = readIdentifier(ch);
String dflt = null;
ch = skipWhite(get_ch());
if (ch == '=') {
// Read default value
ch = skipWhite(get_ch());
if (ch == '"') {
// String
dflt = readString(ch);
dflt = "\"" + dflt + "\"";
} else {
// Read up to comma or close bracket
startCapture(ch);
while ((ch = get_ch()) != -1 && ch != ',' && ch != ')') {}
unget_ch(ch);
dflt = endCapture();
}
} else {
unget_ch(ch);
}
param_list.add(new SVDBMacroDefParam(p, dflt));
}
ch = skipWhite(get_ch());
} while (ch == ',');
m.setParameters(param_list);
if (ch == ')') {
ch = get_ch();
}
}
// Now, read the remainder of the definition
String define = readLine(ch);
if (define == null) {
define = ""; // define this macro as existing
}
/* We should carry-through the single-line comments. However, this is only
* true in the case of a single-line macro. Multi-line macros get to keep
* their single-line comments
*/
int last_comment;
if ((last_comment = define.lastIndexOf("//")) != -1) {
int lr = define.indexOf('\n', last_comment);
if (lr == -1) {
// Nothing beyond this comment
define = define.substring(0, define.indexOf("//"));
}
}
m.setDef(define);
// TODO: need to warn on re-definition?
if (ifdef_enabled()) {
addMacro(m);
SVPreProc2InputData in = currentRealFile();
// Add the macro to the pre-processor version of the file
if (in != null && in.getFileTree() != null && in.getFileTree().getSVDBFile() != null) {
in.getFileTree().getSVDBFile().addChildItem(m);
in.getFileTree().addToMacroSet(m);
}
}
} else if (type.equals("include")) {
ch = skipWhite(get_ch());
if (ch == '"') {
String inc = readString(ch);
if (inc.length() > 2 && ifdef_enabled()) {
// TODO: need to save the include statement in the pre-proc view?
// Process include and switch to new file
if (fIncFileProvider != null) {
Tuple<String, List<SVDBFileTreeMacroList>> defs;
Tuple<String, InputStream> in;
// TODO: for now, assuming accumulated pre-processor state
// doesn't change file content. This isn't precisely correct.
defs = fIncFileProvider.findCachedIncFile(inc);
SVPreProc2InputData curr_in = fInputCurr;
if (defs != null) {
// Add in the macros from the included file
for (SVDBFileTreeMacroList l : defs.second()) {
for (SVDBMacroDef m : l.getMacroList()) {
addMacro(m);
}
}
// TODO: Need to mark as a 'virtual' include?
SVDBInclude svdb_inc = new SVDBInclude(inc);
svdb_inc.setLocation(
new SVDBLocation(
scan_loc.getFileId(), scan_loc.getLineNo(), scan_loc.getLinePos()));
curr_in.getFileTree().getSVDBFile().addChildItem(svdb_inc);
SVDBFileTree ft_i = new SVDBFileTree(defs.first());
ft_i.setParent(curr_in.getFileTree());
curr_in.getFileTree().addIncludedFileTree(ft_i);
for (SVDBFileTreeMacroList ml : defs.second()) {
for (SVDBMacroDef m : ml.getMacroList()) {
ft_i.addToMacroSet(m);
}
}
} else if ((in = fIncFileProvider.findIncFile(inc)) != null && in.second() != null) {
if (fDebugEn) {
fLog.debug("Switching from file " + curr_in.getFileName() + " to " + in.first());
}
// TODO: Add tracking data for new file
// Find the root file path
String rootfile = fInputStack.get(0).getFileName();
fIncFileProvider.addCachedIncFile(in.first(), rootfile);
SVDBInclude svdb_inc = new SVDBInclude(inc);
svdb_inc.setLocation(
new SVDBLocation(
scan_loc.getFileId(), scan_loc.getLineNo(), scan_loc.getLinePos()));
curr_in.getFileTree().getSVDBFile().addChildItem(svdb_inc);
enter_file(in.first(), in.second());
} else {
SVDBLocation location =
new SVDBLocation(
scan_loc.getFileId(), scan_loc.getLineNo(), scan_loc.getLinePos());
SVDBMarker m =
new SVDBMarker(
MarkerType.Error,
MarkerKind.MissingInclude,
"Failed to find include file " + inc);
m.setLocation(location);
// SVPreProc2InputData curr_in = fInputCurr;
curr_in.getFileTree().fMarkers.add(m);
// TODO: add missing-include error
if (fDebugEn) {
fLog.debug("Failed to find include file " + inc);
}
}
}
} else {
inc = "";
}
}
} else if (type.equals("__LINE__")) {
if (ifdef_enabled()) {
output("" + fInputCurr.getLineNo());
}
} else if (type.equals("__FILE__")) {
if (ifdef_enabled()) {
output("\"" + fInputCurr.getFileName() + "\"");
}
} else if (type.equals("pragma")) {
ch = skipWhite(get_ch());
String id = readIdentifier(ch);
if (id != null) {
// Ignore everything in the 'protected' region.
// Not much we can meaningfully show...
if (id.equals("protect")) {
ch = skipWhite(get_ch());
id = readIdentifier(ch);
if (id != null) {
if (id.equals("begin_protected")) {
enter_ifdef(scan_loc, false);
} else if (id.equals("end_protected")) {
leave_ifdef(scan_loc);
}
}
}
}
} else if (type.equals("protected")) {
enter_ifdef(scan_loc, false);
} else if (type.equals("endprotected")) {
leave_ifdef(scan_loc);
} else if (!type.equals("")) {
// Note: type="" occurs when no identifier followed the tick
// macro expansion.
// TODO: is TmpBuffer available?
fTmpBuffer.setLength(0);
fTmpBuffer.append('`');
fTmpBuffer.append(type);
// If we're in a disabled section, don't try to expand
if (ifdef_enabled()) {
// Read the full string
// Add a reference for this macro
add_macro_reference(type);
boolean is_defined = fDefineProvider.isDefined(type, -1);
if (!is_defined) {
SVDBLocation location =
new SVDBLocation(scan_loc.getFileId(), scan_loc.getLineNo(), scan_loc.getLinePos());
SVDBMarker m =
new SVDBMarker(
MarkerType.Error, MarkerKind.UndefinedMacro, "Macro " + type + " undefined");
m.setLocation(location);
fInputCurr.getFileTree().fMarkers.add(m);
}
if (fDefineProvider.hasParameters(type, fInputCurr.getLineNo()) || !is_defined) {
// Try to read the parameter list
ch = get_ch();
// skip up to new-line or non-whitespace
if (!is_defined) {
// For undefined macros, only search up to end-of-line
while (ch != -1 && Character.isWhitespace(ch) && ch != '\n') {
ch = get_ch();
}
} else {
// For defined macros, skip all whitespace
while (ch != -1 && Character.isWhitespace(ch)) {
ch = get_ch();
}
}
if (ch == '(') {
fTmpBuffer.append((char) ch);
// Read the parameter list
int matchLevel = 1, last_ch = -1;
boolean in_string = false;
do {
ch = get_ch();
if (!in_string) {
if (ch == '(') {
matchLevel++;
} else if (ch == ')') {
matchLevel--;
} else if (ch == '\"' && last_ch != '\\') {
in_string = true;
}
} else if (ch == '\"' && last_ch != '\\') {
in_string = false;
}
if (ch != -1) {
fTmpBuffer.append((char) ch);
}
} while (ch != -1 && matchLevel > 0);
} else if (is_defined) {
unget_ch(ch);
} else {
unget_ch(ch);
}
}
if (!is_defined) {
// Leave a breadcrumb for the lexer
output("`undefined");
} else if (fDefineProvider != null) {
try {
String exp =
fDefineProvider.expandMacro(
fTmpBuffer.toString(), fInputCurr.getFileName(), fInputCurr.getLineNo());
if (fDebugEn) {
fLog.debug("Expansion of \"" + fTmpBuffer.toString() + "\" == " + exp);
}
SVPreProc2InputData in =
new SVPreProc2InputData(
this, new StringInputStream(exp), "ANONYMOUS", fInputCurr.getFileId(), false);
fInputStack.push(in);
fInputCurr = in;
} catch (Exception e) {
/*
System.out.println("Exception while expanding \"" +
fTmpBuffer.toString() + "\" @ " +
getLocation().getFileName() + ":" +
getLocation().getLineNo());
*/
e.printStackTrace();
}
}
}
}
}
public SVDBFileTree getFileTree() {
return fInputCurr.getFileTree();
}
public SVPreProcOutput preprocess() {
int ch, last_ch = -1;
int end_comment1 = -1, end_comment2 = -1;
long start, end;
boolean in_string = false;
boolean ifdef_enabled = true;
boolean found_single_line_comment = false;
start = System.currentTimeMillis();
while ((ch = get_ch()) != -1) {
found_single_line_comment = false;
if (!in_string) {
// Handle comment
if (ch == '/') {
int ch2 = get_ch();
if (ch2 == '/') {
output(' ');
found_single_line_comment = true;
beginComment();
while ((ch = get_ch()) != -1 && ch != '\n' && ch != '\r') {
fCommentBuffer.append((char) ch);
}
fCommentBuffer.append('\n');
// Handle
if (ch == '\r') {
ch = get_ch();
if (ch != '\n') {
unget_ch(ch);
}
}
ch = '\n';
last_ch = ' ';
} else if (ch2 == '*') {
end_comment1 = -1;
end_comment2 = -1;
output(' ');
beginComment();
while ((ch = get_ch()) != -1) {
end_comment1 = end_comment2;
end_comment2 = ch;
if (end_comment1 == '*' && end_comment2 == '/') {
endComment();
break;
} else {
fCommentBuffer.append((char) ch);
}
}
ch = ' ';
last_ch = ' ';
} else {
unget_ch(ch2);
}
}
if (!Character.isWhitespace(ch) && fInComment) {
// Send accumlated comment to observer
endComment();
}
if (ch == '`') {
// Processing an ifdef may affect enablement
handle_preproc_directive();
ifdef_enabled = ifdef_enabled();
if (!ifdef_enabled) {
output(' ');
}
} else {
if (ch == '"' && last_ch != '\\') {
// Enter string
in_string = true;
}
if (ifdef_enabled) {
output((char) ch);
}
}
} else {
// In String
if (ch == '"' && last_ch != '\\') {
in_string = false;
}
if (ifdef_enabled) {
output((char) ch);
}
}
// Consecutive back-slashes convert to
// a single backslash. For tracking purposes,
// convert to space
if (last_ch == '\\' && ch == '\\') {
last_ch = ' ';
} else {
last_ch = ch;
}
if (fInComment && !found_single_line_comment && ch == '\n') {
endComment();
}
}
SVPreProcOutput ret = new SVPreProcOutput(fOutput, null, fFileMap, fFileList);
ret.setFileTree(fInputCurr.getFileTree());
// Clean up after any unbalanced pre-processor directives
cleanup_preproc_leftovers();
// Leave final file
fInputCurr.close();
// Finally, save the full pre-processor state to the final file
/*
last_file.fFileTree.fDefinedMacros.clear();
for (Entry<String, SVDBMacroDef> e : fMacroMap.entrySet()) {
if (!e.getKey().equals("__FILE__") && !e.getKey().equals("__LINE__")) {
last_file.fFileTree.fDefinedMacros.put(e.getKey(), e.getValue());
}
}
*/
end = System.currentTimeMillis();
if (fIndexStats != null) {
if (fInputCurr.getInput() instanceof SVFileBuffer) {
fIndexStats.incLastIndexFileReadTime(((SVFileBuffer) fInputCurr.getInput()).getReadTime());
}
fIndexStats.incNumLines(fInputCurr.getLineCount());
fIndexStats.incLastIndexPreProcessTime(end - start);
fIndexStats.incNumProcessedFiles();
}
return ret;
}
public ScanLocation getLocation() {
return fInputCurr.getLocation();
}
