private void enter_file(String filename, InputStream in) { if (!fFileList.contains(filename)) { fFileList.add(filename); } else { // System.out.println("File: " + filename + " already included"); } // Add a marker noting that we're switching to a new file int file_id = 0; if (fFileMapper != null) { file_id = fFileMapper.mapFilePathToId(filename, true); } SVFileBuffer in_b = new SVFileBuffer(in); SVPreProc2InputData in_data = new SVPreProc2InputData(this, in_b, filename, file_id); add_file_change_info(file_id, in_data.getLineNo()); in_data.setFileTree(new SVDBFileTree(new SVDBFile(filename))); // Record the current state of the pre-processor // in_data.fFileTree.fMacroEntryState.putAll(fMacroMap); if (!fInputStack.empty()) { SVPreProc2InputData p_data = fInputCurr; in_data.getFileTree().setParent(p_data.getFileTree()); p_data.getFileTree().addIncludedFileTree(in_data.getFileTree()); } fInputStack.push(in_data); fInputCurr = in_data; }
public SVDBMacroDef findMacro(String name, int lineno) { SVDBMacroDef m = fMacroMap.get(name); // SVPreProc2InputData in = fInputCurr; // TODO: Add macro reference to current file data fInputCurr.addRefMacro(name, m); return m; }
public void unget_ch(int ch) { fInputCurr.unget_ch(ch); /* if (ch != -1 && fCaptureEnabled && fCaptureBuffer.length() > 0) { fCaptureBuffer.deleteCharAt(fCaptureBuffer.length()-1); } */ }
private void endComment() { if (!fInComment) { return; } fInComment = false; String comment = fCommentBuffer.toString(); String title = fDocCommentParser.isDocComment(comment); if (title != null) { SVPreProc2InputData in = fInputCurr; SVDBDocComment doc_comment = new SVDBDocComment(title, comment); SVDBLocation loc = new SVDBLocation( fCommentStart.getFileId(), fCommentStart.getLineNo(), fCommentStart.getLinePos()); doc_comment.setLocation(loc); if (in.getFileTree() != null) { in.getFileTree().getSVDBFile().addChildItem(doc_comment); } } }
private void leave_file() { SVPreProc2InputData old_file = fInputCurr; // Clean up (if needed) after the macro stack // Only cleanup after leaving a file where file content (not expanded macro content) // was present if (fInputCurr.incPos()) { cleanup_preproc_leftovers(); } fInputCurr.leave_file(); if (fIndexStats != null) { // Update stats if (old_file.getInput() instanceof SVFileBuffer) { fIndexStats.incLastIndexFileReadTime(((SVFileBuffer) old_file.getInput()).getReadTime()); } fIndexStats.incNumLines(old_file.getLineCount()); fIndexStats.incNumProcessedFiles(); } /*SVPreProc2InputData curr_in =*/ fInputStack.pop(); SVPreProc2InputData new_file = fInputStack.peek(); fInputCurr = new_file; int file_idx = fFileList.indexOf(new_file.getFileName()); // We will not have registered ANONYMOUS files if (file_idx != -1) { int file_id = 0; if (fFileMapper != null) { file_id = fFileMapper.mapFilePathToId(new_file.getFileName(), true); } // Add a marker noting that we're switching to a new file SVPreProcOutput.FileChangeInfo file_info = new SVPreProcOutput.FileChangeInfo(fOutputLen, file_id, new_file.getLineNo()); fFileMap.add(file_info); } }
private void update_unprocessed_region(ScanLocation scan_loc, boolean enabled_pre) { boolean enabled_post = ifdef_enabled(); // TODO: need to identify exclusions due to multiple includes? // eg: // `ifndef FILE_INCLUDED // `define FILE_INCLUDED // // `endif fInputCurr.update_unprocessed_region(scan_loc, enabled_pre, enabled_post); }
public int get_ch() { int ch = fInputCurr.get_ch(); if (ch == -1) { while (fInputStack.size() > 0) { SVPreProc2InputData in = fInputStack.peek(); ch = in.get_ch(); if (ch == -1 && fInputStack.size() > 1) { leave_file(); continue; } else { // Always leave one on the stack break; } } } return ch; }
private void cleanup_preproc_leftovers() { int file_id = fInputCurr.getFileId(); while (fPreProcLoc.size() > 0 && fPreProcLoc.peek().getFileId() == file_id) { // Leftovers indicates unbalanced directives ScanLocation loc = fPreProcLoc.pop(); fPreProcEn.pop(); SVDBLocation location = new SVDBLocation(loc.getFileId(), loc.getLineNo(), loc.getLinePos()); SVDBMarker m = new SVDBMarker( MarkerType.Error, MarkerKind.UnbalancedDirective, "Unbalanced pre-processor directive"); if (fDebugEn) { fLog.debug("Cleanup pre-proc leftover @ " + loc.getLineNo()); } m.setLocation(location); if (fInputCurr.getFileTree() != null) { fInputCurr.getFileTree().fMarkers.add(m); } } }
private void add_macro_reference(String macro) { SVPreProc2InputData in = fInputCurr; SVDBMacroDef def = fMacroProvider.findMacro(macro, -1); in.addReferencedMacro(macro, def); }
private void handle_preproc_directive() { int ch = -1; while ((ch = get_ch()) != -1 && Character.isWhitespace(ch)) {} ScanLocation scan_loc = getLocation(); String type; if (ch == -1) { type = ""; } else { type = readPreProcIdentifier(ch); if (type == null) { type = ""; } } if (type.equals("ifdef") || type.equals("ifndef") || type.equals("elsif")) { // TODO: line number tracking ch = skipWhite(get_ch()); // TODO: evaluate the expression? String remainder = readPreProcIdentifier(ch); if (remainder != null) { remainder = remainder.trim(); } else { remainder = ""; } // Add a new entry to the referenced macros add_macro_reference(remainder); if (type.equals("ifdef")) { if (fDebugEn) { fLog.debug("ifdef \"" + remainder + "\": " + fDefineProvider.isDefined(remainder, -1)); } boolean active = fDefineProvider.isDefined(remainder, -1); enter_ifdef(scan_loc, active); } else if (type.equals("ifndef")) { boolean active = !fDefineProvider.isDefined(remainder, -1); if (fDebugEn) { fLog.debug("ifndef \"" + remainder + "\": " + active); } enter_ifdef(scan_loc, active); } else { // elsif boolean active = fDefineProvider.isDefined(remainder, -1); enter_elsif(scan_loc, active); } } else if (type.equals("else")) { enter_else(scan_loc); } else if (type.equals("endif")) { leave_ifdef(scan_loc); } else if (fIgnoredDirectives.contains(type)) { // Skip entire line readLine(get_ch()); } else if (type.equals("define")) { SVDBLocation location = new SVDBLocation(scan_loc.getFileId(), scan_loc.getLineNo(), scan_loc.getLinePos()); SVDBMacroDef m = new SVDBMacroDef(); // TODO: save file? // TODO: line numbers ch = skipWhite(get_ch()); m.setName(readIdentifier(ch)); m.setLocation(location); fParamList.clear(); ch = get_ch(); if (ch == '(') { // Has parameters List<SVDBMacroDefParam> param_list = new ArrayList<SVDBMacroDefParam>(); do { ch = skipWhite(get_ch()); if (!(Character.isJavaIdentifierPart(ch))) { break; } else { String p = readIdentifier(ch); String dflt = null; ch = skipWhite(get_ch()); if (ch == '=') { // Read default value ch = skipWhite(get_ch()); if (ch == '"') { // String dflt = readString(ch); dflt = "\"" + dflt + "\""; } else { // Read up to comma or close bracket startCapture(ch); while ((ch = get_ch()) != -1 && ch != ',' && ch != ')') {} unget_ch(ch); dflt = endCapture(); } } else { unget_ch(ch); } param_list.add(new SVDBMacroDefParam(p, dflt)); } ch = skipWhite(get_ch()); } while (ch == ','); m.setParameters(param_list); if (ch == ')') { ch = get_ch(); } } // Now, read the remainder of the definition String define = readLine(ch); if (define == null) { define = ""; // define this macro as existing } /* We should carry-through the single-line comments. However, this is only * true in the case of a single-line macro. Multi-line macros get to keep * their single-line comments */ int last_comment; if ((last_comment = define.lastIndexOf("//")) != -1) { int lr = define.indexOf('\n', last_comment); if (lr == -1) { // Nothing beyond this comment define = define.substring(0, define.indexOf("//")); } } m.setDef(define); // TODO: need to warn on re-definition? if (ifdef_enabled()) { addMacro(m); SVPreProc2InputData in = currentRealFile(); // Add the macro to the pre-processor version of the file if (in != null && in.getFileTree() != null && in.getFileTree().getSVDBFile() != null) { in.getFileTree().getSVDBFile().addChildItem(m); in.getFileTree().addToMacroSet(m); } } } else if (type.equals("include")) { ch = skipWhite(get_ch()); if (ch == '"') { String inc = readString(ch); if (inc.length() > 2 && ifdef_enabled()) { // TODO: need to save the include statement in the pre-proc view? // Process include and switch to new file if (fIncFileProvider != null) { Tuple<String, List<SVDBFileTreeMacroList>> defs; Tuple<String, InputStream> in; // TODO: for now, assuming accumulated pre-processor state // doesn't change file content. This isn't precisely correct. defs = fIncFileProvider.findCachedIncFile(inc); SVPreProc2InputData curr_in = fInputCurr; if (defs != null) { // Add in the macros from the included file for (SVDBFileTreeMacroList l : defs.second()) { for (SVDBMacroDef m : l.getMacroList()) { addMacro(m); } } // TODO: Need to mark as a 'virtual' include? SVDBInclude svdb_inc = new SVDBInclude(inc); svdb_inc.setLocation( new SVDBLocation( scan_loc.getFileId(), scan_loc.getLineNo(), scan_loc.getLinePos())); curr_in.getFileTree().getSVDBFile().addChildItem(svdb_inc); SVDBFileTree ft_i = new SVDBFileTree(defs.first()); ft_i.setParent(curr_in.getFileTree()); curr_in.getFileTree().addIncludedFileTree(ft_i); for (SVDBFileTreeMacroList ml : defs.second()) { for (SVDBMacroDef m : ml.getMacroList()) { ft_i.addToMacroSet(m); } } } else if ((in = fIncFileProvider.findIncFile(inc)) != null && in.second() != null) { if (fDebugEn) { fLog.debug("Switching from file " + curr_in.getFileName() + " to " + in.first()); } // TODO: Add tracking data for new file // Find the root file path String rootfile = fInputStack.get(0).getFileName(); fIncFileProvider.addCachedIncFile(in.first(), rootfile); SVDBInclude svdb_inc = new SVDBInclude(inc); svdb_inc.setLocation( new SVDBLocation( scan_loc.getFileId(), scan_loc.getLineNo(), scan_loc.getLinePos())); curr_in.getFileTree().getSVDBFile().addChildItem(svdb_inc); enter_file(in.first(), in.second()); } else { SVDBLocation location = new SVDBLocation( scan_loc.getFileId(), scan_loc.getLineNo(), scan_loc.getLinePos()); SVDBMarker m = new SVDBMarker( MarkerType.Error, MarkerKind.MissingInclude, "Failed to find include file " + inc); m.setLocation(location); // SVPreProc2InputData curr_in = fInputCurr; curr_in.getFileTree().fMarkers.add(m); // TODO: add missing-include error if (fDebugEn) { fLog.debug("Failed to find include file " + inc); } } } } else { inc = ""; } } } else if (type.equals("__LINE__")) { if (ifdef_enabled()) { output("" + fInputCurr.getLineNo()); } } else if (type.equals("__FILE__")) { if (ifdef_enabled()) { output("\"" + fInputCurr.getFileName() + "\""); } } else if (type.equals("pragma")) { ch = skipWhite(get_ch()); String id = readIdentifier(ch); if (id != null) { // Ignore everything in the 'protected' region. // Not much we can meaningfully show... if (id.equals("protect")) { ch = skipWhite(get_ch()); id = readIdentifier(ch); if (id != null) { if (id.equals("begin_protected")) { enter_ifdef(scan_loc, false); } else if (id.equals("end_protected")) { leave_ifdef(scan_loc); } } } } } else if (type.equals("protected")) { enter_ifdef(scan_loc, false); } else if (type.equals("endprotected")) { leave_ifdef(scan_loc); } else if (!type.equals("")) { // Note: type="" occurs when no identifier followed the tick // macro expansion. // TODO: is TmpBuffer available? fTmpBuffer.setLength(0); fTmpBuffer.append('`'); fTmpBuffer.append(type); // If we're in a disabled section, don't try to expand if (ifdef_enabled()) { // Read the full string // Add a reference for this macro add_macro_reference(type); boolean is_defined = fDefineProvider.isDefined(type, -1); if (!is_defined) { SVDBLocation location = new SVDBLocation(scan_loc.getFileId(), scan_loc.getLineNo(), scan_loc.getLinePos()); SVDBMarker m = new SVDBMarker( MarkerType.Error, MarkerKind.UndefinedMacro, "Macro " + type + " undefined"); m.setLocation(location); fInputCurr.getFileTree().fMarkers.add(m); } if (fDefineProvider.hasParameters(type, fInputCurr.getLineNo()) || !is_defined) { // Try to read the parameter list ch = get_ch(); // skip up to new-line or non-whitespace if (!is_defined) { // For undefined macros, only search up to end-of-line while (ch != -1 && Character.isWhitespace(ch) && ch != '\n') { ch = get_ch(); } } else { // For defined macros, skip all whitespace while (ch != -1 && Character.isWhitespace(ch)) { ch = get_ch(); } } if (ch == '(') { fTmpBuffer.append((char) ch); // Read the parameter list int matchLevel = 1, last_ch = -1; boolean in_string = false; do { ch = get_ch(); if (!in_string) { if (ch == '(') { matchLevel++; } else if (ch == ')') { matchLevel--; } else if (ch == '\"' && last_ch != '\\') { in_string = true; } } else if (ch == '\"' && last_ch != '\\') { in_string = false; } if (ch != -1) { fTmpBuffer.append((char) ch); } } while (ch != -1 && matchLevel > 0); } else if (is_defined) { unget_ch(ch); } else { unget_ch(ch); } } if (!is_defined) { // Leave a breadcrumb for the lexer output("`undefined"); } else if (fDefineProvider != null) { try { String exp = fDefineProvider.expandMacro( fTmpBuffer.toString(), fInputCurr.getFileName(), fInputCurr.getLineNo()); if (fDebugEn) { fLog.debug("Expansion of \"" + fTmpBuffer.toString() + "\" == " + exp); } SVPreProc2InputData in = new SVPreProc2InputData( this, new StringInputStream(exp), "ANONYMOUS", fInputCurr.getFileId(), false); fInputStack.push(in); fInputCurr = in; } catch (Exception e) { /* System.out.println("Exception while expanding \"" + fTmpBuffer.toString() + "\" @ " + getLocation().getFileName() + ":" + getLocation().getLineNo()); */ e.printStackTrace(); } } } } }
public SVDBFileTree getFileTree() { return fInputCurr.getFileTree(); }
public SVPreProcOutput preprocess() { int ch, last_ch = -1; int end_comment1 = -1, end_comment2 = -1; long start, end; boolean in_string = false; boolean ifdef_enabled = true; boolean found_single_line_comment = false; start = System.currentTimeMillis(); while ((ch = get_ch()) != -1) { found_single_line_comment = false; if (!in_string) { // Handle comment if (ch == '/') { int ch2 = get_ch(); if (ch2 == '/') { output(' '); found_single_line_comment = true; beginComment(); while ((ch = get_ch()) != -1 && ch != '\n' && ch != '\r') { fCommentBuffer.append((char) ch); } fCommentBuffer.append('\n'); // Handle if (ch == '\r') { ch = get_ch(); if (ch != '\n') { unget_ch(ch); } } ch = '\n'; last_ch = ' '; } else if (ch2 == '*') { end_comment1 = -1; end_comment2 = -1; output(' '); beginComment(); while ((ch = get_ch()) != -1) { end_comment1 = end_comment2; end_comment2 = ch; if (end_comment1 == '*' && end_comment2 == '/') { endComment(); break; } else { fCommentBuffer.append((char) ch); } } ch = ' '; last_ch = ' '; } else { unget_ch(ch2); } } if (!Character.isWhitespace(ch) && fInComment) { // Send accumlated comment to observer endComment(); } if (ch == '`') { // Processing an ifdef may affect enablement handle_preproc_directive(); ifdef_enabled = ifdef_enabled(); if (!ifdef_enabled) { output(' '); } } else { if (ch == '"' && last_ch != '\\') { // Enter string in_string = true; } if (ifdef_enabled) { output((char) ch); } } } else { // In String if (ch == '"' && last_ch != '\\') { in_string = false; } if (ifdef_enabled) { output((char) ch); } } // Consecutive back-slashes convert to // a single backslash. For tracking purposes, // convert to space if (last_ch == '\\' && ch == '\\') { last_ch = ' '; } else { last_ch = ch; } if (fInComment && !found_single_line_comment && ch == '\n') { endComment(); } } SVPreProcOutput ret = new SVPreProcOutput(fOutput, null, fFileMap, fFileList); ret.setFileTree(fInputCurr.getFileTree()); // Clean up after any unbalanced pre-processor directives cleanup_preproc_leftovers(); // Leave final file fInputCurr.close(); // Finally, save the full pre-processor state to the final file /* last_file.fFileTree.fDefinedMacros.clear(); for (Entry<String, SVDBMacroDef> e : fMacroMap.entrySet()) { if (!e.getKey().equals("__FILE__") && !e.getKey().equals("__LINE__")) { last_file.fFileTree.fDefinedMacros.put(e.getKey(), e.getValue()); } } */ end = System.currentTimeMillis(); if (fIndexStats != null) { if (fInputCurr.getInput() instanceof SVFileBuffer) { fIndexStats.incLastIndexFileReadTime(((SVFileBuffer) fInputCurr.getInput()).getReadTime()); } fIndexStats.incNumLines(fInputCurr.getLineCount()); fIndexStats.incLastIndexPreProcessTime(end - start); fIndexStats.incNumProcessedFiles(); } return ret; }
public ScanLocation getLocation() { return fInputCurr.getLocation(); }