public Lexer(ParserInputSource input, EventHandler eventHandler, boolean parsePython) {
this(
input,
eventHandler,
parsePython,
LineNumberTable.create(input.getContent(), input.getPath()));
}
public Lexer(ParserInputSource input, EventHandler eventHandler) {
this(
input,
eventHandler,
/*parsePython=*/ false,
LineNumberTable.create(input.getContent(), input.getPath()));
}
/**
* Parses some String input without a supporting file, returning statements and comments.
*
* @param inputLines a list of lines of code
*/
@VisibleForTesting
Parser.ParseResult parseFileWithComments(String... inputLines) {
ParserInputSource input = ParserInputSource.create(Joiner.on("\n").join(inputLines), null);
return isSkylark
? Parser.parseFileForSkylark(input, eventHandler, new ValidationEnvironment(this))
: Parser.parseFile(input, eventHandler, /*parsePython=*/ false);
}
/**
* Constructs a lexer which tokenizes the contents of the specified InputBuffer. Any errors during
* lexing are reported on "handler".
*/
public Lexer(
ParserInputSource input,
EventHandler eventHandler,
boolean parsePython,
LineNumberTable lineNumberTable) {
this.buffer = input.getContent();
// Empirical measurements show roughly 1 token per 8 characters in buffer.
this.tokens = Lists.newArrayListWithExpectedSize(buffer.length / 8);
this.pos = 0;
this.parsePython = parsePython;
this.eventHandler = eventHandler;
this.locationInfo = new LocationInfo(input.getPath(), lineNumberTable);
indentStack.push(0);
long startTime = Profiler.nanoTimeMaybe();
tokenize();
Profiler.instance().logSimpleTask(startTime, ProfilerTask.SKYLARK_LEXER, getFilename());
}
