public Lexer(ParserInputSource input, EventHandler eventHandler, boolean parsePython) {
this(
input,
eventHandler,
parsePython,
LineNumberTable.create(input.getContent(), input.getPath()));
}
public Lexer(ParserInputSource input, EventHandler eventHandler) {
this(
input,
eventHandler,
/*parsePython=*/ false,
LineNumberTable.create(input.getContent(), input.getPath()));
}
/**
* Constructs a lexer which tokenizes the contents of the specified InputBuffer. Any errors during
* lexing are reported on "handler".
*/
public Lexer(
ParserInputSource input,
EventHandler eventHandler,
boolean parsePython,
LineNumberTable lineNumberTable) {
this.buffer = input.getContent();
// Empirical measurements show roughly 1 token per 8 characters in buffer.
this.tokens = Lists.newArrayListWithExpectedSize(buffer.length / 8);
this.pos = 0;
this.parsePython = parsePython;
this.eventHandler = eventHandler;
this.locationInfo = new LocationInfo(input.getPath(), lineNumberTable);
indentStack.push(0);
long startTime = Profiler.nanoTimeMaybe();
tokenize();
Profiler.instance().logSimpleTask(startTime, ProfilerTask.SKYLARK_LEXER, getFilename());
}
