protected ParseRequest<TState> adjustParseSpan(OffsetRegion span) {
    int start = span.getStart();
    int end = span.getEnd();

    if (firstDirtyLine != null) {
      int firstDirtyLineOffset =
          snapshot.findLineFromLineNumber(firstDirtyLine).getStart().getOffset();
      start = Math.min(start, firstDirtyLineOffset);
    }

    TState state = null;
    int startLine = snapshot.findLineNumber(start);
    while (startLine > 0) {
      TState lineState = lineStates.get(startLine - 1);
      if (!lineState.getIsMultiLineToken()) {
        state = lineState;
        break;
      }

      startLine--;
    }

    if (startLine == 0) {
      state = getStartState();
    }

    start = snapshot.findLineFromLineNumber(startLine).getStart().getOffset();
    int length = end - start;
    ParseRequest<TState> request = new ParseRequest<>(new OffsetRegion(start, length), state);
    return request;
  }
  protected CharStream createInputStream(OffsetRegion span) throws BadLocationException {
    CharStream input;
    if (span.getLength() > 1000) {
      input = new DocumentSnapshotCharStream(snapshot, span);
    } else {
      input = new DocumentSnapshotCharStream(snapshot);
    }

    input.seek(span.getStart());
    return input;
  }
  protected Collection<TaggedPositionRegion<TokenTag<Token>>> getTagsForToken(Token token) {
    TokenTag<Token> tag = highlightToken(token);
    if (tag != null) {
      return Collections.<TaggedPositionRegion<TokenTag<Token>>>singleton(
          new BaseTaggedPositionRegion<>(
              new SnapshotPositionRegion(
                  snapshot,
                  OffsetRegion.fromBounds(token.getStartIndex(), token.getStopIndex() + 1)),
              tag));
    }

    return Collections.emptyList();
  }
  public List<TaggedPositionRegion<TokenTag<Token>>> getHighlights(int startOffset, int endOffset) {
    List<TaggedPositionRegion<TokenTag<Token>>> tags = new ArrayList<>();
    boolean updateOffsets = true;

    if (endOffset == Integer.MAX_VALUE) {
      endOffset = snapshot.length();
    }

    OffsetRegion span = OffsetRegion.fromBounds(startOffset, endOffset);

    if (failedTimeout) {
      return tags;
    }

    boolean spanExtended = false;

    int extendMultiLineSpanToLine = 0;
    OffsetRegion extendedSpan = span;

    synchronized (lock) {
      OffsetRegion requestedSpan = span;

      ParseRequest<TState> request = adjustParseSpan(span);
      TState startState = request.getState();
      span = request.getRegion();

      CharStream input;
      try {
        input = createInputStream(span);
      } catch (BadLocationException ex) {
        LOGGER.log(Level.WARNING, ex.getMessage(), ex);
        return tags;
      }

      TokenSourceWithStateV4<TState> lexer = createLexer(input, startState);
      lexer.setTokenFactory(new DocumentSnapshotTokenFactory(getEffectiveTokenSource(lexer)));

      Token previousToken = null;
      boolean previousTokenEndsLine = false;

      /* this is held outside the loop because only tokens which end at the end of a line
       * impact its value.
       */
      boolean lineStateChanged = false;

      while (true) {
        // TODO: perform this under a read lock
        Token token = lexer.nextToken();

        // The latter is true for EOF token with span.getEnd() at the end of the document
        boolean inBounds =
            token.getStartIndex() < span.getEnd() || token.getStopIndex() < span.getEnd();

        if (updateOffsets) {
          int startLineCurrent;
          if (token.getType() == Token.EOF) startLineCurrent = snapshot.getLineCount();
          else startLineCurrent = snapshot.findLineNumber(token.getStartIndex());

          // endLinePrevious is the line number the previous token ended on
          int endLinePrevious;
          if (previousToken != null)
            endLinePrevious = snapshot.findLineNumber(previousToken.getStopIndex() + 1);
          else endLinePrevious = snapshot.findLineNumber(span.getStart()) - 1;

          if (startLineCurrent > endLinePrevious + 1
              || (startLineCurrent == endLinePrevious + 1 && !previousTokenEndsLine)) {
            int firstMultilineLine = endLinePrevious;
            if (previousToken == null || previousTokenEndsLine) firstMultilineLine++;

            for (int i = firstMultilineLine; i < startLineCurrent; i++) {
              if (!lineStates.get(i).getIsMultiLineToken() || lineStateChanged)
                extendMultiLineSpanToLine = i + 1;

              if (inBounds) setLineState(i, lineStates.get(i).createMultiLineState());
            }
          }
        }

        if (token.getType() == Token.EOF) break;

        if (updateOffsets && isMultiLineToken(lexer, token)) {
          int startLine = snapshot.findLineNumber(token.getStartIndex());
          int stopLine = snapshot.findLineNumber(token.getStopIndex() + 1);
          for (int i = startLine; i < stopLine; i++) {
            if (!lineStates.get(i).getIsMultiLineToken()) extendMultiLineSpanToLine = i + 1;

            if (inBounds) setLineState(i, lineStates.get(i).createMultiLineState());
          }
        }

        boolean tokenEndsLine = tokenEndsAtEndOfLine(lexer, token);
        if (updateOffsets && tokenEndsLine) {
          TState stateAtEndOfLine = lexer.getCurrentState();
          int line = snapshot.findLineNumber(token.getStopIndex() + 1);
          lineStateChanged =
              lineStates.get(line).getIsMultiLineToken()
                  || !lineStates.get(line).equals(stateAtEndOfLine);

          // even if the state didn't change, we call SetLineState to make sure the
          // _first/_lastChangedLine values get updated.
          // have to check bounds for this one or the editor might not get an update (if the token
          // ends a line)
          if (updateOffsets && inBounds) setLineState(line, stateAtEndOfLine);

          if (lineStateChanged) {
            if (line < snapshot.getLineCount() - 1) {
              /* update the span's end position or the line state change won't be reflected
               * in the editor
               */
              int endPosition =
                  line < snapshot.getLineCount() - 2
                      ? snapshot.findLineFromLineNumber(line + 2).getStart().getOffset()
                      : snapshot.length();
              if (endPosition > extendedSpan.getEnd()) {
                spanExtended = true;
                extendedSpan = OffsetRegion.fromBounds(extendedSpan.getStart(), endPosition);
              }
            }
          }
        }

        if (token.getStartIndex() >= span.getEnd()) {
          break;
        }

        previousToken = token;
        previousTokenEndsLine = tokenEndsLine;

        if (token.getStopIndex() < requestedSpan.getStart()) {
          continue;
        }

        Collection<TaggedPositionRegion<TokenTag<Token>>> tokenClassificationSpans =
            getTagsForToken(token);
        if (tokenClassificationSpans != null) {
          tags.addAll(tokenClassificationSpans);
        }

        if (!inBounds) {
          break;
        }
      }
    }

    if (updateOffsets && extendMultiLineSpanToLine > 0) {
      int endPosition =
          extendMultiLineSpanToLine < snapshot.getLineCount() - 1
              ? snapshot
                  .findLineFromLineNumber(extendMultiLineSpanToLine + 1)
                  .getStart()
                  .getOffset()
              : snapshot.length();
      if (endPosition > extendedSpan.getEnd()) {
        spanExtended = true;
        extendedSpan = OffsetRegion.fromBounds(extendedSpan.getStart(), endPosition);
      }
    }

    if (updateOffsets && spanExtended) {
      /* Subtract 1 from each of these because the spans include the line break on their last
       * line, forcing it to appear as the first position on the following line.
       */
      assert extendedSpan.getEnd() > span.getEnd();
      int firstLine = snapshot.findLineNumber(span.getEnd());
      int lastLine = snapshot.findLineNumber(extendedSpan.getEnd()) - 1;
      // when considering the last line of a document, span and extendedSpan may end on the same
      // line
      forceRehighlightLines(firstLine, Math.max(firstLine, lastLine));
    }

    return tags;
  }