示例#1
0
  @NotNull
  public ASTNode parse(IElementType root, PsiBuilder builder) {
    if (ALLOW_ONLY_ONE_THREAD) {
      try {
        SEMAPHORE.acquire();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
    // builder.setDebugMode(true);
    final PsiBuilder.Marker rootMarker = builder.mark();
    if (!builder.eof()) { // Empty file is not an error
      final GLSLParsing theRealParser = new GLSLParsing(builder);

      theRealParser.parseTranslationUnit();
      while (!builder.eof()) // exhaust the file if unable to parse everything
      builder.advanceLexer();
    }

    rootMarker.done(root);

    if (ALLOW_ONLY_ONE_THREAD) {
      SEMAPHORE.release();
    }
    return builder.getTreeBuilt();
  }
 protected static boolean recoverTo(PsiBuilder b, IElementType toElement, String errorMessage) {
   // recover bad code
   PsiBuilder.Marker errorMarker = b.mark();
   while (!b.eof() && b.getTokenType() != toElement) {
     b.advanceLexer();
     ;
   }
   errorMarker.error(errorMessage);
   return b.eof();
 }
  public static void parseDefine(PsiBuilder builder) {
    PsiBuilder.Marker maker = builder.mark();

    // #define
    advanceLexerAndSkipLines(builder);

    // var name
    doneOneToken(builder, CPsiCompilerVariable.class);

    PsiBuilder.Marker valueMarker = builder.mark();

    while (!builder.eof()) {
      if (builder.getTokenType() == NEW_LINE) break;

      if (builder.getTokenType() == NEXT_LINE) builder.advanceLexer();

      builder.advanceLexer();
    }

    done(valueMarker, CPsiSharpDefineValue.class);

    done(maker, CPsiSharpDefine.class);

    skipLines(builder);
  }
 private static boolean reportError(
     PsiBuilder builder,
     ErrorState state,
     Frame frame,
     IElementType elementType,
     boolean force,
     boolean advance) {
   String expectedText = state.getExpectedText(builder);
   boolean notEmpty = StringUtil.isNotEmpty(expectedText);
   if (force || notEmpty || advance) {
     String gotText =
         builder.eof()
             ? "unexpected end of file"
             : notEmpty
                 ? "got '" + builder.getTokenText() + "'"
                 : "'" + builder.getTokenText() + "' unexpected";
     String message = expectedText + gotText;
     if (advance) {
       PsiBuilder.Marker mark = builder.mark();
       builder.advanceLexer();
       mark.error(message);
     } else if (!force) {
       PsiBuilder.Marker extensionMarker = null;
       IElementType extensionTokenType = null;
       PsiBuilderImpl.ProductionMarker latestDoneMarker =
           elementType == null
               ? null
               : (PsiBuilderImpl.ProductionMarker) builder.getLatestDoneMarker();
       if (latestDoneMarker != null
           && frame.position >= latestDoneMarker.getStartIndex()
           && frame.position <= latestDoneMarker.getEndIndex()) {
         extensionMarker = ((PsiBuilder.Marker) latestDoneMarker).precede();
         extensionTokenType = latestDoneMarker.getTokenType();
         ((PsiBuilder.Marker) latestDoneMarker).drop();
       }
       builder.error(message);
       if (extensionMarker != null) extensionMarker.done(extensionTokenType);
     } else {
       builder.error(message);
     }
     builder.eof(); // skip whitespaces
     frame.errorReportedAt = builder.rawTokenIndex();
     return true;
   }
   return false;
 }
  @NotNull
  public static PsiBuilder.Marker parseArgumentList(final PsiBuilder builder) {
    final PsiBuilder.Marker list = builder.mark();
    builder.advanceLexer();

    boolean first = true;
    while (true) {
      final IElementType tokenType = builder.getTokenType();
      if (first && (ARGS_LIST_END.contains(tokenType) || builder.eof())) break;
      if (!first && !ARGS_LIST_CONTINUE.contains(tokenType)) break;

      boolean hasError = false;
      if (!first) {
        if (builder.getTokenType() == JavaTokenType.COMMA) {
          builder.advanceLexer();
        } else {
          hasError = true;
          error(builder, JavaErrorMessages.message("expected.comma.or.rparen"));
          emptyExpression(builder);
        }
      }
      first = false;

      final PsiBuilder.Marker arg = parse(builder);
      if (arg == null) {
        if (!hasError) {
          error(builder, JavaErrorMessages.message("expected.expression"));
          emptyExpression(builder);
        }
        if (!ARGS_LIST_CONTINUE.contains(builder.getTokenType())) break;
        if (builder.getTokenType() != JavaTokenType.COMMA && !builder.eof()) {
          builder.advanceLexer();
        }
      }
    }

    final boolean closed =
        JavaParserUtil.expectOrError(
            builder, JavaTokenType.RPARENTH, JavaErrorMessages.message("expected.rparen"));

    list.done(JavaElementType.EXPRESSION_LIST);
    if (!closed) {
      list.setCustomEdgeTokenBinders(null, GREEDY_RIGHT_EDGE_PROCESSOR);
    }
    return list;
  }
示例#6
0
 public static boolean parse(final PsiBuilder builder) {
   boolean result = false;
   while (!builder.eof() && SEPARATOR.contains(builder.getTokenType())) {
     builder.advanceLexer();
     result = true;
   }
   return result;
 }
  @Nullable
  private PsiBuilder.Marker parseLambdaAfterParenth(
      final PsiBuilder builder, @Nullable final PsiBuilder.Marker typeList) {
    final boolean isLambda;
    final boolean isTyped;

    final IElementType nextToken1 = builder.lookAhead(1);
    final IElementType nextToken2 = builder.lookAhead(2);
    if (nextToken1 == JavaTokenType.RPARENTH && nextToken2 == JavaTokenType.ARROW) {
      isLambda = true;
      isTyped = false;
    } else if (nextToken1 == JavaTokenType.AT
        || ElementType.MODIFIER_BIT_SET.contains(nextToken1)
        || ElementType.PRIMITIVE_TYPE_BIT_SET.contains(nextToken1)) {
      isLambda = true;
      isTyped = true;
    } else if (nextToken1 == JavaTokenType.IDENTIFIER) {
      if (nextToken2 == JavaTokenType.COMMA
          || nextToken2 == JavaTokenType.RPARENTH && builder.lookAhead(3) == JavaTokenType.ARROW) {
        isLambda = true;
        isTyped = false;
      } else if (nextToken2 == JavaTokenType.ARROW) {
        isLambda = false;
        isTyped = false;
      } else {
        boolean arrow = false;

        final PsiBuilder.Marker marker = builder.mark();
        while (!builder.eof()) {
          builder.advanceLexer();
          final IElementType tokenType = builder.getTokenType();
          if (tokenType == JavaTokenType.ARROW) {
            arrow = true;
            break;
          }
          if (tokenType == JavaTokenType.RPARENTH) {
            arrow = builder.lookAhead(1) == JavaTokenType.ARROW;
            break;
          } else if (tokenType == JavaTokenType.LPARENTH
              || tokenType == JavaTokenType.SEMICOLON
              || tokenType == JavaTokenType.LBRACE
              || tokenType == JavaTokenType.RBRACE) {
            break;
          }
        }
        marker.rollbackTo();

        isLambda = arrow;
        isTyped = true;
      }
    } else {
      isLambda = false;
      isTyped = false;
    }

    return isLambda ? parseLambdaExpression(builder, isTyped, typeList) : null;
  }
  private static void addVariant(PsiBuilder builder, ErrorState state, Object o) {
    builder.eof(); // skip whitespaces
    addVariantInner(state, builder.rawTokenIndex(), o);

    CompletionState completionState = state.completionState;
    if (completionState != null && state.predicateSign) {
      addCompletionVariant(builder, completionState, o);
    }
  }
  public static void parseIf(PsiBuilder builder) {
    PsiBuilder.Marker marker = builder.mark();

    builder.advanceLexer();

    if (builder.getTokenType() == IDENTIFIER) doneOneToken(builder, CPsiCompilerVariable.class);
    else error(builder, "IDENTIFIER.expected");

    builder.advanceLexer();

    if (builder.getTokenType() == NEW_LINE) builder.advanceLexer();

    skipLines(builder);

    if (builder.getTokenType() != S_ENDIF_KEYWORD) {
      PsiBuilder.Marker bodyMarker = builder.mark();

      while (!builder.eof()) {
        parse(builder, 0);

        if (builder.getTokenType() == S_ENDIF_KEYWORD || builder.getTokenType() == S_ELSE_KEYWORD)
          break;
      }

      done(bodyMarker, CPsiSharpIfBody.class);

      if (builder.getTokenType() == S_ELSE_KEYWORD) {
        builder.advanceLexer();

        PsiBuilder.Marker elseBody = builder.mark();

        while (!builder.eof()) {
          parse(builder, 0);

          if (builder.getTokenType() == S_ENDIF_KEYWORD || builder.getTokenType() == S_ELSE_KEYWORD)
            break;
        }
        done(elseBody, CPsiSharpIfBody.class);
      }
    }
    checkMatchesWithoutLines(builder, S_ENDIF_KEYWORD, "S_END_IF.expected");

    done(marker, CPsiSharpIfDef.class);
  }
 private static boolean addVariantSmart(PsiBuilder builder, Object token, boolean force) {
   ErrorState state = ErrorState.get(builder);
   // skip FIRST check in completion mode
   if (state.completionState != null && !force) return false;
   builder.eof();
   if (!state.suppressErrors && state.predicateCount < 2) {
     addVariant(builder, state, token);
   }
   return true;
 }
示例#11
0
  // @todo this is really raw
  public static boolean parseVariableAttributes(PsiBuilder b, int l) {

    PsiBuilder.Marker m = null;
    while (!b.eof() && !VARIABLE_ATTRIBUTE_STOP_TOKENS.contains(b.getTokenType())) {
      if (m == null) m = b.mark();
      b.advanceLexer();
    }
    if (m != null) m.collapse(VAR_ATTRIBUTE);

    return true;
  }
示例#12
0
  /**
   * Smart semi checker decides if we need semi here
   *
   * @param b Perl builder
   * @param l Parsing level
   * @return checking result
   */
  public static boolean statementSemi(PsiBuilder b, int l) {
    IElementType tokenType = b.getTokenType();
    if (tokenType == SEMICOLON) return consumeToken(b, SEMICOLON);
    else if (tokenType == EMBED_MARKER_SEMICOLON) return consumeToken(b, EMBED_MARKER_SEMICOLON);
    else if (tokenType == RIGHT_BRACE || tokenType == REGEX_QUOTE_CLOSE) return true;
    else if (b.eof()) // eof
    return true;

    b.mark().error("Semicolon expected");

    return true;
  }
  // Tries to parse the whole tree
  private void parseTree(PsiBuilder builder) {
    IElementType type;

    while (!builder.eof()) {
      type = builder.getTokenType();
      beforeConsume(builder, type);
      builder.advanceLexer();
      afterConsume(builder, type);
      nextToken = null;
      nextTokenText = null;
    }
    eofCleanup();
  }
 private static void close_frame_impl_(
     ErrorState state,
     Frame frame,
     PsiBuilder builder,
     PsiBuilder.Marker marker,
     IElementType elementType,
     boolean result,
     boolean pinned) {
   if (elementType != null && marker != null) {
     if ((frame.modifiers & _COLLAPSE_) != 0) {
       PsiBuilderImpl.ProductionMarker last =
           result || pinned
               ? (PsiBuilderImpl.ProductionMarker) builder.getLatestDoneMarker()
               : null;
       if (last != null
           && last.getStartIndex() == frame.position
           && state.typeExtends(last.getTokenType(), elementType)) {
         IElementType resultType = last.getTokenType();
         ((PsiBuilder.Marker) last).drop();
         marker.done(resultType);
         return;
       }
     }
     if (result || pinned) {
       if ((frame.modifiers & _UPPER_) != 0) {
         marker.drop();
         frame.parentFrame.elementType = elementType;
       } else if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) {
         marker.done(elementType);
         frame.leftMarker.precede().done(((LighterASTNode) frame.leftMarker).getTokenType());
         frame.leftMarker.drop();
       } else if ((frame.modifiers & _LEFT_) != 0 && frame.leftMarker != null) {
         marker.drop();
         frame.leftMarker.precede().done(elementType);
       } else {
         if (frame.level == 0) builder.eof(); // skip whitespaces
         marker.done(elementType);
       }
     } else {
       close_marker_impl_(frame, marker, null, false);
     }
   } else if (result || pinned) {
     if (marker != null) marker.drop();
     if ((frame.modifiers & _LEFT_INNER_) != 0 && frame.leftMarker != null) {
       frame.leftMarker.precede().done(((LighterASTNode) frame.leftMarker).getTokenType());
       frame.leftMarker.drop();
     }
   } else {
     close_marker_impl_(frame, marker, null, false);
   }
 }
示例#15
0
  public static boolean parseSubPrototype(PsiBuilder b, int l) {
    PsiBuilder.Marker m = null;

    IElementType tokenType = b.getTokenType();
    while (!b.eof() && (tokenType != RIGHT_PAREN)) {
      if (m == null) m = b.mark();

      b.advanceLexer();
      tokenType = b.getTokenType();
    }
    if (m != null) m.collapse(SUB_PROTOTYPE_TOKEN);

    return true;
  }
  protected static boolean parsePerlBlock(
      PsiBuilder b, int l, IElementType closeToken, IElementType blockToken) {
    b.advanceLexer();
    PsiBuilder.Marker abstractBlockMarker = b.mark();

    while (!b.eof() && b.getTokenType() != closeToken) {
      if (!PerlParserImpl.file_item(b, l)) {
        break;
      }
    }
    abstractBlockMarker.done(blockToken);
    abstractBlockMarker.setCustomEdgeTokenBinders(
        WhitespacesBinders.GREEDY_LEFT_BINDER, WhitespacesBinders.GREEDY_RIGHT_BINDER);
    return endOrRecover(b, closeToken);
  }
示例#17
0
  @NotNull
  @Override
  public ASTNode parse(IElementType iElementType, PsiBuilder builder) {

    PsiBuilder.Marker rootMarker = builder.mark();

    // TODO Actual parsing not implemented

    while (!builder.eof()) {
      builder.advanceLexer();
    }

    rootMarker.done(iElementType);
    return builder.getTreeBuilt();
  }
 private static boolean reportError(ErrorState state, PsiBuilder builder_, boolean force) {
   String expectedText = state.getExpectedText(builder_);
   boolean notEmpty = StringUtil.isNotEmpty(expectedText);
   if (force || notEmpty) {
     final String gotText =
         builder_.eof()
             ? "unexpected end of file"
             : notEmpty
                 ? "got '" + builder_.getTokenText() + "'"
                 : "'" + builder_.getTokenText() + "' unexpected";
     builder_.error(expectedText + gotText);
     return true;
   }
   return false;
 }
示例#19
0
  // @todo this is really raw
  public static boolean parseSubAttributes(PsiBuilder b, int l) {

    PsiBuilder.Marker m = null;
    IElementType tokenType = b.getTokenType();
    while (!b.eof()
        && tokenType != LEFT_BRACE
        && tokenType != SEMICOLON
        && tokenType != EMBED_MARKER_SEMICOLON) {
      if (m == null) m = b.mark();
      b.advanceLexer();
      tokenType = b.getTokenType();
    }
    if (m != null) m.collapse(SUB_ATTRIBUTE);

    return true;
  }
  public static void parse(@NotNull PsiBuilder builder, int f) {
    while (!builder.eof()) {
      skipLines(builder);

      if (builder.getTokenType() == S_INCLUDE_KEYWORD) parseInclude(builder);
      else if (builder.getTokenType() == S_DEFINE_KEYWORD) parseDefine(builder);
      else if (builder.getTokenType() == S_IFNDEF_KEYWORD
          || builder.getTokenType() == S_IFDEF_KEYWORD) parseIf(builder);
      else if (builder.getTokenType() == S_ENDIF_KEYWORD
          || builder.getTokenType() == S_ELSE_KEYWORD) {
        if (isSet(f, EAT_LAST_END_IF)) {
          error(builder, "S_IFDEF.or.S_IFNDEF.expected");
          advanceLexerAndSkipLines(builder);
        } else break;
      } else builder.advanceLexer();
    }
  }
示例#21
0
  @NotNull
  public ASTNode parse(IElementType root, PsiBuilder builder) {

    final PsiBuilder.Marker rootMarker = builder.mark();

    while (!builder.eof()) {
      IElementType token = builder.getTokenType();

      if (token != null) {
        builder.mark().done(token);
      }

      builder.advanceLexer();
    }

    rootMarker.done(root);
    return builder.getTreeBuilt();
  }
  private static void addCompletionVariant(
      @NotNull PsiBuilder builder, @NotNull CompletionState completionState, Object o) {
    int offset = builder.getCurrentOffset();
    if (!builder.eof() && offset == builder.rawTokenTypeStart(1))
      return; // suppress for zero-length tokens

    boolean add = false;
    int diff = completionState.offset - offset;
    String text = completionState.convertItem(o);
    int length = text == null ? 0 : text.length();
    if (length == 0) return;
    if (diff == 0) {
      add = true;
    } else if (diff > 0 && diff <= length) {
      CharSequence fragment = builder.getOriginalText().subSequence(offset, completionState.offset);
      add = completionState.prefixMatches(fragment.toString(), text);
    } else if (diff < 0) {
      for (int i = -1; ; i--) {
        IElementType type = builder.rawLookup(i);
        int tokenStart = builder.rawTokenTypeStart(i);
        if (isWhitespaceOrComment(builder, type)) {
          diff = completionState.offset - tokenStart;
        } else if (type != null && tokenStart < completionState.offset) {
          CharSequence fragment =
              builder.getOriginalText().subSequence(tokenStart, completionState.offset);
          if (completionState.prefixMatches(fragment.toString(), text)) {
            diff = completionState.offset - tokenStart;
          }
          break;
        } else break;
      }
      add = diff >= 0 && diff < length;
    }
    add =
        add
            && length > 1
            && !(text.charAt(0) == '<' && text.charAt(length - 1) == '>')
            && !(text.charAt(0) == '\'' && text.charAt(length - 1) == '\'' && length < 5);
    if (add) {
      completionState.addItem(builder, text);
    }
  }
  public static boolean parseAnnotationMemberValueInitializer(
      PsiBuilder builder, GroovyParser parser) {
    if (builder.getTokenType() == mAT) {
      return Annotation.parse(builder, parser);
    } else if (builder.getTokenType() == mLBRACK) {
      PsiBuilder.Marker marker = builder.mark();
      ParserUtils.getToken(builder, mLBRACK);
      while (parseAnnotationMemberValueInitializer(builder, parser)) {
        if (builder.eof() || builder.getTokenType() == mRBRACK) break;
        ParserUtils.getToken(builder, mCOMMA, GroovyBundle.message("comma.expected"));
      }

      ParserUtils.getToken(builder, mRBRACK, GroovyBundle.message("rbrack.expected"));
      marker.done(ANNOTATION_ARRAY_INITIALIZER);
      return true;
    }

    // check
    return ConditionalExpression.parse(builder, parser) && !ParserUtils.getToken(builder, mASSIGN);
  }
 @Override
 public boolean parse(PsiBuilder builder, int level) {
   if (builder.eof()) return false;
   builder.advanceLexer();
   return true;
 }
 public static boolean eof(PsiBuilder builder, int level) {
   return builder.eof();
 }
示例#26
0
  @Nullable
  public static IElementType parseAfterModifiers(
      PsiBuilder builder,
      boolean isInClass,
      boolean isInAnnotation,
      GroovyParser parser,
      PsiBuilder.Marker declMarker,
      boolean modifiersParsed) {
    if (modifiersParsed && mLT == builder.getTokenType()) {
      TypeParameters.parse(builder);
      PsiBuilder.Marker checkMarker = builder.mark(); // point to begin of type or variable

      if (TypeSpec.parse(builder, true)
          == fail) { // if type wasn't recognized trying parse VariableDeclaration
        checkMarker.rollbackTo();
      } else {
        checkMarker.drop();
      }
      IElementType decl =
          VariableDefinitions.parseDefinitions(
              builder, isInClass, false, false, true, modifiersParsed, false, parser);

      if (WRONGWAY.equals(decl)) {
        return WRONGWAY;
      }

      return METHOD_DEFINITION;
    }

    if (modifiersParsed) {

      PsiBuilder.Marker checkMarker = builder.mark(); // point to begin of type or variable

      if (TypeSpec.parse(builder, false)
          == fail) { // if type wasn't recognized trying parse VariableDeclaration
        checkMarker.rollbackTo();

        if (isInAnnotation) {
          builder.error(GroovyBundle.message("type.expected"));
        }

        // current token isn't identifier
        IElementType varDecl =
            VariableDefinitions.parse(builder, isInClass, modifiersParsed, parser);

        if (WRONGWAY.equals(varDecl)) {
          return WRONGWAY;
        }
        return varDecl;
      } else { // type was recognized, identifier here
        // starts after type
        IElementType varDeclarationTop =
            VariableDefinitions.parse(builder, isInClass, modifiersParsed, false, parser);

        if (WRONGWAY.equals(varDeclarationTop)) {
          checkMarker.rollbackTo();

          if (isInAnnotation) {
            builder.error(GroovyBundle.message("type.expected"));
          }

          // starts before "type" identifier, here can't be tuple, because next token is identifier
          // (we are in "type recognized" branch)
          IElementType varDecl =
              VariableDefinitions.parse(builder, isInClass, modifiersParsed, false, parser);

          if (WRONGWAY.equals(varDecl)) {
            return WRONGWAY;
          } else {
            return varDecl;
          }
        } else {
          checkMarker.drop();
          return varDeclarationTop;
        }
      }
    } else {

      // if definition starts with lower case letter than it can be just call expression

      String text = builder.getTokenText();
      if (!builder.eof()
          && !TokenSets.BUILT_IN_TYPE.contains(builder.getTokenType())
          && text != null
          && (Character.isLowerCase((text.charAt(0))) || !Character.isLetter(text.charAt(0)))
          && (ParserUtils.lookAhead(builder, mIDENT, mIDENT)
              || ParserUtils.lookAhead(builder, mIDENT, mLPAREN))) {
        // call expression
        return WRONGWAY;
      }

      boolean typeParsed = false;
      if (!ParserUtils.lookAhead(builder, mIDENT, mLPAREN)) {
        typeParsed = TypeSpec.parse(builder, true) != fail;
        // type specification starts with upper case letter
        if (!typeParsed) {
          builder.error(GroovyBundle.message("type.specification.expected"));
          return WRONGWAY;
        }
      }

      IElementType varDef =
          VariableDefinitions.parseDefinitions(
              builder, isInClass, false, false, false, typeParsed, false, parser);
      if (varDef != WRONGWAY) {
        return varDef;
      } else if (isInClass && typeParsed) {
        return typeParsed ? null : WRONGWAY;
      }

      return WRONGWAY;
    }
  }
  private static void exit_section_impl_(
      ErrorState state,
      Frame frame,
      PsiBuilder builder,
      @Nullable IElementType elementType,
      boolean result,
      boolean pinned,
      @Nullable Parser eatMore) {
    int initialPos = builder.rawTokenIndex();
    boolean willFail = !result && !pinned;
    if (willFail
        && initialPos == frame.position
        && state.lastExpectedVariantPos == frame.position
        && frame.name != null
        && state.variants.size() - frame.variantCount > 1) {
      state.clearVariants(true, frame.variantCount);
      addVariantInner(state, initialPos, frame.name);
    }
    int lastErrorPos = getLastVariantPos(state, initialPos);
    if (!state.suppressErrors && eatMore != null) {
      state.suppressErrors = true;
      final boolean eatMoreFlagOnce = !builder.eof() && eatMore.parse(builder, frame.level + 1);
      boolean eatMoreFlag =
          eatMoreFlagOnce
              || !result && frame.position == initialPos && lastErrorPos > frame.position;

      PsiBuilderImpl.ProductionMarker latestDoneMarker =
          (pinned || result) && (state.altMode || elementType != null) && eatMoreFlagOnce
              ? (PsiBuilderImpl.ProductionMarker) builder.getLatestDoneMarker()
              : null;
      PsiBuilder.Marker extensionMarker = null;
      IElementType extensionTokenType = null;
      // whitespace prefix makes the very first frame offset bigger than marker start offset which
      // is always 0
      if (latestDoneMarker != null
          && frame.position >= latestDoneMarker.getStartIndex()
          && frame.position <= latestDoneMarker.getEndIndex()) {
        extensionMarker = ((PsiBuilder.Marker) latestDoneMarker).precede();
        extensionTokenType = latestDoneMarker.getTokenType();
        ((PsiBuilder.Marker) latestDoneMarker).drop();
      }
      // advance to the last error pos
      // skip tokens until lastErrorPos. parseAsTree might look better here...
      int parenCount = 0;
      while ((eatMoreFlag || parenCount > 0) && builder.rawTokenIndex() < lastErrorPos) {
        if (state.braces != null) {
          if (builder.getTokenType() == state.braces[0].getLeftBraceType()) parenCount++;
          else if (builder.getTokenType() == state.braces[0].getRightBraceType()) parenCount--;
        }
        builder.advanceLexer();
        eatMoreFlag = eatMore.parse(builder, frame.level + 1);
      }
      boolean errorReported =
          frame.errorReportedAt == initialPos || !result && frame.errorReportedAt >= frame.position;
      if (errorReported) {
        if (eatMoreFlag) {
          builder.advanceLexer();
          parseAsTree(state, builder, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore);
        }
      } else if (eatMoreFlag) {
        errorReported = reportError(builder, state, frame, null, true, true);
        parseAsTree(state, builder, frame.level + 1, DUMMY_BLOCK, true, TOKEN_ADVANCER, eatMore);
      } else if (eatMoreFlagOnce
          || (!result && frame.position != builder.rawTokenIndex())
          || frame.errorReportedAt > initialPos) {
        errorReported = reportError(builder, state, frame, null, true, false);
      }
      if (extensionMarker != null) {
        extensionMarker.done(extensionTokenType);
      }
      state.suppressErrors = false;
      if (errorReported || result) {
        state.clearVariants(true, 0);
        state.clearVariants(false, 0);
        state.lastExpectedVariantPos = -1;
      }
    } else if (!result && pinned && frame.errorReportedAt < 0) {
      // do not report if there are errors beyond current position
      if (lastErrorPos == initialPos) {
        // do not force, inner recoverRoot might have skipped some tokens
        reportError(builder, state, frame, elementType, false, false);
      } else if (lastErrorPos > initialPos) {
        // set error pos here as if it is reported for future reference
        frame.errorReportedAt = lastErrorPos;
      }
    }
    // propagate errorReportedAt up the stack to avoid duplicate reporting
    Frame prevFrame = willFail && eatMore == null ? null : state.frameStack.peekLast();
    if (prevFrame != null && prevFrame.errorReportedAt < frame.errorReportedAt) {
      prevFrame.errorReportedAt = frame.errorReportedAt;
    }
  }
 protected boolean eof() {
   return myBuilder.eof();
 }
  @NotNull
  public ASTNode parse(final IElementType root, final PsiBuilder builder) {
    final PsiBuilder.Marker fileMarker = builder.mark();

    final Stack<Pair<Integer, PsiBuilder.Marker>> stack =
        new Stack<Pair<Integer, PsiBuilder.Marker>>();
    stack.push(Pair.create(0, builder.mark()));

    PsiBuilder.Marker startLineMarker = null;
    int currentIndent = 0;
    boolean eolSeen = false;

    while (!builder.eof()) {
      final IElementType type = builder.getTokenType();
      // EOL
      if (type == myEolTokenType) {
        // Handle variant with several EOLs
        if (startLineMarker == null) {
          startLineMarker = builder.mark();
        }
        eolSeen = true;
      } else

      // Indent
      {
        if (type == myIndentTokenType) {
          //noinspection ConstantConditions
          currentIndent = builder.getTokenText().length();
        } else if (eolSeen) {
          if (startLineMarker != null) {
            startLineMarker.rollbackTo();
            startLineMarker = null;
          }
          // Close indentation blocks
          while (!stack.isEmpty() && currentIndent < stack.peek().first) {
            stack.pop().second.done(myBlockElementType);
          }

          if (!stack.isEmpty()) {
            final Pair<Integer, PsiBuilder.Marker> pair = stack.peek();
            if (currentIndent == pair.first) {
              stack.pop().second.done(myBlockElementType);
              passEOLsAndIndents(builder);
              stack.push(Pair.create(currentIndent, builder.mark()));
            }
            if (currentIndent > pair.first) {
              passEOLsAndIndents(builder);
              stack.push(Pair.create(currentIndent, builder.mark()));
            }
          }
          eolSeen = false;
          currentIndent = 0;
        }
      }
      advanceLexer(builder);
    }

    // Close all left opened markers
    if (startLineMarker != null) {
      startLineMarker.drop();
    }
    while (!stack.isEmpty()) {
      stack.pop().second.done(myBlockElementType);
    }

    return buildTree(fileMarker, builder, root);
  }
  @Override
  public boolean parseStatement(PsiBuilder b, int l) {
    IElementType tokenType = b.getTokenType();

    boolean r = false;
    PsiBuilder.Marker m = b.mark();

    if (tokenType == MASON_BLOCK_OPENER) {
      PsiBuilder.Marker statementMarker = b.mark();
      b.advanceLexer();
      if (PerlParserImpl.expr(b, l, -1)) {
        // parseStatement filter
        if (PerlParserUtil.consumeToken(b, MASON_EXPR_FILTER_PIPE)) {
          while (b.getTokenType() == IDENTIFIER) {
            PsiBuilder.Marker fm = b.mark();
            b.advanceLexer();
            fm.collapse(SUB);
            fm.precede().done(METHOD);

            if (!PerlParserUtil.consumeToken(b, OPERATOR_COMMA)) {
              break;
            }
          }
        }
      }
      if (r = endOrRecover(b, MASON_BLOCK_CLOSER)) {
        statementMarker.done(STATEMENT);
      }
    }
    if (tokenType == MASON_CALL_OPENER) {
      PsiBuilder.Marker statementMarker = b.mark();
      b.advanceLexer();
      PerlParserImpl.expr(b, l, -1);
      if (r = endOrRecover(b, MASON_CALL_CLOSER)) {
        statementMarker.done(MASON_CALL_STATEMENT);
      }
    } else if (tokenType == MASON_CLASS_OPENER) {
      r = parsePerlBlock(b, l, MASON_CLASS_CLOSER);
    } else if (tokenType == MASON_INIT_OPENER) {
      r = parsePerlBlock(b, l, MASON_INIT_CLOSER);
    } else if (tokenType == MASON_PERL_OPENER) {
      r = parsePerlBlock(b, l, MASON_PERL_CLOSER);
    } else if (tokenType == MASON_FLAGS_OPENER) {
      b.advanceLexer();
      PsiBuilder.Marker statementMarker = b.mark();

      while (!b.eof() && b.getTokenType() != MASON_FLAGS_CLOSER) {
        if (!PerlParserImpl.expr(b, l, -1)) {
          break;
        }
      }
      statementMarker.done(MASON_FLAGS_STATEMENT);

      r = endOrRecover(b, MASON_FLAGS_CLOSER);
    } else if (tokenType == MASON_DOC_OPENER) {
      b.advanceLexer();
      PerlParserUtil.consumeToken(b, COMMENT_BLOCK);
      r = endOrRecover(b, MASON_DOC_CLOSER);
    } else if (tokenType == MASON_TEXT_OPENER) {
      b.advanceLexer();
      PsiBuilder.Marker stringMarker = b.mark();
      if (PerlParserUtil.consumeToken(b, STRING_CONTENT)) {
        stringMarker.done(MASON_TEXT_BLOCK);
      } else {
        stringMarker.drop();
      }
      r = endOrRecover(b, MASON_TEXT_CLOSER);
    } else if (tokenType == MASON_METHOD_OPENER) {
      r = parseMasonMethod(b, l, MASON_METHOD_CLOSER, MASON_METHOD_DEFINITION);
    } else if (tokenType == MASON_FILTER_OPENER) {
      r = parseMasonMethod(b, l, MASON_FILTER_CLOSER, MASON_FILTER_DEFINITION);
    } else if (tokenType == MASON_OVERRIDE_OPENER) {
      r = parseMasonMethod(b, l, MASON_OVERRIDE_CLOSER, MASON_OVERRIDE_DEFINITION);
    } else if (SIMPLE_MASON_NAMED_BLOCKS.contains(tokenType)) // simple named blocks
    {
      PsiBuilder.Marker statementMarker = b.mark();
      b.advanceLexer();
      IElementType closeToken = RESERVED_OPENER_TO_CLOSER_MAP.get(tokenType);

      if (PerlParserUtil.convertIdentifier(b, l, MASON_METHOD_MODIFIER_NAME)) {
        if (PerlParserUtil.consumeToken(b, MASON_TAG_CLOSER)) {
          PsiBuilder.Marker blockMarker = b.mark();
          PerlParserImpl.block_content(b, l);
          blockMarker.done(BLOCK);
          blockMarker.setCustomEdgeTokenBinders(
              WhitespacesBinders.GREEDY_LEFT_BINDER, WhitespacesBinders.GREEDY_RIGHT_BINDER);

          if (r = PerlParserUtil.consumeToken(b, closeToken)) {
            statementMarker.done(RESERVED_TO_STATEMENT_MAP.get(tokenType));
            statementMarker = null;
          }
        }
      }

      if (statementMarker != null) {
        statementMarker.drop();
      }

      r = r || recoverToGreedy(b, closeToken, "Error");
    }

    if (r) {
      m.drop();
    } else {
      m.rollbackTo();
    }

    return r || super.parseStatement(b, l);
  }