Exemple #1
0
 public void consumeToken(Token token) {
   if (dump) System.out.println("consume token " + token);
   if (!inDecision()) {
     stats.numTokens++;
     return;
   }
   if (lastRealTokenTouchedInDecision == null
       || lastRealTokenTouchedInDecision.getTokenIndex() < token.getTokenIndex()) {
     lastRealTokenTouchedInDecision = token;
   }
   DecisionEvent d = currentDecision();
   // compute lookahead depth
   int thisRefIndex = token.getTokenIndex();
   int numHidden = getNumberOfHiddenTokens(d.startIndex, thisRefIndex);
   int depth = thisRefIndex - d.startIndex - numHidden + 1; // +1 counts consuming start token as 1
   // d.maxk = Math.max(d.maxk, depth);
   if (dump)
     System.out.println(
         "consume "
             + thisRefIndex
             + " "
             + depth
             + " tokens ahead in "
             + d.decision.ruleName
             + "-"
             + d.decision.decision
             + " start index "
             + d.startIndex);
 }
Exemple #2
0
 /** Get num hidden tokens between i..j inclusive */
 public int getNumberOfHiddenTokens(int i, int j) {
   int n = 0;
   TokenStream input = parser.getTokenStream();
   for (int ti = i; ti < input.size() && ti <= j; ti++) {
     Token t = input.get(ti);
     if (t.getChannel() != Token.DEFAULT_CHANNEL) {
       n++;
     }
   }
   return n;
 }
Exemple #3
0
 /** Track refs to lookahead if in a fixed/nonfixed decision. */
 public void LT(int i, Token t) {
   if (inDecision() && i > 0) {
     DecisionEvent d = currentDecision();
     if (dump)
       System.out.println(
           "LT("
               + i
               + ")="
               + t
               + " index "
               + t.getTokenIndex()
               + " relative to "
               + d.decision.ruleName
               + "-"
               + d.decision.decision
               + " start index "
               + d.startIndex);
     if (lastRealTokenTouchedInDecision == null
         || lastRealTokenTouchedInDecision.getTokenIndex() < t.getTokenIndex()) {
       lastRealTokenTouchedInDecision = t;
       if (dump) System.out.println("set last token " + lastRealTokenTouchedInDecision);
     }
     // get starting index off stack
     //			int stackTop = lookaheadStack.size()-1;
     //			Integer startingIndex = (Integer)lookaheadStack.get(stackTop);
     //			// compute lookahead depth
     //			int thisRefIndex = parser.getTokenStream().index();
     //			int numHidden =
     //				getNumberOfHiddenTokens(startingIndex.intValue(), thisRefIndex);
     //			int depth = i + thisRefIndex - startingIndex.intValue() - numHidden;
     //			/*
     //			System.out.println("LT("+i+") @ index "+thisRefIndex+" is depth "+depth+
     //				" max is "+maxLookaheadInCurrentDecision);
     //			*/
     //			if ( depth>maxLookaheadInCurrentDecision ) {
     //				maxLookaheadInCurrentDecision = depth;
     //			}
     //			d.maxk = currentDecision()/
   }
 }
Exemple #4
0
  public void exitDecision(int decisionNumber) {
    DecisionEvent d = decisionStack.pop();
    d.stopTime = System.currentTimeMillis();

    int lastTokenIndex = lastRealTokenTouchedInDecision.getTokenIndex();
    int numHidden = getNumberOfHiddenTokens(d.startIndex, lastTokenIndex);
    int depth =
        lastTokenIndex - d.startIndex - numHidden + 1; // +1 counts consuming start token as 1
    d.k = depth;
    d.decision.maxk = Math.max(d.decision.maxk, depth);

    if (dump)
      System.out.println(
          "exitDecision "
              + decisionNumber
              + " in "
              + d.decision.ruleName
              + " lookahead "
              + d.k
              + " max token "
              + lastRealTokenTouchedInDecision);
    decisionEvents.add(d); // done with decision; track all
  }
 public boolean canBeUsedForSyntaxColoring(org.antlr.runtime3_4_0.Token token) {
   return canBeUsedForSyntaxHighlighting(token.getType());
 }
Exemple #6
0
  public List postProcessAction(List chunks, Token actionToken) {
    /* TODO
      - check for and report TAB usage
    */

    // System.out.println("\n*** Action at " + actionToken.getLine() + ":" +
    // actionToken.getColumn());

    /* First I create a new list of chunks. String chunks are splitted into
       lines and some whitespace my be added at the beginning.

       As a result I get a list of chunks
       - where the first line starts at column 0
       - where every LF is at the end of a string chunk
    */

    List nChunks = new ArrayList();
    for (int i = 0; i < chunks.size(); i++) {
      Object chunk = chunks.get(i);

      if (chunk instanceof String) {
        String text = (String) chunks.get(i);
        if (nChunks.size() == 0 && actionToken.getCharPositionInLine() >= 0) {
          // first chunk and some 'virtual' WS at beginning
          // prepend to this chunk

          String ws = "";
          for (int j = 0; j < actionToken.getCharPositionInLine(); j++) {
            ws += " ";
          }
          text = ws + text;
        }

        List parts = splitLines(text);
        for (int j = 0; j < parts.size(); j++) {
          chunk = parts.get(j);
          nChunks.add(chunk);
        }
      } else {
        if (nChunks.size() == 0 && actionToken.getCharPositionInLine() >= 0) {
          // first chunk and some 'virtual' WS at beginning
          // add as a chunk of its own

          String ws = "";
          for (int j = 0; j <= actionToken.getCharPositionInLine(); j++) {
            ws += " ";
          }
          nChunks.add(ws);
        }

        nChunks.add(chunk);
      }
    }

    int lineNo = actionToken.getLine();
    int col = 0;

    // strip trailing empty lines
    int lastChunk = nChunks.size() - 1;
    while (lastChunk > 0
        && nChunks.get(lastChunk) instanceof String
        && ((String) nChunks.get(lastChunk)).trim().length() == 0) lastChunk--;

    // string leading empty lines
    int firstChunk = 0;
    while (firstChunk <= lastChunk
        && nChunks.get(firstChunk) instanceof String
        && ((String) nChunks.get(firstChunk)).trim().length() == 0
        && ((String) nChunks.get(firstChunk)).endsWith("\n")) {
      lineNo++;
      firstChunk++;
    }

    int indent = -1;
    for (int i = firstChunk; i <= lastChunk; i++) {
      Object chunk = nChunks.get(i);

      // System.out.println(lineNo + ":" + col + " " + quote(chunk.toString()));

      if (chunk instanceof String) {
        String text = (String) chunk;

        if (col == 0) {
          if (indent == -1) {
            // first non-blank line
            // count number of leading whitespaces

            indent = 0;
            for (int j = 0; j < text.length(); j++) {
              if (!Character.isWhitespace(text.charAt(j))) break;

              indent++;
            }
          }

          if (text.length() >= indent) {
            int j;
            for (j = 0; j < indent; j++) {
              if (!Character.isWhitespace(text.charAt(j))) {
                // should do real error reporting here...
                System.err.println("Warning: badly indented line " + lineNo + " in action:");
                System.err.println(text);
                break;
              }
            }

            nChunks.set(i, text.substring(j));
          } else if (text.trim().length() > 0) {
            // should do real error reporting here...
            System.err.println("Warning: badly indented line " + lineNo + " in action:");
            System.err.println(text);
          }
        }

        if (text.endsWith("\n")) {
          lineNo++;
          col = 0;
        } else {
          col += text.length();
        }
      } else {
        // not really correct, but all I need is col to increment...
        col += 1;
      }
    }

    return nChunks;
  }
 protected void retrieveLayoutInformation(
     org.eclipse.emf.ecore.EObject element,
     ASPM.resource.ASPM.grammar.ASPMSyntaxElement syntaxElement,
     Object object,
     boolean ignoreTokensAfterLastVisibleToken) {
   if (disableLayoutRecording || element == null) {
     return;
   }
   // null must be accepted, since the layout information that is found at the end of
   // documents (just before the EOF character) is not associated with a particular
   // syntax element.
   boolean isElementToStore = syntaxElement == null;
   isElementToStore |= syntaxElement instanceof ASPM.resource.ASPM.grammar.ASPMPlaceholder;
   isElementToStore |= syntaxElement instanceof ASPM.resource.ASPM.grammar.ASPMKeyword;
   isElementToStore |= syntaxElement instanceof ASPM.resource.ASPM.grammar.ASPMEnumerationTerminal;
   isElementToStore |= syntaxElement instanceof ASPM.resource.ASPM.grammar.ASPMBooleanTerminal;
   if (!isElementToStore) {
     return;
   }
   ASPM.resource.ASPM.mopp.ASPMLayoutInformationAdapter layoutInformationAdapter =
       getLayoutInformationAdapter(element);
   StringBuilder anonymousText = new StringBuilder();
   for (org.antlr.runtime3_4_0.CommonToken anonymousToken : anonymousTokens) {
     anonymousText.append(anonymousToken.getText());
   }
   int currentPos = getTokenStream().index();
   if (currentPos == 0) {
     return;
   }
   int endPos = currentPos - 1;
   if (ignoreTokensAfterLastVisibleToken) {
     for (; endPos >= this.lastPosition2; endPos--) {
       org.antlr.runtime3_4_0.Token token = getTokenStream().get(endPos);
       int _channel = token.getChannel();
       if (_channel != 99) {
         break;
       }
     }
   }
   StringBuilder hiddenTokenText = new StringBuilder();
   hiddenTokenText.append(anonymousText);
   StringBuilder visibleTokenText = new StringBuilder();
   org.antlr.runtime3_4_0.CommonToken firstToken = null;
   for (int pos = this.lastPosition2; pos <= endPos; pos++) {
     org.antlr.runtime3_4_0.Token token = getTokenStream().get(pos);
     if (firstToken == null) {
       firstToken = (org.antlr.runtime3_4_0.CommonToken) token;
     }
     if (anonymousTokens.contains(token)) {
       continue;
     }
     int _channel = token.getChannel();
     if (_channel == 99) {
       hiddenTokenText.append(token.getText());
     } else {
       visibleTokenText.append(token.getText());
     }
   }
   int offset = -1;
   if (firstToken != null) {
     offset = firstToken.getStartIndex();
   }
   layoutInformationAdapter.addLayoutInformation(
       new ASPM.resource.ASPM.mopp.ASPMLayoutInformation(
           syntaxElement,
           object,
           offset,
           hiddenTokenText.toString(),
           visibleTokenText.toString()));
   this.lastPosition2 = (endPos < 0 ? 0 : endPos + 1);
   anonymousTokens.clear();
 }