コード例 #1
0
 @SuppressWarnings("empty-statement")
 public void testLinesAndOffsetsAreReported() throws IOException {
   LexerSource src = newSource("a\nb\nc\nd\n");
   while (src.read() != RubyLexer.EOF) ;
   assertEquals(4, src.getLine());
   assertEquals(8, src.getOffset());
 }
コード例 #2
0
 public void testCurrentLineGivesAnErrorLocation() throws IOException {
   LexerSource src = newSource("111111\n222222\n333333");
   for (int i = 0; i < 10; i++) {
     src.read();
   }
   assertEquals("222222\n  ^", src.getCurrentLine());
 }
コード例 #3
0
 public void testCaptureLinesWithCarriageReturn() throws IOException {
   List<String> lines = new ArrayList<String>();
   LexerSource src = newSource("1\r\n2\r\n3", lines);
   while (src.read() != RubyLexer.EOF) ;
   assertEquals(3, lines.size());
   assertEquals("1\r\n", lines.get(0));
   assertEquals("2\r\n", lines.get(1));
   assertEquals("3", lines.get(2));
 }
コード例 #4
0
 @SuppressWarnings("empty-statement")
 public void testCaptureLines() throws IOException {
   List<String> lines = new ArrayList<String>();
   LexerSource src = newSource("111111\n222222\n333333", lines);
   while (src.read() != RubyLexer.EOF) ;
   assertEquals(3, lines.size());
   assertEquals("111111\n", lines.get(0));
   assertEquals("222222\n", lines.get(1));
   assertEquals("333333", lines.get(2));
 }
コード例 #5
0
ファイル: StringTerm.java プロジェクト: aslakhellesoy/jruby
  private int parseRegexpFlags(final LexerSource src) throws java.io.IOException {
    char kcode = 0;
    int options = 0;
    int c;
    StringBuilder unknownFlags = new StringBuilder(10);

    for (c = src.read(); c != RubyYaccLexer.EOF && Character.isLetter(c); c = src.read()) {
      switch (c) {
        case 'i':
          options |= ReOptions.RE_OPTION_IGNORECASE;
          break;
        case 'x':
          options |= ReOptions.RE_OPTION_EXTENDED;
          break;
        case 'm':
          options |= ReOptions.RE_OPTION_MULTILINE;
          break;
        case 'o':
          options |= ReOptions.RE_OPTION_ONCE;
          break;
        case 'n':
          kcode = 16;
          break;
        case 'e':
          kcode = 32;
          break;
        case 's':
          kcode = 48;
          break;
        case 'u':
          kcode = 64;
          break;
        case 'j':
          options |= 256; // Regexp engine 'java'
          break;
        default:
          unknownFlags.append((char) c);
          break;
      }
    }
    src.unread(c);
    if (unknownFlags.length() != 0) {
      throw new SyntaxException(
          PID.REGEXP_UNKNOWN_OPTION,
          src.getPosition(),
          "unknown regexp option"
              + (unknownFlags.length() > 1 ? "s" : "")
              + " - "
              + unknownFlags.toString(),
          unknownFlags.toString());
    }
    return options | kcode;
  }
コード例 #6
0
ファイル: StringTerm.java プロジェクト: aslakhellesoy/jruby
  // Was a goto in original ruby lexer
  private void escaped(LexerSource src, ByteList buffer) throws java.io.IOException {
    int c;

    switch (c = src.read()) {
      case '\\':
        parseEscapeIntoBuffer(src, buffer);
        break;
      case RubyYaccLexer.EOF:
        throw new SyntaxException(
            PID.INVALID_ESCAPE_SYNTAX,
            src.getPosition(),
            src.getCurrentLine(),
            "Invalid escape character syntax");
      default:
        buffer.append(c);
    }
  }
コード例 #7
0
ファイル: StringTerm.java プロジェクト: aslakhellesoy/jruby
  public int parseSimpleStringIntoBuffer(LexerSource src, ByteList buffer)
      throws java.io.IOException {
    int c;

    while ((c = src.read()) != RubyYaccLexer.EOF) {
      if (c == end) {
        src.unread(c);
        break;
      } else if (c == '\\') {
        c = src.read();
        if ((c == '\n' || c != end) && c != '\\') buffer.append('\\');
      }

      buffer.append(c);
    }

    return c;
  }
コード例 #8
0
 public void testGetRemainingOutputAsStream() throws IOException {
   LexerSource src = newSource("111111\n222222\n333333\n");
   for (int i = 0; i < 10; i++) {
     src.read();
   }
   final InputStream in = src.getRemainingAsStream();
   assertActionShouldProduce(
       "222\n333333\n",
       new Callable<byte[]>() {
         public byte[] call() throws Exception {
           ByteList buf = new ByteList();
           int c;
           while ((c = in.read()) != -1) {
             buf.append(c);
           }
           return buf.bytes();
         }
       });
 }
コード例 #9
0
 // This one currently fails with InputStreamLexerSource
 public void testCarriageReturnsShouldBeUnreadAutomatically() throws IOException {
   LexerSource src = newSource("abc\r\ndef");
   assertReadShouldProduce(src, "abc\ndef");
   assertEquals(8, src.getOffset());
   src.unreadMany("\ndef");
   assertEquals(3, src.getOffset());
   src.read();
   assertEquals(5, src.getOffset());
   assertReadShouldProduce(src, "def");
 }
コード例 #10
0
 private void assertReadShouldProduce(LexerSource src, String expected) {
   final ByteList actual = new ByteList(expected.length());
   for (int i = 0; i < expected.length(); i++) {
     try {
       int c = src.read();
       if (c == RubyLexer.EOF) {
         break;
       }
       actual.append(c);
     } catch (IOException ex) {
       fail(ex.getMessage());
     }
   }
   assertActionShouldProduce(
       expected,
       new Callable<byte[]>() {
         public byte[] call() throws Exception {
           return actual.bytes();
         }
       });
 }
コード例 #11
0
 public void testCarriageReturnsShouldBeEaten() {
   LexerSource src = newSource("abcd\r\n");
   assertReadShouldProduce(src, "abcd\n");
   assertEquals(6, src.getOffset());
 }
コード例 #12
0
 public void testMatchMarkerShouldNotCountNewlinesAsWhitespace() throws IOException {
   final LexerSource src = newSource("\n=begin\n=end");
   assertFalse(src.matchMarker(new ByteList(safeGetBytes("=begin")), true, false));
 }
コード例 #13
0
ファイル: StringTerm.java プロジェクト: aslakhellesoy/jruby
  private void parseEscapeIntoBuffer(LexerSource src, ByteList buffer) throws java.io.IOException {
    int c;

    switch (c = src.read()) {
      case '\n':
        break; /* just ignore */
      case '0':
      case '1':
      case '2':
      case '3': /* octal constant */
      case '4':
      case '5':
      case '6':
      case '7':
        buffer.append('\\');
        buffer.append(c);
        for (int i = 0; i < 2; i++) {
          c = src.read();
          if (c == RubyYaccLexer.EOF) {
            throw new SyntaxException(
                PID.INVALID_ESCAPE_SYNTAX,
                src.getPosition(),
                src.getCurrentLine(),
                "Invalid escape character syntax");
          }
          if (!RubyYaccLexer.isOctChar(c)) {
            src.unread(c);
            break;
          }
          buffer.append(c);
        }
        break;
      case 'x': /* hex constant */
        buffer.append('\\');
        buffer.append(c);
        c = src.read();
        if (!RubyYaccLexer.isHexChar(c)) {
          throw new SyntaxException(
              PID.INVALID_ESCAPE_SYNTAX,
              src.getPosition(),
              src.getCurrentLine(),
              "Invalid escape character syntax");
        }
        buffer.append(c);
        c = src.read();
        if (RubyYaccLexer.isHexChar(c)) {
          buffer.append(c);
        } else {
          src.unread(c);
        }
        break;
      case 'M':
        if ((c = src.read()) != '-') {
          throw new SyntaxException(
              PID.INVALID_ESCAPE_SYNTAX,
              src.getPosition(),
              src.getCurrentLine(),
              "Invalid escape character syntax");
        }
        buffer.append(new byte[] {'\\', 'M', '-'});
        escaped(src, buffer);
        break;
      case 'C':
        if ((c = src.read()) != '-') {
          throw new SyntaxException(
              PID.INVALID_ESCAPE_SYNTAX,
              src.getPosition(),
              src.getCurrentLine(),
              "Invalid escape character syntax");
        }
        buffer.append(new byte[] {'\\', 'C', '-'});
        escaped(src, buffer);
        break;
      case 'c':
        buffer.append(new byte[] {'\\', 'c'});
        escaped(src, buffer);
        break;
      case RubyYaccLexer.EOF:
        throw new SyntaxException(
            PID.INVALID_ESCAPE_SYNTAX,
            src.getPosition(),
            src.getCurrentLine(),
            "Invalid escape character syntax");
      default:
        if (c != '\\' || c != end) {
          buffer.append('\\');
        }
        buffer.append(c);
    }
  }
コード例 #14
0
ファイル: StringTerm.java プロジェクト: aslakhellesoy/jruby
  public int parseStringIntoBuffer(RubyYaccLexer lexer, LexerSource src, ByteList buffer)
      throws java.io.IOException {
    boolean qwords = (flags & RubyYaccLexer.STR_FUNC_QWORDS) != 0;
    boolean expand = (flags & RubyYaccLexer.STR_FUNC_EXPAND) != 0;
    boolean escape = (flags & RubyYaccLexer.STR_FUNC_ESCAPE) != 0;
    boolean regexp = (flags & RubyYaccLexer.STR_FUNC_REGEXP) != 0;
    int c;

    while ((c = src.read()) != RubyYaccLexer.EOF) {
      if (begin != '\0' && c == begin) {
        nest++;
      } else if (c == end) {
        if (nest == 0) {
          src.unread(c);
          break;
        }
        nest--;
      } else if (c == '#' && expand && !src.peek('\n')) {
        int c2 = src.read();

        if (c2 == '$' || c2 == '@' || c2 == '{') {
          src.unread(c2);
          src.unread(c);
          break;
        }
        src.unread(c2);
      } else if (c == '\\') {
        c = src.read();
        switch (c) {
          case '\n':
            if (qwords) break;
            if (expand) continue;
            buffer.append('\\');
            break;

          case '\\':
            if (escape) buffer.append(c);
            break;

          default:
            if (regexp) {
              src.unread(c);
              parseEscapeIntoBuffer(src, buffer);
              continue;
            } else if (expand) {
              src.unread(c);
              if (escape) buffer.append('\\');
              c = lexer.readEscape();
            } else if (qwords && Character.isWhitespace(c)) {
              /* ignore backslashed spaces in %w */
            } else if (c != end && !(begin != '\0' && c == begin)) {
              buffer.append('\\');
            }
        }
      } else if (qwords && Character.isWhitespace(c)) {
        src.unread(c);
        break;
      }
      buffer.append(c);
    }

    return c;
  }
コード例 #15
0
 public void testReadUntilNonExistentMarkerShouldReturnNull() throws IOException {
   final LexerSource src = newSource("abcd\n1234");
   assertNull(src.readUntil('z'));
 }
コード例 #16
0
ファイル: StringTerm.java プロジェクト: aslakhellesoy/jruby
  public int parseString(RubyYaccLexer lexer, LexerSource src) throws java.io.IOException {
    boolean spaceSeen = false;
    int c;

    // FIXME: How much more obtuse can this be?
    // Heredoc already parsed this and saved string...Do not parse..just return
    if (flags == -1) {
      lexer.setValue(new Token("\"", lexer.getPosition()));
      return Tokens.tSTRING_END;
    }

    c = src.read();
    if ((flags & RubyYaccLexer.STR_FUNC_QWORDS) != 0 && Character.isWhitespace(c)) {
      do {
        c = src.read();
      } while (Character.isWhitespace(c));
      spaceSeen = true;
    }

    if (c == end && nest == 0) {
      if ((flags & RubyYaccLexer.STR_FUNC_QWORDS) != 0) {
        flags = -1;
        lexer.getPosition();
        return ' ';
      }

      if ((flags & RubyYaccLexer.STR_FUNC_REGEXP) != 0) {
        lexer.setValue(
            new RegexpNode(src.getPosition(), ByteList.create(""), parseRegexpFlags(src)));
        return Tokens.tREGEXP_END;
      }

      lexer.setValue(new Token("\"", lexer.getPosition()));
      return Tokens.tSTRING_END;
    }

    if (spaceSeen) {
      src.unread(c);
      lexer.getPosition();
      return ' ';
    }

    // Single-quote fast path
    if (begin == '\0' && flags == 0) {
      ByteList buffer = new ByteList();
      src.unread(c);
      if (parseSimpleStringIntoBuffer(src, buffer) == RubyYaccLexer.EOF) {
        throw new SyntaxException(
            PID.STRING_HITS_EOF,
            src.getPosition(),
            src.getCurrentLine(),
            "unterminated string meets end of file");
      }

      /*
      ByteList buffer;
      src.unread(c);
      if ((buffer = src.readUntil(end)) == null) {
          throw new SyntaxException(src.getPosition(), "unterminated string meets end of file");
      }
      */
      lexer.setValue(new StrNode(lexer.getPosition(), buffer));
      return Tokens.tSTRING_CONTENT;
    }

    ByteList buffer = new ByteList();

    if ((flags & RubyYaccLexer.STR_FUNC_EXPAND) != 0 && c == '#') {
      c = src.read();
      switch (c) {
        case '$':
        case '@':
          src.unread(c);
          lexer.setValue(new Token("#" + c, lexer.getPosition()));
          return Tokens.tSTRING_DVAR;
        case '{':
          lexer.setValue(new Token("#" + c, lexer.getPosition()));
          return Tokens.tSTRING_DBEG;
      }
      buffer.append((byte) '#');
    }
    src.unread(c);

    if (parseStringIntoBuffer(lexer, src, buffer) == RubyYaccLexer.EOF) {
      throw new SyntaxException(
          PID.STRING_HITS_EOF,
          src.getPosition(),
          src.getCurrentLine(),
          "unterminated string meets end of file");
    }

    lexer.setValue(new StrNode(lexer.getPosition(), buffer));
    return Tokens.tSTRING_CONTENT;
  }
コード例 #17
0
 public void testPeekGivesTheNextCharaterWithoutAdvancing() throws IOException {
   LexerSource src = newSource("abc");
   assertTrue(src.peek('a'));
   assertEquals('a', src.read());
 }
コード例 #18
0
 public void testSkipUntilShouldSkipPastNonMatchingChars() throws IOException {
   LexerSource src = newSource("abcd1234");
   src.skipUntil('1');
   assertReadShouldProduce(src, "234");
 }
コード例 #19
0
 public void testWasBeginOfLineBehaviorAtBeginningOfInput() throws IOException {
   final LexerSource src = newSource("=begin");
   src.read();
   assertFalse(src.lastWasBeginOfLine());
   assertTrue(src.wasBeginOfLine());
 }
コード例 #20
0
 public void testWasBeginOfLineTellsIfPreviousCharactersWereNewline() throws IOException {
   final LexerSource src = newSource(".\n=end");
   assertTrue(src.lastWasBeginOfLine());
   assertFalse(src.wasBeginOfLine());
   src.read();
   src.read();
   assertTrue(src.lastWasBeginOfLine());
   assertFalse(src.wasBeginOfLine());
   src.read();
   assertFalse(src.lastWasBeginOfLine());
   assertTrue(src.wasBeginOfLine());
   src.read();
   assertFalse(src.lastWasBeginOfLine());
   assertFalse(src.wasBeginOfLine());
 }
コード例 #21
0
 public void testSkipUntilNonExistentMarkerShouldReturnEOF() throws IOException {
   final LexerSource src = newSource("abcd\n1234");
   assertEquals(RubyLexer.EOF, src.skipUntil('z'));
 }
コード例 #22
0
 public void testUnreadNewCharactersShouldBeAppended() {
   LexerSource src = newSource("abcd");
   src.unreadMany("1234");
   assertReadShouldProduce(src, "1234abcd");
 }
コード例 #23
0
 public void testMatchMarkerShouldMatchAStringInTheInputAndAdvance() throws IOException {
   final LexerSource src = newSource("=begin\n=end");
   assertTrue(src.matchMarker(new ByteList(safeGetBytes("=begin")), false, false));
   assertReadShouldProduce(src, "\n=end");
 }
コード例 #24
0
 public void testMatchMarkerShouldSkipOverLeadingWhitespace() throws IOException {
   final LexerSource src = newSource("   =begin\n=end");
   assertTrue(src.matchMarker(new ByteList(safeGetBytes("=begin")), true, false));
   assertReadShouldProduce(src, "\n=end");
 }
コード例 #25
0
 public void testMatchMarkerShouldNotAdvanceIfItDoesntMatch() throws IOException {
   final LexerSource src = newSource("=begin\n=end");
   assertFalse(src.matchMarker(new ByteList(safeGetBytes("=end")), false, false));
   assertReadShouldProduce(src, "=begin");
 }
コード例 #26
0
 public void testReadUnreadManyReadShouldProduceCorrectSequence() {
   LexerSource src = newSource("abcd1234");
   assertReadShouldProduce(src, "abcd");
   src.unreadMany("abcd");
   assertReadShouldProduce(src, "abcd1234");
 }
コード例 #27
0
 public void testMatchMarkerShouldVerifyEndOfLineCRLFWithIndent() throws IOException {
   final LexerSource src = newSource("  =begin\r\n=end\r\n");
   assertTrue(src.matchMarker(new ByteList(safeGetBytes("=begin")), true, true));
   assertReadShouldProduce(src, "=end");
 }
コード例 #28
0
 public void testUnreadBothNewAndExistingCharacters() throws IOException {
   LexerSource src = newSource("abc");
   src.read();
   src.unreadMany("123a");
   assertReadShouldProduce(src, "123abc");
 }
コード例 #29
0
 public void testMatchMarkerAcrossUnreadBuffers() throws IOException {
   final LexerSource src = newSource("in\n=end");
   src.unreadMany("=beg");
   assertTrue(src.matchMarker(new ByteList(safeGetBytes("=begin")), false, false));
 }