private void expect(String string, Token... tokens) {
    System.out.println("Tokenizing [" + string + "]");
    ExprLexer tokenizer = new ExprLexer(string);
    int expectedIndex = 0;
    while (!tokenizer.isEndOfInput()) {
      String expectedString;
      Token expected;
      if (expectedIndex < tokens.length) {
        expected = tokens[expectedIndex];
        expectedString = expected.toString();
      } else {
        expected = null;
        expectedString = "<NOTHING>";
      }
      Token actual = tokenizer.next();
      System.out.println(
          String.format("Expected: %15s, got %s", expectedString, actual.toString()));
      if (expected != null) {
        assertEquals("tokenStart", expected.getTokenStart(), actual.getTokenStart());
        assertEquals("text", expected.getString(), actual.getString());
        assertEquals("type", expected.getType(), actual.getType());

        if (!expected.equals(actual)) {
          System.err.println("Unexpected result!");
          throw new AssertionError();
        }
      }
      expectedIndex++;
    }
  }
Beispiel #2
0
 public final Enumeration Enumeration() throws ParseException {
   Token t;
   ASTStringNode name;
   ASTStringNode name1;
   ArrayList<ASTStringNode> name1List = new ArrayList<ASTStringNode>();
   Token firstToken = token;
   jj_consume_token(OB);
   t = jj_consume_token(NAME);
   name = new ASTStringNode(t.toString(), new WToken(token));
   label_6:
   while (true) {
     switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) {
       case 32:;
         break;
       default:
         jj_la1[16] = jj_gen;
         break label_6;
     }
     jj_consume_token(32);
     t = jj_consume_token(NAME);
     name1 = new ASTStringNode(t.toString(), new WToken(token));
     name1List.add(name1);
   }
   jj_consume_token(CB);
   {
     if (true) return new Enumeration(name, name1List, firstToken.next, token);
   }
   throw new Error("Missing return statement in function");
 }
Beispiel #3
0
  public static void main(String args[]) {
    Token tok;
    try {

      // Read from the command line for the filename
      Lexer lex = new Lexer(args[0]);

      while (true) {
        tok = lex.nextToken();

        // if token returns error token we print the line for error and exit
        if (tok.getKind() == Tokens.Error) {
          System.out.println(
              "Error in Line " + lex.source.getLineno() + " ErrorValue: " + tok.toString());
          System.exit(0);
        }
        // Printing the information about the line and the token
        String p =
            "Left: "
                + tok.getLeftPosition()
                + "| Right: "
                + tok.getRightPosition()
                + "| Type: "
                + TokenType.tokens.get(tok.getKind())
                + "| Value: ";
        // if ((tok.getKind() == Tokens.Identifier) || (tok.getKind() == Tokens.INTeger))
        p += tok.toString();
        System.out.println(p + "| Line: " + lex.source.getLineno());
      }
    } catch (Exception e) {
    }
  }
Beispiel #4
0
  public static void main(String[] args) {
    String inFile = "C:\\Users\\Sukoon\\Documents\\NetBeansProjects\\cdproj\\src\\Sample.in";
    String outFile = "C:\\Users\\Sukoon\\Desktop\\Sample.out";

    if (args.length > 1) {
      inFile = args[0];
      outFile = args[1];
    }

    Lexer lexer = new Lexer(inFile);

    try {
      BufferedWriter writer = new BufferedWriter(new FileWriter(outFile));

      Token t;

      while ((t = lexer.nextToken()) != null) {
        writer.write(t.toString());
        writer.newLine();
      }

      writer.close();

      System.out.println("Done tokenizing file: " + inFile);
      System.out.println("Output written in file: " + outFile);
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
Beispiel #5
0
 public final AttListDecl AttListDecl() throws ParseException {
   Token t;
   ASTStringNode name;
   AttribDef attribDef;
   ArrayList<AttribDef> attribDefList = new ArrayList<AttribDef>();
   Token firstToken = token;
   jj_consume_token(34);
   t = jj_consume_token(NAME);
   name = new ASTStringNode(t.toString(), new WToken(token));
   label_4:
   while (true) {
     switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) {
       case NAME:;
         break;
       default:
         jj_la1[11] = jj_gen;
         break label_4;
     }
     attribDef = AttribDef();
     attribDefList.add(attribDef);
   }
   jj_consume_token(PCB);
   {
     if (true) return new AttListDecl(name, attribDefList, firstToken.next, token);
   }
   throw new Error("Missing return statement in function");
 }
Beispiel #6
0
 public final DTD DTD() throws ParseException {
   DTDEntry dTDEntry;
   ArrayList<DTDEntry> dTDEntryList = new ArrayList<DTDEntry>();
   Token t;
   ASTStringNode eof;
   Token firstToken = token;
   label_1:
   while (true) {
     switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) {
       case 31:
       case 34:;
         break;
       default:
         jj_la1[0] = jj_gen;
         break label_1;
     }
     dTDEntry = DTDEntry();
     dTDEntryList.add(dTDEntry);
   }
   t = jj_consume_token(0);
   eof = new ASTStringNode(t.toString(), new WToken(token));
   {
     if (true) return new DTD(dTDEntryList, eof, firstToken.next, token);
   }
   throw new Error("Missing return statement in function");
 }
Beispiel #7
0
 /** Prints the content and annotations. */
 public String toString() {
   StringBuffer sb = new StringBuffer();
   sb.append("  SENTENCE begin\n");
   for (Token tok : tokens) sb.append(tok.toString());
   sb.append("  SENTENCE end\n");
   return sb.toString();
 }
Beispiel #8
0
 @Override
 public String toString() {
   StringBuilder sb = new StringBuilder();
   for (Token t : tokens)
     sb.append(t.toString()).append(" String:").append(t.getString()).append("\n");
   return sb.toString();
 }
Beispiel #9
0
 public static final Valor PValorInteiro() throws ParseException {
   Token token;
   token = jj_consume_token(INTEGER_LITERAL);
   {
     if (true) return new ValorInteiro(Integer.parseInt(token.toString()));
   }
   throw new Error("Missing return statement in function");
 }
  public String toString() {
    final StringBuilder result = new StringBuilder();

    // overall format is (cat)token, or:
    //   (cat)'text'[startIdx,endIdx{wordCount}](seqNum.revisNum)
    result.append("(").append(category).append(')').append(token.toString());

    return result.toString();
  }
Beispiel #11
0
 public static final Valor PValorString() throws ParseException {
   Token token;
   token = jj_consume_token(STRING_LITERAL);
   String tokenStr = token.toString();
   tokenStr = tokenStr.substring(1, tokenStr.length() - 1);
   {
     if (true) return new ValorString(tokenStr);
   }
   throw new Error("Missing return statement in function");
 }
Beispiel #12
0
 public final AttribValue AttribValue() throws ParseException {
   Token t;
   ASTStringNode quotedstr;
   Token firstToken = token;
   t = jj_consume_token(QUOTEDSTR);
   quotedstr = new ASTStringNode(t.toString(), new WToken(token));
   {
     if (true) return new AttribValue(quotedstr, firstToken.next, token);
   }
   throw new Error("Missing return statement in function");
 }
Beispiel #13
0
 public final Cp Cp() throws ParseException {
   Token t;
   ASTStringNode name;
   Modifier modifier = null;
   SeqOrChoice seqOrChoice;
   Modifier modifier1 = null;
   Token firstToken = token;
   switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) {
     case PCDATA:
       jj_consume_token(PCDATA);
       {
         if (true) return new Cp1(firstToken.next, token);
       }
       break;
     case NAME:
       t = jj_consume_token(NAME);
       name = new ASTStringNode(t.toString(), new WToken(token));
       switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) {
         case STAR:
         case PLUS:
         case Q:
           modifier = Modifier();
           break;
         default:
           jj_la1[7] = jj_gen;
           ;
       }
       {
         if (true) return new Cp2(name, modifier, firstToken.next, token);
       }
       break;
     case OB:
       seqOrChoice = SeqOrChoice();
       switch ((jj_ntk == -1) ? jj_ntk() : jj_ntk) {
         case STAR:
         case PLUS:
         case Q:
           modifier1 = Modifier();
           break;
         default:
           jj_la1[8] = jj_gen;
           ;
       }
       {
         if (true) return new Cp3(seqOrChoice, modifier1, firstToken.next, token);
       }
       break;
     default:
       jj_la1[9] = jj_gen;
       jj_consume_token(-1);
       throw new ParseException();
   }
   throw new Error("Missing return statement in function");
 }
Beispiel #14
0
 /** This function constructs the tree for debug purposes in a text view. */
 public StringBuilder toString(StringBuilder prefix, boolean isTail, StringBuilder sb) {
   if (right != null) {
     right.toString(
         new StringBuilder().append(prefix).append(isTail ? "│   " : "    "), false, sb);
   }
   sb.append(prefix).append(isTail ? "└── " : "┌── ").append(token.toString()).append("\n");
   if (left != null) {
     left.toString(new StringBuilder().append(prefix).append(isTail ? "    " : "│   "), true, sb);
   }
   return sb;
 }
Beispiel #15
0
 public static void main(String args[]) {
   System.out.println("start.");
   Lexer l = new Lexer();
   while (true) {
     try {
       Token t = l.scan();
       if (t.tag == Tag.NUM) {
         System.out.println(
             "Token : " + t.toString() + " , Tag:" + t.tag + " , value:" + ((Num) t).value);
       } else if (t.tag == Tag.ID) {
         System.out.println(
             "Token : " + t.toString() + " , Tag:" + t.tag + " , lexeme:" + ((Word) t).lexeme);
       } else {
         System.out.println("Token : " + t.toString() + " , Tag:" + t.tag);
       }
     } catch (Exception e) {
       e.printStackTrace();
     }
   }
 }
  /** Writes the data to file */
  public void writeToFile(String path) {
    try (DataOutputStream outputStream =
        new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)))) {
      // First write all unique tokens
      Map<Token, Integer> tokenToId = new HashMap<>();

      Set<Token> tokens = new HashSet<>();
      for (NGram ngram : this.ngramModel.getNgrams().keySet()) {
        if (ngram.length() == 1) {
          tokens.add(ngram.at(0));
        }
      }

      outputStream.writeInt(tokens.size());
      int id = 0;
      for (Token token : tokens) {
        outputStream.writeUTF(token.toString());
        tokenToId.put(token, id);
        id++;
      }

      // Then the Good-Turing parameters
      outputStream.writeInt(this.ngramModel.getGoodTuringEstimation().getTotal());
      outputStream.writeDouble(this.ngramModel.getGoodTuringEstimation().getA());
      outputStream.writeDouble(this.ngramModel.getGoodTuringEstimation().getB());

      // Then the top ranked unigrams
      outputStream.writeInt(this.ngramModel.topUnigrams().size());
      for (NGram ngram : this.ngramModel.topUnigrams()) {
        Token token = ngram.at(0);
        outputStream.writeInt(tokenToId.get(token));
      }

      // Then the n-grams, where the token id points to the previous table
      outputStream.writeInt(this.ngramModel.getNgrams().size());

      for (Map.Entry<NGram, Integer> current : this.ngramModel.getNgrams().entrySet()) {
        outputStream.writeInt(current.getKey().length());

        for (int i = 0; i < current.getKey().length(); i++) {
          Token token = current.getKey().at(i);
          outputStream.writeInt(tokenToId.get(token));
        }

        outputStream.writeInt(current.getValue());
      }

      outputStream.flush();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
Beispiel #17
0
 public final ElementDecl ElementDecl() throws ParseException {
   Token t;
   ASTStringNode name;
   ContentSpec contentSpec;
   Token firstToken = token;
   jj_consume_token(31);
   t = jj_consume_token(NAME);
   name = new ASTStringNode(t.toString(), new WToken(token));
   contentSpec = ContentSpec();
   jj_consume_token(PCB);
   {
     if (true) return new ElementDecl(name, contentSpec, firstToken.next, token);
   }
   throw new Error("Missing return statement in function");
 }
Beispiel #18
0
 public final AttribDef AttribDef() throws ParseException {
   Token t;
   ASTStringNode name;
   AttribType attribType;
   DefaultDecl defaultDecl;
   Token firstToken = token;
   t = jj_consume_token(NAME);
   name = new ASTStringNode(t.toString(), new WToken(token));
   attribType = AttribType();
   defaultDecl = DefaultDecl();
   {
     if (true) return new AttribDef(name, attribType, defaultDecl, firstToken.next, token);
   }
   throw new Error("Missing return statement in function");
 }
Beispiel #19
0
  public static String parse(
      Lexer lexer,
      HashMap<Nonterminal, HashMap<Terminal, Rule>> parsingTable,
      Nonterminal startSymbol) {

    Stack<Token> stack = new Stack<Token>();
    ArrayList<Terminal> inputs = lexer.getTokens();
    int count = 0;

    stack.push(startSymbol);
    while (!stack.isEmpty() && count < inputs.size()) {
      Token top = stack.peek();
      System.out.println(
          "Testing top: " + top.toString() + ", next input: " + inputs.get(count).toString());
      if (top instanceof Nonterminal) {

        Nonterminal nextNonTerm = (Nonterminal) top;
        Rule rules = parsingTable.get(nextNonTerm).get(inputs.get(count));
        if (rules == null)
          return "Invalid. No rule found for "
              + nextNonTerm.toString()
              + " and the input "
              + inputs.get(count);

        stack.pop();

        for (int i = rules.rule.length - 1; i > -1; i--) {
          if (!rules.rule[i].equals(new Terminal(Terminal.TerminalType.EPSILON)))
            stack.push(rules.rule[i]);
        }

      } else if (top instanceof Terminal) {

        Terminal nextTerm = (Terminal) top;
        Terminal nextInput = inputs.get(count);

        if (nextTerm.equals(nextInput)) {
          ++count;
          stack.pop();

        } else return nextTerm + " and " + nextInput + " do not match: invalid";
      }
      System.out.println("Current stack: " + stack.toString());
    }

    if (stack.isEmpty()) return "Valid";
    return "Invalid. Ran out of tokens early or ran over input count";
  }
 protected void onActivityResult(int requestCode, int resultCode, Intent intent) {
   switch (requestCode) {
     case REQUEST_PICK_TOKEN:
       if (resultCode == RESULT_OK && intent != null) {
         Uri name = intent.getData();
         try {
           // URI vs Uri -> Java/Android fail ...
           File f = new File(new URI(name.toString()));
           Token t = Token.fromFile(f);
           SharedPreferences p = PreferenceManager.getDefaultSharedPreferences(getBaseContext());
           SharedPreferences.Editor e = p.edit();
           e.putString("token", t.toString());
           e.commit();
           Toast.makeText(this, "Token successfully imported.", Toast.LENGTH_LONG).show();
         } catch (Exception e) {
           e.printStackTrace();
         }
       }
       break;
     default:
       Log.w(TAG, "Unknown activity " + intent.toString());
   }
 }
Beispiel #21
0
  public void writeDictionary(TokenStream tokenStream, String docId, int docSize) {

    count++;
    while (tokenStream.hasNext()) {
      Node node = new Node();
      node.setDocId(docId);
      Token token = tokenStream.next();
      String tokenText = token.toString();
      int indexPos = token.getIndexPos();
      node.setIndexPos(indexPos);
      node.setDocLength(docSize);

      if (!tokenText.isEmpty()) {
        node.setTokenText(tokenText);
        placeList.add(node);
      }
    }
    if (count == 20000) {
      count = 0;
      sortFiles();
      placeList = new ArrayList<Node>();
      initializeWriter();
    }
  }
 public IllegalTokenNameException(Token token) {
   super(token.toString());
 }
 /**
  * toString
  *
  * @return regresa una representacion en cadena de el nodo.
  */
 public String toString() {
   return tok.toString();
 }
Beispiel #24
0
  public Token next() {
    currentToken = internalNext();
    if (debug) System.out.println(currentToken == null ? "(null)" : currentToken.toString());

    return currentToken;
  }