private List<JavaSymbolName> getParameterNames(
      List<Token> tokens, JavaSymbolName finderName, String plural) {
    List<JavaSymbolName> parameterNames = new ArrayList<JavaSymbolName>();

    for (int i = 0; i < tokens.size(); i++) {
      Token token = tokens.get(i);
      if (token instanceof FieldToken) {
        String fieldName = (((FieldToken) token).getField().getFieldName().getSymbolName());
        parameterNames.add(new JavaSymbolName(fieldName));
      } else {
        if ("Between".equals(token.getValue())) {
          Token field = tokens.get(i - 1);
          if (field instanceof FieldToken) {
            JavaSymbolName fieldName = parameterNames.get(parameterNames.size() - 1);
            // Remove the last field token
            parameterNames.remove(parameterNames.size() - 1);

            // Replace by a min and a max value
            parameterNames.add(
                new JavaSymbolName("min" + fieldName.getSymbolNameCapitalisedFirstLetter()));
            parameterNames.add(
                new JavaSymbolName("max" + fieldName.getSymbolNameCapitalisedFirstLetter()));
          }
        } else if ("IsNull".equals(token.getValue()) || "IsNotNull".equals(token.getValue())) {
          Token field = tokens.get(i - 1);
          if (field instanceof FieldToken) {
            parameterNames.remove(parameterNames.size() - 1);
          }
        }
      }
    }

    return parameterNames;
  }
  /**
   * Parse a set.
   *
   * @param token the initial token.
   * @return the root of the generated parse subtree.
   * @throws Exception if an error occurred.
   */
  private ICodeNode parseSet(Token token) throws Exception {
    ICodeNode rootNode = ICodeFactory.createICodeNode(SETS);
    int tempInteger = 0;
    token = nextToken(); // consume left bracket

    while (token.getType() != RIGHT_BRACKET) {

      if (token.getType() == INTEGER) {
        tempInteger = (int) token.getValue();
        rootNode.addChild(parseTerm(token));
        token = currentToken();
      } else if (token.getType() == IDENTIFIER) {
        ICodeNode var = parseFactor(token);
        rootNode.addChild(var);
        token = currentToken();
      }
      if (token.getType() == COMMA) {
        token = nextToken(); // consume next token
        if (token.getType() == COMMA) {
          errorHandler.flag(token, EXTRA_COMMA, this);
        }
      } else if (token.getType() == RIGHT_BRACKET) { // do nothing
      } else if (token.getType() == SEMICOLON) {
        errorHandler.flag(token, MISSING_CLOSE_SQUARE_BRACKET, this);
        break;
      } else if (token.getType() == DOT_DOT) {
        token = nextToken(); // consume the integer.
        // Create a STRING_CONSTANT node as the root node.
        ICodeNode node = ICodeFactory.createICodeNode(SUBRANGE);
        // denotes the maximum value of the range.
        if (token.getType() != INTEGER) {
          errorHandler.flag(token, MISSING_MAX_VAL_SUBRANGE, this);
        } else {
          int value = (int) token.getValue();
          node.setAttribute(VALUE, value);
        }
        rootNode.addChild(node);
        token = nextToken();

      } else {
        errorHandler.flag(token, MISSING_COMMA, this);
      }
    }
    if (token.getType() != SEMICOLON) {
      token = nextToken(); // consume right bracket
    }
    return rootNode;
  }
Example #3
0
  public void start() throws IOException {
    FileScanner fs = new FileScanner(rules);
    NFAFactory factory = new NFAFactory(fs.getRegexTable(), fs.getTokenTable());
    LinkedHashSet<NFA> nfas = factory.factorize();
    text = readTextFile(input);
    tw = new TableWalkerNFA(nfas, text);

    List<Token> list = tw.parse();
    ArrayList<String> out = new ArrayList<String>();

    for (Token t : list) {
      String str = t.getId();
      if (str.length() > 0) {
        str = str.substring(1);
      }
      str += " ";
      str += t.getValue();
      str += "\r\n";
      out.add(str);
    }

    // Edit this to change where the output is saved.
    FileWriter writer = new FileWriter(output);
    for (String str : out) {
      writer.write(str);
    }
    writer.close();
  }
Example #4
0
    @Override
    public int _handleToken(ParseContext ctx) throws InvalidQueryException {
      Token[] tokens = ctx.getTokens();
      int idx = ctx.getCurrentTokensIndex();
      Token token = tokens[idx];
      RelationalOperator relationalOp = RelationalOperatorFactory.createOperator(token.getValue());

      ctx.addExpression(new RelationalExpression(relationalOp));
      ctx.setCurrentTokensIndex(++idx);

      TokenHandler propertyHandler = new PropertyOperandTokenHandler();
      propertyHandler.handleToken(ctx);

      // handle right operand if applicable to operator
      idx = ctx.getCurrentTokensIndex();
      if (ctx.getCurrentTokensIndex() < tokens.length
          && tokens[idx].getType().equals(Token.TYPE.VALUE_OPERAND)) {
        TokenHandler valueHandler = new ValueOperandTokenHandler();
        valueHandler.handleToken(ctx);
      }

      // skip closing bracket
      idx = ctx.getCurrentTokensIndex();
      if (idx >= tokens.length || tokens[idx].getType() != Token.TYPE.BRACKET_CLOSE) {
        throw new InvalidQueryException("Missing closing bracket for in expression.");
      }
      return 1;
    }
Example #5
0
    @Override
    public int _handleToken(ParseContext ctx) throws InvalidQueryException {
      Token token = ctx.getTokens()[ctx.getCurrentTokensIndex()];
      ctx.getPrecedingExpression().setLeftOperand(token.getValue());

      return 1;
    }
  public fUML.Semantics.Activities.IntermediateActivities.TokenList takeOfferedTokens(
      int maxCount) {
    // Take all the offered tokens, up to the given maximum count of
    // non-null object tokens, and return them.

    TokenList tokens = new TokenList();
    int remainingCount = maxCount;

    while (this.offers.size() > 0 & remainingCount > 0) {
      Offer offer = this.offers.getValue(0);
      TokenList offeredTokens = offer.getOfferedTokens();
      int count = offer.countOfferedValues();
      if (count <= remainingCount) {
        for (int i = 0; i < offeredTokens.size(); i++) {
          tokens.addValue(offeredTokens.getValue(i));
        }
        remainingCount = remainingCount - count;
        this.offers.removeValue(0);
      } else {
        for (int i = 0; i < remainingCount; i++) {
          Token token = offeredTokens.getValue(i);
          if (token.getValue() != null) {
            tokens.addValue(token);
          }
        }
        offer.removeOfferedValues(remainingCount);
        remainingCount = 0;
      }
    }

    return tokens;
  } // takeOfferedTokens
 /**
  * Helper method to determine if a set of tokens contain a value
  *
  * @param tokens set to look in
  * @param value to look for
  * @return boolean <code>true</code> if contained
  */
 static boolean containsTokenWithValue(final Token[] tokens, final Object value) {
   for (final Token token : tokens) {
     if (token.getValue() == value) {
       return true;
     }
   }
   return false;
 }
Example #8
0
 public boolean isNextTokenValue(String expectedValue) {
   Token nextToken = tokenQueue.peek();
   if (nextToken == null) {
     // queue is empty
     return false;
   }
   return nextToken.getValue().equals(expectedValue);
 }
Example #9
0
  /**
   * Checks if this symbol is grammatically equals with providen.
   *
   * @param symbol symbol to compare
   * @return true if symbol is grammatically equal, false otherwise
   */
  boolean grammarEquals(final Token symbol) {
    if (this.token == null) {
      return false;
    }
    final boolean arg2 =
        (this.token.getSymbol() == null
            ? null == symbol.getSymbol()
            : this.token.getSymbol().equals(symbol.getSymbol()));

    boolean arg3 = true;
    if (token.getSymbol() != TokenType.VARIABLE) {
      arg3 =
          this.token.getValue() == null
              ? null == symbol.getValue()
              : this.token.getValue().equals(symbol.getValue());
    }
    return arg2 && arg3;
  }
Example #10
0
  public String toString() {
    String output = "";

    for (Token token : tokenList) {
      output += token.getValue();
    }

    return output;
  }
Example #11
0
    @Override
    public int _handleToken(ParseContext ctx) throws InvalidQueryException {
      Token token = ctx.getTokens()[ctx.getCurrentTokensIndex()];
      RelationalOperator relationalOp = RelationalOperatorFactory.createOperator(token.getValue());
      // todo: use factory to create expression
      ctx.addExpression(new RelationalExpression(relationalOp));

      return 1;
    }
Example #12
0
    @Override
    public int _handleToken(ParseContext ctx) throws InvalidQueryException {
      Token token = ctx.getTokens()[ctx.getCurrentTokensIndex()];
      LogicalOperator logicalOp =
          LogicalOperatorFactory.createOperator(token.getValue(), ctx.getPrecedenceLevel());
      ctx.updateMaxPrecedence(logicalOp.getPrecedence());
      ctx.addExpression(LogicalExpressionFactory.createLogicalExpression(logicalOp));

      return 1;
    }
Example #13
0
  private Token getNagToken(List<Token> tokens) {

    StringBuilder stringBuilder = new StringBuilder();

    for (Token token : tokens) {

      stringBuilder.append(token.getValue());
    }

    return new Token(Type.NAG, stringBuilder.toString());
  }
 /**
  * The internal method to do the formatting.
  *
  * @param tokens the tokens
  * @param years the number of years
  * @param months the number of months
  * @param days the number of days
  * @param hours the number of hours
  * @param minutes the number of minutes
  * @param seconds the number of seconds
  * @param milliseconds the number of millis
  * @param padWithZeros whether to pad
  * @return the formatted string
  */
 static String format(
     final Token[] tokens,
     final long years,
     final long months,
     final long days,
     final long hours,
     final long minutes,
     final long seconds,
     final long milliseconds,
     final boolean padWithZeros) {
   final StringBuilder buffer = new StringBuilder();
   boolean lastOutputSeconds = false;
   for (final Token token : tokens) {
     final Object value = token.getValue();
     final int count = token.getCount();
     if (value instanceof StringBuilder) {
       buffer.append(value.toString());
     } else {
       if (value.equals(y)) {
         buffer.append(paddedValue(years, padWithZeros, count));
         lastOutputSeconds = false;
       } else if (value.equals(M)) {
         buffer.append(paddedValue(months, padWithZeros, count));
         lastOutputSeconds = false;
       } else if (value.equals(d)) {
         buffer.append(paddedValue(days, padWithZeros, count));
         lastOutputSeconds = false;
       } else if (value.equals(H)) {
         buffer.append(paddedValue(hours, padWithZeros, count));
         lastOutputSeconds = false;
       } else if (value.equals(m)) {
         buffer.append(paddedValue(minutes, padWithZeros, count));
         lastOutputSeconds = false;
       } else if (value.equals(s)) {
         buffer.append(paddedValue(seconds, padWithZeros, count));
         lastOutputSeconds = true;
       } else if (value.equals(S)) {
         if (lastOutputSeconds) {
           // ensure at least 3 digits are displayed even if padding is not selected
           final int width = padWithZeros ? Math.max(3, count) : 3;
           buffer.append(paddedValue(milliseconds, true, width));
         } else {
           buffer.append(paddedValue(milliseconds, padWithZeros, count));
         }
         lastOutputSeconds = false;
       }
     }
   }
   return buffer.toString();
 }
  private List<Token> tokenize(
      MemberDetails memberDetails, JavaSymbolName finderName, String plural) {
    String simpleTypeName = getConcreteJavaType(memberDetails).getSimpleTypeName();
    String finder = finderName.getSymbolName();

    // Just in case it starts with findBy we can remove it here
    String findBy = "find" + plural + "By";
    if (finder.startsWith(findBy)) {
      finder = finder.substring(findBy.length());
    }

    // If finder still contains the findBy sequence it is most likely a wrong finder (ie someone
    // pasted the finder string accidentally twice
    if (finder.contains(findBy)) {
      throw new InvalidFinderException(
          "Dynamic finder definition for '"
              + finderName.getSymbolName()
              + "' in "
              + simpleTypeName
              + ".java is invalid");
    }

    SortedSet<FieldToken> fieldTokens = new TreeSet<FieldToken>();
    for (MethodMetadata methodMetadata : getLocatedMutators(memberDetails)) {
      FieldMetadata fieldMetadata =
          BeanInfoUtils.getFieldForPropertyName(
              memberDetails, methodMetadata.getParameterNames().get(0));

      // If we did find a field matching the first parameter name of the mutator method we can add
      // it to the finder ITD
      if (fieldMetadata != null) {
        fieldTokens.add(new FieldToken(fieldMetadata));
      }
    }

    List<Token> tokens = new ArrayList<Token>();

    while (finder.length() > 0) {
      Token token = getFirstToken(fieldTokens, finder, finderName.getSymbolName(), simpleTypeName);
      if (token != null) {
        if (token instanceof FieldToken || token instanceof ReservedToken) {
          tokens.add(token);
        }
        finder = finder.substring(token.getValue().length());
      }
    }

    return tokens;
  }
  private List<JavaType> getParameterTypes(
      List<Token> tokens, JavaSymbolName finderName, String plural) {
    List<JavaType> parameterTypes = new ArrayList<JavaType>();

    for (int i = 0; i < tokens.size(); i++) {
      Token token = tokens.get(i);
      if (token instanceof FieldToken) {
        parameterTypes.add(((FieldToken) token).getField().getFieldType());
      } else {
        if ("Between".equals(token.getValue())) {
          Token field = tokens.get(i - 1);
          if (field instanceof FieldToken) {
            parameterTypes.add(parameterTypes.get(parameterTypes.size() - 1));
          }
        } else if ("IsNull".equals(token.getValue()) || "IsNotNull".equals(token.getValue())) {
          Token field = tokens.get(i - 1);
          if (field instanceof FieldToken) {
            parameterTypes.remove(parameterTypes.size() - 1);
          }
        }
      }
    }
    return parameterTypes;
  }
Example #17
0
    /**
     * Process a token. Handles common token processing functionality then delegates to the
     * individual concrete handlers.
     *
     * @param ctx the current parse context
     * @throws InvalidQueryException if unable to process the token
     */
    public void handleToken(ParseContext ctx) throws InvalidQueryException {
      Token token = ctx.getTokens()[ctx.getCurrentTokensIndex()];
      if (!validate(ctx.getPreviousTokenType())) {
        throw new InvalidQueryException(
            "Unexpected token encountered in query string. Last Token Type="
                + ctx.getPreviousTokenType()
                + ", Current Token[type="
                + token.getType()
                + ", value='"
                + token.getValue()
                + "']");
      }
      ctx.setTokenType(token.getType());

      int idxIncrement = _handleToken(ctx);
      ctx.setCurrentTokensIndex(ctx.getCurrentTokensIndex() + idxIncrement);
    }
Example #18
0
  public RuleWhat(Input input) throws ParserException, IcatException {

    input.consume(Token.Type.SELECT);
    StringBuilder sb = new StringBuilder("SELECT ");
    Token t = input.consume(Token.Type.NAME, Token.Type.DISTINCT);
    String resultValue;
    if (t.getType() == Token.Type.DISTINCT) {
      sb.append("DISTINCT ");
      resultValue = input.consume(Token.Type.NAME).getValue();
    } else {
      resultValue = t.getValue();
    }
    sb.append(resultValue);
    idVar = resultValue.split("\\.")[0].toUpperCase();

    Map<String, Integer> idVarMap = new HashMap<>();
    idVarMap.put(idVar, 0);
    boolean isQuery = false;
    fromClause = new FromClause(input, idVar, idVarMap, isQuery);

    /* Rewind input and skip down to the from clause again */
    input.reset();
    t = input.peek(0);
    while (t.getType() != Token.Type.FROM) {
      input.consume();
      t = input.peek(0);
    }

    idVarMap = new HashMap<>();
    idVarMap.put(idVar, 0);
    isQuery = true;
    crudFromClause = new FromClause(input, idVar, idVarMap, isQuery);

    t = input.peek(0);
    if (t != null && t.getType() == Token.Type.WHERE) {
      whereClause = new WhereClause(input, idVarMap);
      t = input.peek(0);
    }
    if (t != null) {
      throw new ParserException(input, new Type[0]);
    }
    varCount = idVarMap.size();
  }
Example #19
0
  public int getPrecedence(Token token) {
    int precedence = 0;

    // A reserved word can't start an infix expression. Prevents us from
    // parsing a keyword as an identifier.
    if (isReserved(token)) return 0;

    // If we have a prefix parser for this token's name, then that takes
    // precedence. Prevents us from parsing a keyword as an identifier.
    if ((token.getValue() instanceof String) && mPrefixParsers.isReserved(token.getString()))
      return 0;

    InfixParser parser = mInfixParsers.get(token);
    if (parser != null) {
      precedence = parser.getPrecedence();
    }

    return precedence;
  }
Example #20
0
 /**
  * unary ::= NUMBER unary ::= OPEN logicalOr CLOSE unary ::= VARIABLE unary ::= NOT unary unary
  * ::= SUB unary unary ::= ADD unary
  */
 private Value unary() {
   final Value res;
   nextToken();
   if (token.getType() == TokenType.SUB) {
     res = unary();
     return res.setValue(-res.getValue());
   } else if (token.getType() == TokenType.ADD) {
     return unary();
   } else if (token.getType() == TokenType.NOT) {
     res = unary();
     return res.setBooleanValue(!res.getBooleanValue());
   } else if (token.getType() == TokenType.OPEN) {
     res = logicalOr();
     if (token.getType() != TokenType.CLOSE) {
       throw expected(TokenType.CLOSE);
     }
   } else if (token.getType() == TokenType.NUMBER) {
     res = token.getValue();
   } else if (token.getType() == TokenType.VARIABLE) {
     // resolve variable
     res = variables.get(token.getStr());
     if (res == null) {
       throw new IllegalStateException("Unexpected variable name");
     }
   } else {
     throw expected(
         TokenType.SUB,
         TokenType.ADD,
         TokenType.NOT,
         TokenType.OPEN,
         TokenType.NUMBER,
         TokenType.VARIABLE);
   }
   nextToken();
   return res;
 }
  /**
   * Parse a factor.
   *
   * @param token the initial token.
   * @return the root of the generated parse subtree.
   * @throws Exception if an error occurred.
   */
  private ICodeNode parseFactor(Token token) throws Exception {
    TokenType tokenType = token.getType();
    ICodeNode rootNode = null;

    switch ((PascalTokenType) tokenType) {
      case IDENTIFIER:
        {
          // Look up the identifier in the symbol table stack.
          // Flag the identifier as undefined if it's not found.
          String name = token.getText().toLowerCase();
          SymTabEntry id = symTabStack.lookup(name);
          if (id == null) {
            errorHandler.flag(token, IDENTIFIER_UNDEFINED, this);
            id = symTabStack.enterLocal(name);
          }

          rootNode = ICodeFactory.createICodeNode(VARIABLE);
          rootNode.setAttribute(ID, id);
          id.appendLineNumber(token.getLineNumber());

          token = nextToken(); // consume the identifier
          break;
        }

      case INTEGER:
        {
          // Create an INTEGER_CONSTANT node as the root node.
          rootNode = ICodeFactory.createICodeNode(INTEGER_CONSTANT);
          rootNode.setAttribute(VALUE, token.getValue());

          token = nextToken(); // consume the number
          break;
        }

      case REAL:
        {
          // Create an REAL_CONSTANT node as the root node.
          rootNode = ICodeFactory.createICodeNode(REAL_CONSTANT);
          rootNode.setAttribute(VALUE, token.getValue());

          token = nextToken(); // consume the number
          break;
        }

      case STRING:
        {
          String value = (String) token.getValue();

          // Create a STRING_CONSTANT node as the root node.
          rootNode = ICodeFactory.createICodeNode(STRING_CONSTANT);
          rootNode.setAttribute(VALUE, value);

          token = nextToken(); // consume the string
          break;
        }

      case NOT:
        {
          token = nextToken(); // consume the NOT

          // Create a NOT node as the root node.
          rootNode = ICodeFactory.createICodeNode(ICodeNodeTypeImpl.NOT);

          // Parse the factor.  The NOT node adopts the
          // factor node as its child.
          rootNode.addChild(parseFactor(token));

          break;
        }

      case LEFT_PAREN:
        {
          token = nextToken(); // consume the (

          // Parse an expression and make its node the root node.
          rootNode = parseExpression(token);

          // Look for the matching ) token.
          token = currentToken();
          if (token.getType() == RIGHT_PAREN) {
            token = nextToken(); // consume the )
          } else {
            errorHandler.flag(token, MISSING_RIGHT_PAREN, this);
          }

          break;
        }
      case LEFT_BRACKET:
        {
          rootNode = parseSet(token);

          break;
        }

      default:
        {
          errorHandler.flag(token, UNEXPECTED_TOKEN, this);
          break;
        }
    }

    return rootNode;
  }
  private String getJpaQuery(
      List<Token> tokens,
      String simpleTypeName,
      JavaSymbolName finderName,
      String plural,
      String entityName) {
    String typeName = StringUtils.hasText(entityName) ? entityName : simpleTypeName;
    StringBuilder builder = new StringBuilder();
    builder.append("SELECT o FROM ").append(typeName);
    builder.append(" AS o WHERE ");

    FieldToken lastFieldToken = null;
    boolean isNewField = true;
    boolean isFieldApplied = false;

    for (Token token : tokens) {
      if (token instanceof ReservedToken) {
        String reservedToken = token.getValue();
        if (lastFieldToken == null) continue;
        String fieldName = lastFieldToken.getField().getFieldName().getSymbolName();
        boolean setField = true;

        if (!lastFieldToken.getField().getFieldType().isCommonCollectionType()) {
          if (isNewField) {
            if (reservedToken.equalsIgnoreCase("Like")) {
              builder.append("LOWER(").append("o.").append(fieldName).append(')');
            } else {
              builder.append("o.").append(fieldName);
            }
            isNewField = false;
            isFieldApplied = false;
          }
          if (reservedToken.equalsIgnoreCase("And")) {
            if (!isFieldApplied) {
              builder.append(" = :").append(fieldName);
              isFieldApplied = true;
            }
            builder.append(" AND ");
            setField = false;
          } else if (reservedToken.equalsIgnoreCase("Or")) {
            if (!isFieldApplied) {
              builder.append(" = :").append(fieldName);
              isFieldApplied = true;
            }
            builder.append(" OR ");
            setField = false;
          } else if (reservedToken.equalsIgnoreCase("Between")) {
            builder
                .append(" BETWEEN ")
                .append(":min")
                .append(
                    lastFieldToken.getField().getFieldName().getSymbolNameCapitalisedFirstLetter())
                .append(" AND ")
                .append(":max")
                .append(
                    lastFieldToken.getField().getFieldName().getSymbolNameCapitalisedFirstLetter())
                .append(" ");
            setField = false;
            isFieldApplied = true;
          } else if (reservedToken.equalsIgnoreCase("Like")) {
            builder.append(" LIKE ");
            setField = true;
          } else if (reservedToken.equalsIgnoreCase("IsNotNull")) {
            builder.append(" IS NOT NULL ");
            setField = false;
            isFieldApplied = true;
          } else if (reservedToken.equalsIgnoreCase("IsNull")) {
            builder.append(" IS NULL ");
            setField = false;
            isFieldApplied = true;
          } else if (reservedToken.equalsIgnoreCase("Not")) {
            builder.append(" IS NOT ");
          } else if (reservedToken.equalsIgnoreCase("NotEquals")) {
            builder.append(" != ");
          } else if (reservedToken.equalsIgnoreCase("LessThan")) {
            builder.append(" < ");
          } else if (reservedToken.equalsIgnoreCase("LessThanEquals")) {
            builder.append(" <= ");
          } else if (reservedToken.equalsIgnoreCase("GreaterThan")) {
            builder.append(" > ");
          } else if (reservedToken.equalsIgnoreCase("GreaterThanEquals")) {
            builder.append(" >= ");
          } else if (reservedToken.equalsIgnoreCase("Equals")) {
            builder.append(" = ");
          }
          if (setField) {
            if (builder.toString().endsWith("LIKE ")) {
              builder.append("LOWER(:").append(fieldName).append(") ");
            } else {
              builder.append(':').append(fieldName).append(' ');
            }
            isFieldApplied = true;
          }
        }
      } else {
        lastFieldToken = (FieldToken) token;
        isNewField = true;
      }
    }
    if (isNewField) {
      if (lastFieldToken != null
          && !lastFieldToken.getField().getFieldType().isCommonCollectionType()) {
        builder.append("o.").append(lastFieldToken.getField().getFieldName().getSymbolName());
      }
      isFieldApplied = false;
    }
    if (!isFieldApplied) {
      if (lastFieldToken != null
          && !lastFieldToken.getField().getFieldType().isCommonCollectionType()) {
        builder.append(" = :").append(lastFieldToken.getField().getFieldName().getSymbolName());
      }
    }
    return builder.toString().trim();
  }
  /**
   * Parses a classic date format string into Tokens
   *
   * @param format the format to parse, not null
   * @return array of Token[]
   */
  static Token[] lexx(final String format) {
    final ArrayList<Token> list = new ArrayList<Token>(format.length());

    boolean inLiteral = false;
    // Although the buffer is stored in a Token, the Tokens are only
    // used internally, so cannot be accessed by other threads
    StringBuilder buffer = null;
    Token previous = null;
    for (int i = 0; i < format.length(); i++) {
      final char ch = format.charAt(i);
      if (inLiteral && ch != '\'') {
        buffer.append(ch); // buffer can't be null if inLiteral is true
        continue;
      }
      Object value = null;
      switch (ch) {
          // TODO: Need to handle escaping of '
        case '\'':
          if (inLiteral) {
            buffer = null;
            inLiteral = false;
          } else {
            buffer = new StringBuilder();
            list.add(new Token(buffer));
            inLiteral = true;
          }
          break;
        case 'y':
          value = y;
          break;
        case 'M':
          value = M;
          break;
        case 'd':
          value = d;
          break;
        case 'H':
          value = H;
          break;
        case 'm':
          value = m;
          break;
        case 's':
          value = s;
          break;
        case 'S':
          value = S;
          break;
        default:
          if (buffer == null) {
            buffer = new StringBuilder();
            list.add(new Token(buffer));
          }
          buffer.append(ch);
      }

      if (value != null) {
        if (previous != null && previous.getValue().equals(value)) {
          previous.increment();
        } else {
          final Token token = new Token(value);
          list.add(token);
          previous = token;
        }
        buffer = null;
      }
    }
    if (inLiteral) { // i.e. we have not found the end of the literal
      throw new IllegalArgumentException("Unmatched quote in format: " + format);
    }
    return list.toArray(new Token[list.size()]);
  }
Example #24
0
  /**
   * Parse a factor.
   *
   * @param token the initial token.
   * @return the root of the generated parse subtree.
   * @throws Exception if an error occurred.
   */
  private ICodeNode parseFactor(Token token) throws Exception {
    TokenType tokenType = token.getType();
    ICodeNode rootNode = null;

    switch ((PascalTokenType) tokenType) {
      case IDENTIFIER:
        {
          return parseIdentifier(token);
        }

      case INTEGER:
        {
          // Create an INTEGER_CONSTANT node as the root node
          rootNode = ICodeFactory.createICodeNode(INTEGER_CONSTANT);
          rootNode.setAttribute(VALUE, token.getValue());

          token = nextToken(); // consume the number
          rootNode.setTypeSpec(Predefined.integerType);
          break;
        }

      case REAL:
        {
          // Create an REAL_CONSTANT node as the root node.
          rootNode = ICodeFactory.createICodeNode(REAL_CONSTANT);
          rootNode.setAttribute(VALUE, token.getValue());

          token = nextToken(); // consume the number

          rootNode.setTypeSpec(Predefined.realType);
          break;
        }

      case STRING:
        {
          String value = (String) token.getValue();

          // Create a STRING_CONSTANT node as the root node.
          rootNode = ICodeFactory.createICodeNode(STRING_CONSTANT);
          rootNode.setAttribute(VALUE, value);

          TypeSpec resultType =
              value.length() == 1 ? Predefined.charType : TypeFactory.createStringType(value);

          token = nextToken(); // consume the string
          rootNode.setTypeSpec(resultType);
          break;
        }

      case NOT:
        {
          token = nextToken(); // consume the NOT

          // Create a NOT node as the root node.
          rootNode = ICodeFactory.createICodeNode(ICodeNodeTypeImpl.NOT);

          // Parse the factor. The NOT node adopts the factor node as its child.
          ICodeNode factorNode = parseFactor(token);
          rootNode.addChild(factorNode);

          // Type check: the factor must be boolean.
          TypeSpec factorType =
              factorNode != null ? factorNode.getTypeSpec() : Predefined.undefinedType;
          if (!TypeChecker.isBoolean(factorType))
            errorHandler.flag(token, INCOMPATIBLE_TYPES, this);
          rootNode.setTypeSpec(Predefined.booleanType);
          break;
        }

      case LEFT_PAREN:
        {
          token = nextToken(); // consume the (

          // Parse an expression and make its node the root node.
          rootNode = parseExpression(token);
          TypeSpec resultType =
              rootNode != null ? rootNode.getTypeSpec() : Predefined.undefinedType;

          // Look for the matching ) token.
          token = currentToken();
          if (token.getType() == RIGHT_PAREN) token = nextToken(); // consume the )
          else errorHandler.flag(token, MISSING_RIGHT_PAREN, this);
          rootNode.setTypeSpec(resultType);
          break;
        }

      default:
        {
          errorHandler.flag(token, UNEXPECTED_TOKEN, this);
          break;
        }
    }
    return rootNode;
  }
Example #25
0
 /**
  * Adds a token element to the map of tokens to replace.
  *
  * @param token The token to add to the map of replacements. Must not be <code>null</code>.
  */
 public void addConfiguredToken(final Token token) {
   hash.put(token.getKey(), token.getValue());
 }
 /**
  * The internal method to do the formatting.
  *
  * @param tokens the tokens
  * @param years the number of years
  * @param months the number of months
  * @param days the number of days
  * @param hours the number of hours
  * @param minutes the number of minutes
  * @param seconds the number of seconds
  * @param milliseconds the number of millis
  * @param padWithZeros whether to pad
  * @return the formatted string
  */
 static String format(
     Token[] tokens,
     int years,
     int months,
     int days,
     int hours,
     int minutes,
     int seconds,
     int milliseconds,
     boolean padWithZeros) {
   StringBuffer buffer = new StringBuffer();
   boolean lastOutputSeconds = false;
   int sz = tokens.length;
   for (int i = 0; i < sz; i++) {
     Token token = tokens[i];
     Object value = token.getValue();
     int count = token.getCount();
     if (value instanceof StringBuffer) {
       buffer.append(value.toString());
     } else {
       if (value == y) {
         buffer.append(
             padWithZeros
                 ? StringUtils.leftPad(Integer.toString(years), count, '0')
                 : Integer.toString(years));
         lastOutputSeconds = false;
       } else if (value == M) {
         buffer.append(
             padWithZeros
                 ? StringUtils.leftPad(Integer.toString(months), count, '0')
                 : Integer.toString(months));
         lastOutputSeconds = false;
       } else if (value == d) {
         buffer.append(
             padWithZeros
                 ? StringUtils.leftPad(Integer.toString(days), count, '0')
                 : Integer.toString(days));
         lastOutputSeconds = false;
       } else if (value == H) {
         buffer.append(
             padWithZeros
                 ? StringUtils.leftPad(Integer.toString(hours), count, '0')
                 : Integer.toString(hours));
         lastOutputSeconds = false;
       } else if (value == m) {
         buffer.append(
             padWithZeros
                 ? StringUtils.leftPad(Integer.toString(minutes), count, '0')
                 : Integer.toString(minutes));
         lastOutputSeconds = false;
       } else if (value == s) {
         buffer.append(
             padWithZeros
                 ? StringUtils.leftPad(Integer.toString(seconds), count, '0')
                 : Integer.toString(seconds));
         lastOutputSeconds = true;
       } else if (value == S) {
         if (lastOutputSeconds) {
           milliseconds += 1000;
           String str =
               padWithZeros
                   ? StringUtils.leftPad(Integer.toString(milliseconds), count, '0')
                   : Integer.toString(milliseconds);
           buffer.append(str.substring(1));
         } else {
           buffer.append(
               padWithZeros
                   ? StringUtils.leftPad(Integer.toString(milliseconds), count, '0')
                   : Integer.toString(milliseconds));
         }
         lastOutputSeconds = false;
       }
     }
   }
   return buffer.toString();
 }
  /**
   * Parses a classic date format string into Tokens
   *
   * @param format to parse
   * @return array of Token[]
   */
  @SuppressWarnings({"rawtypes", "unchecked"})
  static Token[] lexx(String format) {
    char[] array = format.toCharArray();
    ArrayList list = new ArrayList(array.length);

    boolean inLiteral = false;
    StringBuffer buffer = null;
    Token previous = null;
    int sz = array.length;
    for (int i = 0; i < sz; i++) {
      char ch = array[i];
      if (inLiteral && ch != '\'') {
        buffer.append(ch); // buffer can't be null if inLiteral is true
        continue;
      }
      Object value = null;
      switch (ch) {
          // TODO: Need to handle escaping of '
        case '\'':
          if (inLiteral) {
            buffer = null;
            inLiteral = false;
          } else {
            buffer = new StringBuffer();
            list.add(new Token(buffer));
            inLiteral = true;
          }
          break;
        case 'y':
          value = y;
          break;
        case 'M':
          value = M;
          break;
        case 'd':
          value = d;
          break;
        case 'H':
          value = H;
          break;
        case 'm':
          value = m;
          break;
        case 's':
          value = s;
          break;
        case 'S':
          value = S;
          break;
        default:
          if (buffer == null) {
            buffer = new StringBuffer();
            list.add(new Token(buffer));
          }
          buffer.append(ch);
      }

      if (value != null) {
        if (previous != null && previous.getValue() == value) {
          previous.increment();
        } else {
          Token token = new Token(value);
          list.add(token);
          previous = token;
        }
        buffer = null;
      }
    }
    return (Token[]) list.toArray(new Token[list.size()]);
  }
Example #28
0
  /**
   * オペレータ情報リスト作成
   *
   * @param tknlist in 式を構成するトークンリスト
   * @param explist in/out オペレータ情報リスト
   * @param idxArray in/out トークンリスト参照時の添え字 ※出力パラメータにするために、配列とした
   * @param opinfo in ひとつ前に作成したオペレータ情報
   */
  private int createOperatorList(
      ArrayList<Token> tknlist,
      ArrayList<OperatorInfo> explist,
      int[] idxArray,
      OperatorInfo opinfo) {

    int ret = 0;
    try {
      OperatorInfo nextOpinfo = new OperatorInfo();

      Token tkn = (Token) tknlist.get(idx);
      if (tkn.getValue().equals("(")) {
        idx++;
        explist = new ArrayList<OperatorInfo>();

        OperatorInfo opinfo2 = new OperatorInfo();
        do {
          ret = createOperatorList(tknlist, explist, idxArray, opinfo2);
        } while (ret == 0);

        // 退避していた1階層上のexplistを取得
        explist = (ArrayList) expStack.pop();

        nextOpinfo.setLeft(opinfo2.getLeft());

      } else {
        nextOpinfo.setLeft(new OperatorInfo(tknlist.get(idx++)));
      }
      if (tknlist.size() <= idx) {
        explist.add(nextOpinfo);
        idxArray[0] = idx;
        // 式の評価
        OperatorInfo opinfo_wk = evaluate(explist); // opeの設定は? @@@@@@
        // 出力オペレータ情報を設定
        opinfo.setLeft(opinfo_wk.getLeft());

        return 9;
      }
      ret = 0;
      Token tkn2 = (Token) tknlist.get(idx);
      if (tkn2.getValue().equals(")")) {
        explist.add(nextOpinfo);
        // 式の評価
        OperatorInfo opinfo_wk = evaluate(explist); // opeの設定は? @@@@@@
        // 出力オペレータ情報を設定
        opinfo.setLeft(opinfo_wk.getLeft());

        ret = 9;
        idx++;

      } else {
        nextOpinfo.setOpe(tknlist.get(idx++));
        explist.add(nextOpinfo);
      }
      idxArray[0] = idx;
      // return nextOpinfo ;
    } catch (OtherAppException ex) {
      // LogTrace.logout(3, ex.getMessage());
      ret = 9;
    }
    return ret;
  }
  private void translate() {
    ListIterator<Statement> statementList = statements.listIterator();

    // Loops through all statements in the statement list
    while (statementList.hasNext()) {
      Statement statement = statementList.next();

      statementList.remove();

      // Translated instruction
      StringBuilder translation = new StringBuilder();

      /* At this point in assembly only OpCode and/or Data statements are
       * in the statement list */
      if (statement instanceof OpCodeStatement) {
        // OpCode found
        OpCodeStatement opCode = (OpCodeStatement) statement;

        translation.append(opCode.getOpCode().getValue());

        /** Translate each token into machine code */
        for (Token token : opCode.getTokens()) {
          switch (token.getType()) {
              // ADDRESS tokens are translated into 3 digit addresses
            case ADDRESS:
              StringBuilder address = new StringBuilder(token.getValue());
              while (address.length() < 3) {
                address.insert(0, "0");
              }

              translation.append(address.toString());
              break;

              // CHARACTER tokens are translated into 3 digit ascii values
            case CHARACTER:
              StringBuilder character =
                  new StringBuilder(Integer.toString((int) token.getValue().charAt(1)));

              while (character.length() < 3) {
                character.insert(0, "0");
              }

              translation.append(character);
              break;

              // JMP Condition codes
            case CONDITION:
              StringBuilder code = new StringBuilder();
              code.append(Condition.getByMnemonic(token.getValue()).getValue());
              while (code.length() < 3) {
                code.insert(0, "0");
              }

              translation.append(code.toString());
              break;

              // REGISTER tokens
            case REGISTER:
              String register = token.getValue();

              StringBuilder addressMnemonic = new StringBuilder();
              StringBuilder registerMnemonic = new StringBuilder();

              // Split Register into AddressMode / Register ID
              for (int i = 0; i < register.length(); i++) {
                char nextChar = register.charAt(i);
                if (!Character.isLetterOrDigit(nextChar)) {
                  // AddressModes do not have letter or digit characters
                  addressMnemonic.append(nextChar);
                } else {
                  registerMnemonic.append(nextChar);
                }
              }

              // Get addressMode
              AddressMode addressMode = AddressMode.getByMnemonic(addressMnemonic.toString());

              // IMMEDIATE address modes don't actually use address modes
              switch (addressMode) {
                case IMMEDIATE_DIRECT:
                case IMMEDIATE_INDIRECT:
                  // Ensure correct padding for value
                  StringBuilder value = new StringBuilder(registerMnemonic.toString());
                  while (value.length() < 2) {
                    value.insert(0, "0");
                  }

                  translation.append(value.toString());
                  break;
                default:
                  // All others use Registers
                  translation.append(
                      Register.getByMnemonic(registerMnemonic.toString()).getRegisterID());
              }

              translation.append(addressMode.getValue());

              break;

              // EOL tokens can finally be discarded.
            case EOL:
              break;
          }
        }

        /* OpCode statements can contain optional parameters.
         * If these have not been specified the instruction needs padding */
        while (translation.length() < 7) {
          translation.append("000");
        }

      } else if (statement instanceof DataStatement) {
        // Data found.  Simply add the data it contains in Token format.
        translation.append(((DataStatement) statement).getData());
      } else {
        logError(statement, "Invalid statmenent");
      }

      // Store the instruction
      try {
        Instruction instruction = new Instruction(statement, translation.toString());
        statementList.add(instruction);
        instructions.add(instruction);
      } catch (AssemblyError e) {
        errors.add(e);
      }
    }
  }