Example #1
1
 /**
  * Given a string, return an array of tokens. The separator can be escaped with the '\' character.
  * The '\' character may also be escaped by the '\' character.
  *
  * @param s the string to tokenize.
  * @param separator the separator char.
  * @param maxTokens the maxmimum number of tokens returned. If the max is reached, the remaining
  *     part of s is appended to the end of the last token.
  * @return an array of tokens.
  */
 public static String[] tokenize(String s, char separator, int maxTokens) {
   List tokens = new ArrayList();
   StringBuilder token = new StringBuilder();
   boolean prevIsEscapeChar = false;
   for (int i = 0; i < s.length(); i += Character.charCount(i)) {
     int currentChar = s.codePointAt(i);
     if (prevIsEscapeChar) {
       // Case 1:  escaped character
       token.appendCodePoint(currentChar);
       prevIsEscapeChar = false;
     } else if (currentChar == separator && tokens.size() < maxTokens - 1) {
       // Case 2:  separator
       tokens.add(token.toString());
       token = new StringBuilder();
     } else if (currentChar == '\\') {
       // Case 3:  escape character
       prevIsEscapeChar = true;
     } else {
       // Case 4:  regular character
       token.appendCodePoint(currentChar);
     }
   }
   if (token.length() > 0) {
     tokens.add(token.toString());
   }
   return (String[]) tokens.toArray(new String[] {});
 }
Example #2
0
 /**
  * Escapes a character sequence so that it is valid XML.
  *
  * @param s The character sequence.
  * @return The escaped version of the character sequence.
  */
 public static String escapeXML(CharSequence s) {
   // double quote -- quot
   // ampersand -- amp
   // less than -- lt
   // greater than -- gt
   // apostrophe -- apos
   StringBuilder sb = new StringBuilder(s.length() * 2);
   for (int i = 0; i < s.length(); ) {
     int codePoint = Character.codePointAt(s, i);
     if (codePoint == '<') {
       sb.append(LT);
     } else if (codePoint == '>') {
       sb.append(GT);
     } else if (codePoint == '\"') {
       sb.append(QUOT);
     } else if (codePoint == '&') {
       sb.append(AMP);
     } else if (codePoint == '\'') {
       sb.append(APOS);
     } else {
       sb.appendCodePoint(codePoint);
     }
     i += Character.charCount(codePoint);
   }
   return sb.toString();
 }
Example #3
0
 /** Decodes escapes in an identifier */
 public static String decodeCssIdentifier(CharSequence ident) {
   StringBuilder sb = null;
   int pos = 0;
   for (int i = 0, n = ident.length(); i < n; ) {
     if (ident.charAt(i) == '\\') {
       if (sb == null) {
         sb = new StringBuilder();
       }
       sb.append(ident, pos, i);
       int codepoint = 0;
       while (++i < n && isHexChar(ident.charAt(i))) {
         char ch = ident.charAt(i);
         codepoint <<= 4;
         if (ch >= '0' && ch <= '9') {
           codepoint |= ch - '0';
         } else if (ch >= 'a' && ch <= 'f') {
           codepoint |= ch + 10 - 'a';
         } else {
           codepoint |= ch + 10 - 'A';
         }
       }
       sb.appendCodePoint(codepoint < Character.MAX_CODE_POINT ? codepoint : 0xfffd);
       if (i < n && isSpaceChar(ident.charAt(i))) {
         ++i;
       }
       pos = i;
     } else {
       ++i;
     }
   }
   if (sb == null) {
     return ident.toString();
   }
   return sb.append(ident, pos, ident.length()).toString();
 }
 /* map bits to unicode codepoints */
 private static String mapInt(int codePointTable[], int i) {
   StringBuilder sb = new StringBuilder();
   String binary = Integer.toBinaryString(i);
   for (int j = 0; j < binary.length(); j++)
     sb.appendCodePoint(codePointTable[binary.charAt(j) - '0']);
   return sb.toString();
 }
Example #5
0
 public static String enquoteCString(String value, char delimiter) {
   final int length = value.length();
   StringBuilder result = new StringBuilder();
   result.append(delimiter);
   for (int offset = 0, codepoint; offset < length; offset += Character.charCount(codepoint)) {
     codepoint = value.codePointAt(offset);
     if (codepoint > 0xFF) {
       result.appendCodePoint(codepoint);
     } else if (codepoint == delimiter) {
       result.append("\\" + delimiter);
     } else if (codepoint == '\\') {
       result.append("\\\\");
     } else if (codepoint == '\n') {
       result.append("\\n");
     } else if (codepoint == '\t') {
       result.append("\\t");
     } else if (codepoint == '\r') {
       result.append("\\r");
     } else if (codepoint == '\f') {
       result.append("\\f");
     } else if (codepoint >= 32 && codepoint < 127) {
       result.append((char) codepoint);
     } else if (codepoint <= 0xff) {
       result.append("\\");
       result.append(String.format("%03o", codepoint));
     }
   }
   result.append(delimiter);
   return result.toString();
 }
Example #6
0
    public String read() throws IOException {
      StringBuilder sb = new StringBuilder();
      if (offset == bufferSize) {
        offset = 0;
        bufferSize = in.read(buffer);
      }

      if (bufferSize == -1 || bufferSize == 0) throw new IOException("No new bytes");

      for (;
          buffer[offset] == ' '
              || buffer[offset] == '\t'
              || buffer[offset] == '\n'
              || buffer[offset] == '\r';
          ++offset) {
        if (offset == bufferSize - 1) {
          offset = -1;
          bufferSize = in.read(buffer);
        }
      }
      for (; offset < bufferSize; ++offset) {
        if (buffer[offset] == ' '
            || buffer[offset] == '\t'
            || buffer[offset] == '\n'
            || buffer[offset] == '\r') break;
        if (Character.isValidCodePoint(buffer[offset])) {
          sb.appendCodePoint(buffer[offset]);
        }
        if (offset == bufferSize - 1) {
          offset = -1;
          bufferSize = in.read(buffer);
        }
      }
      return sb.toString();
    }
Example #7
0
 private static String byteArrayToString(byte[] bytes) {
   StringBuilder builder = new StringBuilder();
   for (byte b : bytes) {
     builder.appendCodePoint(b & 0xFF);
   }
   return builder.toString();
 }
Example #8
0
  private void generateChar(int cp) {
    StringBuilder sb = new StringBuilder();
    sb.appendCodePoint(cp);
    String s = sb.toString();

    TessOutput tessed = tesselateString(s);
    int totalVerts = 0;
    for (double[] ds : _vertices) {
      totalVerts += ds.length;
    }

    assert ((totalVerts % 2) == 0);
    assert ((tessed.verts.length % 2) == 0);
    // startIndex is the index of points in the GL buffer this character starts at
    int startIndex = totalVerts / 2;

    // numVerts is the number of vertices in the GL buffer to draw
    int numVerts = tessed.verts.length / 2;

    // Append the verts to the list
    _vertices.add(tessed.verts);

    TessChar tc =
        new TessChar(
            cp,
            startIndex,
            numVerts,
            tessed.bounds.getWidth(),
            tessed.bounds.getHeight(),
            tessed.advances[0]);
    _charMap.put(cp, tc);

    _glBufferDirty = true;
  }
Example #9
0
 /* package for test */ static SuggestedWordInfo getTransformedSuggestedWordInfo(
     final SuggestedWordInfo wordInfo,
     final Locale locale,
     final boolean isAllUpperCase,
     final boolean isFirstCharCapitalized,
     final int trailingSingleQuotesCount) {
   final StringBuilder sb = new StringBuilder(wordInfo.mWord.length());
   if (isAllUpperCase) {
     sb.append(wordInfo.mWord.toUpperCase(locale));
   } else if (isFirstCharCapitalized) {
     sb.append(StringUtils.capitalizeFirstCodePoint(wordInfo.mWord, locale));
   } else {
     sb.append(wordInfo.mWord);
   }
   // Appending quotes is here to help people quote words. However, it's not helpful
   // when they type words with quotes toward the end like "it's" or "didn't", where
   // it's more likely the user missed the last character (or didn't type it yet).
   final int quotesToAppend =
       trailingSingleQuotesCount
           - (-1 == wordInfo.mWord.indexOf(Constants.CODE_SINGLE_QUOTE) ? 0 : 1);
   for (int i = quotesToAppend - 1; i >= 0; --i) {
     sb.appendCodePoint(Constants.CODE_SINGLE_QUOTE);
   }
   return new SuggestedWordInfo(
       sb.toString(),
       wordInfo.mScore,
       wordInfo.mKind,
       wordInfo.mSourceDict,
       wordInfo.mIndexOfTouchPointOfSecondWord,
       wordInfo.mAutoCommitFirstWordConfidence);
 }
Example #10
0
 public static void appendSubscript(StringBuilder sb, int subscript) {
   String digits = Integer.toString(subscript);
   for (int i = 0; i != digits.length(); ++i) {
     int digit = digits.charAt(i) - '0';
     sb.appendCodePoint(0x2080 + digit);
   }
 }
Example #11
0
  protected String consume(BufferedReader br) throws Exception {
    StringBuilder ret = new StringBuilder();

    int ch;
    while ((ch = br.read()) != -1) ret.appendCodePoint(ch);

    return ret.toString();
  }
 /** Reverse escaping done by replaceFileNameDangerousCharacters. */
 public static String getWordListIdFromFileName(final String fname) {
   final StringBuilder sb = new StringBuilder();
   final int fnameLength = fname.length();
   for (int i = 0; i < fnameLength; i = fname.offsetByCodePoints(i, 1)) {
     final int codePoint = fname.codePointAt(i);
     if ('%' != codePoint) {
       sb.appendCodePoint(codePoint);
     } else {
       // + 1 to pass the % sign
       final int encodedCodePoint =
           Integer.parseInt(fname.substring(i + 1, i + 1 + MAX_HEX_DIGITS_FOR_CODEPOINT), 16);
       i += MAX_HEX_DIGITS_FOR_CODEPOINT;
       sb.appendCodePoint(encodedCodePoint);
     }
   }
   return sb.toString();
 }
  public void testRandomRealisticWhiteSpace() throws IOException {
    Map<String, String> map = new HashMap<>();
    Set<String> seen = new HashSet<>();
    int numTerms = atLeast(50);
    boolean ignoreCase = random().nextBoolean();

    for (int i = 0; i < numTerms; i++) {
      String randomRealisticUnicodeString = TestUtil.randomRealisticUnicodeString(random());
      char[] charArray = randomRealisticUnicodeString.toCharArray();
      StringBuilder builder = new StringBuilder();
      for (int j = 0; j < charArray.length; ) {
        int cp = Character.codePointAt(charArray, j, charArray.length);
        if (!Character.isWhitespace(cp)) {
          builder.appendCodePoint(cp);
        }
        j += Character.charCount(cp);
      }
      if (builder.length() > 0) {
        String inputValue = builder.toString();

        // Make sure we don't try to add two inputs that vary only by case:
        String seenInputValue;
        if (ignoreCase) {
          // TODO: can we simply use inputValue.toLowerCase(Locale.ROOT)???
          char[] buffer = inputValue.toCharArray();
          CharacterUtils.toLowerCase(buffer, 0, buffer.length);
          seenInputValue = buffer.toString();
        } else {
          seenInputValue = inputValue;
        }

        if (seen.contains(seenInputValue) == false) {
          seen.add(seenInputValue);
          String value = TestUtil.randomSimpleString(random());
          map.put(inputValue, value.isEmpty() ? "a" : value);
        }
      }
    }
    if (map.isEmpty()) {
      map.put("booked", "books");
    }
    StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder(ignoreCase);
    Set<Entry<String, String>> entrySet = map.entrySet();
    StringBuilder input = new StringBuilder();
    List<String> output = new ArrayList<>();
    for (Entry<String, String> entry : entrySet) {
      builder.add(entry.getKey(), entry.getValue());
      if (random().nextBoolean() || output.isEmpty()) {
        input.append(entry.getKey()).append(" ");
        output.add(entry.getValue());
      }
    }
    Tokenizer tokenizer = new WhitespaceTokenizer();
    tokenizer.setReader(new StringReader(input.toString()));
    TokenStream stream =
        new PorterStemFilter(new StemmerOverrideFilter(tokenizer, builder.build()));
    assertTokenStreamContents(stream, output.toArray(new String[0]));
  }
  /**
   * Evaluate expression in a string.
   *
   * @param s string in witch expression must be replaced
   * @param allowExec allow execution of code
   * @return a string with expression evaluated
   * @throws EoulsanException if an error occurs while parsing the string or executing an expression
   */
  private String evaluateExpressions(final String s, final boolean allowExec)
      throws EoulsanException {

    if (s == null) {
      return null;
    }

    final StringBuilder result = new StringBuilder();

    final int len = s.length();

    for (int i = 0; i < len; i++) {

      final int c0 = s.codePointAt(i);

      // Variable substitution
      if (c0 == '$' && i + 1 < len) {

        final int c1 = s.codePointAt(i + 1);
        if (c1 == '{') {

          final String expr = subStr(s, i + 2, '}');

          final String trimmedExpr = expr.trim();
          if (this.constants.containsKey(trimmedExpr)) {
            result.append(this.constants.get(trimmedExpr));
          }

          i += expr.length() + 2;
          continue;
        }
      }

      // Command substitution
      if (c0 == '`' && allowExec) {
        final String expr = subStr(s, i + 1, '`');
        try {
          final String r = ProcessUtils.execToString(evaluateExpressions(expr, false));

          // remove last '\n' in the result
          if (r.charAt(r.length() - 1) == '\n') {
            result.append(r.substring(0, r.length() - 1));
          } else {
            result.append(r);
          }

        } catch (IOException e) {
          throw new EoulsanException("Error while evaluating expression \"" + expr + "\"", e);
        }
        i += expr.length() + 1;
        continue;
      }

      result.appendCodePoint(c0);
    }

    return result.toString();
  }
 public String ns() {
   int b = skip();
   StringBuilder sb = new StringBuilder();
   while (!(isSpaceChar(b))) { // when nextLine, (isSpaceChar(b) && b != // ' ')
     sb.appendCodePoint(b);
     b = readByte();
   }
   return sb.toString();
 }
  // $ANTLR start "STRING_LITERAL"
  public final void mSTRING_LITERAL() throws RecognitionException {
    try {
      int _type = STRING_LITERAL;
      int _channel = DEFAULT_TOKEN_CHANNEL;
      int c;

      // TWS_VARIABLES.g:51:3: ( '\"' (c=~ ( '\"' | '\\r' | '\\n' ) )* '\"' )
      // TWS_VARIABLES.g:51:5: '\"' (c=~ ( '\"' | '\\r' | '\\n' ) )* '\"'
      {
        match('\"');
        StringBuilder b = new StringBuilder();
        // TWS_VARIABLES.g:53:5: (c=~ ( '\"' | '\\r' | '\\n' ) )*
        loop1:
        while (true) {
          int alt1 = 2;
          int LA1_0 = input.LA(1);
          if (((LA1_0 >= '\u0000' && LA1_0 <= '\t')
              || (LA1_0 >= '\u000B' && LA1_0 <= '\f')
              || (LA1_0 >= '\u000E' && LA1_0 <= '!')
              || (LA1_0 >= '#' && LA1_0 <= '\uFFFF'))) {
            alt1 = 1;
          }

          switch (alt1) {
            case 1:
              // TWS_VARIABLES.g:53:7: c=~ ( '\"' | '\\r' | '\\n' )
              {
                c = input.LA(1);
                if ((input.LA(1) >= '\u0000' && input.LA(1) <= '\t')
                    || (input.LA(1) >= '\u000B' && input.LA(1) <= '\f')
                    || (input.LA(1) >= '\u000E' && input.LA(1) <= '!')
                    || (input.LA(1) >= '#' && input.LA(1) <= '\uFFFF')) {
                  input.consume();
                } else {
                  MismatchedSetException mse = new MismatchedSetException(null, input);
                  recover(mse);
                  throw mse;
                }
                b.appendCodePoint(c);
              }
              break;

            default:
              break loop1;
          }
        }

        match('\"');
        setText(b.toString());
      }

      state.type = _type;
      state.channel = _channel;
    } finally {
      // do for sure before leaving
    }
  }
 /** A..Z, AA..ZZ, AAA..ZZZ ... labeling as described in PDF32000-1:2008, Table 159, Page 375. */
 private static String makeLetterLabel(int num) {
   StringBuilder buf = new StringBuilder();
   int numLetters = num / 26 + Integer.signum(num % 26);
   int letter = num % 26 + 26 * (1 - Integer.signum(num % 26)) + 64;
   for (int i = 0; i < numLetters; i++) {
     buf.appendCodePoint(letter);
   }
   return buf.toString();
 }
 private String ns1() {
   int b = skip();
   StringBuilder sb = new StringBuilder();
   while (!(isSpaceChar(b) && b != ' ')) { // when nextLine,
     sb.appendCodePoint(b);
     b = readByte();
   }
   return sb.toString();
 }
  /**
   * Reads all characters from the current position to the end of this reader into a string.
   *
   * @return a {@link String} containing the characters from the current position to the end of the
   *     reader
   */
  @Nonnull
  public final String readToString() {
    StringBuilder sb = new StringBuilder();
    while (this.getCurrentCodePoint() != -1) {
      sb.appendCodePoint(this.getCurrentCodePoint());
      this.advance();
    }

    return sb.toString();
  }
 public String next() {
   int c = read();
   while (isSpaceChar(c)) c = read();
   StringBuilder res = new StringBuilder();
   do {
     res.appendCodePoint(c);
     c = read();
   } while (!isSpaceChar(c));
   return res.toString();
 }
  /**
   * Builds a Soy string literal for this string value (including the surrounding single quotes).
   * Note that Soy string syntax is a subset of JS string syntax, so the result should also be a
   * valid JS string.
   *
   * <p>Adapted from StringUtil.javaScriptEscape().
   *
   * @param value The string value to escape.
   * @param shouldEscapeToAscii Whether to escape non-ASCII characters as Unicode hex escapes
   *     (backslash + 'u' + 4 hex digits).
   * @return A Soy string literal for this string value (including the surrounding single quotes).
   */
  public static String escapeToSoyString(String value, boolean shouldEscapeToAscii) {

    // StringUtil.javaScriptEscape() is meant to be compatible with JS string syntax, which is a
    // superset of the Soy expression string syntax, so we can't depend on it to properly escape a
    // Soy expression string literal. For example, they switched the default character escaping
    // to octal to save a few bytes, but octal escapes are not allowed in Soy syntax. I'm rewriting
    // the code here in a correct way for Soy.

    int len = value.length();
    StringBuilder out = new StringBuilder(len * 9 / 8);
    out.append('\'');

    int codePoint;
    for (int i = 0; i < len; i += Character.charCount(codePoint)) {
      codePoint = value.codePointAt(i);

      switch (codePoint) {
        case '\n':
          out.append("\\n");
          break;
        case '\r':
          out.append("\\r");
          break;
        case '\t':
          out.append("\\t");
          break;
        case '\b':
          out.append("\\b");
          break;
        case '\f':
          out.append("\\f");
          break;
        case '\\':
          out.append("\\\\");
          break;
        case '\'':
          out.append("\\'");
          break;
        case '"':
          out.append('"');
          break; // note: don't escape double quotes in Soy strings
        default:
          // If shouldEscapeToAscii, then hex escape characters outside the range 0x20 to 0x7F.
          if (shouldEscapeToAscii && (codePoint < 0x20 || codePoint >= 0x7F)) {
            appendHexEscape(out, codePoint);
          } else {
            out.appendCodePoint(codePoint);
          }
          break;
      }
    }

    out.append('\'');
    return out.toString();
  }
Example #22
0
  /** Inserts Korean lead consonants records of name for the given structured name. */
  private void insertKoreanNameConsonantsLookup(long rawContactId, long dataId, String name) {
    int position = 0;
    int consonantLength = 0;
    int character;

    final int stringLength = name.length();
    mStringBuilder.setLength(0);
    do {
      character = name.codePointAt(position++);
      if (character == 0x20) {
        // Skip spaces.
        continue;
      }
      // Exclude characters that are not in Korean leading consonants area
      // and Korean characters area.
      if ((character < 0x1100)
          || (character > 0x1112 && character < 0x3131)
          || (character > 0x314E && character < 0xAC00)
          || (character > 0xD7A3)) {
        break;
      }
      // Decompose and take a only lead-consonant for composed Korean characters.
      if (character >= 0xAC00) {
        // Lead consonant = "Lead consonant base" +
        //      (character - "Korean Character base") /
        //          ("Lead consonant count" * "middle Vowel count")
        character = 0x1100 + (character - 0xAC00) / 588;
      } else if (character >= 0x3131) {
        // Hangul Compatibility Jamo area 0x3131 ~ 0x314E :
        // Convert to Hangul Jamo area 0x1100 ~ 0x1112
        if (character - 0x3131 >= KOREAN_JAUM_CONVERT_MAP_COUNT) {
          // This is not lead-consonant
          break;
        }
        character = KOREAN_JAUM_CONVERT_MAP[character - 0x3131];
        if (character == 0) {
          // This is not lead-consonant
          break;
        }
      }
      mStringBuilder.appendCodePoint(character);
      consonantLength++;
    } while (position < stringLength);

    // At least, insert consonants when Korean characters are two or more.
    // Only one character cases are covered by NAME_COLLATION_KEY
    if (consonantLength > 1) {
      insertNameLookup(
          rawContactId,
          dataId,
          NameLookupType.NAME_CONSONANTS,
          normalizeName(mStringBuilder.toString()));
    }
  }
 public String nextLine() {
   int c = read();
   // while (c != '\n' && c != '\r' && c != '\t' && c != -1)
   // c = read();
   StringBuilder res = new StringBuilder();
   do {
     res.appendCodePoint(c);
     c = read();
   } while (c != '\n' && c != '\r' && c != '\t' && c != -1);
   return res.toString();
 }
    public String nextLine() {
      int c = read();

      StringBuilder result = new StringBuilder();

      do {
        result.appendCodePoint(c);

        c = read();
      } while (!isNewLine(c));

      return result.toString();
    }
  /**
   * Reads next line from the input and:
   *
   * <ul>
   *   <li>Converts ascii-encoded \\uxxxx chars to normal characters.
   *   <li>Converts \r, \n and \t to CR, line feed and tab.
   *   <li>But! Keeps a backspace in '\ ', '\=', '\:' etc (non-trimmable space or
   *       non-key-value-breaking :-) equals).
   *       <ul>
   *         Change from BufferedReader to LinebreakPreservingReader was part of fix for bug 1462566
   */
  protected String getNextLine(LinebreakPreservingReader reader)
      throws IOException, TranslationException {
    String ascii = reader.readLine();
    if (ascii == null) {
      return null;
    }

    StringBuilder result = new StringBuilder();
    for (int cp, len = ascii.length(), i = 0; i < len; i += Character.charCount(cp)) {
      cp = ascii.codePointAt(i);
      if (cp == '\\' && ascii.codePointCount(i, len) > 1) {
        i += Character.charCount(cp);
        cp = ascii.codePointAt(i);
        if (cp != 'u') {
          if (cp == 'n') {
            cp = '\n';
          } else if (cp == 'r') {
            cp = '\r';
          } else if (cp == 't') {
            cp = '\t';
          } else {
            result.append('\\');
          }
        } else if (dontUnescapeULiterals) {
          // Put back the \ we swallowed
          result.append('\\');
        } else {
          // checking if the string is long enough
          if (ascii.codePointCount(i, len) < 1 + 4) {
            throw new TranslationException(OStrings.getString("RBFH_ERROR_ILLEGAL_U_SEQUENCE"));
          }
          int uStart = ascii.offsetByCodePoints(i, 1);
          int uEnd = ascii.offsetByCodePoints(uStart, 4);
          String uStr = ascii.substring(uStart, uEnd);
          try {
            cp = Integer.parseInt(uStr, 16);
            if (!Character.isValidCodePoint(cp)) {
              throw new TranslationException(OStrings.getString("RBFH_ERROR_ILLEGAL_U_SEQUENCE"));
            }
            i = uEnd - Character.charCount(cp);
          } catch (NumberFormatException ex) {
            throw new TranslationException(OStrings.getString("RBFH_ERROR_ILLEGAL_U_SEQUENCE"), ex);
          }
        }
      }
      result.appendCodePoint(cp);
    }

    return result.toString();
  }
Example #26
0
File: Prog.java Project: Godin/re2j
  // prefix() returns a pair of a literal string that all matches for the
  // regexp must start with, and a boolean which is true if the prefix is the
  // entire match.  The string is returned by appending to |prefix|.
  boolean prefix(StringBuilder prefix) {
    Inst i = skipNop(start);

    // Avoid allocation of buffer if prefix is empty.
    if (i.op() != Inst.Op.RUNE || i.runes.length != 1) {
      return i.op == Inst.Op.MATCH; // (append "" to prefix)
    }

    // Have prefix; gather characters.
    while (i.op() == Inst.Op.RUNE && i.runes.length == 1 && (i.arg & RE2.FOLD_CASE) == 0) {
      prefix.appendCodePoint(i.runes[0]); // an int, not a byte.
      i = skipNop(i.out);
    }
    return i.op == Inst.Op.MATCH;
  }
Example #27
0
  default ReadLineResult readLine(
      final boolean eofErrorP, final LispStruct eofValue, final boolean recursiveP) {

    final StringBuilder stringBuilder = new StringBuilder();

    ReadPeekResult readPeekResult = readChar(eofErrorP, eofValue, recursiveP);
    Integer result = readPeekResult.getResult();
    while (!readPeekResult.isEof() && (result != '\n')) {
      stringBuilder.appendCodePoint(result);

      readPeekResult = readChar(eofErrorP, eofValue, recursiveP);
      result = readPeekResult.getResult();
    }
    final String resultString = stringBuilder.toString();
    return new ReadLineResult(resultString, readPeekResult.isEof());
  }
 // TODO: create a unit test for this method
 public static String replaceFileNameDangerousCharacters(final String name) {
   // This assumes '%' is fully available as a non-separator, normal
   // character in a file name. This is probably true for all file systems.
   final StringBuilder sb = new StringBuilder();
   final int nameLength = name.length();
   for (int i = 0; i < nameLength; i = name.offsetByCodePoints(i, 1)) {
     final int codePoint = name.codePointAt(i);
     if (DictionaryInfoUtils.isFileNameCharacter(codePoint)) {
       sb.appendCodePoint(codePoint);
     } else {
       sb.append(
           String.format((Locale) null, "%%%1$0" + MAX_HEX_DIGITS_FOR_CODEPOINT + "x", codePoint));
     }
   }
   return sb.toString();
 }
  /**
   * Makes a human-readable version of a key ID, which is usually 64 bits: lower-case, no leading
   * 0x, space-separated quartets (for keys whose length in hex is divisible by 4)
   *
   * @param idHex - the key id
   * @return - the beautified form
   */
  public static String beautifyKeyId(String idHex) {
    if (idHex.startsWith("0x")) {
      idHex = idHex.substring(2);
    }
    if ((idHex.length() % 4) == 0) {
      StringBuilder sb = new StringBuilder();
      for (int i = 0; i < idHex.length(); i += 4) {
        if (i != 0) {
          sb.appendCodePoint(0x2008); // U+2008 PUNCTUATION SPACE
        }
        sb.append(idHex.substring(i, i + 4).toLowerCase(Locale.US));
      }
      idHex = sb.toString();
    }

    return idHex;
  }
 /**
  * Replace characters in the supplied {@link TextFragment} according to the replacement table
  * supplied by subclasses.
  *
  * @param ctx
  * @param textFragment
  */
 @Override
 public void visitTextFragment(final VisitorContext ctx, TextFragment textFragment) {
   String text = textFragment.getText();
   StringBuilder buf = new StringBuilder();
   int index = 0;
   while (index < text.length()) {
     int codePoint = text.codePointAt(index);
     index += Character.charCount(codePoint);
     String replacement = replacements.get(codePoint);
     if (replacement != null) {
       buf.append(replacement);
     } else {
       buf.appendCodePoint(codePoint);
     }
   }
   ctx.replaceFragment(textFragment, ctx.createTextFragment(buf.toString()));
 }