Beispiel #1
0
  /**
   * Returns whether the following token exists.
   *
   * @return result
   */
  private boolean more() {
    if (all) return tokens.hasNext();

    while (tokens.hasNext()) {
      currToken = tokens.next();
      if (!currToken.isMark() && !currToken.isAttachedWord()) return true;
    }
    return false;
  }
Beispiel #2
0
 /**
  * Returns the token which contains special characters.
  *
  * @return token
  */
 private byte[] getSC() {
   final Morpheme m = tokens.next();
   final String n = m.getSurface();
   if (m.isMark() || m.isAttachedWord()) {
     sc = true;
   } else {
     pos++;
     sc = false;
   }
   return token(n);
 }