示例#1
0
  /*
   * Tests the method public int first()
   */
  public void TestFirst() {
    RuleBasedBreakIterator rbbi = new RuleBasedBreakIterator(".;");
    // Tests when "if (fText == null)" is true
    rbbi.setText((CharacterIterator) null);
    assertEquals("RuleBasedBreakIterator.first()", BreakIterator.DONE, rbbi.first());

    rbbi.setText("abc");
    assertEquals("RuleBasedBreakIterator.first()", 0, rbbi.first());
    assertEquals("RuleBasedBreakIterator.next()", 1, rbbi.next());
  }
示例#2
0
  private void doMultipleSelectionTest(RuleBasedBreakIterator iterator, String testText) {
    logln("Multiple selection test...");
    RuleBasedBreakIterator testIterator = (RuleBasedBreakIterator) iterator.clone();
    int offset = iterator.first();
    int testOffset;
    int count = 0;

    do {
      testOffset = testIterator.first();
      testOffset = testIterator.next(count);
      logln("next(" + count + ") -> " + testOffset);
      if (offset != testOffset)
        errln(
            "next(n) and next() not returning consistent results: for step "
                + count
                + ", next(n) returned "
                + testOffset
                + " and next() had "
                + offset);

      if (offset != RuleBasedBreakIterator.DONE) {
        count++;
        offset = iterator.next();
      }
    } while (offset != RuleBasedBreakIterator.DONE);

    // now do it backwards...
    offset = iterator.last();
    count = 0;

    do {
      testOffset = testIterator.last();
      testOffset = testIterator.next(count);
      logln("next(" + count + ") -> " + testOffset);
      if (offset != testOffset)
        errln(
            "next(n) and next() not returning consistent results: for step "
                + count
                + ", next(n) returned "
                + testOffset
                + " and next() had "
                + offset);

      if (offset != RuleBasedBreakIterator.DONE) {
        count--;
        offset = iterator.previous();
      }
    } while (offset != RuleBasedBreakIterator.DONE);
  }
示例#3
0
  private List<String> _testFirstAndNext(RuleBasedBreakIterator rbbi, String text) {
    int p = rbbi.first();
    int lastP = p;
    List<String> result = new ArrayList<String>();

    if (p != 0) {
      errln("first() returned " + p + " instead of 0");
    }

    while (p != RuleBasedBreakIterator.DONE) {
      p = rbbi.next();
      if (p != RuleBasedBreakIterator.DONE) {
        if (p <= lastP) {
          errln("next() failed to move forward: next() on position " + lastP + " yielded " + p);
        }
        result.add(text.substring(lastP, p));
      } else {
        if (lastP != text.length()) {
          errln(
              "next() returned DONE prematurely: offset was "
                  + lastP
                  + " instead of "
                  + text.length());
        }
      }
      lastP = p;
    }
    return result;
  }
示例#4
0
 @Override
 public int first() {
   working.setText(this.text.getText(), this.text.getStart(), this.text.getLength());
   rules.setText(working);
   workingOffset = 0;
   int first = rules.first();
   return first == BreakIterator.DONE ? BreakIterator.DONE : workingOffset + first;
 }
示例#5
0
  @Override
  public void lexer_open(String text, LocaleId language, Tokens tokens) {

    if (Util.isEmpty(text)) {
      cancel();
      return;
    }
    this.text = text;

    if (iterators.containsKey(language)) {
      iterator = iterators.get(language);
    } else {
      iterator =
          (RuleBasedBreakIterator)
              BreakIterator.getWordInstance(ULocale.createCanonical(language.toString()));
      String defaultRules = iterator.toString();

      // Collect rules for the language, combine with defaultRules
      String newRules = defaultRules;

      for (LexerRule rule : getRules()) {

        boolean isInternal = Util.isEmpty(rule.getPattern());

        if (checkRule(rule, language) && !isInternal) {

          newRules =
              formatRule(
                  newRules,
                  rule.getName(),
                  rule.getDescription(),
                  rule.getPattern(),
                  rule.getLexemId());
        }
      }

      // Recreate iterator for the language(with new rules), store for future reuse
      iterator = new RuleBasedBreakIterator(newRules);
      iterators.put(language, iterator);
    }

    if (iterator == null) return;
    iterator.setText(text);

    // Sets the current iteration position to the beginning of the text
    start = iterator.first();
    end = start;
  }