public void testSupplementaryCharacters() throws IOException {
final String s = _TestUtil.randomUnicodeString(random(), 10);
final int codePointCount = s.codePointCount(0, s.length());
final int minGram = _TestUtil.nextInt(random(), 1, 3);
final int maxGram = _TestUtil.nextInt(random(), minGram, 10);
TokenStream tk = new KeywordTokenizer(new StringReader(s));
tk = new NGramTokenFilter(TEST_VERSION_CURRENT, tk, minGram, maxGram);
final CharTermAttribute termAtt = tk.addAttribute(CharTermAttribute.class);
final OffsetAttribute offsetAtt = tk.addAttribute(OffsetAttribute.class);
tk.reset();
for (int start = 0; start < codePointCount; ++start) {
for (int end = start + minGram; end <= Math.min(codePointCount, start + maxGram); ++end) {
assertTrue(tk.incrementToken());
assertEquals(0, offsetAtt.startOffset());
assertEquals(s.length(), offsetAtt.endOffset());
final int startIndex = Character.offsetByCodePoints(s, 0, start);
final int endIndex = Character.offsetByCodePoints(s, 0, end);
assertEquals(s.substring(startIndex, endIndex), termAtt.toString());
}
}
assertFalse(tk.incrementToken());
}
public void setBatchInputWord(final String word) {
reset();
mIsBatchMode = true;
final int length = word.length();
for (int i = 0; i < length; i = Character.offsetByCodePoints(word, i, 1)) {
final int codePoint = Character.codePointAt(word, i);
// We don't want to override the batch input points that are held in mInputPointers
// (See {@link #add(int,int,int)}).
final Event processedEvent =
processEvent(Event.createEventForCodePointFromUnknownSource(codePoint));
applyProcessedEvent(processedEvent);
}
}
/**
* Approximates whether the text before the cursor looks like a URL.
*
* <p>This is not foolproof, but it should work well in the practice. Essentially it walks
* backward from the cursor until it finds something that's not a letter, digit, or common URL
* symbol like underscore. If it hasn't found a period yet, then it does not look like a URL. If
* the text: - starts with www and contains a period - starts with a slash preceded by either a
* slash, whitespace, or start-of-string Then it looks like a URL and we return true. Otherwise,
* we return false.
*
* <p>Note: this method is called quite often, and should be fast.
*
* <p>TODO: This will return that "abc./def" and ".abc/def" look like URLs to keep down the code
* complexity, but ideally it should not. It's acceptable for now.
*/
public static boolean lastPartLooksLikeURL(final CharSequence text) {
int i = text.length();
if (0 == i) return false;
int wCount = 0;
int slashCount = 0;
boolean hasSlash = false;
boolean hasPeriod = false;
int codePoint = 0;
while (i > 0) {
codePoint = Character.codePointBefore(text, i);
if (codePoint < Constants.CODE_PERIOD || codePoint > 'z') {
// Handwavy heuristic to see if that's a URL character. Anything between period
// and z. This includes all lower- and upper-case ascii letters, period,
// underscore, arrobase, question mark, equal sign. It excludes spaces, exclamation
// marks, double quotes...
// Anything that's not a URL-like character causes us to break from here and
// evaluate normally.
break;
}
if (Constants.CODE_PERIOD == codePoint) {
hasPeriod = true;
}
if (Constants.CODE_SLASH == codePoint) {
hasSlash = true;
if (2 == ++slashCount) {
return true;
}
} else {
slashCount = 0;
}
if ('w' == codePoint) {
++wCount;
} else {
wCount = 0;
}
i = Character.offsetByCodePoints(text, i, -1);
}
// End of the text run.
// If it starts with www and includes a period, then it looks like a URL.
if (wCount >= 3 && hasPeriod) return true;
// If it starts with a slash, and the code point before is whitespace, it looks like an URL.
if (1 == slashCount && (0 == i || Character.isWhitespace(codePoint))) return true;
// If it has both a period and a slash, it looks like an URL.
if (hasPeriod && hasSlash) return true;
// Otherwise, it doesn't look like an URL.
return false;
}
/**
* Copies the codepoints in a CharSequence to an int array.
*
* <p>This method assumes there is enough space in the array to store the code points. The size
* can be measured with Character#codePointCount(CharSequence, int, int) before passing to this
* method. If the int array is too small, an ArrayIndexOutOfBoundsException will be thrown. Also,
* this method makes no effort to be thread-safe. Do not modify the CharSequence while this method
* is running, or the behavior is undefined. This method can optionally downcase code points
* before copying them, but it pays no attention to locale while doing so.
*
* @param destination the int array.
* @param charSequence the CharSequence.
* @param startIndex the start index inside the string in java chars, inclusive.
* @param endIndex the end index inside the string in java chars, exclusive.
* @param downCase if this is true, code points will be downcased before being copied.
* @return the number of copied code points.
*/
public static int copyCodePointsAndReturnCodePointCount(
final int[] destination,
final CharSequence charSequence,
final int startIndex,
final int endIndex,
final boolean downCase) {
int destIndex = 0;
for (int index = startIndex;
index < endIndex;
index = Character.offsetByCodePoints(charSequence, index, 1)) {
final int codePoint = Character.codePointAt(charSequence, index);
// TODO: stop using this, as it's not aware of the locale and does not always do
// the right thing.
destination[destIndex] = downCase ? Character.toLowerCase(codePoint) : codePoint;
destIndex++;
}
return destIndex;
}