private long timeAndCheckReadUnigramsAndBigramsBinary(
final File file,
final List<String> words,
final SparseArray<List<Integer>> bigrams,
final int bufferType,
final boolean checkProbability) {
final TreeMap<Integer, String> resultWords = new TreeMap<>();
final TreeMap<Integer, ArrayList<PendingAttribute>> resultBigrams = new TreeMap<>();
final TreeMap<Integer, Integer> resultFreqs = new TreeMap<>();
long now = -1, diff = -1;
try {
final DictDecoder dictDecoder =
BinaryDictIOUtils.getDictDecoder(file, 0, file.length(), bufferType);
now = System.currentTimeMillis();
dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams);
diff = System.currentTimeMillis() - now;
} catch (IOException e) {
Log.e(TAG, "IOException", e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "UnsupportedFormatException", e);
}
checkWordMap(words, bigrams, resultWords, resultFreqs, resultBigrams, checkProbability);
return diff;
}
// Tests for getTerminalPosition
private String getWordFromBinary(final DictDecoder dictDecoder, final int address) {
if (dictDecoder.getPosition() != 0) dictDecoder.setPosition(0);
DictionaryHeader fileHeader = null;
try {
fileHeader = dictDecoder.readHeader();
} catch (IOException e) {
return null;
} catch (UnsupportedFormatException e) {
return null;
}
if (fileHeader == null) return null;
return BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mBodyOffset, address)
.mWord;
}
private long timeReadingAndCheckDict(
final File file,
final List<String> words,
final SparseArray<List<Integer>> bigrams,
final HashMap<String, List<String>> shortcutMap,
final int bufferType) {
long now, diff = -1;
FusionDictionary dict = null;
try {
final DictDecoder dictDecoder =
BinaryDictIOUtils.getDictDecoder(file, 0, file.length(), bufferType);
now = System.currentTimeMillis();
dict = dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */);
diff = System.currentTimeMillis() - now;
} catch (IOException e) {
Log.e(TAG, "IOException while reading dictionary", e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "Unsupported format", e);
}
checkDictionary(dict, words, bigrams, shortcutMap);
return diff;
}
private long checkGetTerminalPosition(
final DictDecoder dictDecoder, final String word, final boolean contained) {
long diff = -1;
int position = -1;
try {
final long now = System.nanoTime();
position = dictDecoder.getTerminalPosition(word);
diff = System.nanoTime() - now;
} catch (IOException e) {
Log.e(TAG, "IOException while getTerminalPosition", e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "UnsupportedFormatException while getTerminalPosition", e);
}
assertEquals(FormatSpec.NOT_VALID_WORD != position, contained);
if (contained) assertEquals(getWordFromBinary(dictDecoder, position), word);
return diff;
}
private void runGetTerminalPosition(
final ArrayList<String> words,
final SparseArray<List<Integer>> bigrams,
final int bufferType,
final FormatOptions formatOptions,
final String message) {
final String dictName = "testGetTerminalPosition";
final String dictVersion = Long.toString(System.currentTimeMillis());
final File file =
BinaryDictUtils.getDictFile(
dictName, dictVersion, formatOptions, getContext().getCacheDir());
final FusionDictionary dict =
new FusionDictionary(
new PtNodeArray(),
BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions));
addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */);
addBigrams(dict, words, bigrams);
timeWritingDictToFile(file, dict, formatOptions);
final DictDecoder dictDecoder =
BinaryDictIOUtils.getDictDecoder(file, 0, file.length(), DictDecoder.USE_BYTEARRAY);
try {
dictDecoder.openDictBuffer();
} catch (IOException e) {
Log.e(TAG, "IOException while opening the buffer", e);
} catch (UnsupportedFormatException e) {
Log.e(TAG, "IOException while opening the buffer", e);
}
assertTrue("Can't get the buffer", dictDecoder.isDictBufferOpen());
try {
// too long word
final String longWord = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz";
assertEquals(FormatSpec.NOT_VALID_WORD, dictDecoder.getTerminalPosition(longWord));
// null
assertEquals(FormatSpec.NOT_VALID_WORD, dictDecoder.getTerminalPosition(null));
// empty string
assertEquals(FormatSpec.NOT_VALID_WORD, dictDecoder.getTerminalPosition(""));
} catch (IOException e) {
} catch (UnsupportedFormatException e) {
}
// Test a word that is contained within the dictionary.
long sum = 0;
for (int i = 0; i < sWords.size(); ++i) {
final long time = checkGetTerminalPosition(dictDecoder, sWords.get(i), true);
sum += time == -1 ? 0 : time;
}
Log.d(
TAG,
"per search : "
+ (((double) sum) / sWords.size() / 1000000)
+ " : "
+ message
+ " : "
+ outputOptions(bufferType, formatOptions));
// Test a word that isn't contained within the dictionary.
final Random random = new Random((int) System.currentTimeMillis());
final int[] codePointSet =
CodePointUtils.generateCodePointSet(DEFAULT_CODE_POINT_SET_SIZE, random);
for (int i = 0; i < 1000; ++i) {
final String word = CodePointUtils.generateWord(random, codePointSet);
if (sWords.indexOf(word) != -1) continue;
checkGetTerminalPosition(dictDecoder, word, false);
}
}
