private long timeAndCheckReadUnigramsAndBigramsBinary( final File file, final List<String> words, final SparseArray<List<Integer>> bigrams, final int bufferType, final boolean checkProbability) { final TreeMap<Integer, String> resultWords = new TreeMap<>(); final TreeMap<Integer, ArrayList<PendingAttribute>> resultBigrams = new TreeMap<>(); final TreeMap<Integer, Integer> resultFreqs = new TreeMap<>(); long now = -1, diff = -1; try { final DictDecoder dictDecoder = BinaryDictIOUtils.getDictDecoder(file, 0, file.length(), bufferType); now = System.currentTimeMillis(); dictDecoder.readUnigramsAndBigramsBinary(resultWords, resultFreqs, resultBigrams); diff = System.currentTimeMillis() - now; } catch (IOException e) { Log.e(TAG, "IOException", e); } catch (UnsupportedFormatException e) { Log.e(TAG, "UnsupportedFormatException", e); } checkWordMap(words, bigrams, resultWords, resultFreqs, resultBigrams, checkProbability); return diff; }
// Tests for getTerminalPosition private String getWordFromBinary(final DictDecoder dictDecoder, final int address) { if (dictDecoder.getPosition() != 0) dictDecoder.setPosition(0); DictionaryHeader fileHeader = null; try { fileHeader = dictDecoder.readHeader(); } catch (IOException e) { return null; } catch (UnsupportedFormatException e) { return null; } if (fileHeader == null) return null; return BinaryDictDecoderUtils.getWordAtPosition(dictDecoder, fileHeader.mBodyOffset, address) .mWord; }
private long timeReadingAndCheckDict( final File file, final List<String> words, final SparseArray<List<Integer>> bigrams, final HashMap<String, List<String>> shortcutMap, final int bufferType) { long now, diff = -1; FusionDictionary dict = null; try { final DictDecoder dictDecoder = BinaryDictIOUtils.getDictDecoder(file, 0, file.length(), bufferType); now = System.currentTimeMillis(); dict = dictDecoder.readDictionaryBinary(false /* deleteDictIfBroken */); diff = System.currentTimeMillis() - now; } catch (IOException e) { Log.e(TAG, "IOException while reading dictionary", e); } catch (UnsupportedFormatException e) { Log.e(TAG, "Unsupported format", e); } checkDictionary(dict, words, bigrams, shortcutMap); return diff; }
private long checkGetTerminalPosition( final DictDecoder dictDecoder, final String word, final boolean contained) { long diff = -1; int position = -1; try { final long now = System.nanoTime(); position = dictDecoder.getTerminalPosition(word); diff = System.nanoTime() - now; } catch (IOException e) { Log.e(TAG, "IOException while getTerminalPosition", e); } catch (UnsupportedFormatException e) { Log.e(TAG, "UnsupportedFormatException while getTerminalPosition", e); } assertEquals(FormatSpec.NOT_VALID_WORD != position, contained); if (contained) assertEquals(getWordFromBinary(dictDecoder, position), word); return diff; }
private void runGetTerminalPosition( final ArrayList<String> words, final SparseArray<List<Integer>> bigrams, final int bufferType, final FormatOptions formatOptions, final String message) { final String dictName = "testGetTerminalPosition"; final String dictVersion = Long.toString(System.currentTimeMillis()); final File file = BinaryDictUtils.getDictFile( dictName, dictVersion, formatOptions, getContext().getCacheDir()); final FusionDictionary dict = new FusionDictionary( new PtNodeArray(), BinaryDictUtils.makeDictionaryOptions(dictName, dictVersion, formatOptions)); addUnigrams(sWords.size(), dict, sWords, null /* shortcutMap */); addBigrams(dict, words, bigrams); timeWritingDictToFile(file, dict, formatOptions); final DictDecoder dictDecoder = BinaryDictIOUtils.getDictDecoder(file, 0, file.length(), DictDecoder.USE_BYTEARRAY); try { dictDecoder.openDictBuffer(); } catch (IOException e) { Log.e(TAG, "IOException while opening the buffer", e); } catch (UnsupportedFormatException e) { Log.e(TAG, "IOException while opening the buffer", e); } assertTrue("Can't get the buffer", dictDecoder.isDictBufferOpen()); try { // too long word final String longWord = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"; assertEquals(FormatSpec.NOT_VALID_WORD, dictDecoder.getTerminalPosition(longWord)); // null assertEquals(FormatSpec.NOT_VALID_WORD, dictDecoder.getTerminalPosition(null)); // empty string assertEquals(FormatSpec.NOT_VALID_WORD, dictDecoder.getTerminalPosition("")); } catch (IOException e) { } catch (UnsupportedFormatException e) { } // Test a word that is contained within the dictionary. long sum = 0; for (int i = 0; i < sWords.size(); ++i) { final long time = checkGetTerminalPosition(dictDecoder, sWords.get(i), true); sum += time == -1 ? 0 : time; } Log.d( TAG, "per search : " + (((double) sum) / sWords.size() / 1000000) + " : " + message + " : " + outputOptions(bufferType, formatOptions)); // Test a word that isn't contained within the dictionary. final Random random = new Random((int) System.currentTimeMillis()); final int[] codePointSet = CodePointUtils.generateCodePointSet(DEFAULT_CODE_POINT_SET_SIZE, random); for (int i = 0; i < 1000; ++i) { final String word = CodePointUtils.generateWord(random, codePointSet); if (sWords.indexOf(word) != -1) continue; checkGetTerminalPosition(dictDecoder, word, false); } }