public void writeDictionary(TokenStream tokenStream, String docId, int docSize) {
count++;
while (tokenStream.hasNext()) {
Node node = new Node();
node.setDocId(docId);
Token token = tokenStream.next();
String tokenText = token.toString();
int indexPos = token.getIndexPos();
node.setIndexPos(indexPos);
node.setDocLength(docSize);
if (!tokenText.isEmpty()) {
node.setTokenText(tokenText);
placeList.add(node);
}
}
if (count == 20000) {
count = 0;
sortFiles();
placeList = new ArrayList<Node>();
initializeWriter();
}
}
