public void processTerm(String t) {
if (t == null) return;
// current term is a delimiter
if (blockDelimiterTerms.contains(t)) {
// delimiters should also be indexed
if (indexDelimiters) {
final int[] fieldIds = new int[numFields];
int i = 0;
for (String fieldName : termFields) {
fieldIds[i] = fieldNames.get(fieldName);
i++;
}
((BlockFieldDocumentPostingList) termsInDocument).insert(t, fieldIds, blockId);
if (countDelimiters) numOfTokensInDocument++;
}
numOfTokensInBlock = 0;
blockId++;
} else {
// index non-delimiter term
final int[] fieldIds = new int[numFields];
int i = 0;
for (String fieldName : termFields) {
fieldIds[i] = fieldNames.get(fieldName);
i++;
}
((BlockFieldDocumentPostingList) termsInDocument).insert(t, fieldIds, blockId);
numOfTokensInDocument++;
}
}
public void processTerm(String t) {
// null means the term has been filtered out (eg stopwords)
if (t != null) {
// add term to document posting list
for (String fieldName : termFields) {
int tmp = fieldNames.get(fieldName);
if (tmp > 0) {
fields.add(tmp - 1);
}
}
if (ELSE_ENABLED && fields.size() == 0) {
fields.add(ELSE_FIELD_ID);
}
((BlockFieldDocumentPostingList) termsInDocument).insert(t, fields.toArray(), blockId);
numOfTokensInDocument++;
if (++numOfTokensInBlock >= BLOCK_SIZE && blockId < MAX_BLOCKS) {
numOfTokensInBlock = 0;
blockId++;
}
fields.clear();
}
}
