/*
* listTermVectors displays the term vectors for all of the fields
* in a document in an index (specified by reader).
*/
static void listTermVectors(IndexReader reader, String docidString) throws IOException {
System.out.println("\nTermVector: docid " + docidString);
int docid = Integer.parseInt(docidString);
if ((docid < 0) || (docid >= reader.numDocs())) {
System.out.println("ERROR: " + docidString + " is a bad document id.");
return;
}
;
/*
* Iterate over the fields in this document.
*/
Fields fields = reader.getTermVectors(docid);
Iterator<String> fieldIterator = fields.iterator();
while (fieldIterator.hasNext()) {
String fieldName = fieldIterator.next();
System.out.println(" Field: " + fieldName);
Terms terms = fields.terms(fieldName);
termVectorDisplay(terms);
}
;
}
/**
* Prepare a document reconstructor.
*
* @param reader IndexReader to read from.
* @param fieldNames if non-null or not empty, data will be collected only from these fields,
* otherwise data will be collected from all fields
* @param numTerms total number of terms in the index, or -1 if unknown (will be calculated)
* @throws Exception
*/
public DocReconstructor(IndexReader reader, String[] fieldNames, int numTerms) throws Exception {
if (reader == null) {
throw new Exception("IndexReader cannot be null.");
}
this.reader = reader;
if (fieldNames == null || fieldNames.length == 0) {
// collect fieldNames
this.fieldNames = (String[]) reader.getFieldNames(FieldOption.ALL).toArray(new String[0]);
} else {
this.fieldNames = fieldNames;
}
if (numTerms == -1) {
Fields fields = MultiFields.getFields(reader);
numTerms = 0;
FieldsEnum fe = fields.iterator();
String fld = null;
while ((fld = fe.next()) != null) {
TermsEnum te = fe.terms();
while (te.next() != null) {
numTerms++;
}
}
this.numTerms = numTerms;
}
deleted = MultiFields.getDeletedDocs(reader);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
assert index != null;
assert type != null;
assert id != null;
builder.field(FieldStrings._INDEX, index);
builder.field(FieldStrings._TYPE, type);
if (!isArtificial()) {
builder.field(FieldStrings._ID, id);
}
builder.field(FieldStrings._VERSION, docVersion);
builder.field(FieldStrings.FOUND, isExists());
builder.field(FieldStrings.TOOK, tookInMillis);
if (!isExists()) {
return builder;
}
builder.startObject(FieldStrings.TERM_VECTORS);
final CharsRefBuilder spare = new CharsRefBuilder();
Fields theFields = getFields();
Iterator<String> fieldIter = theFields.iterator();
while (fieldIter.hasNext()) {
buildField(builder, spare, theFields, fieldIter);
}
builder.endObject();
return builder;
}
/**
* @param reader
* @param numTerms
* @param field
* @return TermStats[] ordered by terms with highest docFreq first.
* @throws Exception
*/
public static TermStats[] getHighFreqTerms(IndexReader reader, int numTerms, String[] fieldNames)
throws Exception {
TermStatsQueue tiq = null;
TermsEnum te = null;
if (fieldNames != null) {
Fields fields = MultiFields.getFields(reader);
if (fields == null) {
LOG.info("Index with no fields - probably empty or corrupted");
return EMPTY_STATS;
}
tiq = new TermStatsQueue(numTerms);
for (String field : fieldNames) {
Terms terms = fields.terms(field);
if (terms != null) {
te = terms.iterator(te);
fillQueue(te, tiq, field);
}
}
} else {
Fields fields = MultiFields.getFields(reader);
if (fields == null) {
LOG.info("Index with no fields - probably empty or corrupted");
return EMPTY_STATS;
}
tiq = new TermStatsQueue(numTerms);
Iterator<String> fieldIterator = fields.iterator();
while (fieldIterator.hasNext()) {
String field = fieldIterator.next();
Terms terms = fields.terms(field);
if (terms != null) {
te = terms.iterator(te);
fillQueue(te, tiq, field);
}
}
}
TermStats[] result = new TermStats[tiq.size()];
// we want highest first so we read the queue and populate the array
// starting at the end and work backwards
int count = tiq.size() - 1;
while (tiq.size() != 0) {
result[count] = tiq.pop();
count--;
}
return result;
}
@Override
protected ShardTermlistResponse shardOperation(ShardTermlistRequest request)
throws ElasticSearchException {
synchronized (termlistMutex) {
InternalIndexShard indexShard =
(InternalIndexShard)
indicesService.indexServiceSafe(request.index()).shardSafe(request.shardId());
indexShard.store().directory();
Engine.Searcher searcher = indexShard.searcher();
try {
Set<String> set = new CompactHashSet();
Fields fields = MultiFields.getFields(searcher.reader());
if (fields != null) {
for (Iterator<String> it = fields.iterator(); it.hasNext(); ) {
String field = it.next();
if (field.charAt(0) == '_') {
continue;
}
if (request.getField() == null || field.equals(request.getField())) {
Terms terms = fields.terms(field);
if (terms != null) {
TermsEnum termsEnum = terms.iterator(null);
BytesRef text;
while ((text = termsEnum.next()) != null) {
set.add(text.utf8ToString());
System.out.println("field=" + field + "; text=" + text.utf8ToString());
}
}
}
}
}
return new ShardTermlistResponse(request.index(), request.shardId(), set);
} catch (IOException ex) {
throw new ElasticSearchException(ex.getMessage(), ex);
}
}
}
public static void main(String[] args) throws IOException {
IndexReader reader = null;
/*
* Opening the index first simplifies the processing of the
* rest of the command line arguments.
*/
for (int i = 0; i < args.length; i++) {
if (("-index".equals(args[i])) && ((i + 1) < args.length)) {
reader = DirectoryReader.open(FSDirectory.open(new File(args[i + 1])));
if (reader == null) {
System.err.println("Error: Can't open index " + args[i + 1]);
System.exit(1);
}
;
break;
}
;
}
;
if (reader == null) {
System.err.println(usage);
System.exit(1);
}
;
/*
* Process the command line arguments sequentially.
*/
for (int i = 0; i < args.length; i++) {
if ("-index".equals(args[i])) {
/*
* Handled in the previous loop, so just skip the argument.
*/
i++;
} else if ("-list-edocid".equals(args[i])) {
System.out.println("-list-edocid:");
if ((i + 1) >= args.length) {
System.out.println(usage);
break;
}
;
Document d = reader.document(Integer.parseInt(args[i + 1]));
System.out.println(
"Internal docid --> External docid: " + args[i + 1] + " --> " + d.get("externalId"));
i += 1;
} else if ("-list-docids".equals(args[i])) {
System.out.println("-list-docids:");
for (int j = 0; j < reader.numDocs(); j++) {
Document d = reader.document(j);
System.out.println("Internal --> external docid: " + j + " --> " + d.get("externalId"));
}
;
} else if ("-list-fields".equals(args[i])) {
Fields fields = MultiFields.getFields(reader);
System.out.print("\nNumber of fields: ");
if (fields == null) System.out.println("0");
else {
System.out.println(fields.size());
Iterator<String> is = fields.iterator();
while (is.hasNext()) {
System.out.println("\t" + is.next());
}
;
}
;
} else if ("-list-postings".equals(args[i])) {
if ((i + 2) >= args.length) {
System.out.println(usage);
break;
}
;
listPostings(reader, args[i + 1], args[i + 2], Integer.MAX_VALUE);
i += 2;
} else if ("-list-postings-sample".equals(args[i])) {
if ((i + 2) >= args.length) {
System.out.println(usage);
break;
}
;
listPostings(reader, args[i + 1], args[i + 2], 5);
i += 2;
} else if ("-list-stats".equals(args[i])) {
System.out.println("Corpus statistics:");
System.out.println("\tnumdocs\t\t" + reader.numDocs());
System.out.println(
"\turl:\t"
+ "\tnumdocs="
+ reader.getDocCount("url")
+ "\tsumTotalTF="
+ reader.getSumTotalTermFreq("url")
+ "\tavglen="
+ reader.getSumTotalTermFreq("url") / (float) reader.getDocCount("url"));
System.out.println(
"\tkeywords:"
+ "\tnumdocs="
+ reader.getDocCount("keywords")
+ "\tsumTotalTF="
+ reader.getSumTotalTermFreq("keywords")
+ "\tavglen="
+ reader.getSumTotalTermFreq("keywords") / (float) reader.getDocCount("keywords"));
System.out.println(
"\ttitle:\t"
+ "\tnumdocs="
+ reader.getDocCount("title")
+ "\tsumTotalTF="
+ reader.getSumTotalTermFreq("title")
+ "\tavglen="
+ reader.getSumTotalTermFreq("title") / (float) reader.getDocCount("title"));
System.out.println(
"\tbody:\t"
+ "\tnumdocs="
+ reader.getDocCount("body")
+ "\tsumTotalTF="
+ reader.getSumTotalTermFreq("body")
+ "\tavglen="
+ reader.getSumTotalTermFreq("body") / (float) reader.getDocCount("body"));
System.out.println(
"\tinlink:\t"
+ "\tnumdocs="
+ reader.getDocCount("inlink")
+ "\tsumTotalTF="
+ reader.getSumTotalTermFreq("inlink")
+ "\tavglen="
+ reader.getSumTotalTermFreq("inlink") / (float) reader.getDocCount("inlink"));
} else if ("-list-terms".equals(args[i])) {
if ((i + 1) >= args.length) {
System.out.println(usage);
break;
}
;
listTermDictionary(reader, args[i + 1]);
i += 1;
} else if ("-list-termvector".equals(args[i])) {
if ((i + 1) >= args.length) {
System.out.println(usage);
break;
}
;
listTermVectors(reader, args[i + 1]);
i += 1;
} else if ("-list-termvector-field".equals(args[i])) {
if ((i + 2) >= args.length) {
System.out.println(usage);
break;
}
;
listTermVectorField(reader, args[i + 1], args[i + 2]);
i += 2;
} else System.err.println("\nWarning: Unknown argument " + args[i] + " ignored.");
}
;
/*
* Close the index and exit gracefully.
*/
reader.close();
}
public static void verifyEquals(Fields d1, Fields d2) throws IOException {
if (d1 == null) {
assertTrue(d2 == null || d2.size() == 0);
return;
}
assertTrue(d2 != null);
Iterator<String> fieldsEnum2 = d2.iterator();
for (String field1 : d1) {
String field2 = fieldsEnum2.next();
assertEquals(field1, field2);
Terms terms1 = d1.terms(field1);
assertNotNull(terms1);
TermsEnum termsEnum1 = terms1.iterator(null);
Terms terms2 = d2.terms(field2);
assertNotNull(terms2);
TermsEnum termsEnum2 = terms2.iterator(null);
DocsAndPositionsEnum dpEnum1 = null;
DocsAndPositionsEnum dpEnum2 = null;
DocsEnum dEnum1 = null;
DocsEnum dEnum2 = null;
BytesRef term1;
while ((term1 = termsEnum1.next()) != null) {
BytesRef term2 = termsEnum2.next();
assertEquals(term1, term2);
assertEquals(termsEnum1.totalTermFreq(), termsEnum2.totalTermFreq());
dpEnum1 = termsEnum1.docsAndPositions(null, dpEnum1);
dpEnum2 = termsEnum2.docsAndPositions(null, dpEnum2);
if (dpEnum1 != null) {
assertNotNull(dpEnum2);
int docID1 = dpEnum1.nextDoc();
dpEnum2.nextDoc();
// docIDs are not supposed to be equal
// int docID2 = dpEnum2.nextDoc();
// assertEquals(docID1, docID2);
assertTrue(docID1 != DocIdSetIterator.NO_MORE_DOCS);
int freq1 = dpEnum1.freq();
int freq2 = dpEnum2.freq();
assertEquals(freq1, freq2);
OffsetAttribute offsetAtt1 =
dpEnum1.attributes().hasAttribute(OffsetAttribute.class)
? dpEnum1.attributes().getAttribute(OffsetAttribute.class)
: null;
OffsetAttribute offsetAtt2 =
dpEnum2.attributes().hasAttribute(OffsetAttribute.class)
? dpEnum2.attributes().getAttribute(OffsetAttribute.class)
: null;
if (offsetAtt1 != null) {
assertNotNull(offsetAtt2);
} else {
assertNull(offsetAtt2);
}
for (int posUpto = 0; posUpto < freq1; posUpto++) {
int pos1 = dpEnum1.nextPosition();
int pos2 = dpEnum2.nextPosition();
assertEquals(pos1, pos2);
if (offsetAtt1 != null) {
assertEquals(offsetAtt1.startOffset(), offsetAtt2.startOffset());
assertEquals(offsetAtt1.endOffset(), offsetAtt2.endOffset());
}
}
assertEquals(DocIdSetIterator.NO_MORE_DOCS, dpEnum1.nextDoc());
assertEquals(DocIdSetIterator.NO_MORE_DOCS, dpEnum2.nextDoc());
} else {
dEnum1 = TestUtil.docs(random(), termsEnum1, null, dEnum1, DocsEnum.FLAG_FREQS);
dEnum2 = TestUtil.docs(random(), termsEnum2, null, dEnum2, DocsEnum.FLAG_FREQS);
assertNotNull(dEnum1);
assertNotNull(dEnum2);
int docID1 = dEnum1.nextDoc();
dEnum2.nextDoc();
// docIDs are not supposed to be equal
// int docID2 = dEnum2.nextDoc();
// assertEquals(docID1, docID2);
assertTrue(docID1 != DocIdSetIterator.NO_MORE_DOCS);
int freq1 = dEnum1.freq();
int freq2 = dEnum2.freq();
assertEquals(freq1, freq2);
assertEquals(DocIdSetIterator.NO_MORE_DOCS, dEnum1.nextDoc());
assertEquals(DocIdSetIterator.NO_MORE_DOCS, dEnum2.nextDoc());
}
}
assertNull(termsEnum2.next());
}
assertFalse(fieldsEnum2.hasNext());
}
public void verifyEquals(DirectoryReader r1, DirectoryReader r2, String idField)
throws Throwable {
if (VERBOSE) {
System.out.println("\nr1 docs:");
printDocs(r1);
System.out.println("\nr2 docs:");
printDocs(r2);
}
if (r1.numDocs() != r2.numDocs()) {
assert false : "r1.numDocs()=" + r1.numDocs() + " vs r2.numDocs()=" + r2.numDocs();
}
boolean hasDeletes = !(r1.maxDoc() == r2.maxDoc() && r1.numDocs() == r1.maxDoc());
int[] r2r1 = new int[r2.maxDoc()]; // r2 id to r1 id mapping
// create mapping from id2 space to id2 based on idField
final Fields f1 = MultiFields.getFields(r1);
if (f1 == null) {
// make sure r2 is empty
assertNull(MultiFields.getFields(r2));
return;
}
final Terms terms1 = f1.terms(idField);
if (terms1 == null) {
assertTrue(
MultiFields.getFields(r2) == null || MultiFields.getFields(r2).terms(idField) == null);
return;
}
final TermsEnum termsEnum = terms1.iterator(null);
final Bits liveDocs1 = MultiFields.getLiveDocs(r1);
final Bits liveDocs2 = MultiFields.getLiveDocs(r2);
Fields fields = MultiFields.getFields(r2);
if (fields == null) {
// make sure r1 is in fact empty (eg has only all
// deleted docs):
Bits liveDocs = MultiFields.getLiveDocs(r1);
DocsEnum docs = null;
while (termsEnum.next() != null) {
docs = TestUtil.docs(random(), termsEnum, liveDocs, docs, DocsEnum.FLAG_NONE);
while (docs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
fail("r1 is not empty but r2 is");
}
}
return;
}
Terms terms2 = fields.terms(idField);
TermsEnum termsEnum2 = terms2.iterator(null);
DocsEnum termDocs1 = null;
DocsEnum termDocs2 = null;
while (true) {
BytesRef term = termsEnum.next();
// System.out.println("TEST: match id term=" + term);
if (term == null) {
break;
}
termDocs1 = TestUtil.docs(random(), termsEnum, liveDocs1, termDocs1, DocsEnum.FLAG_NONE);
if (termsEnum2.seekExact(term)) {
termDocs2 = TestUtil.docs(random(), termsEnum2, liveDocs2, termDocs2, DocsEnum.FLAG_NONE);
} else {
termDocs2 = null;
}
if (termDocs1.nextDoc() == DocIdSetIterator.NO_MORE_DOCS) {
// This doc is deleted and wasn't replaced
assertTrue(termDocs2 == null || termDocs2.nextDoc() == DocIdSetIterator.NO_MORE_DOCS);
continue;
}
int id1 = termDocs1.docID();
assertEquals(DocIdSetIterator.NO_MORE_DOCS, termDocs1.nextDoc());
assertTrue(termDocs2.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
int id2 = termDocs2.docID();
assertEquals(DocIdSetIterator.NO_MORE_DOCS, termDocs2.nextDoc());
r2r1[id2] = id1;
// verify stored fields are equivalent
try {
verifyEquals(r1.document(id1), r2.document(id2));
} catch (Throwable t) {
System.out.println("FAILED id=" + term + " id1=" + id1 + " id2=" + id2 + " term=" + term);
System.out.println(" d1=" + r1.document(id1));
System.out.println(" d2=" + r2.document(id2));
throw t;
}
try {
// verify term vectors are equivalent
verifyEquals(r1.getTermVectors(id1), r2.getTermVectors(id2));
} catch (Throwable e) {
System.out.println("FAILED id=" + term + " id1=" + id1 + " id2=" + id2);
Fields tv1 = r1.getTermVectors(id1);
System.out.println(" d1=" + tv1);
if (tv1 != null) {
DocsAndPositionsEnum dpEnum = null;
DocsEnum dEnum = null;
for (String field : tv1) {
System.out.println(" " + field + ":");
Terms terms3 = tv1.terms(field);
assertNotNull(terms3);
TermsEnum termsEnum3 = terms3.iterator(null);
BytesRef term2;
while ((term2 = termsEnum3.next()) != null) {
System.out.println(
" " + term2.utf8ToString() + ": freq=" + termsEnum3.totalTermFreq());
dpEnum = termsEnum3.docsAndPositions(null, dpEnum);
if (dpEnum != null) {
assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
final int freq = dpEnum.freq();
System.out.println(" doc=" + dpEnum.docID() + " freq=" + freq);
for (int posUpto = 0; posUpto < freq; posUpto++) {
System.out.println(" pos=" + dpEnum.nextPosition());
}
} else {
dEnum = TestUtil.docs(random(), termsEnum3, null, dEnum, DocsEnum.FLAG_FREQS);
assertNotNull(dEnum);
assertTrue(dEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
final int freq = dEnum.freq();
System.out.println(" doc=" + dEnum.docID() + " freq=" + freq);
}
}
}
}
Fields tv2 = r2.getTermVectors(id2);
System.out.println(" d2=" + tv2);
if (tv2 != null) {
DocsAndPositionsEnum dpEnum = null;
DocsEnum dEnum = null;
for (String field : tv2) {
System.out.println(" " + field + ":");
Terms terms3 = tv2.terms(field);
assertNotNull(terms3);
TermsEnum termsEnum3 = terms3.iterator(null);
BytesRef term2;
while ((term2 = termsEnum3.next()) != null) {
System.out.println(
" " + term2.utf8ToString() + ": freq=" + termsEnum3.totalTermFreq());
dpEnum = termsEnum3.docsAndPositions(null, dpEnum);
if (dpEnum != null) {
assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
final int freq = dpEnum.freq();
System.out.println(" doc=" + dpEnum.docID() + " freq=" + freq);
for (int posUpto = 0; posUpto < freq; posUpto++) {
System.out.println(" pos=" + dpEnum.nextPosition());
}
} else {
dEnum = TestUtil.docs(random(), termsEnum3, null, dEnum, DocsEnum.FLAG_FREQS);
assertNotNull(dEnum);
assertTrue(dEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
final int freq = dEnum.freq();
System.out.println(" doc=" + dEnum.docID() + " freq=" + freq);
}
}
}
}
throw e;
}
}
// System.out.println("TEST: done match id");
// Verify postings
// System.out.println("TEST: create te1");
final Fields fields1 = MultiFields.getFields(r1);
final Iterator<String> fields1Enum = fields1.iterator();
final Fields fields2 = MultiFields.getFields(r2);
final Iterator<String> fields2Enum = fields2.iterator();
String field1 = null, field2 = null;
TermsEnum termsEnum1 = null;
termsEnum2 = null;
DocsEnum docs1 = null, docs2 = null;
// pack both doc and freq into single element for easy sorting
long[] info1 = new long[r1.numDocs()];
long[] info2 = new long[r2.numDocs()];
for (; ; ) {
BytesRef term1 = null, term2 = null;
// iterate until we get some docs
int len1;
for (; ; ) {
len1 = 0;
if (termsEnum1 == null) {
if (!fields1Enum.hasNext()) {
break;
}
field1 = fields1Enum.next();
Terms terms = fields1.terms(field1);
if (terms == null) {
continue;
}
termsEnum1 = terms.iterator(null);
}
term1 = termsEnum1.next();
if (term1 == null) {
// no more terms in this field
termsEnum1 = null;
continue;
}
// System.out.println("TEST: term1=" + term1);
docs1 = TestUtil.docs(random(), termsEnum1, liveDocs1, docs1, DocsEnum.FLAG_FREQS);
while (docs1.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
int d = docs1.docID();
int f = docs1.freq();
info1[len1] = (((long) d) << 32) | f;
len1++;
}
if (len1 > 0) break;
}
// iterate until we get some docs
int len2;
for (; ; ) {
len2 = 0;
if (termsEnum2 == null) {
if (!fields2Enum.hasNext()) {
break;
}
field2 = fields2Enum.next();
Terms terms = fields2.terms(field2);
if (terms == null) {
continue;
}
termsEnum2 = terms.iterator(null);
}
term2 = termsEnum2.next();
if (term2 == null) {
// no more terms in this field
termsEnum2 = null;
continue;
}
// System.out.println("TEST: term1=" + term1);
docs2 = TestUtil.docs(random(), termsEnum2, liveDocs2, docs2, DocsEnum.FLAG_FREQS);
while (docs2.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
int d = r2r1[docs2.docID()];
int f = docs2.freq();
info2[len2] = (((long) d) << 32) | f;
len2++;
}
if (len2 > 0) break;
}
assertEquals(len1, len2);
if (len1 == 0) break; // no more terms
assertEquals(field1, field2);
assertTrue(term1.bytesEquals(term2));
if (!hasDeletes) assertEquals(termsEnum1.docFreq(), termsEnum2.docFreq());
assertEquals("len1=" + len1 + " len2=" + len2 + " deletes?=" + hasDeletes, term1, term2);
// sort info2 to get it into ascending docid
Arrays.sort(info2, 0, len2);
// now compare
for (int i = 0; i < len1; i++) {
assertEquals(
"i="
+ i
+ " len="
+ len1
+ " d1="
+ (info1[i] >>> 32)
+ " f1="
+ (info1[i] & Integer.MAX_VALUE)
+ " d2="
+ (info2[i] >>> 32)
+ " f2="
+ (info2[i] & Integer.MAX_VALUE)
+ " field="
+ field1
+ " term="
+ term1.utf8ToString(),
info1[i],
info2[i]);
}
}
}
/**
* Safe (but, slowish) default method to write every vector field in the document. This default
* implementation requires that the vectors implement both Fields.size and Terms.size.
*/
protected final void addAllDocVectors(Fields vectors, FieldInfos fieldInfos) throws IOException {
if (vectors == null) {
startDocument(0);
return;
}
final int numFields = vectors.size();
if (numFields == -1) {
throw new IllegalStateException("vectors.size() must be implemented (it returned -1)");
}
startDocument(numFields);
final FieldsEnum fieldsEnum = vectors.iterator();
String fieldName;
String lastFieldName = null;
while ((fieldName = fieldsEnum.next()) != null) {
final FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldName);
assert lastFieldName == null || fieldName.compareTo(lastFieldName) > 0
: "lastFieldName=" + lastFieldName + " fieldName=" + fieldName;
lastFieldName = fieldName;
final Terms terms = fieldsEnum.terms();
if (terms == null) {
// FieldsEnum shouldn't lie...
continue;
}
final int numTerms = (int) terms.size();
if (numTerms == -1) {
throw new IllegalStateException("terms.size() must be implemented (it returned -1)");
}
final TermsEnum termsEnum = terms.iterator(null);
DocsAndPositionsEnum docsAndPositionsEnum = null;
boolean startedField = false;
// NOTE: this is tricky, because TermVectors allow
// indexing offsets but NOT positions. So we must
// lazily init the field by checking whether first
// position we see is -1 or not.
int termCount = 0;
while (termsEnum.next() != null) {
termCount++;
final int freq = (int) termsEnum.totalTermFreq();
if (startedField) {
startTerm(termsEnum.term(), freq);
}
// TODO: we need a "query" API where we can ask (via
// flex API) what this term was indexed with...
// Both positions & offsets:
docsAndPositionsEnum = termsEnum.docsAndPositions(null, null, true);
final boolean hasOffsets;
boolean hasPositions = false;
if (docsAndPositionsEnum == null) {
// Fallback: no offsets
docsAndPositionsEnum = termsEnum.docsAndPositions(null, null, false);
hasOffsets = false;
} else {
hasOffsets = true;
}
if (docsAndPositionsEnum != null) {
final int docID = docsAndPositionsEnum.nextDoc();
assert docID != DocIdSetIterator.NO_MORE_DOCS;
assert docsAndPositionsEnum.freq() == freq;
for (int posUpto = 0; posUpto < freq; posUpto++) {
final int pos = docsAndPositionsEnum.nextPosition();
if (!startedField) {
assert numTerms > 0;
hasPositions = pos != -1;
startField(fieldInfo, numTerms, hasPositions, hasOffsets);
startTerm(termsEnum.term(), freq);
startedField = true;
}
final int startOffset;
final int endOffset;
if (hasOffsets) {
startOffset = docsAndPositionsEnum.startOffset();
endOffset = docsAndPositionsEnum.endOffset();
assert startOffset != -1;
assert endOffset != -1;
} else {
startOffset = -1;
endOffset = -1;
}
assert !hasPositions || pos >= 0;
addPosition(pos, startOffset, endOffset);
}
} else {
if (!startedField) {
assert numTerms > 0;
startField(fieldInfo, numTerms, hasPositions, hasOffsets);
startTerm(termsEnum.term(), freq);
startedField = true;
}
}
}
assert termCount == numTerms;
}
}