/** Get an alphabetically sorted list of field names. */
public List<String> getFieldNames() {
HashSet<String> names = new HashSet<String>();
names.addAll(storedFields.keySet());
names.addAll(reconstructedFields.keySet());
ArrayList<String> res = new ArrayList<String>(names.size());
res.addAll(names);
Collections.sort(res);
return res;
}
/**
* Reconstruct document fields.
*
* @param docNum document number. If this document is deleted, but the index is not optimized yet,
* the reconstruction process may still yield the reconstructed field content even from
* deleted documents.
* @return reconstructed document
* @throws Exception
*/
public Reconstructed reconstruct(int docNum) throws Exception {
if (docNum < 0 || docNum > reader.maxDoc()) {
throw new Exception("Document number outside of valid range.");
}
Reconstructed res = new Reconstructed();
if (deleted != null && deleted.get(docNum)) {
throw new Exception("Document is deleted.");
} else {
Document doc = reader.document(docNum);
for (int i = 0; i < fieldNames.length; i++) {
Field[] fs = doc.getFields(fieldNames[i]);
if (fs != null && fs.length > 0) {
res.getStoredFields().put(fieldNames[i], fs);
}
}
}
// collect values from unstored fields
HashSet<String> fields = new HashSet<String>(Arrays.asList(fieldNames));
// try to use term vectors if available
progress.maxValue = fieldNames.length;
progress.curValue = 0;
progress.minValue = 0;
for (int i = 0; i < fieldNames.length; i++) {
TermFreqVector tvf = reader.getTermFreqVector(docNum, fieldNames[i]);
if (tvf != null && tvf.size() > 0 && (tvf instanceof TermPositionVector)) {
TermPositionVector tpv = (TermPositionVector) tvf;
progress.message = "Reading term vectors ...";
progress.curValue = i;
setChanged();
notifyObservers(progress);
BytesRef[] tv = tpv.getTerms();
for (int k = 0; k < tv.length; k++) {
// do we have positions?
int[] posArr = tpv.getTermPositions(k);
if (posArr == null) {
// only offsets
TermVectorOffsetInfo[] offsets = tpv.getOffsets(k);
if (offsets.length == 0) {
continue;
}
// convert offsets into positions
posArr = convertOffsets(offsets);
}
GrowableStringArray gsa = res.getReconstructedFields().get(fieldNames[i]);
if (gsa == null) {
gsa = new GrowableStringArray();
res.getReconstructedFields().put(fieldNames[i], gsa);
}
for (int m = 0; m < posArr.length; m++) {
gsa.append(posArr[m], "|", tv[k].utf8ToString());
}
}
fields.remove(fieldNames[i]); // got what we wanted
}
}
// this loop collects data only from left-over fields
// not yet collected through term vectors
progress.maxValue = fields.size();
progress.curValue = 0;
progress.minValue = 0;
for (String fld : fields) {
progress.message = "Collecting terms in " + fld + " ...";
progress.curValue++;
setChanged();
notifyObservers(progress);
Terms terms = MultiFields.getTerms(reader, fld);
if (terms == null) { // no terms in this field
continue;
}
TermsEnum te = terms.iterator();
while (te.next() != null) {
DocsAndPositionsEnum dpe = te.docsAndPositions(deleted, null);
if (dpe == null) { // no position info for this field
break;
}
int num = dpe.advance(docNum);
if (num != docNum) { // either greater than or NO_MORE_DOCS
continue; // no data for this term in this doc
}
String term = te.term().utf8ToString();
GrowableStringArray gsa = (GrowableStringArray) res.getReconstructedFields().get(fld);
if (gsa == null) {
gsa = new GrowableStringArray();
res.getReconstructedFields().put(fld, gsa);
}
for (int k = 0; k < dpe.freq(); k++) {
int pos = dpe.nextPosition();
gsa.append(pos, "|", term);
}
}
}
progress.message = "Done.";
progress.curValue = 100;
setChanged();
notifyObservers(progress);
return res;
}
/**
* Creates a set of reference objects and stores it in a new index (name "<indexPath>-ro"). Then
* creates ordered lists of reference object positions for each data item in the index with given
* feature. Finally a new index (name "<indexPath>-ms") is created where all the original
* documents as well as the new data are stored.
*
* @param indexPath the path to the original index
* @throws IOException
*/
public void createIndex(String indexPath) throws IOException {
IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(indexPath)));
int numDocs = reader.numDocs();
if (numDocs < numReferenceObjects) {
throw new UnsupportedOperationException("Too few documents in index.");
}
// progress report
progress.setNumDocsAll(numDocs);
progress.setCurrentState(State.RoSelection);
boolean hasDeletions = reader.hasDeletions();
// init reference objects:
IndexWriter iw = LuceneUtils.createIndexWriter(indexPath + "-ro", true);
HashSet<Integer> referenceObjsIds = new HashSet<Integer>(numReferenceObjects);
double numDocsDouble = (double) numDocs;
while (referenceObjsIds.size() < numReferenceObjects) {
referenceObjsIds.add((int) (numDocsDouble * Math.random()));
}
int count = 0;
if (hasDeletions) {
System.err.println(
"WARNING: There are deleted docs in your index. You should "
+ "optimize your index before using this method.");
}
// progress report
progress.setCurrentState(State.RoIndexing);
// find them in the index and put them into a separate index:
for (int i : referenceObjsIds) {
count++;
Document document = reader.document(i);
document.add(new Field("ro-id", count + "", StringField.TYPE_STORED));
iw.addDocument(document);
}
iw.commit();
iw.close();
// progress report
progress.setCurrentState(State.Indexing);
// now find the reference objects for each entry ;)
IndexReader readerRo = DirectoryReader.open(FSDirectory.open(new File(indexPath + "-ro")));
ImageSearcher searcher =
new GenericImageSearcher(numReferenceObjectsUsed, featureClass, featureFieldName);
Map<String, Analyzer> analyzerPerField = new HashMap<String, Analyzer>();
analyzerPerField.put("ro-order", new WhitespaceAnalyzer(LuceneUtils.LUCENE_VERSION));
PerFieldAnalyzerWrapper aWrapper =
new PerFieldAnalyzerWrapper(
new SimpleAnalyzer(LuceneUtils.LUCENE_VERSION), analyzerPerField);
iw =
new IndexWriter(
FSDirectory.open(new File(indexPath)),
new IndexWriterConfig(LuceneUtils.LUCENE_VERSION, aWrapper)
.setOpenMode(IndexWriterConfig.OpenMode.CREATE));
StringBuilder sb = new StringBuilder(256);
// Needed for check whether the document is deleted.
Bits liveDocs = MultiFields.getLiveDocs(reader);
for (int i = 0; i < numDocs; i++) {
if (reader.hasDeletions() && !liveDocs.get(i)) continue; // if it is deleted, just ignore it.
Document document = reader.document(i);
ImageSearchHits hits = searcher.search(document, readerRo);
sb.delete(0, sb.length());
for (int j = 0; j < numReferenceObjectsUsed; j++) {
sb.append(hits.doc(j).getValues("ro-id")[0]);
sb.append(' ');
}
// System.out.println(sb.toString());
document.add(new TextField("ro-order", sb.toString(), Field.Store.YES));
iw.updateDocument(
new Term(
DocumentBuilder.FIELD_NAME_IDENTIFIER,
document.getValues(DocumentBuilder.FIELD_NAME_IDENTIFIER)[0]),
document);
// progress report
progress.setNumDocsProcessed(progress.getNumDocsProcessed() + 1);
}
iw.commit();
iw.close();
// progress report
progress.setCurrentState(State.Idle);
}
protected void validateResponse(
TermVectorResponse esResponse, Fields luceneFields, TestConfig testConfig)
throws IOException {
TestDoc testDoc = testConfig.doc;
HashSet<String> selectedFields =
testConfig.selectedFields == null
? null
: new HashSet<String>(Arrays.asList(testConfig.selectedFields));
Fields esTermVectorFields = esResponse.getFields();
for (TestFieldSetting field : testDoc.fieldSettings) {
Terms esTerms = esTermVectorFields.terms(field.name);
if (selectedFields != null && !selectedFields.contains(field.name)) {
assertNull(esTerms);
continue;
}
assertNotNull(esTerms);
Terms luceneTerms = luceneFields.terms(field.name);
TermsEnum esTermEnum = esTerms.iterator(null);
TermsEnum luceneTermEnum = luceneTerms.iterator(null);
while (esTermEnum.next() != null) {
assertNotNull(luceneTermEnum.next());
assertThat(esTermEnum.totalTermFreq(), equalTo(luceneTermEnum.totalTermFreq()));
DocsAndPositionsEnum esDocsPosEnum = esTermEnum.docsAndPositions(null, null, 0);
DocsAndPositionsEnum luceneDocsPosEnum = luceneTermEnum.docsAndPositions(null, null, 0);
if (luceneDocsPosEnum == null) {
// test we expect that...
assertFalse(field.storedOffset);
assertFalse(field.storedPayloads);
assertFalse(field.storedPositions);
continue;
}
String currentTerm = esTermEnum.term().utf8ToString();
assertThat(
"Token mismatch for field: " + field.name,
currentTerm,
equalTo(luceneTermEnum.term().utf8ToString()));
esDocsPosEnum.nextDoc();
luceneDocsPosEnum.nextDoc();
int freq = esDocsPosEnum.freq();
assertThat(freq, equalTo(luceneDocsPosEnum.freq()));
for (int i = 0; i < freq; i++) {
String failDesc = " (field:" + field.name + " term:" + currentTerm + ")";
int lucenePos = luceneDocsPosEnum.nextPosition();
int esPos = esDocsPosEnum.nextPosition();
if (field.storedPositions && testConfig.requestPositions) {
assertThat("Position test failed" + failDesc, lucenePos, equalTo(esPos));
} else {
assertThat("Missing position test failed" + failDesc, esPos, equalTo(-1));
}
if (field.storedOffset && testConfig.requestOffsets) {
assertThat(
"Offset test failed" + failDesc,
luceneDocsPosEnum.startOffset(),
equalTo(esDocsPosEnum.startOffset()));
assertThat(
"Offset test failed" + failDesc,
luceneDocsPosEnum.endOffset(),
equalTo(esDocsPosEnum.endOffset()));
} else {
assertThat(
"Missing offset test failed" + failDesc, esDocsPosEnum.startOffset(), equalTo(-1));
assertThat(
"Missing offset test failed" + failDesc, esDocsPosEnum.endOffset(), equalTo(-1));
}
if (field.storedPayloads && testConfig.requestPayloads) {
assertThat(
"Payload test failed" + failDesc,
luceneDocsPosEnum.getPayload(),
equalTo(esDocsPosEnum.getPayload()));
} else {
assertThat(
"Missing payload test failed" + failDesc,
esDocsPosEnum.getPayload(),
equalTo(null));
}
}
}
assertNull("Es returned terms are done but lucene isn't", luceneTermEnum.next());
}
}