public void testKeepsLastFilter() throws Throwable {
DuplicateFilter df = new DuplicateFilter(KEY_FIELD);
df.setKeepMode(DuplicateFilter.KeepMode.KM_USE_LAST_OCCURRENCE);
ScoreDoc[] hits = searcher.search(tq, df, 1000).scoreDocs;
assertTrue("Filtered searching should have found some matches", hits.length > 0);
for (ScoreDoc hit : hits) {
StoredDocument d = searcher.doc(hit.doc);
String url = d.get(KEY_FIELD);
DocsEnum td =
_TestUtil.docs(
random(),
reader,
KEY_FIELD,
new BytesRef(url),
MultiFields.getLiveDocs(reader),
null,
0);
int lastDoc = 0;
while (td.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
lastDoc = td.docID();
}
assertEquals("Duplicate urls should return last doc", lastDoc, hit.doc);
}
}
/**
* @param reader
* @param lireFeature
* @return the maximum distance found for normalizing.
* @throws java.io.IOException
*/
@SuppressWarnings("unchecked")
private float[] findSimilar(IndexReader reader, LireFeature[] lireFeature) throws IOException {
float[] maxDistance = new float[lireFeature.length];
float[] overallMaxDistance = new float[lireFeature.length];
for (int i = 0; i < overallMaxDistance.length; i++) {
overallMaxDistance[i] = -1f;
maxDistance[i] = -1f;
}
parDocs = new TreeSet[lireFeature.length];
for (int i = 0; i < parDocs.length; i++) {
parDocs[i] = new TreeSet<SimpleResult>();
}
// Needed for check whether the document is deleted.
Bits liveDocs = MultiFields.getLiveDocs(reader);
// clear result set ...
int docs = reader.numDocs();
for (int i = 0; i < docs; i++) {
if (reader.hasDeletions() && !liveDocs.get(i)) continue; // if it is deleted, just ignore it.
Document d = reader.document(i);
float[] distance = getDistance(d, lireFeature);
// calculate the overall max distance to normalize score afterwards
for (int j = 0; j < distance.length; j++) {
float f = distance[j];
if (overallMaxDistance[j] < f) {
overallMaxDistance[j] = f;
}
// if it is the first document:
if (maxDistance[j] < 0) {
maxDistance[j] = f;
}
// if the array is not full yet:
if (this.parDocs[j].size() < maxHits) {
this.parDocs[j].add(new SimpleResult(f, d));
if (f > maxDistance[j]) {
maxDistance[j] = f;
}
} else if (f < maxDistance[j]) {
// if it is nearer to the sample than at least on of the current set:
// remove the last one ...
this.parDocs[j].remove(this.parDocs[j].last());
// add the new one ...
this.parDocs[j].add(new SimpleResult(f, d));
// and set our new distance border ...
maxDistance[j] = this.parDocs[j].last().getDistance();
}
}
}
return maxDistance;
}
/*
* listPostings displays the first n postings for a term in a
* field in an index (specified by reader). Set n to MAX_VALUE
* to display all postings.
*/
static void listPostings(IndexReader reader, String termString, String field, Integer n)
throws IOException {
System.out.println("\nPostings: " + termString + " " + field);
/*
* Prepare to access the index.
*/
BytesRef termBytes = new BytesRef(termString);
Term term = new Term(field, termBytes);
Bits liveDocs = MultiFields.getLiveDocs(reader);
/*
* Lookup the collection term frequency (ctf).
*/
long df = reader.docFreq(term);
System.out.println("\tdf: " + df);
long ctf = reader.totalTermFreq(term);
System.out.println("\tctf: " + ctf);
if (df < 1) return;
/*
* Lookup the inverted list.
*/
DocsAndPositionsEnum postings =
MultiFields.getTermPositionsEnum(reader, liveDocs, field, termBytes);
/*
* Iterate through the first n postings.
*/
long count = 0;
while ((count < n) && (postings.nextDoc() != DocIdSetIterator.NO_MORE_DOCS)) {
System.out.println("\tdocid: " + postings.docID());
int tf = postings.freq();
System.out.println("\ttf: " + tf);
System.out.print("\tPositions: ");
for (int j = 0; j < tf; j++) {
int pos = postings.nextPosition();
System.out.print(pos + " ");
}
System.out.println("");
count++;
}
;
return;
}
public static long getTotalTermFreq(IndexReader reader, String field, BytesRef termtext)
throws Exception {
// BytesRef br = termtext;
long totalTF = 0;
try {
Bits liveDocs = MultiFields.getLiveDocs(reader);
totalTF = reader.getSumTotalTermFreq(field);
return totalTF;
} catch (Exception e) {
return 0;
}
}
/**
* We assume that the initial indexing has been done and a set of reference objects has been found
* and indexed in the separate directory. However further documents were added and they now need
* to get a ranked list of reference objects. So we (i) get all these new documents missing the
* field "ro-order" and (ii) add this field.
*
* @param indexPath the index to update
* @throws IOException
*/
public void updateIndex(String indexPath) throws IOException {
IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(indexPath)));
int numDocs = reader.numDocs();
boolean hasDeletions = reader.hasDeletions();
int countUpdated = 0;
IndexReader readerRo = DirectoryReader.open(FSDirectory.open(new File(indexPath + "-ro")));
ImageSearcher searcher =
new GenericImageSearcher(numReferenceObjectsUsed, featureClass, featureFieldName);
Map<String, Analyzer> perField = new HashMap<String, Analyzer>(1);
perField.put("ro-order", new WhitespaceAnalyzer(LuceneUtils.LUCENE_VERSION));
PerFieldAnalyzerWrapper aWrapper =
new PerFieldAnalyzerWrapper(new SimpleAnalyzer(LuceneUtils.LUCENE_VERSION), perField);
IndexWriter iw =
new IndexWriter(
FSDirectory.open(new File(indexPath)),
new IndexWriterConfig(LuceneUtils.LUCENE_VERSION, aWrapper)
.setOpenMode(IndexWriterConfig.OpenMode.CREATE));
StringBuilder sb = new StringBuilder(256);
// Needed for check whether the document is deleted.
Bits liveDocs = MultiFields.getLiveDocs(reader);
for (int i = 0; i < numDocs; i++) {
if (reader.hasDeletions() && !liveDocs.get(i)) continue; // if it is deleted, just ignore it.
Document document = reader.document(i);
if (document.getField("ro-order") == null) { // if the field is not here we create it.
ImageSearchHits hits = searcher.search(document, readerRo);
sb.delete(0, sb.length());
for (int j = 0; j < numReferenceObjectsUsed; j++) {
sb.append(hits.doc(j).getValues("ro-id")[0]);
sb.append(' ');
}
// System.out.println(sb.toString());
document.add(new TextField("ro-order", sb.toString(), Field.Store.YES));
iw.updateDocument(
new Term(
DocumentBuilder.FIELD_NAME_IDENTIFIER,
document.getValues(DocumentBuilder.FIELD_NAME_IDENTIFIER)[0]),
document);
countUpdated++;
}
// progress report
progress.setNumDocsProcessed(progress.getNumDocsProcessed() + 1);
// debug:
System.out.println("countUpdated = " + countUpdated);
}
iw.commit();
iw.close();
}
public void tttestGetDistribution() throws IOException {
BufferedWriter bw = new BufferedWriter(new FileWriter("data.csv"));
IndexReader reader = IndexReader.open(FSDirectory.open(new File(indexPath)));
// get the first document:
// if (!IndexReader.indexExists(reader.directory()))
// throw new FileNotFoundException("No index found at this specific location.");
CEDD cedd1 = new CEDD();
FCTH fcth1 = new FCTH();
CEDD cedd2 = new CEDD();
FCTH fcth2 = new FCTH();
JCD jcd1 = new JCD();
JCD jcd2 = new JCD();
String[] cls;
// Needed for check whether the document is deleted.
Bits liveDocs = MultiFields.getLiveDocs(reader);
int docs = reader.numDocs();
for (int i = 0; i < docs; i++) {
if (reader.hasDeletions() && !liveDocs.get(i)) continue; // if it is deleted, just ignore it.
Document doc = reader.document(i);
cls = doc.getValues(DocumentBuilder.FIELD_NAME_CEDD);
if (cls != null && cls.length > 0) cedd1.setStringRepresentation(cls[0]);
cls = doc.getValues(DocumentBuilder.FIELD_NAME_FCTH);
if (cls != null && cls.length > 0) fcth1.setStringRepresentation(cls[0]);
for (int j = i + 1; j < docs; j++) {
if (reader.hasDeletions() && !liveDocs.get(i))
continue; // if it is deleted, just ignore it.
Document doc2 = reader.document(j);
cls = doc2.getValues(DocumentBuilder.FIELD_NAME_CEDD);
if (cls != null && cls.length > 0) cedd2.setStringRepresentation(cls[0]);
cls = doc2.getValues(DocumentBuilder.FIELD_NAME_FCTH);
if (cls != null && cls.length > 0) fcth2.setStringRepresentation(cls[0]);
jcd1.init(cedd1, fcth1);
jcd2.init(cedd2, fcth2);
bw.write(
cedd1.getDistance(cedd2)
+ ";"
+ fcth1.getDistance(fcth2)
+ ";"
+ jcd1.getDistance(jcd2)
+ "\n");
}
if (i % 100 == 0) System.out.println(i + " entries processed ... ");
}
bw.close();
}
private FixedBitSet createExpectedResult(
String queryValue, boolean from, IndexReader topLevelReader, IndexIterationContext context)
throws IOException {
final Map<String, List<RandomDoc>> randomValueDocs;
final Map<String, List<RandomDoc>> linkValueDocuments;
if (from) {
randomValueDocs = context.randomValueFromDocs;
linkValueDocuments = context.toDocuments;
} else {
randomValueDocs = context.randomValueToDocs;
linkValueDocuments = context.fromDocuments;
}
FixedBitSet expectedResult = new FixedBitSet(topLevelReader.maxDoc());
List<RandomDoc> matchingDocs = randomValueDocs.get(queryValue);
if (matchingDocs == null) {
return new FixedBitSet(topLevelReader.maxDoc());
}
for (RandomDoc matchingDoc : matchingDocs) {
for (String linkValue : matchingDoc.linkValues) {
List<RandomDoc> otherMatchingDocs = linkValueDocuments.get(linkValue);
if (otherMatchingDocs == null) {
continue;
}
for (RandomDoc otherSideDoc : otherMatchingDocs) {
DocsEnum docsEnum =
MultiFields.getTermDocsEnum(
topLevelReader,
MultiFields.getLiveDocs(topLevelReader),
"id",
new BytesRef(otherSideDoc.id),
0);
assert docsEnum != null;
int doc = docsEnum.nextDoc();
expectedResult.set(doc);
}
}
}
return expectedResult;
}
/**
* Creates a set of reference objects and stores it in a new index (name "<indexPath>-ro"). Then
* creates ordered lists of reference object positions for each data item in the index with given
* feature. Finally a new index (name "<indexPath>-ms") is created where all the original
* documents as well as the new data are stored.
*
* @param indexPath the path to the original index
* @throws IOException
*/
public void createIndex(String indexPath) throws IOException {
IndexReader reader = DirectoryReader.open(FSDirectory.open(new File(indexPath)));
int numDocs = reader.numDocs();
if (numDocs < numReferenceObjects) {
throw new UnsupportedOperationException("Too few documents in index.");
}
// progress report
progress.setNumDocsAll(numDocs);
progress.setCurrentState(State.RoSelection);
boolean hasDeletions = reader.hasDeletions();
// init reference objects:
IndexWriter iw = LuceneUtils.createIndexWriter(indexPath + "-ro", true);
HashSet<Integer> referenceObjsIds = new HashSet<Integer>(numReferenceObjects);
double numDocsDouble = (double) numDocs;
while (referenceObjsIds.size() < numReferenceObjects) {
referenceObjsIds.add((int) (numDocsDouble * Math.random()));
}
int count = 0;
if (hasDeletions) {
System.err.println(
"WARNING: There are deleted docs in your index. You should "
+ "optimize your index before using this method.");
}
// progress report
progress.setCurrentState(State.RoIndexing);
// find them in the index and put them into a separate index:
for (int i : referenceObjsIds) {
count++;
Document document = reader.document(i);
document.add(new Field("ro-id", count + "", StringField.TYPE_STORED));
iw.addDocument(document);
}
iw.commit();
iw.close();
// progress report
progress.setCurrentState(State.Indexing);
// now find the reference objects for each entry ;)
IndexReader readerRo = DirectoryReader.open(FSDirectory.open(new File(indexPath + "-ro")));
ImageSearcher searcher =
new GenericImageSearcher(numReferenceObjectsUsed, featureClass, featureFieldName);
Map<String, Analyzer> analyzerPerField = new HashMap<String, Analyzer>();
analyzerPerField.put("ro-order", new WhitespaceAnalyzer(LuceneUtils.LUCENE_VERSION));
PerFieldAnalyzerWrapper aWrapper =
new PerFieldAnalyzerWrapper(
new SimpleAnalyzer(LuceneUtils.LUCENE_VERSION), analyzerPerField);
iw =
new IndexWriter(
FSDirectory.open(new File(indexPath)),
new IndexWriterConfig(LuceneUtils.LUCENE_VERSION, aWrapper)
.setOpenMode(IndexWriterConfig.OpenMode.CREATE));
StringBuilder sb = new StringBuilder(256);
// Needed for check whether the document is deleted.
Bits liveDocs = MultiFields.getLiveDocs(reader);
for (int i = 0; i < numDocs; i++) {
if (reader.hasDeletions() && !liveDocs.get(i)) continue; // if it is deleted, just ignore it.
Document document = reader.document(i);
ImageSearchHits hits = searcher.search(document, readerRo);
sb.delete(0, sb.length());
for (int j = 0; j < numReferenceObjectsUsed; j++) {
sb.append(hits.doc(j).getValues("ro-id")[0]);
sb.append(' ');
}
// System.out.println(sb.toString());
document.add(new TextField("ro-order", sb.toString(), Field.Store.YES));
iw.updateDocument(
new Term(
DocumentBuilder.FIELD_NAME_IDENTIFIER,
document.getValues(DocumentBuilder.FIELD_NAME_IDENTIFIER)[0]),
document);
// progress report
progress.setNumDocsProcessed(progress.getNumDocsProcessed() + 1);
}
iw.commit();
iw.close();
// progress report
progress.setCurrentState(State.Idle);
}
public DocumentIterator(Workspace workspace, INDEX indexType) throws Exception {
this.reader = workspace.getIndexReader(indexType);
this.originalFolder = workspace.getOriginalDocFolder(indexType);
this.liveDocs = reader == null ? null : MultiFields.getLiveDocs(reader);
nextDocument = readNextDocument();
}
