public void assertDeleteContent(Store store, DirectoryService service) throws IOException {
store.deleteContent();
assertThat(
Arrays.toString(store.directory().listAll()),
store.directory().listAll().length,
equalTo(0));
assertThat(store.stats().sizeInBytes(), equalTo(0l));
for (Directory dir : service.build()) {
assertThat(dir.listAll().length, equalTo(0));
}
}
public void seek(TermEnum terms) throws IOException {
original.seek(terms);
docFreq = terms.docFreq();
pointer = -1;
if (docFreq > postingMaps.length) { // grow postingsMap
PostingMap[] newMap = new PostingMap[docFreq];
System.arraycopy(postingMaps, 0, newMap, 0, postingMaps.length);
for (int i = postingMaps.length; i < docFreq; i++) {
newMap[i] = new PostingMap();
}
postingMaps = newMap;
}
out.reset();
int i = 0;
while (original.next()) {
PostingMap map = postingMaps[i++];
map.newDoc = oldToNew[original.doc()]; // remap the newDoc id
map.offset = out.getFilePointer(); // save pointer to buffer
final int tf = original.freq(); // buffer tf & positions
out.writeVInt(tf);
int prevPosition = 0;
for (int j = tf; j > 0; j--) { // delta encode positions
int p = original.nextPosition();
out.writeVInt(p - prevPosition);
prevPosition = p;
}
}
out.flush();
docFreq = i; // allow for deletions
Arrays.sort(postingMaps, 0, docFreq); // resort by mapped doc ids
// HeapSorter.sort(postingMaps,docFreq); // TODO MC - due to the lack of space
// NOTE: this might be substantially faster if RAMInputStream were public
// and supported a reset() operation.
in = tempDir.openInput(TEMP_FILE);
}
// private static int[] oldToNew(IndexReader reader, Searcher searcher) throws IOException {
private static DocScore[] newToOld(IndexReader reader, Searcher searcher) throws IOException {
int readerMax = reader.maxDoc();
DocScore[] newToOld = new DocScore[readerMax];
// use site, an indexed, un-tokenized field to get boost
// byte[] boosts = reader.norms("site"); TODO MC
/* TODO MC */
Document docMeta;
Pattern includes = Pattern.compile("\\|");
String value = NutchConfiguration.create().get(INCLUDE_EXTENSIONS_KEY, "");
String includeExtensions[] = includes.split(value);
Hashtable<String, Boolean> validExtensions = new Hashtable<String, Boolean>();
for (int i = 0; i < includeExtensions.length; i++) {
validExtensions.put(includeExtensions[i], true);
System.out.println("extension boosted " + includeExtensions[i]);
}
/* TODO MC */
for (int oldDoc = 0; oldDoc < readerMax; oldDoc++) {
float score;
if (reader.isDeleted(oldDoc)) {
// score = 0.0f;
score = -1f; // TODO MC
} else {
// score = Similarity.decodeNorm(boosts[oldDoc]); TODO MC
/* TODO MC */
docMeta = searcher.doc(oldDoc);
if (validExtensions.get(docMeta.get("subType"))
== null) { // searched extensions will have higher scores
score = -0.5f;
} else {
score = Integer.parseInt(docMeta.get("inlinks"));
/*
if (score==0) {
score=0.001f; // TODO MC - to not erase
}
*/
}
/* TODO MC */
// System.out.println("Score for old document "+oldDoc+" is "+score+" and type
// "+docMeta.get("subType")); // TODO MC debug remove
}
DocScore docScore = new DocScore();
docScore.doc = oldDoc;
docScore.score = score;
newToOld[oldDoc] = docScore;
}
System.out.println("Sorting " + newToOld.length + " documents.");
Arrays.sort(newToOld);
// HeapSorter.sort(newToOld); // TODO MC - due to the lack of space
/* TODO MC
int[] oldToNew = new int[readerMax];
for (int newDoc = 0; newDoc < readerMax; newDoc++) {
DocScore docScore = newToOld[newDoc];
//oldToNew[docScore.oldDoc] = docScore.score > 0.0f ? newDoc : -1; // TODO MC
oldToNew[docScore.oldDoc] = newDoc; // TODO MC
}
*/
/* TODO MC *
for (int newDoc = 0; newDoc < readerMax; newDoc++) {
DocScore docScore = newToOld[newDoc];
System.out.println("Score for new document "+newDoc+" is "+docScore.score); // TODO MC debug remove
}
* TODO MC */
// return oldToNew; TODO MC
return newToOld; // TODO MC
}
protected void validateResponse(
TermVectorResponse esResponse, Fields luceneFields, TestConfig testConfig)
throws IOException {
TestDoc testDoc = testConfig.doc;
HashSet<String> selectedFields =
testConfig.selectedFields == null
? null
: new HashSet<String>(Arrays.asList(testConfig.selectedFields));
Fields esTermVectorFields = esResponse.getFields();
for (TestFieldSetting field : testDoc.fieldSettings) {
Terms esTerms = esTermVectorFields.terms(field.name);
if (selectedFields != null && !selectedFields.contains(field.name)) {
assertNull(esTerms);
continue;
}
assertNotNull(esTerms);
Terms luceneTerms = luceneFields.terms(field.name);
TermsEnum esTermEnum = esTerms.iterator(null);
TermsEnum luceneTermEnum = luceneTerms.iterator(null);
while (esTermEnum.next() != null) {
assertNotNull(luceneTermEnum.next());
assertThat(esTermEnum.totalTermFreq(), equalTo(luceneTermEnum.totalTermFreq()));
DocsAndPositionsEnum esDocsPosEnum = esTermEnum.docsAndPositions(null, null, 0);
DocsAndPositionsEnum luceneDocsPosEnum = luceneTermEnum.docsAndPositions(null, null, 0);
if (luceneDocsPosEnum == null) {
// test we expect that...
assertFalse(field.storedOffset);
assertFalse(field.storedPayloads);
assertFalse(field.storedPositions);
continue;
}
String currentTerm = esTermEnum.term().utf8ToString();
assertThat(
"Token mismatch for field: " + field.name,
currentTerm,
equalTo(luceneTermEnum.term().utf8ToString()));
esDocsPosEnum.nextDoc();
luceneDocsPosEnum.nextDoc();
int freq = esDocsPosEnum.freq();
assertThat(freq, equalTo(luceneDocsPosEnum.freq()));
for (int i = 0; i < freq; i++) {
String failDesc = " (field:" + field.name + " term:" + currentTerm + ")";
int lucenePos = luceneDocsPosEnum.nextPosition();
int esPos = esDocsPosEnum.nextPosition();
if (field.storedPositions && testConfig.requestPositions) {
assertThat("Position test failed" + failDesc, lucenePos, equalTo(esPos));
} else {
assertThat("Missing position test failed" + failDesc, esPos, equalTo(-1));
}
if (field.storedOffset && testConfig.requestOffsets) {
assertThat(
"Offset test failed" + failDesc,
luceneDocsPosEnum.startOffset(),
equalTo(esDocsPosEnum.startOffset()));
assertThat(
"Offset test failed" + failDesc,
luceneDocsPosEnum.endOffset(),
equalTo(esDocsPosEnum.endOffset()));
} else {
assertThat(
"Missing offset test failed" + failDesc, esDocsPosEnum.startOffset(), equalTo(-1));
assertThat(
"Missing offset test failed" + failDesc, esDocsPosEnum.endOffset(), equalTo(-1));
}
if (field.storedPayloads && testConfig.requestPayloads) {
assertThat(
"Payload test failed" + failDesc,
luceneDocsPosEnum.getPayload(),
equalTo(esDocsPosEnum.getPayload()));
} else {
assertThat(
"Missing payload test failed" + failDesc,
esDocsPosEnum.getPayload(),
equalTo(null));
}
}
}
assertNull("Es returned terms are done but lucene isn't", luceneTermEnum.next());
}
}