final void readTIndexed32() { int index = -1; if (interrupted()) index = resumeInt(); if (index < 0) { if (!canReadInteger()) { interruptInt(index); return; } int bits = readInteger(); for (int i = 0; i < _versions.length; i++) ((TIndexed32Read) _versions[i]).setBits(bits); index = 0; } if (_versions[0] instanceof Version32) { Version32 version32 = (Version32) _versions[0]; TGenerated generated = (TGenerated) version32.object(); for (; index < generated.getFieldCount(); index++) { if (Bits.get(version32.getBits(), index)) { version32.readWrite(this, index, _versions); if (interrupted()) { interruptInt(index); return; } } } } }
private static void printDocs(DirectoryReader r) throws Throwable { for (AtomicReaderContext ctx : r.leaves()) { // TODO: improve this AtomicReader sub = ctx.reader(); Bits liveDocs = sub.getLiveDocs(); System.out.println(" " + ((SegmentReader) sub).getSegmentInfo()); for (int docID = 0; docID < sub.maxDoc(); docID++) { StoredDocument doc = sub.document(docID); if (liveDocs == null || liveDocs.get(docID)) { System.out.println(" docID=" + docID + " id:" + doc.get("id")); } else { System.out.println(" DEL docID=" + docID + " id:" + doc.get("id")); } } } }
public void testRandom() throws Exception { int num = atLeast(2); for (int iter = 0; iter < num; iter++) { if (VERBOSE) { System.out.println("TEST: iter=" + iter); } Directory dir = newDirectory(); IndexWriter w = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())) .setMergePolicy(NoMergePolicy.COMPOUND_FILES)); _TestUtil.keepFullyDeletedSegments(w); Map<BytesRef, List<Integer>> docs = new HashMap<BytesRef, List<Integer>>(); Set<Integer> deleted = new HashSet<Integer>(); List<BytesRef> terms = new ArrayList<BytesRef>(); int numDocs = _TestUtil.nextInt(random(), 1, 100 * RANDOM_MULTIPLIER); Document doc = new Document(); Field f = newStringField("field", "", Field.Store.NO); doc.add(f); Field id = newStringField("id", "", Field.Store.NO); doc.add(id); boolean onlyUniqueTerms = random().nextBoolean(); if (VERBOSE) { System.out.println("TEST: onlyUniqueTerms=" + onlyUniqueTerms + " numDocs=" + numDocs); } Set<BytesRef> uniqueTerms = new HashSet<BytesRef>(); for (int i = 0; i < numDocs; i++) { if (!onlyUniqueTerms && random().nextBoolean() && terms.size() > 0) { // re-use existing term BytesRef term = terms.get(random().nextInt(terms.size())); docs.get(term).add(i); f.setStringValue(term.utf8ToString()); } else { String s = _TestUtil.randomUnicodeString(random(), 10); BytesRef term = new BytesRef(s); if (!docs.containsKey(term)) { docs.put(term, new ArrayList<Integer>()); } docs.get(term).add(i); terms.add(term); uniqueTerms.add(term); f.setStringValue(s); } id.setStringValue("" + i); w.addDocument(doc); if (random().nextInt(4) == 1) { w.commit(); } if (i > 0 && random().nextInt(20) == 1) { int delID = random().nextInt(i); deleted.add(delID); w.deleteDocuments(new Term("id", "" + delID)); if (VERBOSE) { System.out.println("TEST: delete " + delID); } } } if (VERBOSE) { List<BytesRef> termsList = new ArrayList<BytesRef>(uniqueTerms); Collections.sort(termsList, BytesRef.getUTF8SortedAsUTF16Comparator()); System.out.println("TEST: terms in UTF16 order:"); for (BytesRef b : termsList) { System.out.println(" " + UnicodeUtil.toHexString(b.utf8ToString()) + " " + b); for (int docID : docs.get(b)) { if (deleted.contains(docID)) { System.out.println(" " + docID + " (deleted)"); } else { System.out.println(" " + docID); } } } } IndexReader reader = w.getReader(); w.close(); if (VERBOSE) { System.out.println("TEST: reader=" + reader); } Bits liveDocs = MultiFields.getLiveDocs(reader); for (int delDoc : deleted) { assertFalse(liveDocs.get(delDoc)); } for (int i = 0; i < 100; i++) { BytesRef term = terms.get(random().nextInt(terms.size())); if (VERBOSE) { System.out.println( "TEST: seek term=" + UnicodeUtil.toHexString(term.utf8ToString()) + " " + term); } DocsEnum docsEnum = _TestUtil.docs(random(), reader, "field", term, liveDocs, null, 0); assertNotNull(docsEnum); for (int docID : docs.get(term)) { if (!deleted.contains(docID)) { assertEquals(docID, docsEnum.nextDoc()); } } assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsEnum.nextDoc()); } reader.close(); dir.close(); } }
@SuppressWarnings({"fallthrough", "null"}) final void readTIndexedN() { for (; ; ) { EntryStep step = EntryStep.INT_INDEX; int intIndex = 0, bit = 0; Bits.Entry entry = null; if (interrupted()) { step = (EntryStep) resume(); intIndex = resumeInt(); entry = (Bits.Entry) resume(); bit = resumeInt(); } switch (step) { case INT_INDEX: { if (!canReadInteger()) { interruptInt(bit); interrupt(entry); interruptInt(intIndex); interrupt(EntryStep.INT_INDEX); return; } intIndex = readInteger(); } case BITS: { int actualIntIndex = intIndex >= 0 ? intIndex : -intIndex - 1; if (!canReadInteger()) { interruptInt(bit); interrupt(entry); interruptInt(intIndex); interrupt(EntryStep.BITS); return; } int bits = readInteger(); for (int i = 0; i < _versions.length; i++) { entry = new Bits.Entry(actualIntIndex, bits); ((TIndexedNRead) _versions[i]).addEntry(entry); } } case VALUES: { if (_versions[0] instanceof VersionN) { VersionN versionN = (VersionN) _versions[0]; int offset = entry.IntIndex << Bits.BITS_PER_UNIT_SHIFT; for (; bit < Bits.BITS_PER_UNIT; bit++) { if (Bits.get(entry.Value, bit)) { versionN.readWrite(this, offset + bit, _versions); if (interrupted()) { interruptInt(bit); interrupt(entry); interruptInt(intIndex); interrupt(EntryStep.VALUES); return; } } } } } } if (intIndex < 0) break; } }
/** @inheritDoc */ public boolean has(int value) { return (value < Bits.getMax(bits)) && Bits.get(bits, value); }