// Delete by query private static long applyQueryDeletes( Iterable<QueryAndLimit> queriesIter, ReadersAndLiveDocs rld, final SegmentReader reader) throws IOException { long delCount = 0; final AtomicReaderContext readerContext = reader.getContext(); boolean any = false; for (QueryAndLimit ent : queriesIter) { Query query = ent.query; int limit = ent.limit; final DocIdSet docs = new QueryWrapperFilter(query).getDocIdSet(readerContext, reader.getLiveDocs()); if (docs != null) { final DocIdSetIterator it = docs.iterator(); if (it != null) { while (true) { int doc = it.nextDoc(); if (doc >= limit) { break; } if (!any) { rld.initWritableLiveDocs(); any = true; } if (rld.delete(doc)) { delCount++; } } } } } return delCount; }
// Delete by Term private synchronized long applyTermDeletes( Iterable<Term> termsIter, ReadersAndLiveDocs rld, SegmentReader reader) throws IOException { long delCount = 0; Fields fields = reader.fields(); if (fields == null) { // This reader has no postings return 0; } TermsEnum termsEnum = null; String currentField = null; DocsEnum docs = null; assert checkDeleteTerm(null); boolean any = false; // System.out.println(Thread.currentThread().getName() + " del terms reader=" + reader); for (Term term : termsIter) { // Since we visit terms sorted, we gain performance // by re-using the same TermsEnum and seeking only // forwards if (!term.field().equals(currentField)) { assert currentField == null || currentField.compareTo(term.field()) < 0; currentField = term.field(); Terms terms = fields.terms(currentField); if (terms != null) { termsEnum = terms.iterator(null); } else { termsEnum = null; } } if (termsEnum == null) { continue; } assert checkDeleteTerm(term); // System.out.println(" term=" + term); if (termsEnum.seekExact(term.bytes(), false)) { // we don't need term frequencies for this DocsEnum docsEnum = termsEnum.docs(rld.getLiveDocs(), docs, DocsEnum.FLAG_NONE); // System.out.println("BDS: got docsEnum=" + docsEnum); if (docsEnum != null) { while (true) { final int docID = docsEnum.nextDoc(); // System.out.println(Thread.currentThread().getName() + " del term=" + term + " doc=" + // docID); if (docID == DocIdSetIterator.NO_MORE_DOCS) { break; } // NOTE: there is no limit check on the docID // when deleting by Term (unlike by Query) // because on flush we apply all Term deletes to // each segment. So all Term deleting here is // against prior segments: if (!any) { rld.initWritableLiveDocs(); any = true; } if (rld.delete(docID)) { delCount++; } } } } } return delCount; }
/** * Resolves the buffered deleted Term/Query/docIDs, into actual deleted docIDs in the liveDocs * MutableBits for each SegmentReader. */ public synchronized ApplyDeletesResult applyDeletes( IndexWriter.ReaderPool readerPool, List<SegmentInfoPerCommit> infos) throws IOException { final long t0 = System.currentTimeMillis(); if (infos.size() == 0) { return new ApplyDeletesResult(false, nextGen++, null); } assert checkDeleteStats(); if (!any()) { if (infoStream.isEnabled("BD")) { infoStream.message("BD", "applyDeletes: no deletes; skipping"); } return new ApplyDeletesResult(false, nextGen++, null); } if (infoStream.isEnabled("BD")) { infoStream.message("BD", "applyDeletes: infos=" + infos + " packetCount=" + deletes.size()); } final long gen = nextGen++; List<SegmentInfoPerCommit> infos2 = new ArrayList<SegmentInfoPerCommit>(); infos2.addAll(infos); Collections.sort(infos2, sortSegInfoByDelGen); CoalescedDeletes coalescedDeletes = null; boolean anyNewDeletes = false; int infosIDX = infos2.size() - 1; int delIDX = deletes.size() - 1; List<SegmentInfoPerCommit> allDeleted = null; while (infosIDX >= 0) { // System.out.println("BD: cycle delIDX=" + delIDX + " infoIDX=" + infosIDX); final FrozenBufferedDeletes packet = delIDX >= 0 ? deletes.get(delIDX) : null; final SegmentInfoPerCommit info = infos2.get(infosIDX); final long segGen = info.getBufferedDeletesGen(); if (packet != null && segGen < packet.delGen()) { // System.out.println(" coalesce"); if (coalescedDeletes == null) { coalescedDeletes = new CoalescedDeletes(); } if (!packet.isSegmentPrivate) { /* * Only coalesce if we are NOT on a segment private del packet: the segment private del packet * must only applied to segments with the same delGen. Yet, if a segment is already deleted * from the SI since it had no more documents remaining after some del packets younger than * its segPrivate packet (higher delGen) have been applied, the segPrivate packet has not been * removed. */ coalescedDeletes.update(packet); } delIDX--; } else if (packet != null && segGen == packet.delGen()) { assert packet.isSegmentPrivate : "Packet and Segments deletegen can only match on a segment private del packet gen=" + segGen; // System.out.println(" eq"); // Lock order: IW -> BD -> RP assert readerPool.infoIsLive(info); final ReadersAndLiveDocs rld = readerPool.get(info, true); final SegmentReader reader = rld.getReader(IOContext.READ); int delCount = 0; final boolean segAllDeletes; try { if (coalescedDeletes != null) { // System.out.println(" del coalesced"); delCount += applyTermDeletes(coalescedDeletes.termsIterable(), rld, reader); delCount += applyQueryDeletes(coalescedDeletes.queriesIterable(), rld, reader); } // System.out.println(" del exact"); // Don't delete by Term here; DocumentsWriterPerThread // already did that on flush: delCount += applyQueryDeletes(packet.queriesIterable(), rld, reader); final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount(); assert fullDelCount <= rld.info.info.getDocCount(); segAllDeletes = fullDelCount == rld.info.info.getDocCount(); } finally { rld.release(reader); readerPool.release(rld); } anyNewDeletes |= delCount > 0; if (segAllDeletes) { if (allDeleted == null) { allDeleted = new ArrayList<SegmentInfoPerCommit>(); } allDeleted.add(info); } if (infoStream.isEnabled("BD")) { infoStream.message( "BD", "seg=" + info + " segGen=" + segGen + " segDeletes=[" + packet + "]; coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "] newDelCount=" + delCount + (segAllDeletes ? " 100% deleted" : "")); } if (coalescedDeletes == null) { coalescedDeletes = new CoalescedDeletes(); } /* * Since we are on a segment private del packet we must not * update the coalescedDeletes here! We can simply advance to the * next packet and seginfo. */ delIDX--; infosIDX--; info.setBufferedDeletesGen(gen); } else { // System.out.println(" gt"); if (coalescedDeletes != null) { // Lock order: IW -> BD -> RP assert readerPool.infoIsLive(info); final ReadersAndLiveDocs rld = readerPool.get(info, true); final SegmentReader reader = rld.getReader(IOContext.READ); int delCount = 0; final boolean segAllDeletes; try { delCount += applyTermDeletes(coalescedDeletes.termsIterable(), rld, reader); delCount += applyQueryDeletes(coalescedDeletes.queriesIterable(), rld, reader); final int fullDelCount = rld.info.getDelCount() + rld.getPendingDeleteCount(); assert fullDelCount <= rld.info.info.getDocCount(); segAllDeletes = fullDelCount == rld.info.info.getDocCount(); } finally { rld.release(reader); readerPool.release(rld); } anyNewDeletes |= delCount > 0; if (segAllDeletes) { if (allDeleted == null) { allDeleted = new ArrayList<SegmentInfoPerCommit>(); } allDeleted.add(info); } if (infoStream.isEnabled("BD")) { infoStream.message( "BD", "seg=" + info + " segGen=" + segGen + " coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "] newDelCount=" + delCount + (segAllDeletes ? " 100% deleted" : "")); } } info.setBufferedDeletesGen(gen); infosIDX--; } } assert checkDeleteStats(); if (infoStream.isEnabled("BD")) { infoStream.message("BD", "applyDeletes took " + (System.currentTimeMillis() - t0) + " msec"); } // assert infos != segmentInfos || !any() : "infos=" + infos + " segmentInfos=" + segmentInfos + // " any=" + any; return new ApplyDeletesResult(anyNewDeletes, gen, allDeleted); }