public CompletionFieldsConsumer(SegmentWriteState state) throws IOException { this.delegatesFieldsConsumer = delegatePostingsFormat.fieldsConsumer(state); String suggestFSTFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION); IndexOutput output = null; boolean success = false; try { output = state.directory.createOutput(suggestFSTFile, state.context); CodecUtil.writeIndexHeader( output, CODEC_NAME, SUGGEST_VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); /* * we write the delegate postings format name so we can load it * without getting an instance in the ctor */ output.writeString(delegatePostingsFormat.getName()); output.writeString(writeProvider.getName()); this.suggestFieldsConsumer = writeProvider.consumer(output); success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(output); } } }
@Test @Override public void testIndexWritingAndFinding() throws IOException, InterruptedException { verifyBoth(cache0, cache1); IndexOutput indexOutput = dirA.createOutput(filename, IOContext.DEFAULT); indexOutput.writeString("no need to write, nobody ever will read this"); indexOutput.flush(); indexOutput.close(); assertFileExistsHavingRLCount(filename, 1, true); IndexInput firstOpenOnB = dirB.openInput(filename, IOContext.DEFAULT); assertFileExistsHavingRLCount(filename, 2, true); dirA.deleteFile(filename); assertFileExistsHavingRLCount(filename, 1, false); // Lucene does use clone() - lock implementation ignores it as a clone is // cast on locked segments and released before the close on the parent object IndexInput cloneOfFirstOpenOnB = (IndexInput) firstOpenOnB.clone(); assertFileExistsHavingRLCount(filename, 1, false); cloneOfFirstOpenOnB.close(); assertFileExistsHavingRLCount(filename, 1, false); IndexInput firstOpenOnA = dirA.openInput(filename, IOContext.DEFAULT); assertFileExistsHavingRLCount(filename, 2, false); IndexInput secondOpenOnA = dirA.openInput(filename, IOContext.DEFAULT); assertFileExistsHavingRLCount(filename, 2, false); firstOpenOnA.close(); assertFileExistsHavingRLCount(filename, 2, false); secondOpenOnA.close(); assertFileExistsHavingRLCount(filename, 1, false); firstOpenOnB.close(); assertFileNotExists(filename); dirA.close(); dirB.close(); verifyBoth(cache0, cache1); }
@Override public void write( Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException { final String fileName = IndexFileNames.segmentFileName( segmentInfo.name, segmentSuffix, Lucene46FieldInfosFormat.EXTENSION); try (IndexOutput output = directory.createOutput(fileName, context)) { CodecUtil.writeHeader( output, Lucene46FieldInfosFormat.CODEC_NAME, Lucene46FieldInfosFormat.FORMAT_CURRENT); output.writeVInt(infos.size()); for (FieldInfo fi : infos) { IndexOptions indexOptions = fi.getIndexOptions(); byte bits = 0x0; if (fi.hasVectors()) bits |= Lucene46FieldInfosFormat.STORE_TERMVECTOR; if (fi.omitsNorms()) bits |= Lucene46FieldInfosFormat.OMIT_NORMS; if (fi.hasPayloads()) bits |= Lucene46FieldInfosFormat.STORE_PAYLOADS; if (fi.getIndexOptions() != IndexOptions.NONE) { bits |= Lucene46FieldInfosFormat.IS_INDEXED; assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !fi.hasPayloads(); if (indexOptions == IndexOptions.DOCS) { bits |= Lucene46FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS; } else if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) { bits |= Lucene46FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS; } else if (indexOptions == IndexOptions.DOCS_AND_FREQS) { bits |= Lucene46FieldInfosFormat.OMIT_POSITIONS; } } output.writeString(fi.name); output.writeVInt(fi.number); output.writeByte(bits); // pack the DV types in one byte final byte dv = docValuesByte(fi.getDocValuesType()); final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : DocValuesType.NONE); assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0; byte val = (byte) (0xff & ((nrm << 4) | dv)); output.writeByte(val); output.writeLong(fi.getDocValuesGen()); output.writeStringStringMap(fi.attributes()); } CodecUtil.writeFooter(output); } }
@Override public void close() throws IOException { delegateFieldsConsumer.close(); // Now we are done accumulating values for these fields List<Entry<FieldInfo, FuzzySet>> nonSaturatedBlooms = new ArrayList<Map.Entry<FieldInfo, FuzzySet>>(); for (Entry<FieldInfo, FuzzySet> entry : bloomFilters.entrySet()) { FuzzySet bloomFilter = entry.getValue(); if (!bloomFilterFactory.isSaturated(bloomFilter, entry.getKey())) { nonSaturatedBlooms.add(entry); } } String bloomFileName = IndexFileNames.segmentFileName( state.segmentInfo.name, state.segmentSuffix, BLOOM_EXTENSION); IndexOutput bloomOutput = null; try { bloomOutput = state.directory.createOutput(bloomFileName, state.context); CodecUtil.writeHeader(bloomOutput, BLOOM_CODEC_NAME, BLOOM_CODEC_VERSION); // remember the name of the postings format we will delegate to bloomOutput.writeString(delegatePostingsFormat.getName()); // First field in the output file is the number of fields+blooms saved bloomOutput.writeInt(nonSaturatedBlooms.size()); for (Entry<FieldInfo, FuzzySet> entry : nonSaturatedBlooms) { FieldInfo fieldInfo = entry.getKey(); FuzzySet bloomFilter = entry.getValue(); bloomOutput.writeInt(fieldInfo.number); saveAppropriatelySizedBloomFilter(bloomOutput, bloomFilter, fieldInfo); } } finally { IOUtils.close(bloomOutput); } // We are done with large bitsets so no need to keep them hanging around bloomFilters.clear(); }
@Override public void writeString(String s) throws IOException { delegate.writeString(s); }
private void write(Directory directory) throws IOException { long nextGeneration = getNextPendingGeneration(); String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS, "", nextGeneration); // Always advance the generation on write: generation = nextGeneration; IndexOutput segnOutput = null; boolean success = false; try { segnOutput = directory.createOutput(segmentFileName, IOContext.DEFAULT); CodecUtil.writeIndexHeader( segnOutput, "segments", VERSION_CURRENT, StringHelper.randomId(), Long.toString(nextGeneration, Character.MAX_RADIX)); segnOutput.writeVInt(Version.LATEST.major); segnOutput.writeVInt(Version.LATEST.minor); segnOutput.writeVInt(Version.LATEST.bugfix); segnOutput.writeLong(version); segnOutput.writeInt(counter); // write counter segnOutput.writeInt(size()); if (size() > 0) { Version minSegmentVersion = null; // We do a separate loop up front so we can write the minSegmentVersion before // any SegmentInfo; this makes it cleaner to throw IndexFormatTooOldExc at read time: for (SegmentCommitInfo siPerCommit : this) { Version segmentVersion = siPerCommit.info.getVersion(); if (minSegmentVersion == null || segmentVersion.onOrAfter(minSegmentVersion) == false) { minSegmentVersion = segmentVersion; } } segnOutput.writeVInt(minSegmentVersion.major); segnOutput.writeVInt(minSegmentVersion.minor); segnOutput.writeVInt(minSegmentVersion.bugfix); } // write infos for (SegmentCommitInfo siPerCommit : this) { SegmentInfo si = siPerCommit.info; segnOutput.writeString(si.name); byte segmentID[] = si.getId(); // TODO: remove this in lucene 6, we don't need to include 4.x segments in commits anymore if (segmentID == null) { segnOutput.writeByte((byte) 0); } else { if (segmentID.length != StringHelper.ID_LENGTH) { throw new IllegalStateException( "cannot write segment: invalid id segment=" + si.name + "id=" + StringHelper.idToString(segmentID)); } segnOutput.writeByte((byte) 1); segnOutput.writeBytes(segmentID, segmentID.length); } segnOutput.writeString(si.getCodec().getName()); segnOutput.writeLong(siPerCommit.getDelGen()); int delCount = siPerCommit.getDelCount(); if (delCount < 0 || delCount > si.maxDoc()) { throw new IllegalStateException( "cannot write segment: invalid maxDoc segment=" + si.name + " maxDoc=" + si.maxDoc() + " delCount=" + delCount); } segnOutput.writeInt(delCount); segnOutput.writeLong(siPerCommit.getFieldInfosGen()); segnOutput.writeLong(siPerCommit.getDocValuesGen()); segnOutput.writeSetOfStrings(siPerCommit.getFieldInfosFiles()); final Map<Integer, Set<String>> dvUpdatesFiles = siPerCommit.getDocValuesUpdatesFiles(); segnOutput.writeInt(dvUpdatesFiles.size()); for (Entry<Integer, Set<String>> e : dvUpdatesFiles.entrySet()) { segnOutput.writeInt(e.getKey()); segnOutput.writeSetOfStrings(e.getValue()); } } segnOutput.writeMapOfStrings(userData); CodecUtil.writeFooter(segnOutput); segnOutput.close(); directory.sync(Collections.singleton(segmentFileName)); success = true; } finally { if (success) { pendingCommit = true; } else { // We hit an exception above; try to close the file // but suppress any exception: IOUtils.closeWhileHandlingException(segnOutput); // Try not to leave a truncated segments_N file in // the index: IOUtils.deleteFilesIgnoringExceptions(directory, segmentFileName); } } }