@Override public void write(Directory dir, SegmentInfo si, IOContext context) throws IOException { String dataFile = IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION); int numFiles = si.files().size(); String names[] = si.files().toArray(new String[numFiles]); Arrays.sort(names); long startOffsets[] = new long[numFiles]; long endOffsets[] = new long[numFiles]; BytesRefBuilder scratch = new BytesRefBuilder(); try (IndexOutput out = dir.createOutput(dataFile, context)) { for (int i = 0; i < names.length; i++) { // write header for file SimpleTextUtil.write(out, HEADER); SimpleTextUtil.write(out, names[i], scratch); SimpleTextUtil.writeNewline(out); // write bytes for file startOffsets[i] = out.getFilePointer(); try (IndexInput in = dir.openInput(names[i], IOContext.READONCE)) { out.copyBytes(in, in.length()); } endOffsets[i] = out.getFilePointer(); } long tocPos = out.getFilePointer(); // write CFS table SimpleTextUtil.write(out, TABLE); SimpleTextUtil.write(out, Integer.toString(numFiles), scratch); SimpleTextUtil.writeNewline(out); for (int i = 0; i < names.length; i++) { SimpleTextUtil.write(out, TABLENAME); SimpleTextUtil.write(out, names[i], scratch); SimpleTextUtil.writeNewline(out); SimpleTextUtil.write(out, TABLESTART); SimpleTextUtil.write(out, Long.toString(startOffsets[i]), scratch); SimpleTextUtil.writeNewline(out); SimpleTextUtil.write(out, TABLEEND); SimpleTextUtil.write(out, Long.toString(endOffsets[i]), scratch); SimpleTextUtil.writeNewline(out); } DecimalFormat df = new DecimalFormat(OFFSETPATTERN, DecimalFormatSymbols.getInstance(Locale.ROOT)); SimpleTextUtil.write(out, TABLEPOS); SimpleTextUtil.write(out, df.format(tocPos), scratch); SimpleTextUtil.writeNewline(out); } }
private void checkFieldSummary( SegmentInfo info, IndexInput in, TermsReader field, TermsReader previous) throws IOException { // #docs with field must be <= #docs if (field.docCount < 0 || field.docCount > info.getDocCount()) { throw new CorruptIndexException( "invalid docCount: " + field.docCount + " maxDoc: " + info.getDocCount() + " (resource=" + in + ")"); } // #postings must be >= #docs with field if (field.sumDocFreq < field.docCount) { throw new CorruptIndexException( "invalid sumDocFreq: " + field.sumDocFreq + " docCount: " + field.docCount + " (resource=" + in + ")"); } // #positions must be >= #postings if (field.sumTotalTermFreq != -1 && field.sumTotalTermFreq < field.sumDocFreq) { throw new CorruptIndexException( "invalid sumTotalTermFreq: " + field.sumTotalTermFreq + " sumDocFreq: " + field.sumDocFreq + " (resource=" + in + ")"); } if (previous != null) { throw new CorruptIndexException( "duplicate fields: " + field.fieldInfo.name + " (resource=" + in + ")"); } }
public SimpleTextTermVectorsReader(Directory directory, SegmentInfo si, IOContext context) throws IOException { boolean success = false; try { in = directory.openInput( IndexFileNames.segmentFileName(si.name, "", VECTORS_EXTENSION), context); success = true; } finally { if (!success) { try { close(); } catch (Throwable t) { } // ensure we throw our original exception } } readIndex(si.maxDoc()); }
public SimpleTextStoredFieldsReader( Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { this.fieldInfos = fn; boolean success = false; try { in = directory.openInput( IndexFileNames.segmentFileName( si.name, "", SimpleTextStoredFieldsWriter.FIELDS_EXTENSION), context); success = true; } finally { if (!success) { try { close(); } catch (Throwable t) { } // ensure we throw our original exception } } readIndex(si.maxDoc()); }
@Override public synchronized IndexReader reopen() throws CorruptIndexException, IOException { IndexReader newInner = null; SegmentInfos sinfos = new SegmentInfos(); sinfos.read(_dir); int size = sinfos.size(); if (in instanceof MultiReader) { // setup current reader list List<IndexReader> boboReaderList = new LinkedList<IndexReader>(); ReaderUtil.gatherSubReaders((List<IndexReader>) boboReaderList, in); Map<String, BoboIndexReader> readerMap = new HashMap<String, BoboIndexReader>(); for (IndexReader reader : boboReaderList) { BoboIndexReader boboReader = (BoboIndexReader) reader; SegmentReader sreader = (SegmentReader) (boboReader.in); readerMap.put(sreader.getSegmentName(), boboReader); } ArrayList<BoboIndexReader> currentReaders = new ArrayList<BoboIndexReader>(size); boolean isNewReader = false; for (int i = 0; i < size; ++i) { SegmentInfo sinfo = (SegmentInfo) sinfos.info(i); BoboIndexReader breader = readerMap.remove(sinfo.name); if (breader != null) { // should use SegmentReader.reopen // TODO: see LUCENE-2559 BoboIndexReader newReader = (BoboIndexReader) breader.reopen(true); if (newReader != breader) { isNewReader = true; } if (newReader != null) { currentReaders.add(newReader); } } else { isNewReader = true; SegmentReader newSreader = SegmentReader.get(true, sinfo, 1); breader = BoboIndexReader.getInstanceAsSubReader( newSreader, this._facetHandlers, this._runtimeFacetHandlerFactories); breader._dir = _dir; currentReaders.add(breader); } } isNewReader = isNewReader || (readerMap.size() != 0); if (!isNewReader) { return this; } else { MultiReader newMreader = new MultiReader( currentReaders.toArray(new BoboIndexReader[currentReaders.size()]), false); BoboIndexReader newReader = BoboIndexReader.getInstanceAsSubReader( newMreader, this._facetHandlers, this._runtimeFacetHandlerFactories); newReader._dir = _dir; return newReader; } } else if (in instanceof SegmentReader) { // should use SegmentReader.reopen // TODO: see LUCENE-2559 SegmentReader sreader = (SegmentReader) in; int numDels = sreader.numDeletedDocs(); SegmentInfo sinfo = null; boolean sameSeg = false; // get SegmentInfo instance for (int i = 0; i < size; ++i) { SegmentInfo sinfoTmp = (SegmentInfo) sinfos.info(i); if (sinfoTmp.name.equals(sreader.getSegmentName())) { int numDels2 = sinfoTmp.getDelCount(); sameSeg = numDels == numDels2; sinfo = sinfoTmp; break; } } if (sinfo == null) { // segment no longer exists return null; } if (sameSeg) { return this; } else { SegmentReader newSreader = SegmentReader.get(true, sinfo, 1); return BoboIndexReader.getInstanceAsSubReader( newSreader, this._facetHandlers, this._runtimeFacetHandlerFactories); } } else { // should not reach here, a catch-all default case IndexReader reader = in.reopen(true); if (in != reader) { return BoboIndexReader.getInstance( newInner, _facetHandlers, _runtimeFacetHandlerFactories, _workArea); } else { return this; } } }