public BloomFilteredFieldsProducer(SegmentReadState state) throws IOException { String bloomFileName = IndexFileNames.segmentFileName( state.segmentInfo.name, state.segmentSuffix, BLOOM_EXTENSION); IndexInput bloomIn = null; boolean success = false; try { bloomIn = state.directory.openInput(bloomFileName, state.context); CodecUtil.checkHeader(bloomIn, BLOOM_CODEC_NAME, BLOOM_CODEC_VERSION, BLOOM_CODEC_VERSION); // // Load the hash function used in the BloomFilter // hashFunction = HashFunction.forName(bloomIn.readString()); // Load the delegate postings format PostingsFormat delegatePostingsFormat = PostingsFormat.forName(bloomIn.readString()); this.delegateFieldsProducer = delegatePostingsFormat.fieldsProducer(state); int numBlooms = bloomIn.readInt(); for (int i = 0; i < numBlooms; i++) { int fieldNum = bloomIn.readInt(); FuzzySet bloom = FuzzySet.deserialize(bloomIn); FieldInfo fieldInfo = state.fieldInfos.fieldInfo(fieldNum); bloomsByFieldName.put(fieldInfo.name, bloom); } IOUtils.close(bloomIn); success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(bloomIn, delegateFieldsProducer); } } }
private void verifyData(CoherenceDirectory dir, String fileName) throws IOException { byte[] test = new byte[] {1, 2, 3, 4, 5, 6, 7, 8}; assertTrue(dir.fileExists(fileName)); assertEquals(38, dir.fileLength(fileName)); IndexInput indexInput = dir.openInput(fileName); indexInput.readBytes(test, 0, 5); assertEquals(8, test[0]); assertEquals(-1, indexInput.readInt()); assertEquals(10, indexInput.readLong()); assertEquals(0, indexInput.readInt()); assertEquals(0, indexInput.readInt()); indexInput.readBytes(test, 0, 8); assertEquals((byte) 1, test[0]); assertEquals((byte) 8, test[7]); indexInput.readBytes(test, 0, 5); assertEquals((byte) 1, test[0]); assertEquals((byte) 5, test[4]); indexInput.seek(28); assertEquals((byte) 4, indexInput.readByte()); indexInput.seek(30); assertEquals((byte) 6, indexInput.readByte()); indexInput.close(); }
private void readFields(IndexInput meta, FieldInfos infos) throws IOException { for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) { FieldInfo info = infos.fieldInfo(fieldNumber); if (info == null) { throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta); } else if (!info.hasNorms()) { throw new CorruptIndexException("Invalid field: " + info.name, meta); } NormsEntry entry = new NormsEntry(); entry.docsWithFieldOffset = meta.readLong(); entry.numDocsWithField = meta.readInt(); entry.bytesPerNorm = meta.readByte(); switch (entry.bytesPerNorm) { case 0: case 1: case 2: case 4: case 8: break; default: throw new CorruptIndexException( "Invalid bytesPerValue: " + entry.bytesPerNorm + ", field: " + info.name, meta); } entry.normsOffset = meta.readLong(); norms.put(info.number, entry); } }
@Override public void init(IndexInput termsIn) throws IOException { // Make sure we are talking to the matching past writer CodecUtil.checkHeader( termsIn, SepPostingsWriter.CODEC, SepPostingsWriter.VERSION_START, SepPostingsWriter.VERSION_START); skipInterval = termsIn.readInt(); maxSkipLevels = termsIn.readInt(); skipMinimum = termsIn.readInt(); }
/** * Creates an array with content retrieved from the given IndexInput. * * @param in an IndexInput, positioned at the start of Packed64-content. * @param valueCount the number of elements. * @param bitsPerValue the number of bits available for any given value. * @throws java.io.IOException if the values for the backing array could not be retrieved. */ public Packed32(IndexInput in, int valueCount, int bitsPerValue) throws IOException { super(valueCount, bitsPerValue); int size = size(bitsPerValue, valueCount); blocks = new int[size + 1]; // +1 due to non-conditional tricks // TODO: find a faster way to bulk-read ints... for (int i = 0; i < size; i++) { blocks[i] = in.readInt(); } if (size % 2 == 1) { in.readInt(); // Align to long } updateCached(); }
static Map<String, String> readChecksums(Directory[] dirs, Map<String, String> defaultValue) throws IOException { long lastFound = -1; Directory lastDir = null; for (Directory dir : dirs) { for (String name : dir.listAll()) { if (!isChecksum(name)) { continue; } long current = Long.parseLong(name.substring(CHECKSUMS_PREFIX.length())); if (current > lastFound) { lastFound = current; lastDir = dir; } } } if (lastFound == -1) { return defaultValue; } IndexInput indexInput = lastDir.openInput(CHECKSUMS_PREFIX + lastFound, IOContext.READONCE); try { indexInput.readInt(); // version return indexInput.readStringStringMap(); } catch (Exception e) { // failed to load checksums, ignore and return an empty map return defaultValue; } finally { indexInput.close(); } }
public static void IndexInputTest() throws Exception { String path = "D:\\Lucene Document"; directory = FSDirectory.getDirectory(path); IndexInput indexInput = directory.openInput("segments.gen"); int version = indexInput.readInt(); System.out.println(version); System.out.println(indexInput.readString()); }
private void verifyData(ByteBufferDirectory dir) throws IOException { byte[] test = new byte[] {1, 2, 3, 4, 5, 6, 7, 8}; assertThat(dir.fileExists("value1"), equalTo(true)); assertThat(dir.fileLength("value1"), equalTo(38l)); IndexInput indexInput = dir.openInput("value1", IOContext.DEFAULT); indexInput.readBytes(test, 0, 5); assertThat(test[0], equalTo((byte) 8)); assertThat(indexInput.readInt(), equalTo(-1)); assertThat(indexInput.readLong(), equalTo((long) 10)); assertThat(indexInput.readInt(), equalTo(0)); assertThat(indexInput.readInt(), equalTo(0)); indexInput.readBytes(test, 0, 8); assertThat(test[0], equalTo((byte) 1)); assertThat(test[7], equalTo((byte) 8)); indexInput.readBytes(test, 0, 5); assertThat(test[0], equalTo((byte) 1)); assertThat(test[4], equalTo((byte) 5)); indexInput.seek(28); assertThat(indexInput.readByte(), equalTo((byte) 4)); indexInput.seek(30); assertThat(indexInput.readByte(), equalTo((byte) 6)); indexInput.seek(0); indexInput.readBytes(test, 0, 5); assertThat(test[0], equalTo((byte) 8)); indexInput.close(); indexInput = dir.openInput("value1", IOContext.DEFAULT); // iterate over all the data for (int i = 0; i < 38; i++) { indexInput.readByte(); } indexInput.close(); }
@Override public int readInt() throws IOException { return delegate.readInt(); }
public Object run(IndexCommit commit) throws CorruptIndexException, IOException { if (commit != null) { if (directory != commit.getDirectory()) throw new IOException("the specified commit does not match the specified Directory"); return doBody(commit.getSegmentsFileName()); } String segmentFileName = null; long lastGen = -1; long gen = 0; int genLookaheadCount = 0; IOException exc = null; int retryCount = 0; boolean useFirstMethod = true; // Loop until we succeed in calling doBody() without // hitting an IOException. An IOException most likely // means a commit was in process and has finished, in // the time it took us to load the now-old infos files // (and segments files). It's also possible it's a // true error (corrupt index). To distinguish these, // on each retry we must see "forward progress" on // which generation we are trying to load. If we // don't, then the original error is real and we throw // it. // We have three methods for determining the current // generation. We try the first two in parallel (when // useFirstMethod is true), and fall back to the third // when necessary. while (true) { if (useFirstMethod) { // List the directory and use the highest // segments_N file. This method works well as long // as there is no stale caching on the directory // contents (NOTE: NFS clients often have such stale // caching): String[] files = null; long genA = -1; files = directory.listAll(); if (files != null) { genA = getCurrentSegmentGeneration(files); } if (infoStream != null) { message("directory listing genA=" + genA); } // Also open segments.gen and read its // contents. Then we take the larger of the two // gens. This way, if either approach is hitting // a stale cache (NFS) we have a better chance of // getting the right generation. long genB = -1; for (int i = 0; i < defaultGenFileRetryCount; i++) { IndexInput genInput = null; try { genInput = directory.openInput(IndexFileNames.SEGMENTS_GEN); } catch (FileNotFoundException e) { if (infoStream != null) { message("segments.gen open: FileNotFoundException " + e); } break; } catch (IOException e) { if (infoStream != null) { message("segments.gen open: IOException " + e); } } if (genInput != null) { try { int version = genInput.readInt(); if (version == FORMAT_LOCKLESS) { long gen0 = genInput.readLong(); long gen1 = genInput.readLong(); if (infoStream != null) { message("fallback check: " + gen0 + "; " + gen1); } if (gen0 == gen1) { // The file is consistent. genB = gen0; break; } } } catch (IOException err2) { // will retry } finally { genInput.close(); } } try { Thread.sleep(defaultGenFileRetryPauseMsec); } catch (InterruptedException ie) { throw new ThreadInterruptedException(ie); } } if (infoStream != null) { message(IndexFileNames.SEGMENTS_GEN + " check: genB=" + genB); } // Pick the larger of the two gen's: if (genA > genB) gen = genA; else gen = genB; if (gen == -1) { // Neither approach found a generation throw new IndexNotFoundException( "no segments* file found in " + directory + ": files: " + Arrays.toString(files)); } } if (useFirstMethod && lastGen == gen && retryCount >= 2) { // Give up on first method -- this is 3rd cycle on // listing directory and checking gen file to // attempt to locate the segments file. useFirstMethod = false; } // Second method: since both directory cache and // file contents cache seem to be stale, just // advance the generation. if (!useFirstMethod) { if (genLookaheadCount < defaultGenLookaheadCount) { gen++; genLookaheadCount++; if (infoStream != null) { message("look ahead increment gen to " + gen); } } else { // All attempts have failed -- throw first exc: throw exc; } } else if (lastGen == gen) { // This means we're about to try the same // segments_N last tried. retryCount++; } else { // Segment file has advanced since our last loop // (we made "progress"), so reset retryCount: retryCount = 0; } lastGen = gen; segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen); try { Object v = doBody(segmentFileName); if (infoStream != null) { message("success on " + segmentFileName); } return v; } catch (IOException err) { // Save the original root cause: if (exc == null) { exc = err; } if (infoStream != null) { message( "primary Exception on '" + segmentFileName + "': " + err + "'; will retry: retryCount=" + retryCount + "; gen = " + gen); } if (gen > 1 && useFirstMethod && retryCount == 1) { // This is our second time trying this same segments // file (because retryCount is 1), and, there is // possibly a segments_(N-1) (because gen > 1). // So, check if the segments_(N-1) exists and // try it if so: String prevSegmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen - 1); final boolean prevExists; prevExists = directory.fileExists(prevSegmentFileName); if (prevExists) { if (infoStream != null) { message("fallback to prior segment file '" + prevSegmentFileName + "'"); } try { Object v = doBody(prevSegmentFileName); if (infoStream != null) { message("success on fallback " + prevSegmentFileName); } return v; } catch (IOException err2) { if (infoStream != null) { message( "secondary Exception on '" + prevSegmentFileName + "': " + err2 + "'; will retry"); } } } } } } }