/** Close all streams. */ final void close() throws IOException { try { closeDocument(); } finally { // make an effort to close all streams we can but remember and re-throw // the first exception encountered in this process IOException keep = null; if (tvx != null) try { tvx.close(); } catch (IOException e) { if (keep == null) keep = e; } if (tvd != null) try { tvd.close(); } catch (IOException e) { if (keep == null) keep = e; } if (tvf != null) try { tvf.close(); } catch (IOException e) { if (keep == null) keep = e; } if (keep != null) throw (IOException) keep.fillInStackTrace(); } }
public void writeChecksums() throws IOException { String checksumName = CHECKSUMS_PREFIX + System.currentTimeMillis(); ImmutableMap<String, StoreFileMetaData> files = list(); synchronized (mutex) { Map<String, String> checksums = new HashMap<String, String>(); for (StoreFileMetaData metaData : files.values()) { if (metaData.checksum() != null) { checksums.put(metaData.name(), metaData.checksum()); } } IndexOutput output = directory.createOutput(checksumName, IOContext.DEFAULT, true); output.writeInt(0); // version output.writeStringStringMap(checksums); output.close(); } for (StoreFileMetaData metaData : files.values()) { if (metaData.name().startsWith(CHECKSUMS_PREFIX) && !checksumName.equals(metaData.name())) { try { directory.deleteFileChecksum(metaData.name()); } catch (Exception e) { // ignore } } } }
// Make sure we don't somehow use more than 1 descriptor // when reading a CFS with many subs: public void testManySubFiles() throws IOException { final Directory d = newFSDirectory(_TestUtil.getTempDir("CFSManySubFiles")); final int FILE_COUNT = 10000; for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) { IndexOutput out = d.createOutput("file." + fileIdx); out.writeByte((byte) fileIdx); out.close(); } final CompoundFileWriter cfw = new CompoundFileWriter(d, "c.cfs"); for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) { cfw.addFile("file." + fileIdx); } cfw.close(); final IndexInput[] ins = new IndexInput[FILE_COUNT]; final CompoundFileReader cfr = new CompoundFileReader(d, "c.cfs"); for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) { ins[fileIdx] = cfr.openInput("file." + fileIdx); } for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) { assertEquals((byte) fileIdx, ins[fileIdx].readByte()); } for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) { ins[fileIdx].close(); } cfr.close(); d.close(); }
private void testOn(Directory dir, int writeSize, int readSize, Cache cache) throws IOException { if (cache != null) cache .clear(); // needed to make sure no chunks are left over in case of Infinispan // implementation final String filename = "chunkTest"; IndexOutput indexOutput = dir.createOutput(filename); byte[] toWrite = fillBytes(writeSize); indexOutput.writeBytes(toWrite, writeSize); indexOutput.close(); if (cache != null) { AssertJUnit.assertEquals( writeSize, DirectoryIntegrityCheck.deepCountFileSize(new FileCacheKey(INDEXNAME, filename), cache)); } AssertJUnit.assertEquals(writeSize, indexOutput.length()); byte[] results = new byte[readSize]; IndexInput openInput = dir.openInput(filename); try { openInput.readBytes(results, 0, readSize); for (int i = 0; i < writeSize && i < readSize; i++) { AssertJUnit.assertEquals(results[i], toWrite[i]); } if (readSize > writeSize) AssertJUnit.fail("should have thrown an IOException for reading past EOF"); } catch (IOException ioe) { if (readSize <= writeSize) AssertJUnit.fail("should not have thrown an IOException" + ioe.getMessage()); } }
public void multipleFlushTest() throws IOException { final String filename = "longFile.writtenInMultipleFlushes"; final int bufferSize = 300; Cache cache = cacheManager.getCache(); cache.clear(); Directory dir = DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME) .chunkSize(13) .create(); byte[] manyBytes = fillBytes(bufferSize); IndexOutput indexOutput = dir.createOutput(filename); for (int i = 0; i < 10; i++) { indexOutput.writeBytes(manyBytes, bufferSize); indexOutput.flush(); } indexOutput.close(); IndexInput input = dir.openInput(filename); final int finalSize = (10 * bufferSize); AssertJUnit.assertEquals(finalSize, input.length()); final byte[] resultingBuffer = new byte[finalSize]; input.readBytes(resultingBuffer, 0, finalSize); int index = 0; for (int i = 0; i < 10; i++) { for (int j = 0; j < bufferSize; j++) AssertJUnit.assertEquals(resultingBuffer[index++], manyBytes[j]); } }
private void demo_FSIndexInputBug(Directory fsdir, String file) throws IOException { // Setup the test file - we need more than 1024 bytes IndexOutput os = fsdir.createOutput(file); for (int i = 0; i < 2000; i++) { os.writeByte((byte) i); } os.close(); IndexInput in = fsdir.openInput(file); // This read primes the buffer in IndexInput in.readByte(); // Close the file in.close(); // ERROR: this call should fail, but succeeds because the buffer // is still filled in.readByte(); // ERROR: this call should fail, but succeeds for some reason as well in.seek(1099); try { // OK: this call correctly fails. We are now past the 1024 internal // buffer, so an actual IO is attempted, which fails in.readByte(); fail("expected readByte() to throw exception"); } catch (IOException e) { // expected exception } }
public void close() throws IOException { try { final long dirStart = out.getFilePointer(); final int fieldCount = fields.size(); int nonNullFieldCount = 0; for (int i = 0; i < fieldCount; i++) { FSTFieldWriter field = fields.get(i); if (field.fst != null) { nonNullFieldCount++; } } out.writeVInt(nonNullFieldCount); for (int i = 0; i < fieldCount; i++) { FSTFieldWriter field = fields.get(i); if (field.fst != null) { out.writeVInt(field.fieldInfo.number); out.writeVLong(field.indexStart); } } writeTrailer(dirStart); } finally { out.close(); } }
@Test @Override public void testIndexWritingAndFinding() throws IOException, InterruptedException { verifyBoth(cache0, cache1); IndexOutput indexOutput = dirA.createOutput(filename, IOContext.DEFAULT); indexOutput.writeString("no need to write, nobody ever will read this"); indexOutput.flush(); indexOutput.close(); assertFileExistsHavingRLCount(filename, 1, true); IndexInput firstOpenOnB = dirB.openInput(filename, IOContext.DEFAULT); assertFileExistsHavingRLCount(filename, 2, true); dirA.deleteFile(filename); assertFileExistsHavingRLCount(filename, 1, false); // Lucene does use clone() - lock implementation ignores it as a clone is // cast on locked segments and released before the close on the parent object IndexInput cloneOfFirstOpenOnB = (IndexInput) firstOpenOnB.clone(); assertFileExistsHavingRLCount(filename, 1, false); cloneOfFirstOpenOnB.close(); assertFileExistsHavingRLCount(filename, 1, false); IndexInput firstOpenOnA = dirA.openInput(filename, IOContext.DEFAULT); assertFileExistsHavingRLCount(filename, 2, false); IndexInput secondOpenOnA = dirA.openInput(filename, IOContext.DEFAULT); assertFileExistsHavingRLCount(filename, 2, false); firstOpenOnA.close(); assertFileExistsHavingRLCount(filename, 2, false); secondOpenOnA.close(); assertFileExistsHavingRLCount(filename, 1, false); firstOpenOnB.close(); assertFileNotExists(filename); dirA.close(); dirB.close(); verifyBoth(cache0, cache1); }
public void testWriteChunks() throws Exception { final int BUFFER_SIZE = 64; Cache cache = cacheManager.getCache(); Directory dir = DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME) .chunkSize(BUFFER_SIZE) .create(); IndexOutput io = dir.createOutput("MyNewFile.txt"); io.writeByte((byte) 66); io.writeByte((byte) 69); io.flush(); io.close(); assert dir.fileExists("MyNewFile.txt"); assert null != cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 0, BUFFER_SIZE)); // test contents by reading: byte[] buf = new byte[9]; IndexInput ii = dir.openInput("MyNewFile.txt"); ii.readBytes(buf, 0, (int) ii.length()); ii.close(); assert new String(new byte[] {66, 69}).equals(new String(buf).trim()); String testText = "This is some rubbish again that will span more than one chunk - one hopes. Who knows, maybe even three or four chunks."; io = dir.createOutput("MyNewFile.txt"); io.seek(0); io.writeBytes(testText.getBytes(), 0, testText.length()); io.close(); // now compare. byte[] chunk1 = (byte[]) cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 0, BUFFER_SIZE)); byte[] chunk2 = (byte[]) cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 1, BUFFER_SIZE)); assert null != chunk1; assert null != chunk2; assert testText.equals(new String(chunk1) + new String(chunk2).trim()); dir.close(); DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME); }
/** Creates a file of the specified size with random data. */ private void createRandomFile(Directory dir, String name, int size) throws IOException { IndexOutput os = dir.createOutput(name); for (int i = 0; i < size; i++) { byte b = (byte) (Math.random() * 256); os.writeByte(b); } os.close(); }
/** * It creates a file with fixed size using a RepeatableLongByteSequence object to generate a * repeatable content * * @param dir The Directory containing the file to create * @param fileName The file name to create * @param contentFileSize The size content file to create * @throws IOException */ private void createFileWithRepeatableContent( Directory dir, String fileName, final int contentFileSize) throws IOException { IndexOutput indexOutput = dir.createOutput(fileName); RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence(); for (int i = 0; i < contentFileSize; i++) { indexOutput.writeByte(bytesGenerator.nextByte()); } indexOutput.close(); }
/** * Creates a file of the specified size with sequential data. The first byte is written as the * start byte provided. All subsequent bytes are computed as start + offset where offset is the * number of the byte. */ private void createSequenceFile(Directory dir, String name, byte start, int size) throws IOException { IndexOutput os = dir.createOutput(name); for (int i = 0; i < size; i++) { os.writeByte(start); start++; } os.close(); }
public void testEncodeDecode() throws IOException { final int iterations = RandomInts.randomIntBetween(random(), 1, 1000); final float acceptableOverheadRatio = random().nextFloat(); final int[] values = new int[(iterations - 1) * BLOCK_SIZE + ForUtil.MAX_DATA_SIZE]; for (int i = 0; i < iterations; ++i) { final int bpv = random().nextInt(32); if (bpv == 0) { final int value = RandomInts.randomIntBetween(random(), 0, Integer.MAX_VALUE); for (int j = 0; j < BLOCK_SIZE; ++j) { values[i * BLOCK_SIZE + j] = value; } } else { for (int j = 0; j < BLOCK_SIZE; ++j) { values[i * BLOCK_SIZE + j] = RandomInts.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv)); } } } final Directory d = new RAMDirectory(); final long endPointer; { // encode IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT); final ForUtil forUtil = new ForUtil(acceptableOverheadRatio, out); for (int i = 0; i < iterations; ++i) { forUtil.writeBlock( Arrays.copyOfRange(values, i * BLOCK_SIZE, values.length), new byte[MAX_ENCODED_SIZE], out); } endPointer = out.getFilePointer(); out.close(); } { // decode IndexInput in = d.openInput("test.bin", IOContext.READONCE); final ForUtil forUtil = new ForUtil(in); for (int i = 0; i < iterations; ++i) { if (random().nextBoolean()) { forUtil.skipBlock(in); continue; } final int[] restored = new int[MAX_DATA_SIZE]; forUtil.readBlock(in, new byte[MAX_ENCODED_SIZE], restored); assertArrayEquals( Arrays.copyOfRange(values, i * BLOCK_SIZE, (i + 1) * BLOCK_SIZE), Arrays.copyOf(restored, BLOCK_SIZE)); } assertEquals(endPointer, in.getFilePointer()); in.close(); } }
final void finishCommit(Directory dir) throws IOException { if (pendingSegnOutput == null) throw new IllegalStateException("prepareCommit was not called"); boolean success = false; try { pendingSegnOutput.finishCommit(); pendingSegnOutput.close(); pendingSegnOutput = null; success = true; } finally { if (!success) rollbackCommit(dir); } // NOTE: if we crash here, we have left a segments_N // file in the directory in a possibly corrupt state (if // some bytes made it to stable storage and others // didn't). But, the segments_N file includes checksum // at the end, which should catch this case. So when a // reader tries to read it, it will throw a // CorruptIndexException, which should cause the retry // logic in SegmentInfos to kick in and load the last // good (previous) segments_N-1 file. final String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation); success = false; try { dir.sync(Collections.singleton(fileName)); success = true; } finally { if (!success) { try { dir.deleteFile(fileName); } catch (Throwable t) { // Suppress so we keep throwing the original exception } } } lastGeneration = generation; try { IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN); try { genOutput.writeInt(FORMAT_LOCKLESS); genOutput.writeLong(generation); genOutput.writeLong(generation); } finally { genOutput.close(); } } catch (ThreadInterruptedException t) { throw t; } catch (Throwable t) { // It's OK if we fail to write this file since it's // used only as one of the retry fallbacks. } }
@Override public void close() { try { output.flush(); output.close(); super.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } }
@Override public void messageReceived(final RecoveryFileChunkRequest request, TransportChannel channel) throws Exception { try (RecoveriesCollection.StatusRef statusRef = onGoingRecoveries.getStatusSafe(request.recoveryId(), request.shardId())) { final RecoveryStatus recoveryStatus = statusRef.status(); final Store store = recoveryStatus.store(); recoveryStatus.state().getTranslog().totalOperations(request.totalTranslogOps()); final RecoveryState.Index indexState = recoveryStatus.state().getIndex(); if (request.sourceThrottleTimeInNanos() != RecoveryState.Index.UNKNOWN) { indexState.addSourceThrottling(request.sourceThrottleTimeInNanos()); } IndexOutput indexOutput; if (request.position() == 0) { indexOutput = recoveryStatus.openAndPutIndexOutput(request.name(), request.metadata(), store); } else { indexOutput = recoveryStatus.getOpenIndexOutput(request.name()); } BytesReference content = request.content(); if (!content.hasArray()) { content = content.toBytesArray(); } RateLimiter rl = recoverySettings.rateLimiter(); if (rl != null) { long bytes = bytesSinceLastPause.addAndGet(content.length()); if (bytes > rl.getMinPauseCheckBytes()) { // Time to pause bytesSinceLastPause.addAndGet(-bytes); long throttleTimeInNanos = rl.pause(bytes); indexState.addTargetThrottling(throttleTimeInNanos); recoveryStatus.indexShard().recoveryStats().addThrottleTime(throttleTimeInNanos); } } indexOutput.writeBytes(content.array(), content.arrayOffset(), content.length()); indexState.addRecoveredBytesToFile(request.name(), content.length()); if (indexOutput.getFilePointer() >= request.length() || request.lastChunk()) { try { Store.verify(indexOutput); } finally { // we are done indexOutput.close(); } // write the checksum recoveryStatus.legacyChecksums().add(request.metadata()); final String temporaryFileName = recoveryStatus.getTempNameForFile(request.name()); assert Arrays.asList(store.directory().listAll()).contains(temporaryFileName); store.directory().sync(Collections.singleton(temporaryFileName)); IndexOutput remove = recoveryStatus.removeOpenIndexOutputs(request.name()); assert remove == null || remove == indexOutput; // remove maybe null if we got finished } } channel.sendResponse(TransportResponse.Empty.INSTANCE); }
@Test public void testNoDocs() throws IOException { AnalyzingCompletionLookupProvider provider = new AnalyzingCompletionLookupProvider(true, false, true, true); RAMDirectory dir = new RAMDirectory(); IndexOutput output = dir.createOutput("foo.txt", IOContext.DEFAULT); FieldsConsumer consumer = provider.consumer(output); consumer.write( new Fields() { @Override public Iterator<String> iterator() { return Arrays.asList("foo").iterator(); } @Override public Terms terms(String field) throws IOException { return null; } @Override public int size() { return 1; } }); consumer.close(); output.close(); IndexInput input = dir.openInput("foo.txt", IOContext.DEFAULT); LookupFactory load = provider.load(input); PostingsFormat format = new Elasticsearch090PostingsFormat(); NamedAnalyzer analyzer = new NamedAnalyzer("foo", new StandardAnalyzer()); assertNull( load.getLookup( new CompletionFieldMapper( new Names("foo"), analyzer, analyzer, format, null, true, true, true, Integer.MAX_VALUE, indexSettings, AbstractFieldMapper.MultiFields.empty(), null, ContextMapping.EMPTY_MAPPING), new CompletionSuggestionContext(null))); dir.close(); }
/** * This test that writes larger than the size of the buffer output will correctly increment the * file pointer. */ public void testLargeWrites() throws IOException { IndexOutput os = dir.createOutput("testBufferStart.txt"); byte[] largeBuf = new byte[2048]; for (int i = 0; i < largeBuf.length; i++) { largeBuf[i] = (byte) (Math.random() * 256); } long currentPos = os.getFilePointer(); os.writeBytes(largeBuf, largeBuf.length); try { assertEquals(currentPos + largeBuf.length, os.getFilePointer()); } finally { os.close(); } }
private void insertData(ByteBufferDirectory dir, int bufferSizeInBytes) throws IOException { byte[] test = new byte[] {1, 2, 3, 4, 5, 6, 7, 8}; IndexOutput indexOutput = dir.createOutput("value1", IOContext.DEFAULT); indexOutput.writeBytes(new byte[] {2, 4, 6, 7, 8}, 5); indexOutput.writeInt(-1); indexOutput.writeLong(10); indexOutput.writeInt(0); indexOutput.writeInt(0); indexOutput.writeBytes(test, 8); indexOutput.writeBytes(test, 5); indexOutput.seek(0); indexOutput.writeByte((byte) 8); if (bufferSizeInBytes > 4) { indexOutput.seek(2); indexOutput.writeBytes(new byte[] {1, 2}, 2); } indexOutput.close(); }
private void insertData(CoherenceDirectory dir, String fileName) throws IOException { byte[] test = new byte[] {1, 2, 3, 4, 5, 6, 7, 8}; IndexOutput indexOutput = dir.createOutput(fileName); indexOutput.writeBytes(new byte[] {2, 4, 6, 7, 8}, 5); indexOutput.writeInt(-1); indexOutput.writeLong(10); indexOutput.writeInt(0); indexOutput.writeInt(0); indexOutput.writeBytes(test, 8); indexOutput.writeBytes(test, 5); indexOutput.seek(0); indexOutput.writeByte((byte) 8); if (dir.getBucketSize() > 4) { indexOutput.seek(2); indexOutput.writeBytes(new byte[] {1, 2}, 2); } indexOutput.close(); }
@Test public void testVerifyingIndexOutput() throws IOException { Directory dir = newDirectory(); IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT); int iters = scaledRandomIntBetween(10, 100); for (int i = 0; i < iters; i++) { BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024)); output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length); } CodecUtil.writeFooter(output); output.close(); IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT); String checksum = Store.digestToString(CodecUtil.retrieveChecksum(indexInput)); indexInput.seek(0); BytesRef ref = new BytesRef(scaledRandomIntBetween(1, 1024)); long length = indexInput.length(); IndexOutput verifyingOutput = new Store.LuceneVerifyingIndexOutput( new StoreFileMetaData("foo1.bar", length, checksum), dir.createOutput("foo1.bar", IOContext.DEFAULT)); while (length > 0) { if (random().nextInt(10) == 0) { verifyingOutput.writeByte(indexInput.readByte()); length--; } else { int min = (int) Math.min(length, ref.bytes.length); indexInput.readBytes(ref.bytes, ref.offset, min); verifyingOutput.writeBytes(ref.bytes, ref.offset, min); length -= min; } } Store.verify(verifyingOutput); verifyingOutput.writeByte((byte) 0x0); try { Store.verify(verifyingOutput); fail("should be a corrupted index"); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) { // ok } IOUtils.close(indexInput, verifyingOutput, dir); }
@Test public void testWriteChunksDefaultChunks() throws Exception { Cache cache = cacheManager.getCache(); Directory dir = DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME).create(); final String testText = "This is some rubbish"; final byte[] testTextAsBytes = testText.getBytes(); IndexOutput io = dir.createOutput("MyNewFile.txt"); io.writeByte((byte) 1); io.writeByte((byte) 2); io.writeByte((byte) 3); io.writeBytes(testTextAsBytes, testTextAsBytes.length); io.close(); DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME); FileCacheKey fileCacheKey = new FileCacheKey(INDEXNAME, "MyNewFile.txt"); assert null != cache.get(fileCacheKey); FileMetadata metadata = (FileMetadata) cache.get(fileCacheKey); AssertJUnit.assertEquals(testTextAsBytes.length + 3, metadata.getSize()); assert null != cache.get( new ChunkCacheKey( INDEXNAME, "MyNewFile.txt", 0, DirectoryBuilderImpl.DEFAULT_BUFFER_SIZE)); // test contents by reading: IndexInput ii = dir.openInput("MyNewFile.txt"); assert ii.readByte() == 1; assert ii.readByte() == 2; assert ii.readByte() == 3; byte[] buf = new byte[testTextAsBytes.length]; ii.readBytes(buf, 0, testTextAsBytes.length); ii.close(); assert testText.equals(new String(buf).trim()); dir.close(); DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME); }
@Override public void close() throws IOException { out.close(); String checksum = null; IndexOutput underlying = out; // TODO: cut over to lucene's CRC // *WARNING*: lucene has classes in same o.a.l.store package with very similar names, // but using CRC, not Adler! if (underlying instanceof BufferedChecksumIndexOutput) { Checksum digest = ((BufferedChecksumIndexOutput) underlying).digest(); assert digest instanceof Adler32; checksum = Long.toString(digest.getValue(), Character.MAX_RADIX); } synchronized (mutex) { StoreFileMetaData md = new StoreFileMetaData( name, metaData.directory().fileLength(name), checksum, metaData.directory()); filesMetadata = ImmutableOpenMap.builder(filesMetadata).fPut(name, md).build(); files = filesMetadata.keys().toArray(String.class); } }
/** * closes temporary file, compresses data and removes temporary file. * * @throws IOException */ @Override public void close() throws IOException { byte[] buffer = new byte[chunkSize]; tempOut.close(); // directory with offsets offsets of compressed chunks with // real position in decompressed stream IndexInput in = tempDirectory.openInput(tmpName); long len = closeLength = in.length(); // write length of the file at the begining for easier retreval output.writeLong(-1); // write configuration writeConfig(); int toRead; // read all data and compresse it in variable block chunks while (len > 0) { if (len > buffer.length) { toRead = buffer.length; } else { toRead = (int) len; } // just for safety --- can be improoved long bufferPos = in.getFilePointer(); // read original data in.readBytes(buffer, 0, toRead); writeChunk(buffer, bufferPos, toRead); len -= toRead; } // now let's crate directory entry of all chunks and their's original // position in inflated stream in.close(); if (tempDirectory.fileExists(tmpName)) { tempDirectory.deleteFile(tmpName); } super.close(); }
public void multipleFlushTest() throws IOException { final String filename = "longFile.writtenInMultipleFlushes"; final int bufferSize = 300; Cache cache = cacheManager.getCache(); cache.clear(); InfinispanDirectory dir = new InfinispanDirectory(cache, cache, cache, INDEXNAME, 13); byte[] manyBytes = fillBytes(bufferSize); IndexOutput indexOutput = dir.createOutput(filename); for (int i = 0; i < 10; i++) { indexOutput.writeBytes(manyBytes, bufferSize); indexOutput.flush(); } indexOutput.close(); IndexInput input = dir.openInput(filename); final int finalSize = (10 * bufferSize); assert input.length() == finalSize; final byte[] resultingBuffer = new byte[finalSize]; input.readBytes(resultingBuffer, 0, finalSize); int index = 0; for (int i = 0; i < 10; i++) { for (int j = 0; j < bufferSize; j++) assert resultingBuffer[index++] == manyBytes[j]; } }
@Override public void close() throws IOException { out.close(); String checksum = null; IndexOutput underlying = out; if (underlying instanceof BufferedChecksumIndexOutput) { checksum = Long.toString( ((BufferedChecksumIndexOutput) underlying).digest().getValue(), Character.MAX_RADIX); } else if (underlying instanceof ChecksumIndexOutput) { checksum = Long.toString( ((ChecksumIndexOutput) underlying).digest().getValue(), Character.MAX_RADIX); } synchronized (mutex) { StoreFileMetaData md = new StoreFileMetaData( name, metaData.directory().fileLength(name), checksum, metaData.directory()); filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, md).immutableMap(); files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); } }
@Test public void testVerifyingIndexInput() throws IOException { Directory dir = newDirectory(); IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT); int iters = scaledRandomIntBetween(10, 100); for (int i = 0; i < iters; i++) { BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024)); output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length); } CodecUtil.writeFooter(output); output.close(); // Check file IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT); long checksum = CodecUtil.retrieveChecksum(indexInput); indexInput.seek(0); IndexInput verifyingIndexInput = new Store.VerifyingIndexInput(dir.openInput("foo.bar", IOContext.DEFAULT)); readIndexInputFullyWithRandomSeeks(verifyingIndexInput); Store.verify(verifyingIndexInput); assertThat(checksum, equalTo(((ChecksumIndexInput) verifyingIndexInput).getChecksum())); IOUtils.close(indexInput, verifyingIndexInput); // Corrupt file and check again corruptFile(dir, "foo.bar", "foo1.bar"); verifyingIndexInput = new Store.VerifyingIndexInput(dir.openInput("foo1.bar", IOContext.DEFAULT)); readIndexInputFullyWithRandomSeeks(verifyingIndexInput); try { Store.verify(verifyingIndexInput); fail("should be a corrupted index"); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) { // ok } IOUtils.close(verifyingIndexInput); IOUtils.close(dir); }
@Override public void close() throws IOException { delegate.close(); openOutputs.remove(IndexOutputDelegate.this); }
@Override public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { // we pull this before the seed intentionally: because its not consumed at runtime // (the skipInterval is written into postings header) int skipInterval = _TestUtil.nextInt(seedRandom, 2, 10); if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: skipInterval=" + skipInterval); } final long seed = seedRandom.nextLong(); if (LuceneTestCase.VERBOSE) { System.out.println( "MockRandomCodec: writing to seg=" + state.segmentName + " formatID=" + state.segmentSuffix + " seed=" + seed); } final String seedFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, SEED_EXT); final IndexOutput out = state.directory.createOutput(seedFileName, state.context); try { out.writeLong(seed); } finally { out.close(); } final Random random = new Random(seed); random.nextInt(); // consume a random for buffersize PostingsWriterBase postingsWriter; if (random.nextBoolean()) { postingsWriter = new SepPostingsWriter(state, new MockIntStreamFactory(random), skipInterval); } else { if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: writing Standard postings"); } postingsWriter = new Lucene40PostingsWriter(state, skipInterval); } if (random.nextBoolean()) { final int totTFCutoff = _TestUtil.nextInt(random, 1, 20); if (LuceneTestCase.VERBOSE) { System.out.println( "MockRandomCodec: writing pulsing postings with totTFCutoff=" + totTFCutoff); } postingsWriter = new PulsingPostingsWriter(totTFCutoff, postingsWriter); } final FieldsConsumer fields; if (random.nextBoolean()) { // Use BlockTree terms dict if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: writing BlockTree terms dict"); } // TODO: would be nice to allow 1 but this is very // slow to write final int minTermsInBlock = _TestUtil.nextInt(random, 2, 100); final int maxTermsInBlock = Math.max(2, (minTermsInBlock - 1) * 2 + random.nextInt(100)); boolean success = false; try { fields = new BlockTreeTermsWriter(state, postingsWriter, minTermsInBlock, maxTermsInBlock); success = true; } finally { if (!success) { postingsWriter.close(); } } } else { if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: writing Block terms dict"); } boolean success = false; final TermsIndexWriterBase indexWriter; try { if (random.nextBoolean()) { state.termIndexInterval = _TestUtil.nextInt(random, 1, 100); if (LuceneTestCase.VERBOSE) { System.out.println( "MockRandomCodec: fixed-gap terms index (tii=" + state.termIndexInterval + ")"); } indexWriter = new FixedGapTermsIndexWriter(state); } else { final VariableGapTermsIndexWriter.IndexTermSelector selector; final int n2 = random.nextInt(3); if (n2 == 0) { final int tii = _TestUtil.nextInt(random, 1, 100); selector = new VariableGapTermsIndexWriter.EveryNTermSelector(tii); if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: variable-gap terms index (tii=" + tii + ")"); } } else if (n2 == 1) { final int docFreqThresh = _TestUtil.nextInt(random, 2, 100); final int tii = _TestUtil.nextInt(random, 1, 100); selector = new VariableGapTermsIndexWriter.EveryNOrDocFreqTermSelector(docFreqThresh, tii); } else { final long seed2 = random.nextLong(); final int gap = _TestUtil.nextInt(random, 2, 40); if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: random-gap terms index (max gap=" + gap + ")"); } selector = new VariableGapTermsIndexWriter.IndexTermSelector() { final Random rand = new Random(seed2); @Override public boolean isIndexTerm(BytesRef term, TermStats stats) { return rand.nextInt(gap) == gap / 2; } @Override public void newField(FieldInfo fieldInfo) {} }; } indexWriter = new VariableGapTermsIndexWriter(state, selector); } success = true; } finally { if (!success) { postingsWriter.close(); } } success = false; try { fields = new BlockTermsWriter(indexWriter, state, postingsWriter); success = true; } finally { if (!success) { try { postingsWriter.close(); } finally { indexWriter.close(); } } } } return fields; }
@Override public void close() throws IOException { dataOut.close(); }