public TermVectorsWriter(Directory directory, String segment, FieldInfos fieldInfos)
throws IOException {
// Open files for TermVector storage
tvx = directory.createOutput(segment + TVX_EXTENSION);
tvx.writeInt(FORMAT_VERSION);
tvd = directory.createOutput(segment + TVD_EXTENSION);
tvd.writeInt(FORMAT_VERSION);
tvf = directory.createOutput(segment + TVF_EXTENSION);
tvf.writeInt(FORMAT_VERSION);
this.fieldInfos = fieldInfos;
fields = new Vector(fieldInfos.size());
terms = new Vector();
}
private void testOn(Directory dir, int writeSize, int readSize, Cache cache) throws IOException {
if (cache != null)
cache
.clear(); // needed to make sure no chunks are left over in case of Infinispan
// implementation
final String filename = "chunkTest";
IndexOutput indexOutput = dir.createOutput(filename);
byte[] toWrite = fillBytes(writeSize);
indexOutput.writeBytes(toWrite, writeSize);
indexOutput.close();
if (cache != null) {
AssertJUnit.assertEquals(
writeSize,
DirectoryIntegrityCheck.deepCountFileSize(new FileCacheKey(INDEXNAME, filename), cache));
}
AssertJUnit.assertEquals(writeSize, indexOutput.length());
byte[] results = new byte[readSize];
IndexInput openInput = dir.openInput(filename);
try {
openInput.readBytes(results, 0, readSize);
for (int i = 0; i < writeSize && i < readSize; i++) {
AssertJUnit.assertEquals(results[i], toWrite[i]);
}
if (readSize > writeSize)
AssertJUnit.fail("should have thrown an IOException for reading past EOF");
} catch (IOException ioe) {
if (readSize <= writeSize)
AssertJUnit.fail("should not have thrown an IOException" + ioe.getMessage());
}
}
public IndexOutput createOutput(String name, IOContext context, boolean raw)
throws IOException {
Directory directory;
if (isChecksum(name)) {
directory = distributor.primary();
} else {
directory = distributor.any();
}
IndexOutput out = directory.createOutput(name, context);
synchronized (mutex) {
StoreFileMetaData metaData = new StoreFileMetaData(name, -1, null, directory);
filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, metaData).immutableMap();
files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]);
boolean computeChecksum = !raw;
if (computeChecksum) {
// don't compute checksum for segment based files
if ("segments.gen".equals(name) || name.startsWith("segments")) {
computeChecksum = false;
}
}
if (computeChecksum) {
out = new BufferedChecksumIndexOutput(out, new Adler32());
}
return new StoreIndexOutput(metaData, out, name);
}
}
// Make sure we don't somehow use more than 1 descriptor
// when reading a CFS with many subs:
public void testManySubFiles() throws IOException {
final Directory d = newFSDirectory(_TestUtil.getTempDir("CFSManySubFiles"));
final int FILE_COUNT = 10000;
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
IndexOutput out = d.createOutput("file." + fileIdx);
out.writeByte((byte) fileIdx);
out.close();
}
final CompoundFileWriter cfw = new CompoundFileWriter(d, "c.cfs");
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
cfw.addFile("file." + fileIdx);
}
cfw.close();
final IndexInput[] ins = new IndexInput[FILE_COUNT];
final CompoundFileReader cfr = new CompoundFileReader(d, "c.cfs");
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
ins[fileIdx] = cfr.openInput("file." + fileIdx);
}
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
assertEquals((byte) fileIdx, ins[fileIdx].readByte());
}
for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
ins[fileIdx].close();
}
cfr.close();
d.close();
}
public void multipleFlushTest() throws IOException {
final String filename = "longFile.writtenInMultipleFlushes";
final int bufferSize = 300;
Cache cache = cacheManager.getCache();
cache.clear();
Directory dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
.chunkSize(13)
.create();
byte[] manyBytes = fillBytes(bufferSize);
IndexOutput indexOutput = dir.createOutput(filename);
for (int i = 0; i < 10; i++) {
indexOutput.writeBytes(manyBytes, bufferSize);
indexOutput.flush();
}
indexOutput.close();
IndexInput input = dir.openInput(filename);
final int finalSize = (10 * bufferSize);
AssertJUnit.assertEquals(finalSize, input.length());
final byte[] resultingBuffer = new byte[finalSize];
input.readBytes(resultingBuffer, 0, finalSize);
int index = 0;
for (int i = 0; i < 10; i++) {
for (int j = 0; j < bufferSize; j++)
AssertJUnit.assertEquals(resultingBuffer[index++], manyBytes[j]);
}
}
private void demo_FSIndexInputBug(Directory fsdir, String file) throws IOException {
// Setup the test file - we need more than 1024 bytes
IndexOutput os = fsdir.createOutput(file);
for (int i = 0; i < 2000; i++) {
os.writeByte((byte) i);
}
os.close();
IndexInput in = fsdir.openInput(file);
// This read primes the buffer in IndexInput
in.readByte();
// Close the file
in.close();
// ERROR: this call should fail, but succeeds because the buffer
// is still filled
in.readByte();
// ERROR: this call should fail, but succeeds for some reason as well
in.seek(1099);
try {
// OK: this call correctly fails. We are now past the 1024 internal
// buffer, so an actual IO is attempted, which fails
in.readByte();
fail("expected readByte() to throw exception");
} catch (IOException e) {
// expected exception
}
}
public void testWriteChunks() throws Exception {
final int BUFFER_SIZE = 64;
Cache cache = cacheManager.getCache();
Directory dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
.chunkSize(BUFFER_SIZE)
.create();
IndexOutput io = dir.createOutput("MyNewFile.txt");
io.writeByte((byte) 66);
io.writeByte((byte) 69);
io.flush();
io.close();
assert dir.fileExists("MyNewFile.txt");
assert null != cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 0, BUFFER_SIZE));
// test contents by reading:
byte[] buf = new byte[9];
IndexInput ii = dir.openInput("MyNewFile.txt");
ii.readBytes(buf, 0, (int) ii.length());
ii.close();
assert new String(new byte[] {66, 69}).equals(new String(buf).trim());
String testText =
"This is some rubbish again that will span more than one chunk - one hopes. Who knows, maybe even three or four chunks.";
io = dir.createOutput("MyNewFile.txt");
io.seek(0);
io.writeBytes(testText.getBytes(), 0, testText.length());
io.close();
// now compare.
byte[] chunk1 =
(byte[]) cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 0, BUFFER_SIZE));
byte[] chunk2 =
(byte[]) cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 1, BUFFER_SIZE));
assert null != chunk1;
assert null != chunk2;
assert testText.equals(new String(chunk1) + new String(chunk2).trim());
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
/** Creates a file of the specified size with random data. */
private void createRandomFile(Directory dir, String name, int size) throws IOException {
IndexOutput os = dir.createOutput(name);
for (int i = 0; i < size; i++) {
byte b = (byte) (Math.random() * 256);
os.writeByte(b);
}
os.close();
}
/**
* Creates a file of the specified size with sequential data. The first byte is written as the
* start byte provided. All subsequent bytes are computed as start + offset where offset is the
* number of the byte.
*/
private void createSequenceFile(Directory dir, String name, byte start, int size)
throws IOException {
IndexOutput os = dir.createOutput(name);
for (int i = 0; i < size; i++) {
os.writeByte(start);
start++;
}
os.close();
}
/**
* It creates a file with fixed size using a RepeatableLongByteSequence object to generate a
* repeatable content
*
* @param dir The Directory containing the file to create
* @param fileName The file name to create
* @param contentFileSize The size content file to create
* @throws IOException
*/
private void createFileWithRepeatableContent(
Directory dir, String fileName, final int contentFileSize) throws IOException {
IndexOutput indexOutput = dir.createOutput(fileName);
RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
for (int i = 0; i < contentFileSize; i++) {
indexOutput.writeByte(bytesGenerator.nextByte());
}
indexOutput.close();
}
final void finishCommit(Directory dir) throws IOException {
if (pendingSegnOutput == null) throw new IllegalStateException("prepareCommit was not called");
boolean success = false;
try {
pendingSegnOutput.finishCommit();
pendingSegnOutput.close();
pendingSegnOutput = null;
success = true;
} finally {
if (!success) rollbackCommit(dir);
}
// NOTE: if we crash here, we have left a segments_N
// file in the directory in a possibly corrupt state (if
// some bytes made it to stable storage and others
// didn't). But, the segments_N file includes checksum
// at the end, which should catch this case. So when a
// reader tries to read it, it will throw a
// CorruptIndexException, which should cause the retry
// logic in SegmentInfos to kick in and load the last
// good (previous) segments_N-1 file.
final String fileName =
IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation);
success = false;
try {
dir.sync(Collections.singleton(fileName));
success = true;
} finally {
if (!success) {
try {
dir.deleteFile(fileName);
} catch (Throwable t) {
// Suppress so we keep throwing the original exception
}
}
}
lastGeneration = generation;
try {
IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN);
try {
genOutput.writeInt(FORMAT_LOCKLESS);
genOutput.writeLong(generation);
genOutput.writeLong(generation);
} finally {
genOutput.close();
}
} catch (ThreadInterruptedException t) {
throw t;
} catch (Throwable t) {
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
}
}
public void testEncodeDecode() throws IOException {
final int iterations = RandomInts.randomIntBetween(random(), 1, 1000);
final float acceptableOverheadRatio = random().nextFloat();
final int[] values = new int[(iterations - 1) * BLOCK_SIZE + ForUtil.MAX_DATA_SIZE];
for (int i = 0; i < iterations; ++i) {
final int bpv = random().nextInt(32);
if (bpv == 0) {
final int value = RandomInts.randomIntBetween(random(), 0, Integer.MAX_VALUE);
for (int j = 0; j < BLOCK_SIZE; ++j) {
values[i * BLOCK_SIZE + j] = value;
}
} else {
for (int j = 0; j < BLOCK_SIZE; ++j) {
values[i * BLOCK_SIZE + j] =
RandomInts.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));
}
}
}
final Directory d = new RAMDirectory();
final long endPointer;
{
// encode
IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);
final ForUtil forUtil = new ForUtil(acceptableOverheadRatio, out);
for (int i = 0; i < iterations; ++i) {
forUtil.writeBlock(
Arrays.copyOfRange(values, i * BLOCK_SIZE, values.length),
new byte[MAX_ENCODED_SIZE],
out);
}
endPointer = out.getFilePointer();
out.close();
}
{
// decode
IndexInput in = d.openInput("test.bin", IOContext.READONCE);
final ForUtil forUtil = new ForUtil(in);
for (int i = 0; i < iterations; ++i) {
if (random().nextBoolean()) {
forUtil.skipBlock(in);
continue;
}
final int[] restored = new int[MAX_DATA_SIZE];
forUtil.readBlock(in, new byte[MAX_ENCODED_SIZE], restored);
assertArrayEquals(
Arrays.copyOfRange(values, i * BLOCK_SIZE, (i + 1) * BLOCK_SIZE),
Arrays.copyOf(restored, BLOCK_SIZE));
}
assertEquals(endPointer, in.getFilePointer());
in.close();
}
}
@Override
public void write(Directory dir, SegmentInfo si, IOContext context) throws IOException {
String dataFile = IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION);
int numFiles = si.files().size();
String names[] = si.files().toArray(new String[numFiles]);
Arrays.sort(names);
long startOffsets[] = new long[numFiles];
long endOffsets[] = new long[numFiles];
BytesRefBuilder scratch = new BytesRefBuilder();
try (IndexOutput out = dir.createOutput(dataFile, context)) {
for (int i = 0; i < names.length; i++) {
// write header for file
SimpleTextUtil.write(out, HEADER);
SimpleTextUtil.write(out, names[i], scratch);
SimpleTextUtil.writeNewline(out);
// write bytes for file
startOffsets[i] = out.getFilePointer();
try (IndexInput in = dir.openInput(names[i], IOContext.READONCE)) {
out.copyBytes(in, in.length());
}
endOffsets[i] = out.getFilePointer();
}
long tocPos = out.getFilePointer();
// write CFS table
SimpleTextUtil.write(out, TABLE);
SimpleTextUtil.write(out, Integer.toString(numFiles), scratch);
SimpleTextUtil.writeNewline(out);
for (int i = 0; i < names.length; i++) {
SimpleTextUtil.write(out, TABLENAME);
SimpleTextUtil.write(out, names[i], scratch);
SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, TABLESTART);
SimpleTextUtil.write(out, Long.toString(startOffsets[i]), scratch);
SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, TABLEEND);
SimpleTextUtil.write(out, Long.toString(endOffsets[i]), scratch);
SimpleTextUtil.writeNewline(out);
}
DecimalFormat df =
new DecimalFormat(OFFSETPATTERN, DecimalFormatSymbols.getInstance(Locale.ROOT));
SimpleTextUtil.write(out, TABLEPOS);
SimpleTextUtil.write(out, df.format(tocPos), scratch);
SimpleTextUtil.writeNewline(out);
}
}
@Test
public void testVerifyingIndexOutput() throws IOException {
Directory dir = newDirectory();
IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT);
int iters = scaledRandomIntBetween(10, 100);
for (int i = 0; i < iters; i++) {
BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
}
CodecUtil.writeFooter(output);
output.close();
IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT);
String checksum = Store.digestToString(CodecUtil.retrieveChecksum(indexInput));
indexInput.seek(0);
BytesRef ref = new BytesRef(scaledRandomIntBetween(1, 1024));
long length = indexInput.length();
IndexOutput verifyingOutput =
new Store.LuceneVerifyingIndexOutput(
new StoreFileMetaData("foo1.bar", length, checksum),
dir.createOutput("foo1.bar", IOContext.DEFAULT));
while (length > 0) {
if (random().nextInt(10) == 0) {
verifyingOutput.writeByte(indexInput.readByte());
length--;
} else {
int min = (int) Math.min(length, ref.bytes.length);
indexInput.readBytes(ref.bytes, ref.offset, min);
verifyingOutput.writeBytes(ref.bytes, ref.offset, min);
length -= min;
}
}
Store.verify(verifyingOutput);
verifyingOutput.writeByte((byte) 0x0);
try {
Store.verify(verifyingOutput);
fail("should be a corrupted index");
} catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
// ok
}
IOUtils.close(indexInput, verifyingOutput, dir);
}
/**
* Sets some otherwise hard-to-test properties: random segment names, ID values, document count,
* etc and round-trips
*/
public void testRandom() throws Exception {
Codec codec = getCodec();
Version[] versions = getVersions();
for (int i = 0; i < 10; i++) {
Directory dir = newDirectory();
Version version = versions[random().nextInt(versions.length)];
String name =
"_" + Integer.toString(random().nextInt(Integer.MAX_VALUE), Character.MAX_RADIX);
int docCount = TestUtil.nextInt(random(), 1, IndexWriter.MAX_DOCS);
boolean isCompoundFile = random().nextBoolean();
Set<String> files = new HashSet<>();
int numFiles = random().nextInt(10);
for (int j = 0; j < numFiles; j++) {
String file = IndexFileNames.segmentFileName(name, "", Integer.toString(j));
files.add(file);
dir.createOutput(file, IOContext.DEFAULT).close();
}
Map<String, String> diagnostics = new HashMap<>();
int numDiags = random().nextInt(10);
for (int j = 0; j < numDiags; j++) {
diagnostics.put(
TestUtil.randomUnicodeString(random()), TestUtil.randomUnicodeString(random()));
}
byte id[] = new byte[StringHelper.ID_LENGTH];
random().nextBytes(id);
Map<String, String> attributes = new HashMap<>();
int numAttributes = random().nextInt(10);
for (int j = 0; j < numAttributes; j++) {
attributes.put(
TestUtil.randomUnicodeString(random()), TestUtil.randomUnicodeString(random()));
}
SegmentInfo info =
new SegmentInfo(
dir,
version,
name,
docCount,
isCompoundFile,
codec,
diagnostics,
id,
attributes,
null);
info.setFiles(files);
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
SegmentInfo info2 = codec.segmentInfoFormat().read(dir, name, id, IOContext.DEFAULT);
assertEquals(info, info2);
dir.close();
}
}
/**
* creates compressed output.
*
* @param pCompressed compressed directory to inform when file is closed
* @param pTempDir temporary directory that can be the asame as compressed one
* @param pOut actual IndexOutput from compresed directory (nested storage)
* @param pName name of output
* @param pChunkSize chunk size
* @param pDeflateMethod deflate method
* @param deflateCount number of times to deflate data
* @throws IOException
*/
public TransformedIndexOutput(
TransformedDirectory pCompressed,
Directory pTempDir,
IndexOutput pOut,
String pName,
int pChunkSize,
StoreDataTransformer pTransformer)
throws IOException {
super(pName, pOut, pTransformer, pCompressed);
this.tempDirectory = pTempDir;
this.chunkSize = pChunkSize;
tmpName = pName + ".plain." + pCompressed.nextSequence() + ".tmp";
tempOut = pTempDir.createOutput(tmpName);
}
@Override
public void write(
Directory directory,
SegmentInfo segmentInfo,
String segmentSuffix,
FieldInfos infos,
IOContext context)
throws IOException {
final String fileName =
IndexFileNames.segmentFileName(
segmentInfo.name, segmentSuffix, Lucene46FieldInfosFormat.EXTENSION);
try (IndexOutput output = directory.createOutput(fileName, context)) {
CodecUtil.writeHeader(
output, Lucene46FieldInfosFormat.CODEC_NAME, Lucene46FieldInfosFormat.FORMAT_CURRENT);
output.writeVInt(infos.size());
for (FieldInfo fi : infos) {
IndexOptions indexOptions = fi.getIndexOptions();
byte bits = 0x0;
if (fi.hasVectors()) bits |= Lucene46FieldInfosFormat.STORE_TERMVECTOR;
if (fi.omitsNorms()) bits |= Lucene46FieldInfosFormat.OMIT_NORMS;
if (fi.hasPayloads()) bits |= Lucene46FieldInfosFormat.STORE_PAYLOADS;
if (fi.getIndexOptions() != IndexOptions.NONE) {
bits |= Lucene46FieldInfosFormat.IS_INDEXED;
assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0
|| !fi.hasPayloads();
if (indexOptions == IndexOptions.DOCS) {
bits |= Lucene46FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS;
} else if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
bits |= Lucene46FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS;
} else if (indexOptions == IndexOptions.DOCS_AND_FREQS) {
bits |= Lucene46FieldInfosFormat.OMIT_POSITIONS;
}
}
output.writeString(fi.name);
output.writeVInt(fi.number);
output.writeByte(bits);
// pack the DV types in one byte
final byte dv = docValuesByte(fi.getDocValuesType());
final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : DocValuesType.NONE);
assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
byte val = (byte) (0xff & ((nrm << 4) | dv));
output.writeByte(val);
output.writeLong(fi.getDocValuesGen());
output.writeStringStringMap(fi.attributes());
}
CodecUtil.writeFooter(output);
}
}
/**
* This test that writes larger than the size of the buffer output will correctly increment the
* file pointer.
*/
public void testLargeWrites() throws IOException {
IndexOutput os = dir.createOutput("testBufferStart.txt");
byte[] largeBuf = new byte[2048];
for (int i = 0; i < largeBuf.length; i++) {
largeBuf[i] = (byte) (Math.random() * 256);
}
long currentPos = os.getFilePointer();
os.writeBytes(largeBuf, largeBuf.length);
try {
assertEquals(currentPos + largeBuf.length, os.getFilePointer());
} finally {
os.close();
}
}
private final void write(Directory directory) throws IOException {
String segmentFileName = getNextSegmentFileName();
// Always advance the generation on write:
if (generation == -1) {
generation = 1;
} else {
generation++;
}
ChecksumIndexOutput segnOutput =
new ChecksumIndexOutput(directory.createOutput(segmentFileName));
boolean success = false;
try {
segnOutput.writeInt(CURRENT_FORMAT); // write FORMAT
segnOutput.writeLong(version);
segnOutput.writeInt(counter); // write counter
segnOutput.writeInt(size()); // write infos
for (SegmentInfo si : this) {
si.write(segnOutput);
}
segnOutput.writeStringStringMap(userData);
segnOutput.prepareCommit();
pendingSegnOutput = segnOutput;
success = true;
} finally {
if (!success) {
// We hit an exception above; try to close the file
// but suppress any exception:
IOUtils.closeSafely(true, segnOutput);
try {
// Try not to leave a truncated segments_N file in
// the index:
directory.deleteFile(segmentFileName);
} catch (Throwable t) {
// Suppress so we keep throwing the original exception
}
}
}
}
@Test
public void testVerifyingIndexOutputWithBogusInput() throws IOException {
Directory dir = newDirectory();
int length = scaledRandomIntBetween(10, 1024);
IndexOutput verifyingOutput =
new Store.LuceneVerifyingIndexOutput(
new StoreFileMetaData("foo1.bar", length, ""),
dir.createOutput("foo1.bar", IOContext.DEFAULT));
try {
while (length > 0) {
verifyingOutput.writeByte((byte) random().nextInt());
length--;
}
fail("should be a corrupted index");
} catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
// ok
}
IOUtils.close(verifyingOutput, dir);
}
private void corruptFile(Directory dir, String fileIn, String fileOut) throws IOException {
IndexInput input = dir.openInput(fileIn, IOContext.READONCE);
IndexOutput output = dir.createOutput(fileOut, IOContext.DEFAULT);
long len = input.length();
byte[] b = new byte[1024];
long broken = randomInt((int) len);
long pos = 0;
while (pos < len) {
int min = (int) Math.min(input.length() - pos, b.length);
input.readBytes(b, 0, min);
if (broken >= pos && broken < pos + min) {
// Flip one byte
int flipPos = (int) (broken - pos);
b[flipPos] = (byte) (b[flipPos] ^ 42);
}
output.writeBytes(b, min);
pos += min;
}
IOUtils.close(input, output);
}
@Test
public void testWriteChunksDefaultChunks() throws Exception {
Cache cache = cacheManager.getCache();
Directory dir = DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME).create();
final String testText = "This is some rubbish";
final byte[] testTextAsBytes = testText.getBytes();
IndexOutput io = dir.createOutput("MyNewFile.txt");
io.writeByte((byte) 1);
io.writeByte((byte) 2);
io.writeByte((byte) 3);
io.writeBytes(testTextAsBytes, testTextAsBytes.length);
io.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
FileCacheKey fileCacheKey = new FileCacheKey(INDEXNAME, "MyNewFile.txt");
assert null != cache.get(fileCacheKey);
FileMetadata metadata = (FileMetadata) cache.get(fileCacheKey);
AssertJUnit.assertEquals(testTextAsBytes.length + 3, metadata.getSize());
assert null
!= cache.get(
new ChunkCacheKey(
INDEXNAME, "MyNewFile.txt", 0, DirectoryBuilderImpl.DEFAULT_BUFFER_SIZE));
// test contents by reading:
IndexInput ii = dir.openInput("MyNewFile.txt");
assert ii.readByte() == 1;
assert ii.readByte() == 2;
assert ii.readByte() == 3;
byte[] buf = new byte[testTextAsBytes.length];
ii.readBytes(buf, 0, testTextAsBytes.length);
ii.close();
assert testText.equals(new String(buf).trim());
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
public IndexOutput createOutput(String name, IOContext context, boolean raw)
throws IOException {
ensureOpen();
Directory directory;
// we want to write the segments gen file to the same directory *all* the time
// to make sure we don't create multiple copies of it
if (isChecksum(name) || IndexFileNames.SEGMENTS_GEN.equals(name)) {
directory = distributor.primary();
} else {
directory = distributor.any();
}
IndexOutput out = directory.createOutput(name, context);
boolean success = false;
try {
synchronized (mutex) {
StoreFileMetaData metaData = new StoreFileMetaData(name, -1, null, directory);
filesMetadata = ImmutableOpenMap.builder(filesMetadata).fPut(name, metaData).build();
files = filesMetadata.keys().toArray(String.class);
boolean computeChecksum = !raw;
if (computeChecksum) {
// don't compute checksum for segment based files
if (IndexFileNames.SEGMENTS_GEN.equals(name)
|| name.startsWith(IndexFileNames.SEGMENTS)) {
computeChecksum = false;
}
}
if (computeChecksum) {
out = new BufferedChecksumIndexOutput(out, new Adler32());
}
final StoreIndexOutput storeIndexOutput = new StoreIndexOutput(metaData, out, name);
success = true;
return storeIndexOutput;
}
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(out);
}
}
}
@Override
public IntIndexOutput createOutput(Directory dir, String fileName, IOContext context)
throws IOException {
IndexOutput out = dir.createOutput(fileName, context);
boolean success = false;
try {
FixedIntBlockIndexOutput ret =
new FixedIntBlockIndexOutput(out, blockSize) {
@Override
protected void flushBlock() throws IOException {
for (int i = 0; i < buffer.length; i++) {
out.writeVInt(buffer[i]);
}
}
};
success = true;
return ret;
} finally {
if (!success) {
IOUtils.closeWhileHandlingException(out);
}
}
}
@Test
public void testVerifyingIndexInput() throws IOException {
Directory dir = newDirectory();
IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT);
int iters = scaledRandomIntBetween(10, 100);
for (int i = 0; i < iters; i++) {
BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
}
CodecUtil.writeFooter(output);
output.close();
// Check file
IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT);
long checksum = CodecUtil.retrieveChecksum(indexInput);
indexInput.seek(0);
IndexInput verifyingIndexInput =
new Store.VerifyingIndexInput(dir.openInput("foo.bar", IOContext.DEFAULT));
readIndexInputFullyWithRandomSeeks(verifyingIndexInput);
Store.verify(verifyingIndexInput);
assertThat(checksum, equalTo(((ChecksumIndexInput) verifyingIndexInput).getChecksum()));
IOUtils.close(indexInput, verifyingIndexInput);
// Corrupt file and check again
corruptFile(dir, "foo.bar", "foo1.bar");
verifyingIndexInput =
new Store.VerifyingIndexInput(dir.openInput("foo1.bar", IOContext.DEFAULT));
readIndexInputFullyWithRandomSeeks(verifyingIndexInput);
try {
Store.verify(verifyingIndexInput);
fail("should be a corrupted index");
} catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
// ok
}
IOUtils.close(verifyingIndexInput);
IOUtils.close(dir);
}
// TODO ADD more unittests
private void writeData(
Directory dir, Completion090PostingsFormat.CompletionLookupProvider provider)
throws IOException {
IndexOutput output = dir.createOutput("foo.txt", IOContext.DEFAULT);
FieldsConsumer consumer = provider.consumer(output);
final List<TermPosAndPayload> terms = new ArrayList<>();
terms.add(
new TermPosAndPayload(
"foofightersgenerator",
256 - 2,
provider.buildPayload(
new BytesRef("Generator - Foo Fighters"), 9, new BytesRef("id:10"))));
terms.add(
new TermPosAndPayload(
"generator",
256 - 1,
provider.buildPayload(
new BytesRef("Generator - Foo Fighters"), 9, new BytesRef("id:10"))));
Fields fields =
new Fields() {
@Override
public Iterator<String> iterator() {
return Arrays.asList("foo").iterator();
}
@Override
public Terms terms(String field) throws IOException {
if (field.equals("foo")) {
return new Terms() {
@Override
public TermsEnum iterator(TermsEnum reuse) throws IOException {
final Iterator<TermPosAndPayload> iterator = terms.iterator();
return new TermsEnum() {
private TermPosAndPayload current = null;
@Override
public SeekStatus seekCeil(BytesRef text) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public void seekExact(long ord) throws IOException {
throw new UnsupportedOperationException();
}
@Override
public BytesRef term() throws IOException {
return current == null ? null : current.term;
}
@Override
public long ord() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public int docFreq() throws IOException {
return current == null ? 0 : 1;
}
@Override
public long totalTermFreq() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public PostingsEnum postings(Bits liveDocs, PostingsEnum reuse, int flags)
throws IOException {
final TermPosAndPayload data = current;
return new PostingsEnum() {
boolean done = false;
@Override
public int nextPosition() throws IOException {
return current.pos;
}
@Override
public int startOffset() throws IOException {
return 0;
}
@Override
public int endOffset() throws IOException {
return 0;
}
@Override
public BytesRef getPayload() throws IOException {
return current.payload;
}
@Override
public int freq() throws IOException {
return 1;
}
@Override
public int docID() {
if (done) {
return NO_MORE_DOCS;
}
return 0;
}
@Override
public int nextDoc() throws IOException {
if (done) {
return NO_MORE_DOCS;
}
done = true;
return 0;
}
@Override
public int advance(int target) throws IOException {
if (done) {
return NO_MORE_DOCS;
}
done = true;
return 0;
}
@Override
public long cost() {
return 0;
}
};
}
@Override
public BytesRef next() throws IOException {
if (iterator.hasNext()) {
current = iterator.next();
return current.term;
}
current = null;
return null;
}
};
}
@Override
public long size() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public long getSumTotalTermFreq() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public long getSumDocFreq() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public int getDocCount() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public boolean hasFreqs() {
throw new UnsupportedOperationException();
}
@Override
public boolean hasOffsets() {
throw new UnsupportedOperationException();
}
@Override
public boolean hasPositions() {
throw new UnsupportedOperationException();
}
@Override
public boolean hasPayloads() {
throw new UnsupportedOperationException();
}
};
}
return null;
}
@Override
public int size() {
return 0;
}
};
consumer.write(fields);
consumer.close();
output.close();
}
private void write(Directory directory) throws IOException {
long nextGeneration = getNextPendingGeneration();
String segmentFileName =
IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS, "", nextGeneration);
// Always advance the generation on write:
generation = nextGeneration;
IndexOutput segnOutput = null;
boolean success = false;
try {
segnOutput = directory.createOutput(segmentFileName, IOContext.DEFAULT);
CodecUtil.writeIndexHeader(
segnOutput,
"segments",
VERSION_CURRENT,
StringHelper.randomId(),
Long.toString(nextGeneration, Character.MAX_RADIX));
segnOutput.writeVInt(Version.LATEST.major);
segnOutput.writeVInt(Version.LATEST.minor);
segnOutput.writeVInt(Version.LATEST.bugfix);
segnOutput.writeLong(version);
segnOutput.writeInt(counter); // write counter
segnOutput.writeInt(size());
if (size() > 0) {
Version minSegmentVersion = null;
// We do a separate loop up front so we can write the minSegmentVersion before
// any SegmentInfo; this makes it cleaner to throw IndexFormatTooOldExc at read time:
for (SegmentCommitInfo siPerCommit : this) {
Version segmentVersion = siPerCommit.info.getVersion();
if (minSegmentVersion == null || segmentVersion.onOrAfter(minSegmentVersion) == false) {
minSegmentVersion = segmentVersion;
}
}
segnOutput.writeVInt(minSegmentVersion.major);
segnOutput.writeVInt(minSegmentVersion.minor);
segnOutput.writeVInt(minSegmentVersion.bugfix);
}
// write infos
for (SegmentCommitInfo siPerCommit : this) {
SegmentInfo si = siPerCommit.info;
segnOutput.writeString(si.name);
byte segmentID[] = si.getId();
// TODO: remove this in lucene 6, we don't need to include 4.x segments in commits anymore
if (segmentID == null) {
segnOutput.writeByte((byte) 0);
} else {
if (segmentID.length != StringHelper.ID_LENGTH) {
throw new IllegalStateException(
"cannot write segment: invalid id segment="
+ si.name
+ "id="
+ StringHelper.idToString(segmentID));
}
segnOutput.writeByte((byte) 1);
segnOutput.writeBytes(segmentID, segmentID.length);
}
segnOutput.writeString(si.getCodec().getName());
segnOutput.writeLong(siPerCommit.getDelGen());
int delCount = siPerCommit.getDelCount();
if (delCount < 0 || delCount > si.maxDoc()) {
throw new IllegalStateException(
"cannot write segment: invalid maxDoc segment="
+ si.name
+ " maxDoc="
+ si.maxDoc()
+ " delCount="
+ delCount);
}
segnOutput.writeInt(delCount);
segnOutput.writeLong(siPerCommit.getFieldInfosGen());
segnOutput.writeLong(siPerCommit.getDocValuesGen());
segnOutput.writeSetOfStrings(siPerCommit.getFieldInfosFiles());
final Map<Integer, Set<String>> dvUpdatesFiles = siPerCommit.getDocValuesUpdatesFiles();
segnOutput.writeInt(dvUpdatesFiles.size());
for (Entry<Integer, Set<String>> e : dvUpdatesFiles.entrySet()) {
segnOutput.writeInt(e.getKey());
segnOutput.writeSetOfStrings(e.getValue());
}
}
segnOutput.writeMapOfStrings(userData);
CodecUtil.writeFooter(segnOutput);
segnOutput.close();
directory.sync(Collections.singleton(segmentFileName));
success = true;
} finally {
if (success) {
pendingCommit = true;
} else {
// We hit an exception above; try to close the file
// but suppress any exception:
IOUtils.closeWhileHandlingException(segnOutput);
// Try not to leave a truncated segments_N file in
// the index:
IOUtils.deleteFilesIgnoringExceptions(directory, segmentFileName);
}
}
}
@Test
public void testWriteUsingSeekMethod() throws IOException {
final int BUFFER_SIZE = 64;
Cache cache = cacheManager.getCache();
Directory dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
.chunkSize(BUFFER_SIZE)
.create();
String fileName = "SomeText.txt";
IndexOutput io = dir.createOutput(fileName);
RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
// It writes repeatable text
final int REPEATABLE_BUFFER_SIZE = 1501;
for (int i = 0; i < REPEATABLE_BUFFER_SIZE; i++) {
io.writeByte(bytesGenerator.nextByte());
}
io.flush();
assert io.length() == REPEATABLE_BUFFER_SIZE;
// Text to write on file with repeatable text
final String someText = "This is some text";
final byte[] someTextAsBytes = someText.getBytes();
// 4 points in random order where writing someText: at begin of file, at end of file, within a
// single chunk,
// between 2 chunks
final int[] pointers = {0, 635, REPEATABLE_BUFFER_SIZE, 135};
for (int i = 0; i < pointers.length; i++) {
io.seek(pointers[i]);
io.writeBytes(someTextAsBytes, someTextAsBytes.length);
}
io.close();
bytesGenerator.reset();
final long finalSize = REPEATABLE_BUFFER_SIZE + someTextAsBytes.length;
assert io.length() == finalSize;
assert io.length()
== DirectoryIntegrityCheck.deepCountFileSize(new FileCacheKey(INDEXNAME, fileName), cache);
int indexPointer = 0;
Arrays.sort(pointers);
byte[] buffer = null;
int chunkIndex = -1;
// now testing the stream is equal to the produced repeatable but including the edits at pointed
// positions
for (int i = 0; i < REPEATABLE_BUFFER_SIZE + someTextAsBytes.length; i++) {
if (i % BUFFER_SIZE == 0) {
buffer =
(byte[]) cache.get(new ChunkCacheKey(INDEXNAME, fileName, ++chunkIndex, BUFFER_SIZE));
}
byte predictableByte = bytesGenerator.nextByte();
if (i < pointers[indexPointer]) {
// Assert predictable text
AssertJUnit.assertEquals(predictableByte, buffer[i % BUFFER_SIZE]);
} else if (pointers[indexPointer] <= i
&& i < pointers[indexPointer] + someTextAsBytes.length) {
// Assert someText
AssertJUnit.assertEquals(
someTextAsBytes[i - pointers[indexPointer]], buffer[i % BUFFER_SIZE]);
}
if (i == pointers[indexPointer] + someTextAsBytes.length) {
// Change pointer
indexPointer++;
}
}
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
public void testCheckIntegrity() throws IOException {
Directory dir = newDirectory();
long luceneFileLength = 0;
try (IndexOutput output = dir.createOutput("lucene_checksum.bin", IOContext.DEFAULT)) {
int iters = scaledRandomIntBetween(10, 100);
for (int i = 0; i < iters; i++) {
BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
luceneFileLength += bytesRef.length;
}
CodecUtil.writeFooter(output);
luceneFileLength += CodecUtil.footerLength();
}
final Adler32 adler32 = new Adler32();
long legacyFileLength = 0;
try (IndexOutput output = dir.createOutput("legacy.bin", IOContext.DEFAULT)) {
int iters = scaledRandomIntBetween(10, 100);
for (int i = 0; i < iters; i++) {
BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
adler32.update(bytesRef.bytes, bytesRef.offset, bytesRef.length);
legacyFileLength += bytesRef.length;
}
}
final long luceneChecksum;
final long adler32LegacyChecksum = adler32.getValue();
try (IndexInput indexInput = dir.openInput("lucene_checksum.bin", IOContext.DEFAULT)) {
assertEquals(luceneFileLength, indexInput.length());
luceneChecksum = CodecUtil.retrieveChecksum(indexInput);
}
{ // positive check
StoreFileMetaData lucene =
new StoreFileMetaData(
"lucene_checksum.bin",
luceneFileLength,
Store.digestToString(luceneChecksum),
Version.LUCENE_4_8_0);
StoreFileMetaData legacy =
new StoreFileMetaData(
"legacy.bin", legacyFileLength, Store.digestToString(adler32LegacyChecksum));
assertTrue(legacy.hasLegacyChecksum());
assertFalse(lucene.hasLegacyChecksum());
assertTrue(Store.checkIntegrityNoException(lucene, dir));
assertTrue(Store.checkIntegrityNoException(legacy, dir));
}
{ // negative check - wrong checksum
StoreFileMetaData lucene =
new StoreFileMetaData(
"lucene_checksum.bin",
luceneFileLength,
Store.digestToString(luceneChecksum + 1),
Version.LUCENE_4_8_0);
StoreFileMetaData legacy =
new StoreFileMetaData(
"legacy.bin", legacyFileLength, Store.digestToString(adler32LegacyChecksum + 1));
assertTrue(legacy.hasLegacyChecksum());
assertFalse(lucene.hasLegacyChecksum());
assertFalse(Store.checkIntegrityNoException(lucene, dir));
assertFalse(Store.checkIntegrityNoException(legacy, dir));
}
{ // negative check - wrong length
StoreFileMetaData lucene =
new StoreFileMetaData(
"lucene_checksum.bin",
luceneFileLength + 1,
Store.digestToString(luceneChecksum),
Version.LUCENE_4_8_0);
StoreFileMetaData legacy =
new StoreFileMetaData(
"legacy.bin", legacyFileLength + 1, Store.digestToString(adler32LegacyChecksum));
assertTrue(legacy.hasLegacyChecksum());
assertFalse(lucene.hasLegacyChecksum());
assertFalse(Store.checkIntegrityNoException(lucene, dir));
assertFalse(Store.checkIntegrityNoException(legacy, dir));
}
{ // negative check - wrong file
StoreFileMetaData lucene =
new StoreFileMetaData(
"legacy.bin",
luceneFileLength,
Store.digestToString(luceneChecksum),
Version.LUCENE_4_8_0);
StoreFileMetaData legacy =
new StoreFileMetaData(
"lucene_checksum.bin", legacyFileLength, Store.digestToString(adler32LegacyChecksum));
assertTrue(legacy.hasLegacyChecksum());
assertFalse(lucene.hasLegacyChecksum());
assertFalse(Store.checkIntegrityNoException(lucene, dir));
assertFalse(Store.checkIntegrityNoException(legacy, dir));
}
dir.close();
}
@Override
public IndexOutput createOutput(String name, IOContext context) throws IOException {
return fsDir.createOutput(name, context);
}
