private void assertSameSeekBehavior(String msg, IndexInput expected, IndexInput actual)
throws IOException {
// seek to 0
long point = 0;
assertSameStreams(msg + ", seek(0)", expected, actual, point);
// seek to middle
point = expected.length() / 2l;
assertSameStreams(msg + ", seek(mid)", expected, actual, point);
// seek to end - 2
point = expected.length() - 2;
assertSameStreams(msg + ", seek(end-2)", expected, actual, point);
// seek to end - 1
point = expected.length() - 1;
assertSameStreams(msg + ", seek(end-1)", expected, actual, point);
// seek to the end
point = expected.length();
assertSameStreams(msg + ", seek(end)", expected, actual, point);
// seek past end
point = expected.length() + 1;
assertSameStreams(msg + ", seek(end+1)", expected, actual, point);
}
public void testReadPastEOF() throws IOException {
setUp_2();
CompoundFileReader cr = new CompoundFileReader(dir, "f.comp");
IndexInput is = cr.openInput("f2");
is.seek(is.length() - 10);
byte b[] = new byte[100];
is.readBytes(b, 0, 10);
try {
is.readByte();
fail("Single byte read past end of file");
} catch (IOException e) {
/* success */
// System.out.println("SUCCESS: single byte read past end of file: " + e);
}
is.seek(is.length() - 10);
try {
is.readBytes(b, 0, 50);
fail("Block read past end of file");
} catch (IOException e) {
/* success */
// System.out.println("SUCCESS: block read past end of file: " + e);
}
is.close();
cr.close();
}
private void seekDir(IndexInput in) throws IOException {
if (version >= FSTTermsWriter.TERMS_VERSION_CHECKSUM) {
in.seek(in.length() - CodecUtil.footerLength() - 8);
} else {
in.seek(in.length() - 8);
}
in.seek(in.readLong());
}
@Test
public void testReadRandomSampleFile() throws IOException {
final int BUFFER_SIZE = 64;
Cache cache = cacheManager.getCache();
InfinispanDirectory dir = new InfinispanDirectory(cache, cache, cache, INDEXNAME, BUFFER_SIZE);
final int FILE_SIZE = 1000;
assert BUFFER_SIZE < FILE_SIZE;
createFileWithRepeatableContent(dir, "RandomSampleFile.txt", FILE_SIZE);
IndexInput indexInput = dir.openInput("RandomSampleFile.txt");
assert indexInput.length() == FILE_SIZE;
RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
Random r = new Random();
long seekPoint = 0;
// Now it reads some random byte and it compares to the expected byte
for (int i = 0; i < FILE_SIZE; i++) {
if (seekPoint == i) {
byte expectedByte = bytesGenerator.nextByte();
byte actualByte = indexInput.readByte();
assert expectedByte == actualByte;
seekPoint = indexInput.getFilePointer() + r.nextInt(10);
indexInput.seek(seekPoint);
} else {
bytesGenerator.nextByte();
}
}
indexInput.close();
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
public void multipleFlushTest() throws IOException {
final String filename = "longFile.writtenInMultipleFlushes";
final int bufferSize = 300;
Cache cache = cacheManager.getCache();
cache.clear();
Directory dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
.chunkSize(13)
.create();
byte[] manyBytes = fillBytes(bufferSize);
IndexOutput indexOutput = dir.createOutput(filename);
for (int i = 0; i < 10; i++) {
indexOutput.writeBytes(manyBytes, bufferSize);
indexOutput.flush();
}
indexOutput.close();
IndexInput input = dir.openInput(filename);
final int finalSize = (10 * bufferSize);
AssertJUnit.assertEquals(finalSize, input.length());
final byte[] resultingBuffer = new byte[finalSize];
input.readBytes(resultingBuffer, 0, finalSize);
int index = 0;
for (int i = 0; i < 10; i++) {
for (int j = 0; j < bufferSize; j++)
AssertJUnit.assertEquals(resultingBuffer[index++], manyBytes[j]);
}
}
/**
* Used to verify that IndexInput.readBytes method reads correctly the whole file content
* comparing the result with the expected sequence of bytes
*
* @param dir The Directory containing the file to verify
* @param fileName The file name to read
* @param contentFileSizeExpected The size content file expected
* @param arrayLengthToRead The length of byte array to read
* @throws IOException
*/
private void assertReadBytesWorkingCorrectly(
Directory dir,
String fileName,
final int contentFileSizeExpected,
final int arrayLengthToRead)
throws IOException {
IndexInput indexInput = dir.openInput(fileName);
AssertJUnit.assertEquals(contentFileSizeExpected, indexInput.length());
RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
byte[] readBytes = new byte[arrayLengthToRead];
byte[] expectedBytes = new byte[arrayLengthToRead];
long toRead = contentFileSizeExpected;
while (toRead > 0) {
// the condition is satisfied when the file is close to the end
if (toRead < arrayLengthToRead) {
readBytes = new byte[(int) toRead];
expectedBytes = new byte[(int) toRead];
}
int nextBytesToRead = (int) Math.min(toRead, arrayLengthToRead);
bytesGenerator.nextBytes(expectedBytes);
indexInput.readBytes(readBytes, 0, nextBytesToRead);
assert Arrays.equals(expectedBytes, readBytes);
toRead = toRead - nextBytesToRead;
}
indexInput.close();
}
FieldsReader(Directory d, String segment, FieldInfos fn) throws IOException {
fieldInfos = fn;
cloneableFieldsStream = d.openInput(segment + ".fdt");
fieldsStream = (IndexInput) cloneableFieldsStream.clone();
indexStream = d.openInput(segment + ".fdx");
size = (int) (indexStream.length() / 8);
}
private void assertSameStreams(String msg, IndexInput expected, IndexInput actual, long seekTo)
throws IOException {
if (seekTo >= 0 && seekTo < expected.length()) {
expected.seek(seekTo);
actual.seek(seekTo);
assertSameStreams(msg + ", seek(mid)", expected, actual);
}
}
private void assertSameStreams(String msg, IndexInput expected, IndexInput test)
throws IOException {
assertNotNull(msg + " null expected", expected);
assertNotNull(msg + " null test", test);
assertEquals(msg + " length", expected.length(), test.length());
assertEquals(msg + " position", expected.getFilePointer(), test.getFilePointer());
byte expectedBuffer[] = new byte[512];
byte testBuffer[] = new byte[expectedBuffer.length];
long remainder = expected.length() - expected.getFilePointer();
while (remainder > 0) {
int readLen = (int) Math.min(remainder, expectedBuffer.length);
expected.readBytes(expectedBuffer, 0, readLen);
test.readBytes(testBuffer, 0, readLen);
assertEqualArrays(msg + ", remainder " + remainder, expectedBuffer, testBuffer, 0, readLen);
remainder -= readLen;
}
}
@Override
public void write(Directory dir, SegmentInfo si, IOContext context) throws IOException {
String dataFile = IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION);
int numFiles = si.files().size();
String names[] = si.files().toArray(new String[numFiles]);
Arrays.sort(names);
long startOffsets[] = new long[numFiles];
long endOffsets[] = new long[numFiles];
BytesRefBuilder scratch = new BytesRefBuilder();
try (IndexOutput out = dir.createOutput(dataFile, context)) {
for (int i = 0; i < names.length; i++) {
// write header for file
SimpleTextUtil.write(out, HEADER);
SimpleTextUtil.write(out, names[i], scratch);
SimpleTextUtil.writeNewline(out);
// write bytes for file
startOffsets[i] = out.getFilePointer();
try (IndexInput in = dir.openInput(names[i], IOContext.READONCE)) {
out.copyBytes(in, in.length());
}
endOffsets[i] = out.getFilePointer();
}
long tocPos = out.getFilePointer();
// write CFS table
SimpleTextUtil.write(out, TABLE);
SimpleTextUtil.write(out, Integer.toString(numFiles), scratch);
SimpleTextUtil.writeNewline(out);
for (int i = 0; i < names.length; i++) {
SimpleTextUtil.write(out, TABLENAME);
SimpleTextUtil.write(out, names[i], scratch);
SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, TABLESTART);
SimpleTextUtil.write(out, Long.toString(startOffsets[i]), scratch);
SimpleTextUtil.writeNewline(out);
SimpleTextUtil.write(out, TABLEEND);
SimpleTextUtil.write(out, Long.toString(endOffsets[i]), scratch);
SimpleTextUtil.writeNewline(out);
}
DecimalFormat df =
new DecimalFormat(OFFSETPATTERN, DecimalFormatSymbols.getInstance(Locale.ROOT));
SimpleTextUtil.write(out, TABLEPOS);
SimpleTextUtil.write(out, df.format(tocPos), scratch);
SimpleTextUtil.writeNewline(out);
}
}
/**
* Used to verify that IndexInput.readByte method reads correctly the whole file content comparing
* the result with the expected sequence of bytes
*
* @param dir The Directory containing the file to verify
* @param fileName The file name to read
* @param contentFileSizeExpected The size content file expected
* @throws IOException
*/
private void assertReadByteWorkingCorrectly(
Directory dir, String fileName, final int contentFileSizeExpected) throws IOException {
IndexInput indexInput = dir.openInput(fileName);
AssertJUnit.assertEquals(contentFileSizeExpected, indexInput.length());
RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
for (int i = 0; i < contentFileSizeExpected; i++) {
AssertJUnit.assertEquals(bytesGenerator.nextByte(), indexInput.readByte());
}
indexInput.close();
}
private void corruptFile(Directory dir, String fileIn, String fileOut) throws IOException {
IndexInput input = dir.openInput(fileIn, IOContext.READONCE);
IndexOutput output = dir.createOutput(fileOut, IOContext.DEFAULT);
long len = input.length();
byte[] b = new byte[1024];
long broken = randomInt((int) len);
long pos = 0;
while (pos < len) {
int min = (int) Math.min(input.length() - pos, b.length);
input.readBytes(b, 0, min);
if (broken >= pos && broken < pos + min) {
// Flip one byte
int flipPos = (int) (broken - pos);
b[flipPos] = (byte) (b[flipPos] ^ 42);
}
output.writeBytes(b, min);
pos += min;
}
IOUtils.close(input, output);
}
private void readIndexInputFullyWithRandomSeeks(IndexInput indexInput) throws IOException {
BytesRef ref = new BytesRef(scaledRandomIntBetween(1, 1024));
long pos = 0;
while (pos < indexInput.length()) {
assertEquals(pos, indexInput.getFilePointer());
int op = random().nextInt(5);
if (op == 0) {
int shift = 100 - randomIntBetween(0, 200);
pos = Math.min(indexInput.length() - 1, Math.max(0, pos + shift));
indexInput.seek(pos);
} else if (op == 1) {
indexInput.readByte();
pos++;
} else {
int min = (int) Math.min(indexInput.length() - pos, ref.bytes.length);
indexInput.readBytes(ref.bytes, ref.offset, min);
pos += min;
}
}
}
/**
* Used to verify that IndexInput.readByte method reads correctly the whole file content comparing
* the result with the expected sequence of bytes
*
* @param dir The Directory containing the file to verify
* @param fileName The file name to read
* @param contentFileSizeExpected The size content file expected
* @throws IOException
*/
private void assertReadByteWorkingCorrectly(
InfinispanDirectory dir, String fileName, final int contentFileSizeExpected)
throws IOException {
IndexInput indexInput = dir.openInput(fileName);
assert indexInput.length() == contentFileSizeExpected;
RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
for (int i = 0; i < contentFileSizeExpected; i++) {
assert bytesGenerator.nextByte() == indexInput.readByte();
}
indexInput.close();
}
public void testWriteChunks() throws Exception {
final int BUFFER_SIZE = 64;
Cache cache = cacheManager.getCache();
Directory dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
.chunkSize(BUFFER_SIZE)
.create();
IndexOutput io = dir.createOutput("MyNewFile.txt");
io.writeByte((byte) 66);
io.writeByte((byte) 69);
io.flush();
io.close();
assert dir.fileExists("MyNewFile.txt");
assert null != cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 0, BUFFER_SIZE));
// test contents by reading:
byte[] buf = new byte[9];
IndexInput ii = dir.openInput("MyNewFile.txt");
ii.readBytes(buf, 0, (int) ii.length());
ii.close();
assert new String(new byte[] {66, 69}).equals(new String(buf).trim());
String testText =
"This is some rubbish again that will span more than one chunk - one hopes. Who knows, maybe even three or four chunks.";
io = dir.createOutput("MyNewFile.txt");
io.seek(0);
io.writeBytes(testText.getBytes(), 0, testText.length());
io.close();
// now compare.
byte[] chunk1 =
(byte[]) cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 0, BUFFER_SIZE));
byte[] chunk2 =
(byte[]) cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 1, BUFFER_SIZE));
assert null != chunk1;
assert null != chunk2;
assert testText.equals(new String(chunk1) + new String(chunk2).trim());
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
/**
* Loads the actual byte array from a segment, in the range of a specific chunkSize. Not that
* while the chunkSize is specified in this case, it's likely derived from the invocations of
* other loading methods.
*/
private byte[] loadIntern(final ChunkCacheKey key) throws IOException {
final String fileName = key.getFileName();
final long chunkId = key.getChunkId(); // needs to be long to upcast following operations
int bufferSize = key.getBufferSize();
final long seekTo = chunkId * bufferSize;
final byte[] buffer;
final IndexInput input = directory.openInput(fileName, IOContext.READ);
final long length = input.length();
try {
if (seekTo != 0) {
input.seek(seekTo);
}
bufferSize = (int) Math.min(length - seekTo, (long) bufferSize);
buffer = new byte[bufferSize];
input.readBytes(buffer, 0, bufferSize);
} finally {
input.close();
}
return buffer;
}
@Test
public void testVerifyingIndexOutput() throws IOException {
Directory dir = newDirectory();
IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT);
int iters = scaledRandomIntBetween(10, 100);
for (int i = 0; i < iters; i++) {
BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
}
CodecUtil.writeFooter(output);
output.close();
IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT);
String checksum = Store.digestToString(CodecUtil.retrieveChecksum(indexInput));
indexInput.seek(0);
BytesRef ref = new BytesRef(scaledRandomIntBetween(1, 1024));
long length = indexInput.length();
IndexOutput verifyingOutput =
new Store.LuceneVerifyingIndexOutput(
new StoreFileMetaData("foo1.bar", length, checksum),
dir.createOutput("foo1.bar", IOContext.DEFAULT));
while (length > 0) {
if (random().nextInt(10) == 0) {
verifyingOutput.writeByte(indexInput.readByte());
length--;
} else {
int min = (int) Math.min(length, ref.bytes.length);
indexInput.readBytes(ref.bytes, ref.offset, min);
verifyingOutput.writeBytes(ref.bytes, ref.offset, min);
length -= min;
}
}
Store.verify(verifyingOutput);
verifyingOutput.writeByte((byte) 0x0);
try {
Store.verify(verifyingOutput);
fail("should be a corrupted index");
} catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
// ok
}
IOUtils.close(indexInput, verifyingOutput, dir);
}
/**
* closes temporary file, compresses data and removes temporary file.
*
* @throws IOException
*/
@Override
public void close() throws IOException {
byte[] buffer = new byte[chunkSize];
tempOut.close();
// directory with offsets offsets of compressed chunks with
// real position in decompressed stream
IndexInput in = tempDirectory.openInput(tmpName);
long len = closeLength = in.length();
// write length of the file at the begining for easier retreval
output.writeLong(-1);
// write configuration
writeConfig();
int toRead;
// read all data and compresse it in variable block chunks
while (len > 0) {
if (len > buffer.length) {
toRead = buffer.length;
} else {
toRead = (int) len;
}
// just for safety --- can be improoved
long bufferPos = in.getFilePointer();
// read original data
in.readBytes(buffer, 0, toRead);
writeChunk(buffer, bufferPos, toRead);
len -= toRead;
}
// now let's crate directory entry of all chunks and their's original
// position in inflated stream
in.close();
if (tempDirectory.fileExists(tmpName)) {
tempDirectory.deleteFile(tmpName);
}
super.close();
}
public void multipleFlushTest() throws IOException {
final String filename = "longFile.writtenInMultipleFlushes";
final int bufferSize = 300;
Cache cache = cacheManager.getCache();
cache.clear();
InfinispanDirectory dir = new InfinispanDirectory(cache, cache, cache, INDEXNAME, 13);
byte[] manyBytes = fillBytes(bufferSize);
IndexOutput indexOutput = dir.createOutput(filename);
for (int i = 0; i < 10; i++) {
indexOutput.writeBytes(manyBytes, bufferSize);
indexOutput.flush();
}
indexOutput.close();
IndexInput input = dir.openInput(filename);
final int finalSize = (10 * bufferSize);
assert input.length() == finalSize;
final byte[] resultingBuffer = new byte[finalSize];
input.readBytes(resultingBuffer, 0, finalSize);
int index = 0;
for (int i = 0; i < 10; i++) {
for (int j = 0; j < bufferSize; j++) assert resultingBuffer[index++] == manyBytes[j];
}
}
@Test(enabled = false)
public void testReadChunks() throws Exception {
final int BUFFER_SIZE = 64;
Cache cache = cacheManager.getCache();
Directory dir =
DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
.chunkSize(BUFFER_SIZE)
.create();
// create file headers
FileMetadata file1 = new FileMetadata(5);
FileCacheKey key1 = new FileCacheKey(INDEXNAME, "Hello.txt");
cache.put(key1, file1);
FileMetadata file2 = new FileMetadata(5);
FileCacheKey key2 = new FileCacheKey(INDEXNAME, "World.txt");
cache.put(key2, file2);
// byte array for Hello.txt
String helloText = "Hello world. This is some text.";
cache.put(new ChunkCacheKey(INDEXNAME, "Hello.txt", 0, BUFFER_SIZE), helloText.getBytes());
// byte array for World.txt - should be in at least 2 chunks.
String worldText =
"This String should contain more than sixty four characters but less than one hundred and twenty eight.";
assert worldText.getBytes().length > BUFFER_SIZE;
assert worldText.getBytes().length < (2 * BUFFER_SIZE);
byte[] buf = new byte[BUFFER_SIZE];
System.arraycopy(worldText.getBytes(), 0, buf, 0, BUFFER_SIZE);
cache.put(new ChunkCacheKey(INDEXNAME, "World.txt", 0, BUFFER_SIZE), buf);
String part1 = new String(buf);
buf = new byte[BUFFER_SIZE];
System.arraycopy(worldText.getBytes(), BUFFER_SIZE, buf, 0, worldText.length() - BUFFER_SIZE);
cache.put(new ChunkCacheKey(INDEXNAME, "World.txt", 1, BUFFER_SIZE), buf);
String part2 = new String(buf);
// make sure the generated bytes do add up!
AssertJUnit.assertEquals(part1 + part2.trim(), worldText);
file1.setSize(helloText.length());
file2.setSize(worldText.length());
Set<String> s = new HashSet<String>();
s.add("Hello.txt");
s.add("World.txt");
Set other = new HashSet(Arrays.asList(dir.listAll()));
// ok, file listing works.
AssertJUnit.assertEquals(s, other);
IndexInput ii = dir.openInput("Hello.txt");
assert ii.length() == helloText.length();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (int i = 0; i < ii.length(); i++) {
baos.write(ii.readByte());
}
assert new String(baos.toByteArray()).equals(helloText);
ii = dir.openInput("World.txt");
assert ii.length() == worldText.length();
baos = new ByteArrayOutputStream();
for (int i = 0; i < ii.length(); i++) {
baos.write(ii.readByte());
}
assert new String(baos.toByteArray()).equals(worldText);
// now with buffered reading
ii = dir.openInput("Hello.txt");
assert ii.length() == helloText.length();
baos = new ByteArrayOutputStream();
long toRead = ii.length();
while (toRead > 0) {
buf = new byte[19]; // suitably arbitrary
int bytesRead = (int) Math.min(toRead, 19);
ii.readBytes(buf, 0, bytesRead);
toRead = toRead - bytesRead;
baos.write(buf, 0, bytesRead);
}
assert new String(baos.toByteArray()).equals(helloText);
ii = dir.openInput("World.txt");
assert ii.length() == worldText.length();
baos = new ByteArrayOutputStream();
toRead = ii.length();
while (toRead > 0) {
buf = new byte[19]; // suitably arbitrary
int bytesRead = (int) Math.min(toRead, 19);
ii.readBytes(buf, 0, bytesRead);
toRead = toRead - bytesRead;
baos.write(buf, 0, bytesRead);
}
assert new String(baos.toByteArray()).equals(worldText);
dir.deleteFile("Hello.txt");
assert null == cache.get(new FileCacheKey(INDEXNAME, "Hello.txt"));
assert null == cache.get(new ChunkCacheKey(INDEXNAME, "Hello.txt", 0, BUFFER_SIZE));
Object ob1 = cache.get(new FileCacheKey(INDEXNAME, "World.txt"));
Object ob2 = cache.get(new ChunkCacheKey(INDEXNAME, "World.txt", 0, BUFFER_SIZE));
Object ob3 = cache.get(new ChunkCacheKey(INDEXNAME, "World.txt", 1, BUFFER_SIZE));
((DirectoryExtensions) dir).renameFile("World.txt", "HelloWorld.txt");
assert null == cache.get(new FileCacheKey(INDEXNAME, "Hello.txt"));
assert null == cache.get(new ChunkCacheKey(INDEXNAME, "Hello.txt", 0, BUFFER_SIZE));
assert null == cache.get(new ChunkCacheKey(INDEXNAME, "Hello.txt", 1, BUFFER_SIZE));
assert cache.get(new FileCacheKey(INDEXNAME, "HelloWorld.txt")).equals(ob1);
assert cache.get(new ChunkCacheKey(INDEXNAME, "HelloWorld.txt", 0, BUFFER_SIZE)).equals(ob2);
assert cache.get(new ChunkCacheKey(INDEXNAME, "HelloWorld.txt", 1, BUFFER_SIZE)).equals(ob3);
// test that contents survives a move
ii = dir.openInput("HelloWorld.txt");
assert ii.length() == worldText.length();
baos = new ByteArrayOutputStream();
toRead = ii.length();
while (toRead > 0) {
buf = new byte[19]; // suitably arbitrary
int bytesRead = (int) Math.min(toRead, 19);
ii.readBytes(buf, 0, bytesRead);
toRead = toRead - bytesRead;
baos.write(buf, 0, bytesRead);
}
assert new String(baos.toByteArray()).equals(worldText);
dir.close();
DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
}
public boolean hasNext() {
return input.getFilePointer() < input.length();
}
/** Seek {@code input} to the directory offset. */
private void seekDir(IndexInput input, long dirOffset) throws IOException {
input.seek(input.length() - CodecUtil.footerLength() - 8);
dirOffset = input.readLong();
input.seek(dirOffset);
}
public void testCheckIntegrity() throws IOException {
Directory dir = newDirectory();
long luceneFileLength = 0;
try (IndexOutput output = dir.createOutput("lucene_checksum.bin", IOContext.DEFAULT)) {
int iters = scaledRandomIntBetween(10, 100);
for (int i = 0; i < iters; i++) {
BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
luceneFileLength += bytesRef.length;
}
CodecUtil.writeFooter(output);
luceneFileLength += CodecUtil.footerLength();
}
final Adler32 adler32 = new Adler32();
long legacyFileLength = 0;
try (IndexOutput output = dir.createOutput("legacy.bin", IOContext.DEFAULT)) {
int iters = scaledRandomIntBetween(10, 100);
for (int i = 0; i < iters; i++) {
BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
adler32.update(bytesRef.bytes, bytesRef.offset, bytesRef.length);
legacyFileLength += bytesRef.length;
}
}
final long luceneChecksum;
final long adler32LegacyChecksum = adler32.getValue();
try (IndexInput indexInput = dir.openInput("lucene_checksum.bin", IOContext.DEFAULT)) {
assertEquals(luceneFileLength, indexInput.length());
luceneChecksum = CodecUtil.retrieveChecksum(indexInput);
}
{ // positive check
StoreFileMetaData lucene =
new StoreFileMetaData(
"lucene_checksum.bin",
luceneFileLength,
Store.digestToString(luceneChecksum),
Version.LUCENE_4_8_0);
StoreFileMetaData legacy =
new StoreFileMetaData(
"legacy.bin", legacyFileLength, Store.digestToString(adler32LegacyChecksum));
assertTrue(legacy.hasLegacyChecksum());
assertFalse(lucene.hasLegacyChecksum());
assertTrue(Store.checkIntegrityNoException(lucene, dir));
assertTrue(Store.checkIntegrityNoException(legacy, dir));
}
{ // negative check - wrong checksum
StoreFileMetaData lucene =
new StoreFileMetaData(
"lucene_checksum.bin",
luceneFileLength,
Store.digestToString(luceneChecksum + 1),
Version.LUCENE_4_8_0);
StoreFileMetaData legacy =
new StoreFileMetaData(
"legacy.bin", legacyFileLength, Store.digestToString(adler32LegacyChecksum + 1));
assertTrue(legacy.hasLegacyChecksum());
assertFalse(lucene.hasLegacyChecksum());
assertFalse(Store.checkIntegrityNoException(lucene, dir));
assertFalse(Store.checkIntegrityNoException(legacy, dir));
}
{ // negative check - wrong length
StoreFileMetaData lucene =
new StoreFileMetaData(
"lucene_checksum.bin",
luceneFileLength + 1,
Store.digestToString(luceneChecksum),
Version.LUCENE_4_8_0);
StoreFileMetaData legacy =
new StoreFileMetaData(
"legacy.bin", legacyFileLength + 1, Store.digestToString(adler32LegacyChecksum));
assertTrue(legacy.hasLegacyChecksum());
assertFalse(lucene.hasLegacyChecksum());
assertFalse(Store.checkIntegrityNoException(lucene, dir));
assertFalse(Store.checkIntegrityNoException(legacy, dir));
}
{ // negative check - wrong file
StoreFileMetaData lucene =
new StoreFileMetaData(
"legacy.bin",
luceneFileLength,
Store.digestToString(luceneChecksum),
Version.LUCENE_4_8_0);
StoreFileMetaData legacy =
new StoreFileMetaData(
"lucene_checksum.bin", legacyFileLength, Store.digestToString(adler32LegacyChecksum));
assertTrue(legacy.hasLegacyChecksum());
assertFalse(lucene.hasLegacyChecksum());
assertFalse(Store.checkIntegrityNoException(lucene, dir));
assertFalse(Store.checkIntegrityNoException(legacy, dir));
}
dir.close();
}
@Override
public Directory getCompoundReader(Directory dir, SegmentInfo si, IOContext context)
throws IOException {
String dataFile = IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION);
final IndexInput in = dir.openInput(dataFile, context);
BytesRefBuilder scratch = new BytesRefBuilder();
// first get to TOC:
DecimalFormat df =
new DecimalFormat(OFFSETPATTERN, DecimalFormatSymbols.getInstance(Locale.ROOT));
long pos = in.length() - TABLEPOS.length - OFFSETPATTERN.length() - 1;
in.seek(pos);
SimpleTextUtil.readLine(in, scratch);
assert StringHelper.startsWith(scratch.get(), TABLEPOS);
long tablePos = -1;
try {
tablePos = df.parse(stripPrefix(scratch, TABLEPOS)).longValue();
} catch (ParseException e) {
throw new CorruptIndexException(
"can't parse CFS trailer, got: " + scratch.get().utf8ToString(), in);
}
// seek to TOC and read it
in.seek(tablePos);
SimpleTextUtil.readLine(in, scratch);
assert StringHelper.startsWith(scratch.get(), TABLE);
int numEntries = Integer.parseInt(stripPrefix(scratch, TABLE));
final String fileNames[] = new String[numEntries];
final long startOffsets[] = new long[numEntries];
final long endOffsets[] = new long[numEntries];
for (int i = 0; i < numEntries; i++) {
SimpleTextUtil.readLine(in, scratch);
assert StringHelper.startsWith(scratch.get(), TABLENAME);
fileNames[i] = si.name + IndexFileNames.stripSegmentName(stripPrefix(scratch, TABLENAME));
if (i > 0) {
// files must be unique and in sorted order
assert fileNames[i].compareTo(fileNames[i - 1]) > 0;
}
SimpleTextUtil.readLine(in, scratch);
assert StringHelper.startsWith(scratch.get(), TABLESTART);
startOffsets[i] = Long.parseLong(stripPrefix(scratch, TABLESTART));
SimpleTextUtil.readLine(in, scratch);
assert StringHelper.startsWith(scratch.get(), TABLEEND);
endOffsets[i] = Long.parseLong(stripPrefix(scratch, TABLEEND));
}
return new Directory() {
private int getIndex(String name) throws IOException {
int index = Arrays.binarySearch(fileNames, name);
if (index < 0) {
throw new FileNotFoundException(
"No sub-file found (fileName="
+ name
+ " files: "
+ Arrays.toString(fileNames)
+ ")");
}
return index;
}
@Override
public String[] listAll() throws IOException {
ensureOpen();
return fileNames.clone();
}
@Override
public long fileLength(String name) throws IOException {
ensureOpen();
int index = getIndex(name);
return endOffsets[index] - startOffsets[index];
}
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
int index = getIndex(name);
return in.slice(name, startOffsets[index], endOffsets[index] - startOffsets[index]);
}
@Override
public void close() throws IOException {
in.close();
}
// write methods: disabled
@Override
public IndexOutput createOutput(String name, IOContext context) {
throw new UnsupportedOperationException();
}
@Override
public void sync(Collection<String> names) {
throw new UnsupportedOperationException();
}
@Override
public void deleteFile(String name) {
throw new UnsupportedOperationException();
}
@Override
public void renameFile(String source, String dest) {
throw new UnsupportedOperationException();
}
@Override
public Lock makeLock(String name) {
throw new UnsupportedOperationException();
}
};
}
@Override
public long length() {
return delegate.length();
}
private boolean doRestore() throws Exception {
Path backupPath = Paths.get(backupLocation).resolve(backupName);
SimpleDateFormat dateFormat = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT);
String restoreIndexName = "restore." + dateFormat.format(new Date());
String restoreIndexPath = core.getDataDir() + restoreIndexName;
Directory restoreIndexDir = null;
Directory indexDir = null;
try (Directory backupDir = FSDirectory.open(backupPath)) {
final Version version =
IndexFetcher.checkOldestVersion(SegmentInfos.readLatestCommit(backupDir));
restoreIndexDir =
core.getDirectoryFactory()
.get(
restoreIndexPath,
DirectoryFactory.DirContext.DEFAULT,
core.getSolrConfig().indexConfig.lockType);
// Prefer local copy.
indexDir =
core.getDirectoryFactory()
.get(
core.getIndexDir(),
DirectoryFactory.DirContext.DEFAULT,
core.getSolrConfig().indexConfig.lockType);
// Move all files from backupDir to restoreIndexDir
for (String filename : backupDir.listAll()) {
checkInterrupted();
log.info("Copying file {} to restore directory ", filename);
try (IndexInput indexInput = backupDir.openInput(filename, IOContext.READONCE)) {
Long checksum = null;
try {
checksum = CodecUtil.retrieveChecksum(indexInput);
} catch (Exception e) {
log.warn("Could not read checksum from index file: " + filename, e);
}
long length = indexInput.length();
IndexFetcher.CompareResult compareResult =
IndexFetcher.compareFile(indexDir, version, filename, length, checksum);
if (!compareResult.equal
|| (!compareResult.checkSummed
&& (filename.endsWith(".si")
|| filename.endsWith(".liv")
|| filename.startsWith("segments_")))) {
restoreIndexDir.copyFrom(backupDir, filename, filename, IOContext.READONCE);
} else {
// prefer local copy
restoreIndexDir.copyFrom(indexDir, filename, filename, IOContext.READONCE);
}
} catch (Exception e) {
throw new SolrException(
SolrException.ErrorCode.UNKNOWN, "Exception while restoring the backup index", e);
}
}
log.debug("Switching directories");
IndexFetcher.modifyIndexProps(core, restoreIndexName);
boolean success;
try {
core.getUpdateHandler().newIndexWriter(false);
openNewSearcher();
success = true;
log.info("Successfully restored to the backup index");
} catch (Exception e) {
// Rollback to the old index directory. Delete the restore index directory and mark the
// restore as failed.
log.warn("Could not switch to restored index. Rolling back to the current index");
Directory dir = null;
try {
dir =
core.getDirectoryFactory()
.get(
core.getDataDir(),
DirectoryFactory.DirContext.META_DATA,
core.getSolrConfig().indexConfig.lockType);
dir.deleteFile(IndexFetcher.INDEX_PROPERTIES);
} finally {
if (dir != null) {
core.getDirectoryFactory().release(dir);
}
}
core.getDirectoryFactory().doneWithDirectory(restoreIndexDir);
core.getDirectoryFactory().remove(restoreIndexDir);
core.getUpdateHandler().newIndexWriter(false);
openNewSearcher();
throw new SolrException(
SolrException.ErrorCode.UNKNOWN, "Exception while restoring the backup index", e);
}
if (success) {
core.getDirectoryFactory().doneWithDirectory(indexDir);
core.getDirectoryFactory().remove(indexDir);
}
return true;
} finally {
if (restoreIndexDir != null) {
core.getDirectoryFactory().release(restoreIndexDir);
}
if (indexDir != null) {
core.getDirectoryFactory().release(indexDir);
}
}
}
