/** * Closes the {@code segment} and deletes the backing file. * * @param segment the {@link LogSegment} that needs to be closed and whose backing file needs to * be deleted. * @throws IOException */ private void closeSegmentAndDeleteFile(LogSegment segment) throws IOException { segment.close(); assertFalse("File channel is not closed", segment.getView().getSecond().isOpen()); File segmentFile = new File(tempDir, segment.getName()); assertTrue( "The segment file [" + segmentFile.getAbsolutePath() + "] could not be deleted", segmentFile.delete()); }
private void testRecoveryWithBadMessageSize(List<Record> records, int size) throws IOException { LogSegment seg = this.store.log().lastSegment(); writeToOffset(seg.file(), seg.file().length(), ByteBuffer.allocate(4).putInt(size).array()); // now add some message bytes, but not enough if (size > 0) writeToOffset( seg.file(), seg.file().length(), ByteBuffer.allocate(Math.min(size - 1, 256)).array()); this.store.close(); this.store = new HashStore(config); assertEquals("Same records should be present after close and re-open", records, records(store)); }
/** * Using the given {@code appender}'s {@link Appender#append(LogSegment, ByteBuffer)} function, * tests for various cases for append operations. * * @param appender the {@link Appender} to use * @throws IOException */ private void doAppendTest(Appender appender) throws IOException { String currSegmentName = "log_current"; LogSegment segment = getSegment(currSegmentName, STANDARD_SEGMENT_SIZE, true); try { long writeStartOffset = segment.getStartOffset(); byte[] bufOne = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 2); byte[] bufTwo = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 3); appender.append(segment, ByteBuffer.wrap(bufOne)); assertEquals( "End offset is not as expected", writeStartOffset + bufOne.length, segment.getEndOffset()); appender.append(segment, ByteBuffer.wrap(bufTwo)); assertEquals( "End offset is not as expected", writeStartOffset + bufOne.length + bufTwo.length, segment.getEndOffset()); // try to do a write that won't fit ByteBuffer failBuf = ByteBuffer.wrap( TestUtils.getRandomBytes((int) (STANDARD_SEGMENT_SIZE - writeStartOffset + 1))); long writeOverFlowCount = metrics.overflowWriteError.getCount(); try { appender.append(segment, failBuf); fail("Append should have failed because data won't fit in the segment"); } catch (IllegalArgumentException e) { assertEquals( "Write overflow should have been reported", writeOverFlowCount + 1, metrics.overflowWriteError.getCount()); assertEquals("Position of buffer has changed", 0, failBuf.position()); } // read and ensure data matches readAndEnsureMatch(segment, writeStartOffset, bufOne); readAndEnsureMatch(segment, writeStartOffset + bufOne.length, bufTwo); segment.close(); // ensure that append fails. ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes(1)); try { appender.append(segment, buffer); fail("Append should have failed because segments are closed"); } catch (ClosedChannelException e) { assertEquals("Position of buffer has changed", 0, buffer.position()); } } finally { closeSegmentAndDeleteFile(segment); } }
@Test public void testRecoveryTotalCorruption() throws IOException { List<Record> records = StoreTestUtils.randomRecords(10); putAll(records); // mangle log file LogSegment seg = this.store.log().segmentFor(0); writeToOffset(seg.file(), 0, "Hayduke lives!".getBytes()); this.store.close(); this.store = new HashStore(config); if (!config.saveIndexOnClose()) assertEquals( "No records should be present after mangling", Collections.emptyList(), records(store)); }
private void writeHeader(LogSegment segment, byte[] buf) throws IOException { FileChannel channel = segment.getView().getSecond(); ByteBuffer buffer = ByteBuffer.wrap(buf); while (buffer.hasRemaining()) { channel.write(buffer, 0); } }
/** * Gets a view of the given {@code segment} and verifies the ref count and the data obtained from * the view against {@code expectedRefCount} and {@code dataInSegment} respectively. * * @param segment the {@link LogSegment} to get a view from. * @param writeStartOffset the offset at which write was started on the segment. * @param offset the offset for which a view is required. * @param dataInSegment the entire data in the {@link LogSegment}. * @param expectedRefCount the expected return value of {@link LogSegment#refCount()} once the * view is obtained from the {@code segment} * @throws IOException */ private void getAndVerifyView( LogSegment segment, long writeStartOffset, int offset, byte[] dataInSegment, long expectedRefCount) throws IOException { Random random = new Random(); Pair<File, FileChannel> view = segment.getView(); assertNotNull("File object received in view is null", view.getFirst()); assertNotNull("FileChannel object received in view is null", view.getSecond()); assertEquals("Ref count is not as expected", expectedRefCount, segment.refCount()); int sizeToRead = random.nextInt(dataInSegment.length - offset + 1); ByteBuffer buffer = ByteBuffer.wrap(new byte[sizeToRead]); view.getSecond().read(buffer, writeStartOffset + offset); assertArrayEquals( "Data read from file does not match data written", Arrays.copyOfRange(dataInSegment, offset, offset + sizeToRead), buffer.array()); }
/** * Verifies getting and closing views and makes sure that data and ref counts are consistent. * * @throws IOException */ @Test public void viewAndRefCountTest() throws IOException { String segmentName = "log_current"; LogSegment segment = getSegment(segmentName, STANDARD_SEGMENT_SIZE, true); try { long startOffset = segment.getStartOffset(); int readSize = 100; int viewCount = 5; byte[] data = appendRandomData(segment, readSize * viewCount); for (int i = 0; i < viewCount; i++) { getAndVerifyView(segment, startOffset, i * readSize, data, i + 1); } for (int i = 0; i < viewCount; i++) { segment.closeView(); assertEquals("Ref count is not as expected", viewCount - i - 1, segment.refCount()); } } finally { closeSegmentAndDeleteFile(segment); } }
/** * Tests setting end offset - makes sure legal values are set correctly and illegal values are * rejected. * * @throws IOException */ @Test public void endOffsetTest() throws IOException { String segmentName = "log_current"; LogSegment segment = getSegment(segmentName, STANDARD_SEGMENT_SIZE, true); try { long writeStartOffset = segment.getStartOffset(); int segmentSize = 500; appendRandomData(segment, segmentSize); assertEquals( "End offset is not as expected", writeStartOffset + segmentSize, segment.getEndOffset()); // should be able to set end offset to something >= initial offset and <= file size int[] offsetsToSet = {(int) (writeStartOffset), segmentSize / 2, segmentSize}; for (int offset : offsetsToSet) { segment.setEndOffset(offset); assertEquals("End offset is not equal to what was set", offset, segment.getEndOffset()); assertEquals( "File channel positioning is incorrect", offset, segment.getView().getSecond().position()); } // cannot set end offset to illegal values (< initial offset or > file size) int[] invalidOffsets = {(int) (writeStartOffset - 1), (int) (segment.sizeInBytes() + 1)}; for (int offset : invalidOffsets) { try { segment.setEndOffset(offset); fail("Setting log end offset an invalid offset [" + offset + "] should have failed"); } catch (IllegalArgumentException e) { // expected. Nothing to do. } } } finally { closeSegmentAndDeleteFile(segment); } }
/** * Tests appending and reading to make sure data is written and the data read is consistent with * the data written. * * @throws IOException */ @Test public void basicWriteAndReadTest() throws IOException { String segmentName = "log_current"; LogSegment segment = getSegment(segmentName, STANDARD_SEGMENT_SIZE, true); try { assertEquals( "Name of segment is inconsistent with what was provided", segmentName, segment.getName()); assertEquals( "Capacity of segment is inconsistent with what was provided", STANDARD_SEGMENT_SIZE, segment.getCapacityInBytes()); assertEquals( "Start offset is not equal to header size", LogSegment.HEADER_SIZE, segment.getStartOffset()); int writeSize = 100; byte[] buf = TestUtils.getRandomBytes(3 * writeSize); long writeStartOffset = segment.getStartOffset(); // append with buffer int written = segment.appendFrom(ByteBuffer.wrap(buf, 0, writeSize)); assertEquals("Size written did not match size of buffer provided", writeSize, written); assertEquals( "End offset is not as expected", writeStartOffset + writeSize, segment.getEndOffset()); readAndEnsureMatch(segment, writeStartOffset, Arrays.copyOfRange(buf, 0, writeSize)); // append with channel segment.appendFrom( Channels.newChannel( new ByteBufferInputStream(ByteBuffer.wrap(buf, writeSize, writeSize))), writeSize); assertEquals( "End offset is not as expected", writeStartOffset + 2 * writeSize, segment.getEndOffset()); readAndEnsureMatch( segment, writeStartOffset + writeSize, Arrays.copyOfRange(buf, writeSize, 2 * writeSize)); // use writeFrom segment.writeFrom( Channels.newChannel( new ByteBufferInputStream(ByteBuffer.wrap(buf, 2 * writeSize, writeSize))), segment.getEndOffset(), writeSize); assertEquals( "End offset is not as expected", writeStartOffset + 3 * writeSize, segment.getEndOffset()); readAndEnsureMatch( segment, writeStartOffset + 2 * writeSize, Arrays.copyOfRange(buf, 2 * writeSize, buf.length)); readAndEnsureMatch(segment, writeStartOffset, buf); // check file size and end offset (they will not match) assertEquals( "End offset is not equal to the cumulative bytes written", writeStartOffset + 3 * writeSize, segment.getEndOffset()); assertEquals( "Size in bytes is not equal to size of the file", STANDARD_SEGMENT_SIZE, segment.sizeInBytes()); // ensure flush doesn't throw any errors. segment.flush(); // close and reopen segment and ensure persistence. segment.close(); segment = new LogSegment(segmentName, new File(tempDir, segmentName), metrics); segment.setEndOffset(writeStartOffset + buf.length); readAndEnsureMatch(segment, writeStartOffset, buf); } finally { closeSegmentAndDeleteFile(segment); } }
/** * Reads data starting from {@code offsetToStartRead} of {@code segment} and matches it with * {@code original}. * * @param segment the {@link LogSegment} to read from. * @param offsetToStartRead the offset in {@code segment} to start reading from. * @param original the byte array to compare against. * @throws IOException */ private void readAndEnsureMatch(LogSegment segment, long offsetToStartRead, byte[] original) throws IOException { ByteBuffer readBuf = ByteBuffer.wrap(new byte[original.length]); segment.readInto(readBuf, offsetToStartRead); assertArrayEquals("Data read does not match data written", original, readBuf.array()); }
/** * Appends random data of size {@code size} to given {@code segment}. * * @param segment the {@link LogSegment} to append data to. * @param size the size of the data that should be appended. * @return the data that was appended. * @throws IOException */ private byte[] appendRandomData(LogSegment segment, int size) throws IOException { byte[] buf = TestUtils.getRandomBytes(size); segment.appendFrom(ByteBuffer.wrap(buf)); return buf; }
private byte[] getHeader(LogSegment segment) throws IOException { FileChannel channel = segment.getView().getSecond(); ByteBuffer header = ByteBuffer.allocate(LogSegment.HEADER_SIZE); channel.read(header, 0); return header.array(); }
/** * Tests for bad construction cases of {@link LogSegment}. * * @throws IOException */ @Test public void badConstructionTest() throws IOException { // try to construct with a file that does not exist. String name = "log_non_existent"; File file = new File(tempDir, name); try { new LogSegment(name, file, STANDARD_SEGMENT_SIZE, metrics, true); fail("Construction should have failed because the backing file does not exist"); } catch (IllegalArgumentException e) { // expected. Nothing to do. } try { new LogSegment(name, file, metrics); fail("Construction should have failed because the backing file does not exist"); } catch (IllegalArgumentException e) { // expected. Nothing to do. } // try to construct with a file that is a directory name = tempDir.getName(); file = new File(tempDir.getParent(), name); try { new LogSegment(name, file, STANDARD_SEGMENT_SIZE, metrics, true); fail("Construction should have failed because the backing file does not exist"); } catch (IllegalArgumentException e) { // expected. Nothing to do. } name = tempDir.getName(); file = new File(tempDir.getParent(), name); try { new LogSegment(name, file, metrics); fail("Construction should have failed because the backing file does not exist"); } catch (IllegalArgumentException e) { // expected. Nothing to do. } // unknown version LogSegment segment = getSegment("dummy_log", STANDARD_SEGMENT_SIZE, true); file = segment.getView().getFirst(); byte[] header = getHeader(segment); byte savedByte = header[0]; // mess with version header[0] = (byte) (header[0] + 10); writeHeader(segment, header); try { new LogSegment(name, file, metrics); fail("Construction should have failed because version is unknown"); } catch (IllegalArgumentException e) { // expected. Nothing to do. } // bad CRC // fix version but mess with data after version header[0] = savedByte; header[2] = header[2] == (byte) 1 ? (byte) 0 : (byte) 1; writeHeader(segment, header); try { new LogSegment(name, file, metrics); fail("Construction should have failed because crc check should have failed"); } catch (IllegalStateException e) { // expected. Nothing to do. } closeSegmentAndDeleteFile(segment); }
/** * Tests for special constructor cases. * * @throws IOException */ @Test public void constructorTest() throws IOException { LogSegment segment = getSegment("log_current", STANDARD_SEGMENT_SIZE, false); assertEquals( "Start offset should be 0 when no headers are written", 0, segment.getStartOffset()); }
/** * Tests {@link LogSegment#writeFrom(ReadableByteChannel, long, long)} for various cases. * * @throws IOException */ @Test public void writeFromTest() throws IOException { String currSegmentName = "log_current"; LogSegment segment = getSegment(currSegmentName, STANDARD_SEGMENT_SIZE, true); try { long writeStartOffset = segment.getStartOffset(); byte[] bufOne = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 3); byte[] bufTwo = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 2); segment.writeFrom( Channels.newChannel(new ByteBufferInputStream(ByteBuffer.wrap(bufOne))), writeStartOffset, bufOne.length); assertEquals( "End offset is not as expected", writeStartOffset + bufOne.length, segment.getEndOffset()); readAndEnsureMatch(segment, writeStartOffset, bufOne); // overwrite using bufTwo segment.writeFrom( Channels.newChannel(new ByteBufferInputStream(ByteBuffer.wrap(bufTwo))), writeStartOffset, bufTwo.length); assertEquals( "End offset is not as expected", writeStartOffset + bufTwo.length, segment.getEndOffset()); readAndEnsureMatch(segment, writeStartOffset, bufTwo); // overwrite using bufOne segment.writeFrom( Channels.newChannel(new ByteBufferInputStream(ByteBuffer.wrap(bufOne))), writeStartOffset, bufOne.length); // end offset should not have changed assertEquals( "End offset is not as expected", writeStartOffset + bufTwo.length, segment.getEndOffset()); readAndEnsureMatch(segment, writeStartOffset, bufOne); readAndEnsureMatch( segment, writeStartOffset + bufOne.length, Arrays.copyOfRange(bufTwo, bufOne.length, bufTwo.length)); // write at random locations for (int i = 0; i < 10; i++) { long offset = writeStartOffset + Utils.getRandomLong( TestUtils.RANDOM, segment.getCapacityInBytes() - bufOne.length - writeStartOffset); segment.writeFrom( Channels.newChannel(new ByteBufferInputStream(ByteBuffer.wrap(bufOne))), offset, bufOne.length); readAndEnsureMatch(segment, offset, bufOne); } // try to overwrite using a channel that won't fit ByteBuffer failBuf = ByteBuffer.wrap( TestUtils.getRandomBytes((int) (STANDARD_SEGMENT_SIZE - writeStartOffset + 1))); long writeOverFlowCount = metrics.overflowWriteError.getCount(); try { segment.writeFrom( Channels.newChannel(new ByteBufferInputStream(failBuf)), writeStartOffset, failBuf.remaining()); fail("WriteFrom should have failed because data won't fit"); } catch (IndexOutOfBoundsException e) { assertEquals( "Write overflow should have been reported", writeOverFlowCount + 1, metrics.overflowWriteError.getCount()); assertEquals("Position of buffer has changed", 0, failBuf.position()); } // data cannot be written at invalid offsets. long[] invalidOffsets = { writeStartOffset - 1, STANDARD_SEGMENT_SIZE, STANDARD_SEGMENT_SIZE + 1 }; ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes(1)); for (long invalidOffset : invalidOffsets) { try { segment.writeFrom( Channels.newChannel(new ByteBufferInputStream(buffer)), invalidOffset, buffer.remaining()); fail("WriteFrom should have failed because offset provided for write is invalid"); } catch (IndexOutOfBoundsException e) { assertEquals("Position of buffer has changed", 0, buffer.position()); } } segment.close(); // ensure that writeFrom fails. try { segment.writeFrom( Channels.newChannel(new ByteBufferInputStream(buffer)), writeStartOffset, buffer.remaining()); fail("WriteFrom should have failed because segments are closed"); } catch (ClosedChannelException e) { assertEquals("Position of buffer has changed", 0, buffer.position()); } } finally { closeSegmentAndDeleteFile(segment); } }
/** * Tests {@link LogSegment#readInto(ByteBuffer, long)} for various cases. * * @throws IOException */ @Test public void readTest() throws IOException { Random random = new Random(); String segmentName = "log_current"; LogSegment segment = getSegment(segmentName, STANDARD_SEGMENT_SIZE, true); try { long writeStartOffset = segment.getStartOffset(); byte[] data = appendRandomData(segment, 2 * STANDARD_SEGMENT_SIZE / 3); readAndEnsureMatch(segment, writeStartOffset, data); int readCount = 10; for (int i = 0; i < readCount; i++) { int position = random.nextInt(data.length); int size = random.nextInt(data.length - position); readAndEnsureMatch( segment, writeStartOffset + position, Arrays.copyOfRange(data, position, position + size)); } // error scenarios ByteBuffer readBuf = ByteBuffer.wrap(new byte[data.length]); // data cannot be read at invalid offsets. long[] invalidOffsets = { writeStartOffset - 1, segment.getEndOffset(), segment.getEndOffset() + 1 }; ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes(1)); for (long invalidOffset : invalidOffsets) { try { segment.readInto(readBuf, invalidOffset); fail("Should have failed to read because position provided is invalid"); } catch (IndexOutOfBoundsException e) { assertEquals("Position of buffer has changed", 0, buffer.position()); } } // position + buffer.remaining() > endOffset. long readOverFlowCount = metrics.overflowReadError.getCount(); try { segment.readInto(readBuf, writeStartOffset + 1); fail("Should have failed to read because position + buffer.remaining() > endOffset"); } catch (IndexOutOfBoundsException e) { assertEquals( "Read overflow should have been reported", readOverFlowCount + 1, metrics.overflowReadError.getCount()); assertEquals("Position of buffer has changed", 0, readBuf.position()); } segment.close(); // read after close buffer = ByteBuffer.allocate(1); try { segment.readInto(buffer, writeStartOffset); fail("Should have failed to read because segment is closed"); } catch (ClosedChannelException e) { assertEquals("Position of buffer has changed", 0, buffer.position()); } } finally { closeSegmentAndDeleteFile(segment); } }