/**
* Using the given {@code appender}'s {@link Appender#append(LogSegment, ByteBuffer)} function,
* tests for various cases for append operations.
*
* @param appender the {@link Appender} to use
* @throws IOException
*/
private void doAppendTest(Appender appender) throws IOException {
String currSegmentName = "log_current";
LogSegment segment = getSegment(currSegmentName, STANDARD_SEGMENT_SIZE, true);
try {
long writeStartOffset = segment.getStartOffset();
byte[] bufOne = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 2);
byte[] bufTwo = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 3);
appender.append(segment, ByteBuffer.wrap(bufOne));
assertEquals(
"End offset is not as expected",
writeStartOffset + bufOne.length,
segment.getEndOffset());
appender.append(segment, ByteBuffer.wrap(bufTwo));
assertEquals(
"End offset is not as expected",
writeStartOffset + bufOne.length + bufTwo.length,
segment.getEndOffset());
// try to do a write that won't fit
ByteBuffer failBuf =
ByteBuffer.wrap(
TestUtils.getRandomBytes((int) (STANDARD_SEGMENT_SIZE - writeStartOffset + 1)));
long writeOverFlowCount = metrics.overflowWriteError.getCount();
try {
appender.append(segment, failBuf);
fail("Append should have failed because data won't fit in the segment");
} catch (IllegalArgumentException e) {
assertEquals(
"Write overflow should have been reported",
writeOverFlowCount + 1,
metrics.overflowWriteError.getCount());
assertEquals("Position of buffer has changed", 0, failBuf.position());
}
// read and ensure data matches
readAndEnsureMatch(segment, writeStartOffset, bufOne);
readAndEnsureMatch(segment, writeStartOffset + bufOne.length, bufTwo);
segment.close();
// ensure that append fails.
ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes(1));
try {
appender.append(segment, buffer);
fail("Append should have failed because segments are closed");
} catch (ClosedChannelException e) {
assertEquals("Position of buffer has changed", 0, buffer.position());
}
} finally {
closeSegmentAndDeleteFile(segment);
}
}
/**
* Tests appending and reading to make sure data is written and the data read is consistent with
* the data written.
*
* @throws IOException
*/
@Test
public void basicWriteAndReadTest() throws IOException {
String segmentName = "log_current";
LogSegment segment = getSegment(segmentName, STANDARD_SEGMENT_SIZE, true);
try {
assertEquals(
"Name of segment is inconsistent with what was provided", segmentName, segment.getName());
assertEquals(
"Capacity of segment is inconsistent with what was provided",
STANDARD_SEGMENT_SIZE,
segment.getCapacityInBytes());
assertEquals(
"Start offset is not equal to header size",
LogSegment.HEADER_SIZE,
segment.getStartOffset());
int writeSize = 100;
byte[] buf = TestUtils.getRandomBytes(3 * writeSize);
long writeStartOffset = segment.getStartOffset();
// append with buffer
int written = segment.appendFrom(ByteBuffer.wrap(buf, 0, writeSize));
assertEquals("Size written did not match size of buffer provided", writeSize, written);
assertEquals(
"End offset is not as expected", writeStartOffset + writeSize, segment.getEndOffset());
readAndEnsureMatch(segment, writeStartOffset, Arrays.copyOfRange(buf, 0, writeSize));
// append with channel
segment.appendFrom(
Channels.newChannel(
new ByteBufferInputStream(ByteBuffer.wrap(buf, writeSize, writeSize))),
writeSize);
assertEquals(
"End offset is not as expected",
writeStartOffset + 2 * writeSize,
segment.getEndOffset());
readAndEnsureMatch(
segment, writeStartOffset + writeSize, Arrays.copyOfRange(buf, writeSize, 2 * writeSize));
// use writeFrom
segment.writeFrom(
Channels.newChannel(
new ByteBufferInputStream(ByteBuffer.wrap(buf, 2 * writeSize, writeSize))),
segment.getEndOffset(),
writeSize);
assertEquals(
"End offset is not as expected",
writeStartOffset + 3 * writeSize,
segment.getEndOffset());
readAndEnsureMatch(
segment,
writeStartOffset + 2 * writeSize,
Arrays.copyOfRange(buf, 2 * writeSize, buf.length));
readAndEnsureMatch(segment, writeStartOffset, buf);
// check file size and end offset (they will not match)
assertEquals(
"End offset is not equal to the cumulative bytes written",
writeStartOffset + 3 * writeSize,
segment.getEndOffset());
assertEquals(
"Size in bytes is not equal to size of the file",
STANDARD_SEGMENT_SIZE,
segment.sizeInBytes());
// ensure flush doesn't throw any errors.
segment.flush();
// close and reopen segment and ensure persistence.
segment.close();
segment = new LogSegment(segmentName, new File(tempDir, segmentName), metrics);
segment.setEndOffset(writeStartOffset + buf.length);
readAndEnsureMatch(segment, writeStartOffset, buf);
} finally {
closeSegmentAndDeleteFile(segment);
}
}
/**
* Tests {@link LogSegment#writeFrom(ReadableByteChannel, long, long)} for various cases.
*
* @throws IOException
*/
@Test
public void writeFromTest() throws IOException {
String currSegmentName = "log_current";
LogSegment segment = getSegment(currSegmentName, STANDARD_SEGMENT_SIZE, true);
try {
long writeStartOffset = segment.getStartOffset();
byte[] bufOne = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 3);
byte[] bufTwo = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 2);
segment.writeFrom(
Channels.newChannel(new ByteBufferInputStream(ByteBuffer.wrap(bufOne))),
writeStartOffset,
bufOne.length);
assertEquals(
"End offset is not as expected",
writeStartOffset + bufOne.length,
segment.getEndOffset());
readAndEnsureMatch(segment, writeStartOffset, bufOne);
// overwrite using bufTwo
segment.writeFrom(
Channels.newChannel(new ByteBufferInputStream(ByteBuffer.wrap(bufTwo))),
writeStartOffset,
bufTwo.length);
assertEquals(
"End offset is not as expected",
writeStartOffset + bufTwo.length,
segment.getEndOffset());
readAndEnsureMatch(segment, writeStartOffset, bufTwo);
// overwrite using bufOne
segment.writeFrom(
Channels.newChannel(new ByteBufferInputStream(ByteBuffer.wrap(bufOne))),
writeStartOffset,
bufOne.length);
// end offset should not have changed
assertEquals(
"End offset is not as expected",
writeStartOffset + bufTwo.length,
segment.getEndOffset());
readAndEnsureMatch(segment, writeStartOffset, bufOne);
readAndEnsureMatch(
segment,
writeStartOffset + bufOne.length,
Arrays.copyOfRange(bufTwo, bufOne.length, bufTwo.length));
// write at random locations
for (int i = 0; i < 10; i++) {
long offset =
writeStartOffset
+ Utils.getRandomLong(
TestUtils.RANDOM,
segment.getCapacityInBytes() - bufOne.length - writeStartOffset);
segment.writeFrom(
Channels.newChannel(new ByteBufferInputStream(ByteBuffer.wrap(bufOne))),
offset,
bufOne.length);
readAndEnsureMatch(segment, offset, bufOne);
}
// try to overwrite using a channel that won't fit
ByteBuffer failBuf =
ByteBuffer.wrap(
TestUtils.getRandomBytes((int) (STANDARD_SEGMENT_SIZE - writeStartOffset + 1)));
long writeOverFlowCount = metrics.overflowWriteError.getCount();
try {
segment.writeFrom(
Channels.newChannel(new ByteBufferInputStream(failBuf)),
writeStartOffset,
failBuf.remaining());
fail("WriteFrom should have failed because data won't fit");
} catch (IndexOutOfBoundsException e) {
assertEquals(
"Write overflow should have been reported",
writeOverFlowCount + 1,
metrics.overflowWriteError.getCount());
assertEquals("Position of buffer has changed", 0, failBuf.position());
}
// data cannot be written at invalid offsets.
long[] invalidOffsets = {
writeStartOffset - 1, STANDARD_SEGMENT_SIZE, STANDARD_SEGMENT_SIZE + 1
};
ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes(1));
for (long invalidOffset : invalidOffsets) {
try {
segment.writeFrom(
Channels.newChannel(new ByteBufferInputStream(buffer)),
invalidOffset,
buffer.remaining());
fail("WriteFrom should have failed because offset provided for write is invalid");
} catch (IndexOutOfBoundsException e) {
assertEquals("Position of buffer has changed", 0, buffer.position());
}
}
segment.close();
// ensure that writeFrom fails.
try {
segment.writeFrom(
Channels.newChannel(new ByteBufferInputStream(buffer)),
writeStartOffset,
buffer.remaining());
fail("WriteFrom should have failed because segments are closed");
} catch (ClosedChannelException e) {
assertEquals("Position of buffer has changed", 0, buffer.position());
}
} finally {
closeSegmentAndDeleteFile(segment);
}
}
/**
* Appends random data of size {@code size} to given {@code segment}.
*
* @param segment the {@link LogSegment} to append data to.
* @param size the size of the data that should be appended.
* @return the data that was appended.
* @throws IOException
*/
private byte[] appendRandomData(LogSegment segment, int size) throws IOException {
byte[] buf = TestUtils.getRandomBytes(size);
segment.appendFrom(ByteBuffer.wrap(buf));
return buf;
}
/**
* Tests {@link LogSegment#readInto(ByteBuffer, long)} for various cases.
*
* @throws IOException
*/
@Test
public void readTest() throws IOException {
Random random = new Random();
String segmentName = "log_current";
LogSegment segment = getSegment(segmentName, STANDARD_SEGMENT_SIZE, true);
try {
long writeStartOffset = segment.getStartOffset();
byte[] data = appendRandomData(segment, 2 * STANDARD_SEGMENT_SIZE / 3);
readAndEnsureMatch(segment, writeStartOffset, data);
int readCount = 10;
for (int i = 0; i < readCount; i++) {
int position = random.nextInt(data.length);
int size = random.nextInt(data.length - position);
readAndEnsureMatch(
segment,
writeStartOffset + position,
Arrays.copyOfRange(data, position, position + size));
}
// error scenarios
ByteBuffer readBuf = ByteBuffer.wrap(new byte[data.length]);
// data cannot be read at invalid offsets.
long[] invalidOffsets = {
writeStartOffset - 1, segment.getEndOffset(), segment.getEndOffset() + 1
};
ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes(1));
for (long invalidOffset : invalidOffsets) {
try {
segment.readInto(readBuf, invalidOffset);
fail("Should have failed to read because position provided is invalid");
} catch (IndexOutOfBoundsException e) {
assertEquals("Position of buffer has changed", 0, buffer.position());
}
}
// position + buffer.remaining() > endOffset.
long readOverFlowCount = metrics.overflowReadError.getCount();
try {
segment.readInto(readBuf, writeStartOffset + 1);
fail("Should have failed to read because position + buffer.remaining() > endOffset");
} catch (IndexOutOfBoundsException e) {
assertEquals(
"Read overflow should have been reported",
readOverFlowCount + 1,
metrics.overflowReadError.getCount());
assertEquals("Position of buffer has changed", 0, readBuf.position());
}
segment.close();
// read after close
buffer = ByteBuffer.allocate(1);
try {
segment.readInto(buffer, writeStartOffset);
fail("Should have failed to read because segment is closed");
} catch (ClosedChannelException e) {
assertEquals("Position of buffer has changed", 0, buffer.position());
}
} finally {
closeSegmentAndDeleteFile(segment);
}
}