Example #1
0
 /**
  * Closes the {@code segment} and deletes the backing file.
  *
  * @param segment the {@link LogSegment} that needs to be closed and whose backing file needs to
  *     be deleted.
  * @throws IOException
  */
 private void closeSegmentAndDeleteFile(LogSegment segment) throws IOException {
   segment.close();
   assertFalse("File channel is not closed", segment.getView().getSecond().isOpen());
   File segmentFile = new File(tempDir, segment.getName());
   assertTrue(
       "The segment file [" + segmentFile.getAbsolutePath() + "] could not be deleted",
       segmentFile.delete());
 }
Example #2
0
 private void testRecoveryWithBadMessageSize(List<Record> records, int size) throws IOException {
   LogSegment seg = this.store.log().lastSegment();
   writeToOffset(seg.file(), seg.file().length(), ByteBuffer.allocate(4).putInt(size).array());
   // now add some message bytes, but not enough
   if (size > 0)
     writeToOffset(
         seg.file(), seg.file().length(), ByteBuffer.allocate(Math.min(size - 1, 256)).array());
   this.store.close();
   this.store = new HashStore(config);
   assertEquals("Same records should be present after close and re-open", records, records(store));
 }
Example #3
0
  /**
   * Using the given {@code appender}'s {@link Appender#append(LogSegment, ByteBuffer)} function,
   * tests for various cases for append operations.
   *
   * @param appender the {@link Appender} to use
   * @throws IOException
   */
  private void doAppendTest(Appender appender) throws IOException {
    String currSegmentName = "log_current";
    LogSegment segment = getSegment(currSegmentName, STANDARD_SEGMENT_SIZE, true);
    try {
      long writeStartOffset = segment.getStartOffset();
      byte[] bufOne = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 2);
      byte[] bufTwo = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 3);

      appender.append(segment, ByteBuffer.wrap(bufOne));
      assertEquals(
          "End offset is not as expected",
          writeStartOffset + bufOne.length,
          segment.getEndOffset());

      appender.append(segment, ByteBuffer.wrap(bufTwo));
      assertEquals(
          "End offset is not as expected",
          writeStartOffset + bufOne.length + bufTwo.length,
          segment.getEndOffset());

      // try to do a write that won't fit
      ByteBuffer failBuf =
          ByteBuffer.wrap(
              TestUtils.getRandomBytes((int) (STANDARD_SEGMENT_SIZE - writeStartOffset + 1)));
      long writeOverFlowCount = metrics.overflowWriteError.getCount();
      try {
        appender.append(segment, failBuf);
        fail("Append should have failed because data won't fit in the segment");
      } catch (IllegalArgumentException e) {
        assertEquals(
            "Write overflow should have been reported",
            writeOverFlowCount + 1,
            metrics.overflowWriteError.getCount());
        assertEquals("Position of buffer has changed", 0, failBuf.position());
      }

      // read and ensure data matches
      readAndEnsureMatch(segment, writeStartOffset, bufOne);
      readAndEnsureMatch(segment, writeStartOffset + bufOne.length, bufTwo);

      segment.close();
      // ensure that append fails.
      ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes(1));
      try {
        appender.append(segment, buffer);
        fail("Append should have failed because segments are closed");
      } catch (ClosedChannelException e) {
        assertEquals("Position of buffer has changed", 0, buffer.position());
      }
    } finally {
      closeSegmentAndDeleteFile(segment);
    }
  }
Example #4
0
 @Test
 public void testRecoveryTotalCorruption() throws IOException {
   List<Record> records = StoreTestUtils.randomRecords(10);
   putAll(records);
   // mangle log file
   LogSegment seg = this.store.log().segmentFor(0);
   writeToOffset(seg.file(), 0, "Hayduke lives!".getBytes());
   this.store.close();
   this.store = new HashStore(config);
   if (!config.saveIndexOnClose())
     assertEquals(
         "No records should be present after mangling", Collections.emptyList(), records(store));
 }
Example #5
0
 private void writeHeader(LogSegment segment, byte[] buf) throws IOException {
   FileChannel channel = segment.getView().getSecond();
   ByteBuffer buffer = ByteBuffer.wrap(buf);
   while (buffer.hasRemaining()) {
     channel.write(buffer, 0);
   }
 }
Example #6
0
 /**
  * Gets a view of the given {@code segment} and verifies the ref count and the data obtained from
  * the view against {@code expectedRefCount} and {@code dataInSegment} respectively.
  *
  * @param segment the {@link LogSegment} to get a view from.
  * @param writeStartOffset the offset at which write was started on the segment.
  * @param offset the offset for which a view is required.
  * @param dataInSegment the entire data in the {@link LogSegment}.
  * @param expectedRefCount the expected return value of {@link LogSegment#refCount()} once the
  *     view is obtained from the {@code segment}
  * @throws IOException
  */
 private void getAndVerifyView(
     LogSegment segment,
     long writeStartOffset,
     int offset,
     byte[] dataInSegment,
     long expectedRefCount)
     throws IOException {
   Random random = new Random();
   Pair<File, FileChannel> view = segment.getView();
   assertNotNull("File object received in view is null", view.getFirst());
   assertNotNull("FileChannel object received in view is null", view.getSecond());
   assertEquals("Ref count is not as expected", expectedRefCount, segment.refCount());
   int sizeToRead = random.nextInt(dataInSegment.length - offset + 1);
   ByteBuffer buffer = ByteBuffer.wrap(new byte[sizeToRead]);
   view.getSecond().read(buffer, writeStartOffset + offset);
   assertArrayEquals(
       "Data read from file does not match data written",
       Arrays.copyOfRange(dataInSegment, offset, offset + sizeToRead),
       buffer.array());
 }
Example #7
0
  /**
   * Verifies getting and closing views and makes sure that data and ref counts are consistent.
   *
   * @throws IOException
   */
  @Test
  public void viewAndRefCountTest() throws IOException {
    String segmentName = "log_current";
    LogSegment segment = getSegment(segmentName, STANDARD_SEGMENT_SIZE, true);
    try {
      long startOffset = segment.getStartOffset();
      int readSize = 100;
      int viewCount = 5;
      byte[] data = appendRandomData(segment, readSize * viewCount);

      for (int i = 0; i < viewCount; i++) {
        getAndVerifyView(segment, startOffset, i * readSize, data, i + 1);
      }

      for (int i = 0; i < viewCount; i++) {
        segment.closeView();
        assertEquals("Ref count is not as expected", viewCount - i - 1, segment.refCount());
      }
    } finally {
      closeSegmentAndDeleteFile(segment);
    }
  }
Example #8
0
  /**
   * Tests setting end offset - makes sure legal values are set correctly and illegal values are
   * rejected.
   *
   * @throws IOException
   */
  @Test
  public void endOffsetTest() throws IOException {
    String segmentName = "log_current";
    LogSegment segment = getSegment(segmentName, STANDARD_SEGMENT_SIZE, true);
    try {
      long writeStartOffset = segment.getStartOffset();
      int segmentSize = 500;
      appendRandomData(segment, segmentSize);
      assertEquals(
          "End offset is not as expected", writeStartOffset + segmentSize, segment.getEndOffset());

      // should be able to set end offset to something >= initial offset and <= file size
      int[] offsetsToSet = {(int) (writeStartOffset), segmentSize / 2, segmentSize};
      for (int offset : offsetsToSet) {
        segment.setEndOffset(offset);
        assertEquals("End offset is not equal to what was set", offset, segment.getEndOffset());
        assertEquals(
            "File channel positioning is incorrect",
            offset,
            segment.getView().getSecond().position());
      }

      // cannot set end offset to illegal values (< initial offset or > file size)
      int[] invalidOffsets = {(int) (writeStartOffset - 1), (int) (segment.sizeInBytes() + 1)};
      for (int offset : invalidOffsets) {
        try {
          segment.setEndOffset(offset);
          fail("Setting log end offset an invalid offset [" + offset + "] should have failed");
        } catch (IllegalArgumentException e) {
          // expected. Nothing to do.
        }
      }
    } finally {
      closeSegmentAndDeleteFile(segment);
    }
  }
Example #9
0
  /**
   * Tests appending and reading to make sure data is written and the data read is consistent with
   * the data written.
   *
   * @throws IOException
   */
  @Test
  public void basicWriteAndReadTest() throws IOException {
    String segmentName = "log_current";
    LogSegment segment = getSegment(segmentName, STANDARD_SEGMENT_SIZE, true);
    try {
      assertEquals(
          "Name of segment is inconsistent with what was provided", segmentName, segment.getName());
      assertEquals(
          "Capacity of segment is inconsistent with what was provided",
          STANDARD_SEGMENT_SIZE,
          segment.getCapacityInBytes());
      assertEquals(
          "Start offset is not equal to header size",
          LogSegment.HEADER_SIZE,
          segment.getStartOffset());
      int writeSize = 100;
      byte[] buf = TestUtils.getRandomBytes(3 * writeSize);
      long writeStartOffset = segment.getStartOffset();
      // append with buffer
      int written = segment.appendFrom(ByteBuffer.wrap(buf, 0, writeSize));
      assertEquals("Size written did not match size of buffer provided", writeSize, written);
      assertEquals(
          "End offset is not as expected", writeStartOffset + writeSize, segment.getEndOffset());
      readAndEnsureMatch(segment, writeStartOffset, Arrays.copyOfRange(buf, 0, writeSize));

      // append with channel
      segment.appendFrom(
          Channels.newChannel(
              new ByteBufferInputStream(ByteBuffer.wrap(buf, writeSize, writeSize))),
          writeSize);
      assertEquals(
          "End offset is not as expected",
          writeStartOffset + 2 * writeSize,
          segment.getEndOffset());
      readAndEnsureMatch(
          segment, writeStartOffset + writeSize, Arrays.copyOfRange(buf, writeSize, 2 * writeSize));

      // use writeFrom
      segment.writeFrom(
          Channels.newChannel(
              new ByteBufferInputStream(ByteBuffer.wrap(buf, 2 * writeSize, writeSize))),
          segment.getEndOffset(),
          writeSize);
      assertEquals(
          "End offset is not as expected",
          writeStartOffset + 3 * writeSize,
          segment.getEndOffset());
      readAndEnsureMatch(
          segment,
          writeStartOffset + 2 * writeSize,
          Arrays.copyOfRange(buf, 2 * writeSize, buf.length));

      readAndEnsureMatch(segment, writeStartOffset, buf);
      // check file size and end offset (they will not match)
      assertEquals(
          "End offset is not equal to the cumulative bytes written",
          writeStartOffset + 3 * writeSize,
          segment.getEndOffset());
      assertEquals(
          "Size in bytes is not equal to size of the file",
          STANDARD_SEGMENT_SIZE,
          segment.sizeInBytes());

      // ensure flush doesn't throw any errors.
      segment.flush();
      // close and reopen segment and ensure persistence.
      segment.close();
      segment = new LogSegment(segmentName, new File(tempDir, segmentName), metrics);
      segment.setEndOffset(writeStartOffset + buf.length);
      readAndEnsureMatch(segment, writeStartOffset, buf);
    } finally {
      closeSegmentAndDeleteFile(segment);
    }
  }
Example #10
0
 /**
  * Reads data starting from {@code offsetToStartRead} of {@code segment} and matches it with
  * {@code original}.
  *
  * @param segment the {@link LogSegment} to read from.
  * @param offsetToStartRead the offset in {@code segment} to start reading from.
  * @param original the byte array to compare against.
  * @throws IOException
  */
 private void readAndEnsureMatch(LogSegment segment, long offsetToStartRead, byte[] original)
     throws IOException {
   ByteBuffer readBuf = ByteBuffer.wrap(new byte[original.length]);
   segment.readInto(readBuf, offsetToStartRead);
   assertArrayEquals("Data read does not match data written", original, readBuf.array());
 }
Example #11
0
 /**
  * Appends random data of size {@code size} to given {@code segment}.
  *
  * @param segment the {@link LogSegment} to append data to.
  * @param size the size of the data that should be appended.
  * @return the data that was appended.
  * @throws IOException
  */
 private byte[] appendRandomData(LogSegment segment, int size) throws IOException {
   byte[] buf = TestUtils.getRandomBytes(size);
   segment.appendFrom(ByteBuffer.wrap(buf));
   return buf;
 }
Example #12
0
 private byte[] getHeader(LogSegment segment) throws IOException {
   FileChannel channel = segment.getView().getSecond();
   ByteBuffer header = ByteBuffer.allocate(LogSegment.HEADER_SIZE);
   channel.read(header, 0);
   return header.array();
 }
Example #13
0
  /**
   * Tests for bad construction cases of {@link LogSegment}.
   *
   * @throws IOException
   */
  @Test
  public void badConstructionTest() throws IOException {
    // try to construct with a file that does not exist.
    String name = "log_non_existent";
    File file = new File(tempDir, name);
    try {
      new LogSegment(name, file, STANDARD_SEGMENT_SIZE, metrics, true);
      fail("Construction should have failed because the backing file does not exist");
    } catch (IllegalArgumentException e) {
      // expected. Nothing to do.
    }

    try {
      new LogSegment(name, file, metrics);
      fail("Construction should have failed because the backing file does not exist");
    } catch (IllegalArgumentException e) {
      // expected. Nothing to do.
    }

    // try to construct with a file that is a directory
    name = tempDir.getName();
    file = new File(tempDir.getParent(), name);
    try {
      new LogSegment(name, file, STANDARD_SEGMENT_SIZE, metrics, true);
      fail("Construction should have failed because the backing file does not exist");
    } catch (IllegalArgumentException e) {
      // expected. Nothing to do.
    }

    name = tempDir.getName();
    file = new File(tempDir.getParent(), name);
    try {
      new LogSegment(name, file, metrics);
      fail("Construction should have failed because the backing file does not exist");
    } catch (IllegalArgumentException e) {
      // expected. Nothing to do.
    }

    // unknown version
    LogSegment segment = getSegment("dummy_log", STANDARD_SEGMENT_SIZE, true);
    file = segment.getView().getFirst();
    byte[] header = getHeader(segment);
    byte savedByte = header[0];
    // mess with version
    header[0] = (byte) (header[0] + 10);
    writeHeader(segment, header);
    try {
      new LogSegment(name, file, metrics);
      fail("Construction should have failed because version is unknown");
    } catch (IllegalArgumentException e) {
      // expected. Nothing to do.
    }

    // bad CRC
    // fix version but mess with data after version
    header[0] = savedByte;
    header[2] = header[2] == (byte) 1 ? (byte) 0 : (byte) 1;
    writeHeader(segment, header);
    try {
      new LogSegment(name, file, metrics);
      fail("Construction should have failed because crc check should have failed");
    } catch (IllegalStateException e) {
      // expected. Nothing to do.
    }
    closeSegmentAndDeleteFile(segment);
  }
Example #14
0
 /**
  * Tests for special constructor cases.
  *
  * @throws IOException
  */
 @Test
 public void constructorTest() throws IOException {
   LogSegment segment = getSegment("log_current", STANDARD_SEGMENT_SIZE, false);
   assertEquals(
       "Start offset should be 0 when no headers are written", 0, segment.getStartOffset());
 }
Example #15
0
  /**
   * Tests {@link LogSegment#writeFrom(ReadableByteChannel, long, long)} for various cases.
   *
   * @throws IOException
   */
  @Test
  public void writeFromTest() throws IOException {
    String currSegmentName = "log_current";
    LogSegment segment = getSegment(currSegmentName, STANDARD_SEGMENT_SIZE, true);
    try {
      long writeStartOffset = segment.getStartOffset();
      byte[] bufOne = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 3);
      byte[] bufTwo = TestUtils.getRandomBytes(STANDARD_SEGMENT_SIZE / 2);

      segment.writeFrom(
          Channels.newChannel(new ByteBufferInputStream(ByteBuffer.wrap(bufOne))),
          writeStartOffset,
          bufOne.length);
      assertEquals(
          "End offset is not as expected",
          writeStartOffset + bufOne.length,
          segment.getEndOffset());
      readAndEnsureMatch(segment, writeStartOffset, bufOne);

      // overwrite using bufTwo
      segment.writeFrom(
          Channels.newChannel(new ByteBufferInputStream(ByteBuffer.wrap(bufTwo))),
          writeStartOffset,
          bufTwo.length);
      assertEquals(
          "End offset is not as expected",
          writeStartOffset + bufTwo.length,
          segment.getEndOffset());
      readAndEnsureMatch(segment, writeStartOffset, bufTwo);

      // overwrite using bufOne
      segment.writeFrom(
          Channels.newChannel(new ByteBufferInputStream(ByteBuffer.wrap(bufOne))),
          writeStartOffset,
          bufOne.length);
      // end offset should not have changed
      assertEquals(
          "End offset is not as expected",
          writeStartOffset + bufTwo.length,
          segment.getEndOffset());
      readAndEnsureMatch(segment, writeStartOffset, bufOne);
      readAndEnsureMatch(
          segment,
          writeStartOffset + bufOne.length,
          Arrays.copyOfRange(bufTwo, bufOne.length, bufTwo.length));

      // write at random locations
      for (int i = 0; i < 10; i++) {
        long offset =
            writeStartOffset
                + Utils.getRandomLong(
                    TestUtils.RANDOM,
                    segment.getCapacityInBytes() - bufOne.length - writeStartOffset);
        segment.writeFrom(
            Channels.newChannel(new ByteBufferInputStream(ByteBuffer.wrap(bufOne))),
            offset,
            bufOne.length);
        readAndEnsureMatch(segment, offset, bufOne);
      }

      // try to overwrite using a channel that won't fit
      ByteBuffer failBuf =
          ByteBuffer.wrap(
              TestUtils.getRandomBytes((int) (STANDARD_SEGMENT_SIZE - writeStartOffset + 1)));
      long writeOverFlowCount = metrics.overflowWriteError.getCount();
      try {
        segment.writeFrom(
            Channels.newChannel(new ByteBufferInputStream(failBuf)),
            writeStartOffset,
            failBuf.remaining());
        fail("WriteFrom should have failed because data won't fit");
      } catch (IndexOutOfBoundsException e) {
        assertEquals(
            "Write overflow should have been reported",
            writeOverFlowCount + 1,
            metrics.overflowWriteError.getCount());
        assertEquals("Position of buffer has changed", 0, failBuf.position());
      }

      // data cannot be written at invalid offsets.
      long[] invalidOffsets = {
        writeStartOffset - 1, STANDARD_SEGMENT_SIZE, STANDARD_SEGMENT_SIZE + 1
      };
      ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes(1));
      for (long invalidOffset : invalidOffsets) {
        try {
          segment.writeFrom(
              Channels.newChannel(new ByteBufferInputStream(buffer)),
              invalidOffset,
              buffer.remaining());
          fail("WriteFrom should have failed because offset provided for write is invalid");
        } catch (IndexOutOfBoundsException e) {
          assertEquals("Position of buffer has changed", 0, buffer.position());
        }
      }

      segment.close();
      // ensure that writeFrom fails.
      try {
        segment.writeFrom(
            Channels.newChannel(new ByteBufferInputStream(buffer)),
            writeStartOffset,
            buffer.remaining());
        fail("WriteFrom should have failed because segments are closed");
      } catch (ClosedChannelException e) {
        assertEquals("Position of buffer has changed", 0, buffer.position());
      }
    } finally {
      closeSegmentAndDeleteFile(segment);
    }
  }
Example #16
0
  /**
   * Tests {@link LogSegment#readInto(ByteBuffer, long)} for various cases.
   *
   * @throws IOException
   */
  @Test
  public void readTest() throws IOException {
    Random random = new Random();
    String segmentName = "log_current";
    LogSegment segment = getSegment(segmentName, STANDARD_SEGMENT_SIZE, true);
    try {
      long writeStartOffset = segment.getStartOffset();
      byte[] data = appendRandomData(segment, 2 * STANDARD_SEGMENT_SIZE / 3);
      readAndEnsureMatch(segment, writeStartOffset, data);
      int readCount = 10;
      for (int i = 0; i < readCount; i++) {
        int position = random.nextInt(data.length);
        int size = random.nextInt(data.length - position);
        readAndEnsureMatch(
            segment,
            writeStartOffset + position,
            Arrays.copyOfRange(data, position, position + size));
      }

      // error scenarios
      ByteBuffer readBuf = ByteBuffer.wrap(new byte[data.length]);
      // data cannot be read at invalid offsets.
      long[] invalidOffsets = {
        writeStartOffset - 1, segment.getEndOffset(), segment.getEndOffset() + 1
      };
      ByteBuffer buffer = ByteBuffer.wrap(TestUtils.getRandomBytes(1));
      for (long invalidOffset : invalidOffsets) {
        try {
          segment.readInto(readBuf, invalidOffset);
          fail("Should have failed to read because position provided is invalid");
        } catch (IndexOutOfBoundsException e) {
          assertEquals("Position of buffer has changed", 0, buffer.position());
        }
      }

      // position + buffer.remaining() > endOffset.
      long readOverFlowCount = metrics.overflowReadError.getCount();
      try {
        segment.readInto(readBuf, writeStartOffset + 1);
        fail("Should have failed to read because position + buffer.remaining() > endOffset");
      } catch (IndexOutOfBoundsException e) {
        assertEquals(
            "Read overflow should have been reported",
            readOverFlowCount + 1,
            metrics.overflowReadError.getCount());
        assertEquals("Position of buffer has changed", 0, readBuf.position());
      }

      segment.close();
      // read after close
      buffer = ByteBuffer.allocate(1);
      try {
        segment.readInto(buffer, writeStartOffset);
        fail("Should have failed to read because segment is closed");
      } catch (ClosedChannelException e) {
        assertEquals("Position of buffer has changed", 0, buffer.position());
      }
    } finally {
      closeSegmentAndDeleteFile(segment);
    }
  }