コード例 #1
0
  @Test
  public void testGetRemoteEditLog() throws IOException {
    StorageDirectory sd =
        FSImageTestUtil.mockStorageDirectory(
            NameNodeDirType.EDITS,
            false,
            NNStorage.getFinalizedEditsFileName(1, 100),
            NNStorage.getFinalizedEditsFileName(101, 200),
            NNStorage.getInProgressEditsFileName(201),
            NNStorage.getFinalizedEditsFileName(1001, 1100));

    FileJournalManager fjm = new FileJournalManager(sd);
    assertEquals("[1,100],[101,200],[1001,1100]", getLogsAsString(fjm, 1));
    assertEquals("[101,200],[1001,1100]", getLogsAsString(fjm, 101));
    assertEquals("[1001,1100]", getLogsAsString(fjm, 201));
    try {
      assertEquals("[]", getLogsAsString(fjm, 150));
      fail("Did not throw when asking for a txn in the middle of a log");
    } catch (IOException ioe) {
      GenericTestUtils.assertExceptionContains("150 which is in the middle", ioe);
    }
    assertEquals(
        "Asking for a newer log than exists should return empty list",
        "",
        getLogsAsString(fjm, 9999));
  }
コード例 #2
0
  /**
   * Test that we can load an edits directory with a corrupt inprogress file. The corrupt inprogress
   * file should be moved to the side.
   */
  @Test
  public void testManyLogsWithCorruptInprogress() throws IOException {
    File f = new File(TestEditLog.TEST_DIR + "/filejournaltest5");
    NNStorage storage =
        setupEdits(Collections.<URI>singletonList(f.toURI()), 10, new AbortSpec(10, 0));
    StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

    File[] files =
        new File(f, "current")
            .listFiles(
                new FilenameFilter() {
                  public boolean accept(File dir, String name) {
                    if (name.startsWith("edits_inprogress")) {
                      return true;
                    }
                    return false;
                  }
                });
    assertEquals(files.length, 1);

    corruptAfterStartSegment(files[0]);

    FileJournalManager jm = new FileJournalManager(sd);
    assertEquals(10 * TXNS_PER_ROLL + 1, jm.getNumberOfTransactions(1));
  }
コード例 #3
0
  /**
   * Test that we can read from a stream created by FileJournalManager. Create a single edits
   * directory, failing it on the final roll. Then try loading from the point of the 3rd roll.
   * Verify that we read the correct number of transactions from this point.
   */
  @Test
  public void testReadFromStream() throws IOException {
    File f = new File(TestEditLog.TEST_DIR + "/filejournaltest1");
    // abort after 10th roll
    NNStorage storage =
        setupEdits(Collections.<URI>singletonList(f.toURI()), 10, new AbortSpec(10, 0));
    StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

    FileJournalManager jm = new FileJournalManager(sd);
    long expectedTotalTxnCount = TXNS_PER_ROLL * 10 + TXNS_PER_FAIL;
    assertEquals(expectedTotalTxnCount, jm.getNumberOfTransactions(1));

    long skippedTxns = (3 * TXNS_PER_ROLL); // skip first 3 files
    long startingTxId = skippedTxns + 1;

    long numTransactionsToLoad = jm.getNumberOfTransactions(startingTxId);
    long numLoaded = 0;
    while (numLoaded < numTransactionsToLoad) {
      EditLogInputStream editIn = jm.getInputStream(startingTxId);
      FSEditLogLoader.EditLogValidation val = FSEditLogLoader.validateEditLog(editIn);
      long count = val.getNumTransactions();

      editIn.close();
      startingTxId += count;
      numLoaded += count;
    }

    assertEquals(expectedTotalTxnCount - skippedTxns, numLoaded);
  }
コード例 #4
0
  /** Checks the JournalManager.isSegmentInProgress() */
  @Test
  public void testInprogressSegment() throws IOException {
    File f = new File(TestEditLog.TEST_DIR + "/testInprogressSegment");
    // abort after the 5th roll
    NNStorage storage =
        setupEdits(Collections.<URI>singletonList(f.toURI()), 5, new AbortSpec(5, 0));
    StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

    FileJournalManager jm = new FileJournalManager(sd);
    assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(0));

    boolean isOneInProgress = false;
    boolean isOneNotInProgress = false;

    for (RemoteEditLog rel : jm.getRemoteEditLogs(0)) {
      if (rel.inProgress()) {
        isOneInProgress = true;
        assertTrue(jm.isSegmentInProgress(rel.getStartTxId()));
      } else {
        isOneNotInProgress = true;
        assertFalse(jm.isSegmentInProgress(rel.getStartTxId()));
      }
    }
    assertTrue(isOneInProgress);
    assertTrue(isOneNotInProgress);
  }
コード例 #5
0
  /**
   * Try to make a request with a start transaction id which doesn't match the start ID of some log
   * segment. This should fail as edit logs must currently be treated as indevisable units.
   */
  @Test(expected = IOException.class)
  public void testAskForTransactionsMidfile() throws IOException {
    File f = new File(TestEditLog.TEST_DIR + "/filejournaltest2");
    NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10);
    StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

    FileJournalManager jm = new FileJournalManager(sd);
    jm.getNumberOfTransactions(2);
  }
コード例 #6
0
  /**
   * Test that inprogress files are handled correct. Set up a single edits directory. Fail on after
   * the last roll. Then verify that the logs have the expected number of transactions.
   */
  @Test
  public void testInprogressRecovery() throws IOException {
    File f = new File(TestEditLog.TEST_DIR + "/filejournaltest0");
    // abort after the 5th roll
    NNStorage storage =
        setupEdits(Collections.<URI>singletonList(f.toURI()), 5, new AbortSpec(5, 0));
    StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

    FileJournalManager jm = new FileJournalManager(sd);
    assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1));
  }
コード例 #7
0
  /**
   * Test the normal operation of loading transactions from file journal manager. 3 edits
   * directories are setup without any failures. Test that we read in the expected number of
   * transactions.
   */
  @Test
  public void testNormalOperation() throws IOException {
    File f1 = new File(TestEditLog.TEST_DIR + "/normtest0");
    File f2 = new File(TestEditLog.TEST_DIR + "/normtest1");
    File f3 = new File(TestEditLog.TEST_DIR + "/normtest2");

    List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI());
    NNStorage storage = setupEdits(editUris, 5);

    long numJournals = 0;
    for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) {
      FileJournalManager jm = new FileJournalManager(sd);
      assertEquals(6 * TXNS_PER_ROLL, jm.getNumberOfTransactions(1));
      numJournals++;
    }
    assertEquals(3, numJournals);
  }
コード例 #8
0
  /**
   * Test that we receive the correct number of transactions when we count the number of
   * transactions around gaps. Set up a single edits directory, with no failures. Delete the 4th
   * logfile. Test that getNumberOfTransactions returns the correct number of transactions before
   * this gap and after this gap. Also verify that if you try to count on the gap that an exception
   * is thrown.
   */
  @Test
  public void testManyLogsWithGaps() throws IOException {
    File f = new File(TestEditLog.TEST_DIR + "/filejournaltest3");
    NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10);
    StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

    final long startGapTxId = 3 * TXNS_PER_ROLL; // 30
    final long endGapTxId = 4 * TXNS_PER_ROLL - 1; // 39
    File[] files =
        new File(f, "current")
            .listFiles(
                new FilenameFilter() {
                  public boolean accept(File dir, String name) {
                    if (name.startsWith(
                        NNStorage.getFinalizedEditsFileName(startGapTxId, endGapTxId))) {
                      return true;
                    }
                    return false;
                  }
                });
    assertEquals(1, files.length);
    assertTrue(files[0].delete());

    FileJournalManager jm = new FileJournalManager(sd);
    assertEquals(startGapTxId, jm.getNumberOfTransactions(0));

    try {
      jm.getNumberOfTransactions(startGapTxId);
      fail("Should have thrown an exception by now");
    } catch (IOException ioe) {
      assertTrue(true);
    }

    // rolled 10 times so there should be 11 files.
    assertEquals(
        11 * TXNS_PER_ROLL - endGapTxId - 1, // 110 - 39 - 1
        jm.getNumberOfTransactions(endGapTxId + 1));
  }
コード例 #9
0
  /**
   * Test a mixture of inprogress files and finalised. Set up 3 edits directories and fail the
   * second on the last roll. Verify that reading the transactions, reads from the finalised
   * directories.
   */
  @Test
  public void testInprogressRecoveryMixed() throws IOException {
    File f1 = new File(TestEditLog.TEST_DIR + "/mixtest0");
    File f2 = new File(TestEditLog.TEST_DIR + "/mixtest1");
    File f3 = new File(TestEditLog.TEST_DIR + "/mixtest2");

    List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI());

    // abort after the 5th roll
    NNStorage storage = setupEdits(editUris, 5, new AbortSpec(5, 1));
    Iterator<StorageDirectory> dirs = storage.dirIterator(NameNodeDirType.EDITS);
    StorageDirectory sd = dirs.next();
    FileJournalManager jm = new FileJournalManager(sd);
    assertEquals(6 * TXNS_PER_ROLL, jm.getNumberOfTransactions(1));

    sd = dirs.next();
    jm = new FileJournalManager(sd);
    assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(1));

    sd = dirs.next();
    jm = new FileJournalManager(sd);
    assertEquals(6 * TXNS_PER_ROLL, jm.getNumberOfTransactions(1));
  }