コード例 #1
0
  /** Checks the JournalManager.isSegmentInProgress() */
  @Test
  public void testInprogressSegment() throws IOException {
    File f = new File(TestEditLog.TEST_DIR + "/testInprogressSegment");
    // abort after the 5th roll
    NNStorage storage =
        setupEdits(Collections.<URI>singletonList(f.toURI()), 5, new AbortSpec(5, 0));
    StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();

    FileJournalManager jm = new FileJournalManager(sd);
    assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL, jm.getNumberOfTransactions(0));

    boolean isOneInProgress = false;
    boolean isOneNotInProgress = false;

    for (RemoteEditLog rel : jm.getRemoteEditLogs(0)) {
      if (rel.inProgress()) {
        isOneInProgress = true;
        assertTrue(jm.isSegmentInProgress(rel.getStartTxId()));
      } else {
        isOneNotInProgress = true;
        assertFalse(jm.isSegmentInProgress(rel.getStartTxId()));
      }
    }
    assertTrue(isOneInProgress);
    assertTrue(isOneNotInProgress);
  }
コード例 #2
0
ファイル: JournalSet.java プロジェクト: bharathravi/hadoop
  /**
   * Return a manifest of what finalized edit logs are available. All available edit logs are
   * returned starting from the transaction id passed.
   *
   * @param fromTxId Starting transaction id to read the logs.
   * @return RemoteEditLogManifest object.
   */
  public synchronized RemoteEditLogManifest getEditLogManifest(long fromTxId) {
    // Collect RemoteEditLogs available from each FileJournalManager
    List<RemoteEditLog> allLogs = Lists.newArrayList();
    for (JournalAndStream j : journals) {
      if (j.getManager() instanceof FileJournalManager) {
        FileJournalManager fjm = (FileJournalManager) j.getManager();
        try {
          allLogs.addAll(fjm.getRemoteEditLogs(fromTxId));
        } catch (Throwable t) {
          LOG.warn("Cannot list edit logs in " + fjm, t);
        }
      }
    }

    // Group logs by their starting txid
    ImmutableListMultimap<Long, RemoteEditLog> logsByStartTxId =
        Multimaps.index(allLogs, RemoteEditLog.GET_START_TXID);
    long curStartTxId = fromTxId;

    List<RemoteEditLog> logs = Lists.newArrayList();
    while (true) {
      ImmutableList<RemoteEditLog> logGroup = logsByStartTxId.get(curStartTxId);
      if (logGroup.isEmpty()) {
        // we have a gap in logs - for example because we recovered some old
        // storage directory with ancient logs. Clear out any logs we've
        // accumulated so far, and then skip to the next segment of logs
        // after the gap.
        SortedSet<Long> startTxIds = Sets.newTreeSet(logsByStartTxId.keySet());
        startTxIds = startTxIds.tailSet(curStartTxId);
        if (startTxIds.isEmpty()) {
          break;
        } else {
          if (LOG.isDebugEnabled()) {
            LOG.debug(
                "Found gap in logs at "
                    + curStartTxId
                    + ": "
                    + "not returning previous logs in manifest.");
          }
          logs.clear();
          curStartTxId = startTxIds.first();
          continue;
        }
      }

      // Find the one that extends the farthest forward
      RemoteEditLog bestLog = Collections.max(logGroup);
      logs.add(bestLog);
      // And then start looking from after that point
      curStartTxId = bestLog.getEndTxId() + 1;
    }
    RemoteEditLogManifest ret = new RemoteEditLogManifest(logs);

    if (LOG.isDebugEnabled()) {
      LOG.debug("Generated manifest for logs since " + fromTxId + ":" + ret);
    }
    return ret;
  }