Exemple #1
0
  /**
   * If clocks in sync, then acquires the oldest expired lock Else, on first call, just remembers
   * the oldest expired lock, on next call check if the lock is updated. if not updated then
   * acquires the lock
   *
   * @return a lock object
   * @throws IOException
   */
  private FileLock getOldestExpiredLock() throws IOException {
    // 1 - acquire lock on dir
    DirLock dirlock = DirLock.tryLock(hdfs, lockDirPath);
    if (dirlock == null) {
      dirlock = DirLock.takeOwnershipIfStale(hdfs, lockDirPath, lockTimeoutSec);
      if (dirlock == null) {
        LOG.debug("Spout {} could not take over ownership of DirLock for {}", spoutId, lockDirPath);
        return null;
      }
      LOG.debug(
          "Spout {} now took over ownership of abandoned DirLock for {}", spoutId, lockDirPath);
    } else {
      LOG.debug("Spout {} now owns DirLock for {}", spoutId, lockDirPath);
    }

    try {
      // 2 - if clocks are in sync then simply take ownership of the oldest expired lock
      if (clocksInSync) {
        return FileLock.acquireOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec, spoutId);
      }

      // 3 - if clocks are not in sync ..
      if (lastExpiredLock == null) {
        // just make a note of the oldest expired lock now and check if its still unmodified after
        // lockTimeoutSec
        lastExpiredLock = FileLock.locateOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec);
        lastExpiredLockTime = System.currentTimeMillis();
        return null;
      }
      // see if lockTimeoutSec time has elapsed since we last selected the lock file
      if (hasExpired(lastExpiredLockTime)) {
        return null;
      }

      // If lock file has expired, then own it
      FileLock.LogEntry lastEntry = FileLock.getLastEntry(hdfs, lastExpiredLock.getKey());
      if (lastEntry.equals(lastExpiredLock.getValue())) {
        FileLock result =
            FileLock.takeOwnership(hdfs, lastExpiredLock.getKey(), lastEntry, spoutId);
        lastExpiredLock = null;
        return result;
      } else {
        // if lock file has been updated since last time, then leave this lock file alone
        lastExpiredLock = null;
        return null;
      }
    } finally {
      dirlock.release();
      LOG.debug("Released DirLock {}, SpoutID {} ", dirlock.getLockFile(), spoutId);
    }
  }
Exemple #2
0
 @Override
 public void fail(Object msgId) {
   LOG.trace("Fail received for msg id {} on spout {}", msgId, spoutId);
   super.fail(msgId);
   if (ackEnabled) {
     HdfsUtils.Pair<MessageId, List<Object>> item =
         HdfsUtils.Pair.of(msgId, inflight.remove(msgId));
     retryList.add(item);
   }
 }