private void rereadNameSpaceEntry(final PinTask task) throws CacheException {
    /* Ensure that task is still valid and stays valid for the
     * duration of the name space lookup.
     */
    refreshTimeout(task, getExpirationTimeForNameSpaceLookup());

    /* We allow the set of provided attributes to be incomplete
     * and thus add attributes required by pool manager.
     */
    Set<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class);
    attributes.addAll(task.getFileAttributes().getDefinedAttributes());
    attributes.addAll(PoolMgrSelectReadPoolMsg.getRequiredAttributes());

    _pnfsStub.send(
        new PnfsGetFileAttributes(task.getPnfsId(), attributes),
        PnfsGetFileAttributes.class,
        new AbstractMessageCallback<PnfsGetFileAttributes>() {
          @Override
          public void success(PnfsGetFileAttributes msg) {
            try {
              task.setFileAttributes(msg.getFileAttributes());

              /* Ensure that task is still valid
               * and stays valid for the duration
               * of the pool selection.
               */
              refreshTimeout(task, getExpirationTimeForPoolSelection());
              selectReadPool(task);
            } catch (CacheException e) {
              fail(task, e.getRc(), e.getMessage());
            } catch (RuntimeException e) {
              fail(task, CacheException.UNEXPECTED_SYSTEM_EXCEPTION, e.toString());
            }
          }

          @Override
          public void failure(int rc, Object error) {
            fail(task, rc, error.toString());
          }

          @Override
          public void noroute(CellPath path) {
            /* PnfsManager is unreachable. We
             * expect this to be a transient
             * problem and retry in a moment.
             */
            retry(task, RETRY_DELAY);
          }

          @Override
          public void timeout(CellPath path) {
            /* PnfsManager did not respond. We
             * expect this to be a transient
             * problem and retry in a moment.
             */
            retry(task, SMALL_DELAY);
          }
        });
  }
/**
 * Processes pin requests.
 *
 * <p>A pin request goes through several steps to pin a file on a pool:
 *
 * <p>- Create DB entry in state PINNING - Optionally read the name space entry - Select a read pool
 * (which may involve staging) - Update DB entry with the pool name - Create sticky flag on pool -
 * Update DB entry to state PINNED
 *
 * <p>If during any step the entry is no longer in PINNING then the operation is aborted.
 *
 * <p>If a DB error occurs it is considered fatal and the pinning operation is not completed. The DB
 * entry will stay in PINNING until either explicitly unpinned or it expires.
 *
 * <p>Database operations are blocking. Communication with PoolManager and pools is asynchronous.
 */
public class PinRequestProcessor implements CellMessageReceiver {
  private static final Logger _log = LoggerFactory.getLogger(PinRequestProcessor.class);

  /** The delay we use after a pin request failed and before retrying the request. */
  private static final long RETRY_DELAY = SECONDS.toMillis(30);

  /**
   * The delay we use after transient failures that should be retried immediately. The small delay
   * prevents tight retry loops.
   */
  private static final long SMALL_DELAY = MILLISECONDS.toMillis(10);

  /** Safety margin added to the lifetime of the sticky bit to account for clock drift. */
  private static final long CLOCK_DRIFT_MARGIN = MINUTES.toMillis(30);

  private static final Set<FileAttribute> REQUIRED_ATTRIBUTES =
      PoolMgrSelectReadPoolMsg.getRequiredAttributes();

  private ScheduledExecutorService _executor;
  private PinDao _dao;
  private CellStub _poolStub;
  private CellStub _pnfsStub;
  private CellStub _poolManagerStub;
  private CheckStagePermission _checkStagePermission;
  private long _maxLifetime;
  private TimeUnit _maxLifetimeUnit;

  private PoolMonitor _poolMonitor;

  @Required
  public void setExecutor(ScheduledExecutorService executor) {
    _executor = executor;
  }

  @Required
  public void setDao(PinDao dao) {
    _dao = dao;
  }

  @Required
  public void setPoolStub(CellStub stub) {
    _poolStub = stub;
  }

  @Required
  public void setPnfsStub(CellStub stub) {
    _pnfsStub = stub;
  }

  @Required
  public void setPoolManagerStub(CellStub stub) {
    _poolManagerStub = stub;
  }

  @Required
  public void setStagePermission(CheckStagePermission checker) {
    _checkStagePermission = checker;
  }

  @Required
  public void setMaxLifetime(long maxLifetime) {
    _maxLifetime = maxLifetime;
  }

  @Required
  public void setPoolMonitor(PoolMonitor poolMonitor) {
    _poolMonitor = poolMonitor;
  }

  public long getMaxLifetime() {
    return _maxLifetime;
  }

  @Required
  public void setMaxLifetimeUnit(TimeUnit unit) {
    _maxLifetimeUnit = unit;
  }

  public TimeUnit getMaxLifetimeUnit() {
    return _maxLifetimeUnit;
  }

  private void enforceLifetimeLimit(PinManagerPinMessage message) {
    if (_maxLifetime > -1) {
      long millis = _maxLifetimeUnit.toMillis(_maxLifetime);
      long requestedLifetime = message.getLifetime();
      if (requestedLifetime == -1) {
        message.setLifetime(millis);
      } else {
        message.setLifetime(Math.min(millis, requestedLifetime));
      }
    }
  }

  public MessageReply<PinManagerPinMessage> messageArrived(PinManagerPinMessage message)
      throws CacheException {
    MessageReply<PinManagerPinMessage> reply = new MessageReply<>();

    enforceLifetimeLimit(message);

    PinTask task = createTask(message, reply);
    if (task != null) {
      if (!task.getFileAttributes().isDefined(REQUIRED_ATTRIBUTES)) {
        rereadNameSpaceEntry(task);
      } else {
        selectReadPool(task);
      }
    }

    return reply;
  }

  protected EnumSet<RequestContainerV5.RequestState> checkStaging(PinTask task) {
    try {
      Subject subject = task.getSubject();
      StorageInfo info = task.getFileAttributes().getStorageInfo();
      return _checkStagePermission.canPerformStaging(subject, info)
          ? RequestContainerV5.allStates
          : RequestContainerV5.allStatesExceptStage;
    } catch (PatternSyntaxException | IOException ex) {
      _log.error("Failed to check stage permission: " + ex);
    }
    return RequestContainerV5.allStatesExceptStage;
  }

  private void retry(final PinTask task, long delay) {
    if (!task.isValidIn(delay)) {
      fail(task, CacheException.TIMEOUT, "Pin request TTL exceeded");
    } else {
      _executor.schedule(
          new Runnable() {
            @Override
            public void run() {
              try {
                rereadNameSpaceEntry(task);
              } catch (CacheException e) {
                fail(task, e.getRc(), e.getMessage());
              } catch (RuntimeException e) {
                fail(task, CacheException.UNEXPECTED_SYSTEM_EXCEPTION, e.toString());
              }
            }
          },
          delay,
          MILLISECONDS);
    }
  }

  private void fail(PinTask task, int rc, String error) {
    try {
      task.fail(rc, error);
      clearPin(task);
    } catch (RuntimeException e) {
      _log.error(e.toString());
    }
  }

  private void rereadNameSpaceEntry(final PinTask task) throws CacheException {
    /* Ensure that task is still valid and stays valid for the
     * duration of the name space lookup.
     */
    refreshTimeout(task, getExpirationTimeForNameSpaceLookup());

    /* We allow the set of provided attributes to be incomplete
     * and thus add attributes required by pool manager.
     */
    Set<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class);
    attributes.addAll(task.getFileAttributes().getDefinedAttributes());
    attributes.addAll(PoolMgrSelectReadPoolMsg.getRequiredAttributes());

    _pnfsStub.send(
        new PnfsGetFileAttributes(task.getPnfsId(), attributes),
        PnfsGetFileAttributes.class,
        new AbstractMessageCallback<PnfsGetFileAttributes>() {
          @Override
          public void success(PnfsGetFileAttributes msg) {
            try {
              task.setFileAttributes(msg.getFileAttributes());

              /* Ensure that task is still valid
               * and stays valid for the duration
               * of the pool selection.
               */
              refreshTimeout(task, getExpirationTimeForPoolSelection());
              selectReadPool(task);
            } catch (CacheException e) {
              fail(task, e.getRc(), e.getMessage());
            } catch (RuntimeException e) {
              fail(task, CacheException.UNEXPECTED_SYSTEM_EXCEPTION, e.toString());
            }
          }

          @Override
          public void failure(int rc, Object error) {
            fail(task, rc, error.toString());
          }

          @Override
          public void noroute(CellPath path) {
            /* PnfsManager is unreachable. We
             * expect this to be a transient
             * problem and retry in a moment.
             */
            retry(task, RETRY_DELAY);
          }

          @Override
          public void timeout(CellPath path) {
            /* PnfsManager did not respond. We
             * expect this to be a transient
             * problem and retry in a moment.
             */
            retry(task, SMALL_DELAY);
          }
        });
  }

  private void selectReadPool(final PinTask task) throws CacheException {
    try {
      PoolSelector poolSelector =
          _poolMonitor.getPoolSelector(task.getFileAttributes(), task.getProtocolInfo(), null);

      PoolInfo pool = poolSelector.selectPinPool();
      setPool(task, pool.getName());
      setStickyFlag(task, pool.getName(), pool.getAddress());
    } catch (FileNotOnlineCacheException e) {
      askPoolManager(task);
    }
  }

  private void askPoolManager(final PinTask task) {
    PoolMgrSelectReadPoolMsg msg =
        new PoolMgrSelectReadPoolMsg(
            task.getFileAttributes(),
            task.getProtocolInfo(),
            task.getReadPoolSelectionContext(),
            checkStaging(task));
    msg.setSubject(task.getSubject());
    msg.setSkipCostUpdate(true);
    _poolManagerStub.send(
        msg,
        PoolMgrSelectReadPoolMsg.class,
        new AbstractMessageCallback<PoolMgrSelectReadPoolMsg>() {
          @Override
          public void success(PoolMgrSelectReadPoolMsg msg) {
            try {
              /* Pool manager expects us
               * to keep some state
               * between retries.
               */
              task.setReadPoolSelectionContext(msg.getContext());

              /* Store the pool name in
               * the DB so we know what to
               * clean up if something
               * fails.
               */
              String poolName = msg.getPoolName();
              CellAddressCore poolAddress = msg.getPoolAddress();
              task.getFileAttributes().getLocations().add(poolName);
              setPool(task, poolName);

              setStickyFlag(task, poolName, poolAddress);
            } catch (CacheException e) {
              fail(task, e.getRc(), e.getMessage());
            } catch (RuntimeException e) {
              fail(task, CacheException.UNEXPECTED_SYSTEM_EXCEPTION, e.toString());
            }
          }

          @Override
          public void failure(int rc, Object error) {
            /* Pool manager expects us to
             * keep some state between
             * retries.
             */
            task.setReadPoolSelectionContext(getReply().getContext());
            switch (rc) {
              case CacheException.OUT_OF_DATE:
                /* Pool manager asked for a
                 * refresh of the request.
                 * Retry right away.
                 */
                retry(task, 0);
                break;
              case CacheException.FILE_NOT_IN_REPOSITORY:
              case CacheException.PERMISSION_DENIED:
                fail(task, rc, error.toString());
                break;
              default:
                /* Ideally we would delegate the retry to the door,
                 * but for the time being the retry is dealed with
                 * by pin manager.
                 */
                retry(task, RETRY_DELAY);
                break;
            }
          }

          @Override
          public void noroute(CellPath path) {
            /* Pool manager is
             * unreachable. We expect this
             * to be transient and retry in
             * a moment.
             */
            retry(task, RETRY_DELAY);
          }

          @Override
          public void timeout(CellPath path) {
            /* Pool manager did not
             * respond. We expect this to be
             * transient and retry in a
             * moment.
             */
            retry(task, SMALL_DELAY);
          }
        });
  }

  private void setStickyFlag(
      final PinTask task, final String poolName, CellAddressCore poolAddress) {
    /* The pin lifetime should be from the moment the file is
     * actually pinned. Due to staging and pool to pool transfers
     * this may be much later than when the pin was requested.
     */
    Date pinExpiration = task.freezeExpirationTime();

    /* To allow for some drift in clocks we add a safety margin to
     * the lifetime of the sticky bit.
     */
    long poolExpiration =
        (pinExpiration == null) ? -1 : pinExpiration.getTime() + CLOCK_DRIFT_MARGIN;

    PoolSetStickyMessage msg =
        new PoolSetStickyMessage(
            poolName, task.getPnfsId(), true, task.getSticky(), poolExpiration);
    _poolStub.send(
        new CellPath(poolAddress),
        msg,
        PoolSetStickyMessage.class,
        new AbstractMessageCallback<PoolSetStickyMessage>() {
          @Override
          public void success(PoolSetStickyMessage msg) {
            try {
              setToPinned(task);
              task.success();
            } catch (CacheException e) {
              fail(task, e.getRc(), e.getMessage());
            } catch (RuntimeException e) {
              fail(task, CacheException.UNEXPECTED_SYSTEM_EXCEPTION, e.toString());
            }
          }

          @Override
          public void failure(int rc, Object error) {
            switch (rc) {
              case CacheException.POOL_DISABLED:
                /* Pool manager had outdated
                 * information about the pool. Give
                 * it a chance to be updated and
                 * then retry.
                 */
                retry(task, RETRY_DELAY);
                break;
              case CacheException.FILE_NOT_IN_REPOSITORY:
                /* Pnfs manager had stale location
                 * information. The pool clears
                 * this information as a result of
                 * this error, so we retry in a
                 * moment.
                 */
                retry(task, SMALL_DELAY);
                break;
              default:
                fail(task, rc, error.toString());
                break;
            }
          }

          @Override
          public void noroute(CellPath path) {
            /* The pool must have gone down. Give
             * pool manager a moment to notice this
             * and then retry.
             */
            retry(task, RETRY_DELAY);
          }

          @Override
          public void timeout(CellPath path) {
            /* No response from pool. Typically this is
             * because the pool is overloaded.
             */
            fail(task, CacheException.TIMEOUT, "No reply from " + path);
          }
        });
  }

  private Date getExpirationTimeForNameSpaceLookup() {
    long now = System.currentTimeMillis();
    long timeout = _pnfsStub.getTimeoutInMillis();
    return new Date(now + 2 * (timeout + RETRY_DELAY));
  }

  private Date getExpirationTimeForPoolSelection() {
    long now = System.currentTimeMillis();
    long timeout = _poolManagerStub.getTimeoutInMillis();
    return new Date(now + 2 * (timeout + RETRY_DELAY));
  }

  private Date getExpirationTimeForSettingFlag() {
    long now = System.currentTimeMillis();
    long timeout = _poolStub.getTimeoutInMillis();
    return new Date(now + 2 * timeout);
  }

  @Transactional
  protected PinTask createTask(
      PinManagerPinMessage message, MessageReply<PinManagerPinMessage> reply) {
    PnfsId pnfsId = message.getFileAttributes().getPnfsId();

    if (message.getRequestId() != null) {
      Pin pin = _dao.getPin(pnfsId, message.getRequestId());
      if (pin != null) {
        /* In this case the request is a resubmission. If the
         * previous pin completed then use it. Otherwise abort the
         * previous pin and create a new one.
         */
        if (pin.getState() == PINNED) {
          message.setPin(pin);
          reply.reply(message);
          return null;
        }

        pin.setState(UNPINNING);
        pin.setRequestId(null);
        _dao.storePin(pin);
      }
    }

    Pin pin = new Pin(message.getSubject(), pnfsId);
    pin.setRequestId(message.getRequestId());
    pin.setSticky("PinManager-" + UUID.randomUUID().toString());
    pin.setExpirationTime(getExpirationTimeForPoolSelection());

    return new PinTask(message, reply, _dao.storePin(pin));
  }

  /**
   * Load the pin belonging to the PinTask.
   *
   * @throw CacheException if the pin no longer exists or is no longer in PINNING.
   */
  protected Pin loadPinBelongingTo(PinTask task) throws CacheException {
    Pin pin = _dao.getPin(task.getPinId(), task.getSticky(), PINNING);
    if (pin == null) {
      throw new CacheException("Operation was aborted");
    }
    return pin;
  }

  @Transactional(isolation = REPEATABLE_READ)
  protected void refreshTimeout(PinTask task, Date date) throws CacheException {
    Pin pin = loadPinBelongingTo(task);
    pin.setExpirationTime(date);
    task.setPin(_dao.storePin(pin));
  }

  @Transactional(isolation = REPEATABLE_READ)
  protected void setPool(PinTask task, String pool) throws CacheException {
    Pin pin = loadPinBelongingTo(task);
    pin.setExpirationTime(getExpirationTimeForSettingFlag());
    pin.setPool(pool);
    task.setPin(_dao.storePin(pin));
  }

  @Transactional(isolation = REPEATABLE_READ)
  protected void setToPinned(PinTask task) throws CacheException {
    Pin pin = loadPinBelongingTo(task);
    pin.setExpirationTime(task.getExpirationTime());
    pin.setState(PINNED);
    task.setPin(_dao.storePin(pin));
  }

  @Transactional
  protected void clearPin(PinTask task) {
    if (task.getPool() != null) {
      /* If the pin record expired or the pin was explicitly
       * unpinned, then the unpin processor may already have
       * submitted a request to the pool to clear the sticky
       * flag. Although out of order delivery of messages is
       * unlikely, if it would happen then we have a race
       * between the set sticky and clear sticky messages. To
       * cover this case we delete the old record and create a
       * fresh one in UNPINNING.
       */
      _dao.deletePin(task.getPin());
      Pin pin = new Pin(task.getSubject(), task.getPnfsId());
      pin.setState(UNPINNING);
      _dao.storePin(pin);
    } else {
      /* We didn't create a sticky flag yet, so there is no
       * reason to keep the record. It will expire by itself,
       * but we delete the record now to avoid that we get
       * tickets from admins wondering why they have records
       * staying in PINNING.
       */
      _dao.deletePin(task.getPin());
    }
  }
}
  private void askPoolManager(final PinTask task) {
    PoolMgrSelectReadPoolMsg msg =
        new PoolMgrSelectReadPoolMsg(
            task.getFileAttributes(),
            task.getProtocolInfo(),
            task.getReadPoolSelectionContext(),
            checkStaging(task));
    msg.setSubject(task.getSubject());
    msg.setSkipCostUpdate(true);
    _poolManagerStub.send(
        msg,
        PoolMgrSelectReadPoolMsg.class,
        new AbstractMessageCallback<PoolMgrSelectReadPoolMsg>() {
          @Override
          public void success(PoolMgrSelectReadPoolMsg msg) {
            try {
              /* Pool manager expects us
               * to keep some state
               * between retries.
               */
              task.setReadPoolSelectionContext(msg.getContext());

              /* Store the pool name in
               * the DB so we know what to
               * clean up if something
               * fails.
               */
              String poolName = msg.getPoolName();
              CellAddressCore poolAddress = msg.getPoolAddress();
              task.getFileAttributes().getLocations().add(poolName);
              setPool(task, poolName);

              setStickyFlag(task, poolName, poolAddress);
            } catch (CacheException e) {
              fail(task, e.getRc(), e.getMessage());
            } catch (RuntimeException e) {
              fail(task, CacheException.UNEXPECTED_SYSTEM_EXCEPTION, e.toString());
            }
          }

          @Override
          public void failure(int rc, Object error) {
            /* Pool manager expects us to
             * keep some state between
             * retries.
             */
            task.setReadPoolSelectionContext(getReply().getContext());
            switch (rc) {
              case CacheException.OUT_OF_DATE:
                /* Pool manager asked for a
                 * refresh of the request.
                 * Retry right away.
                 */
                retry(task, 0);
                break;
              case CacheException.FILE_NOT_IN_REPOSITORY:
              case CacheException.PERMISSION_DENIED:
                fail(task, rc, error.toString());
                break;
              default:
                /* Ideally we would delegate the retry to the door,
                 * but for the time being the retry is dealed with
                 * by pin manager.
                 */
                retry(task, RETRY_DELAY);
                break;
            }
          }

          @Override
          public void noroute(CellPath path) {
            /* Pool manager is
             * unreachable. We expect this
             * to be transient and retry in
             * a moment.
             */
            retry(task, RETRY_DELAY);
          }

          @Override
          public void timeout(CellPath path) {
            /* Pool manager did not
             * respond. We expect this to be
             * transient and retry in a
             * moment.
             */
            retry(task, SMALL_DELAY);
          }
        });
  }