@Transactional(isolation = REPEATABLE_READ)
protected void setToPinned(PinTask task) throws CacheException {
Pin pin = loadPinBelongingTo(task);
pin.setExpirationTime(task.getExpirationTime());
pin.setState(PINNED);
task.setPin(_dao.storePin(pin));
}
@Transactional
protected void clearPin(PinTask task) {
if (task.getPool() != null) {
/* If the pin record expired or the pin was explicitly
* unpinned, then the unpin processor may already have
* submitted a request to the pool to clear the sticky
* flag. Although out of order delivery of messages is
* unlikely, if it would happen then we have a race
* between the set sticky and clear sticky messages. To
* cover this case we delete the old record and create a
* fresh one in UNPINNING.
*/
_dao.deletePin(task.getPin());
Pin pin = new Pin(task.getSubject(), task.getPnfsId());
pin.setState(UNPINNING);
_dao.storePin(pin);
} else {
/* We didn't create a sticky flag yet, so there is no
* reason to keep the record. It will expire by itself,
* but we delete the record now to avoid that we get
* tickets from admins wondering why they have records
* staying in PINNING.
*/
_dao.deletePin(task.getPin());
}
}
/**
* Load the pin belonging to the PinTask.
*
* @throw CacheException if the pin no longer exists or is no longer in PINNING.
*/
protected Pin loadPinBelongingTo(PinTask task) throws CacheException {
Pin pin = _dao.getPin(task.getPinId(), task.getSticky(), PINNING);
if (pin == null) {
throw new CacheException("Operation was aborted");
}
return pin;
}
private void rereadNameSpaceEntry(final PinTask task) throws CacheException {
/* Ensure that task is still valid and stays valid for the
* duration of the name space lookup.
*/
refreshTimeout(task, getExpirationTimeForNameSpaceLookup());
/* We allow the set of provided attributes to be incomplete
* and thus add attributes required by pool manager.
*/
Set<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class);
attributes.addAll(task.getFileAttributes().getDefinedAttributes());
attributes.addAll(PoolMgrSelectReadPoolMsg.getRequiredAttributes());
_pnfsStub.send(
new PnfsGetFileAttributes(task.getPnfsId(), attributes),
PnfsGetFileAttributes.class,
new AbstractMessageCallback<PnfsGetFileAttributes>() {
@Override
public void success(PnfsGetFileAttributes msg) {
try {
task.setFileAttributes(msg.getFileAttributes());
/* Ensure that task is still valid
* and stays valid for the duration
* of the pool selection.
*/
refreshTimeout(task, getExpirationTimeForPoolSelection());
selectReadPool(task);
} catch (CacheException e) {
fail(task, e.getRc(), e.getMessage());
} catch (RuntimeException e) {
fail(task, CacheException.UNEXPECTED_SYSTEM_EXCEPTION, e.toString());
}
}
@Override
public void failure(int rc, Object error) {
fail(task, rc, error.toString());
}
@Override
public void noroute(CellPath path) {
/* PnfsManager is unreachable. We
* expect this to be a transient
* problem and retry in a moment.
*/
retry(task, RETRY_DELAY);
}
@Override
public void timeout(CellPath path) {
/* PnfsManager did not respond. We
* expect this to be a transient
* problem and retry in a moment.
*/
retry(task, SMALL_DELAY);
}
});
}
private void selectReadPool(final PinTask task) throws CacheException {
try {
PoolSelector poolSelector =
_poolMonitor.getPoolSelector(task.getFileAttributes(), task.getProtocolInfo(), null);
PoolInfo pool = poolSelector.selectPinPool();
setPool(task, pool.getName());
setStickyFlag(task, pool.getName(), pool.getAddress());
} catch (FileNotOnlineCacheException e) {
askPoolManager(task);
}
}
protected EnumSet<RequestContainerV5.RequestState> checkStaging(PinTask task) {
try {
Subject subject = task.getSubject();
StorageInfo info = task.getFileAttributes().getStorageInfo();
return _checkStagePermission.canPerformStaging(subject, info)
? RequestContainerV5.allStates
: RequestContainerV5.allStatesExceptStage;
} catch (PatternSyntaxException | IOException ex) {
_log.error("Failed to check stage permission: " + ex);
}
return RequestContainerV5.allStatesExceptStage;
}
@Transactional(isolation = REPEATABLE_READ)
protected void setPool(PinTask task, String pool) throws CacheException {
Pin pin = loadPinBelongingTo(task);
pin.setExpirationTime(getExpirationTimeForSettingFlag());
pin.setPool(pool);
task.setPin(_dao.storePin(pin));
}
private void fail(PinTask task, int rc, String error) {
try {
task.fail(rc, error);
clearPin(task);
} catch (RuntimeException e) {
_log.error(e.toString());
}
}
public MessageReply<PinManagerPinMessage> messageArrived(PinManagerPinMessage message)
throws CacheException {
MessageReply<PinManagerPinMessage> reply = new MessageReply<>();
enforceLifetimeLimit(message);
PinTask task = createTask(message, reply);
if (task != null) {
if (!task.getFileAttributes().isDefined(REQUIRED_ATTRIBUTES)) {
rereadNameSpaceEntry(task);
} else {
selectReadPool(task);
}
}
return reply;
}
private void retry(final PinTask task, long delay) {
if (!task.isValidIn(delay)) {
fail(task, CacheException.TIMEOUT, "Pin request TTL exceeded");
} else {
_executor.schedule(
new Runnable() {
@Override
public void run() {
try {
rereadNameSpaceEntry(task);
} catch (CacheException e) {
fail(task, e.getRc(), e.getMessage());
} catch (RuntimeException e) {
fail(task, CacheException.UNEXPECTED_SYSTEM_EXCEPTION, e.toString());
}
}
},
delay,
MILLISECONDS);
}
}
@Transactional(isolation = REPEATABLE_READ)
protected void refreshTimeout(PinTask task, Date date) throws CacheException {
Pin pin = loadPinBelongingTo(task);
pin.setExpirationTime(date);
task.setPin(_dao.storePin(pin));
}
private void setStickyFlag(
final PinTask task, final String poolName, CellAddressCore poolAddress) {
/* The pin lifetime should be from the moment the file is
* actually pinned. Due to staging and pool to pool transfers
* this may be much later than when the pin was requested.
*/
Date pinExpiration = task.freezeExpirationTime();
/* To allow for some drift in clocks we add a safety margin to
* the lifetime of the sticky bit.
*/
long poolExpiration =
(pinExpiration == null) ? -1 : pinExpiration.getTime() + CLOCK_DRIFT_MARGIN;
PoolSetStickyMessage msg =
new PoolSetStickyMessage(
poolName, task.getPnfsId(), true, task.getSticky(), poolExpiration);
_poolStub.send(
new CellPath(poolAddress),
msg,
PoolSetStickyMessage.class,
new AbstractMessageCallback<PoolSetStickyMessage>() {
@Override
public void success(PoolSetStickyMessage msg) {
try {
setToPinned(task);
task.success();
} catch (CacheException e) {
fail(task, e.getRc(), e.getMessage());
} catch (RuntimeException e) {
fail(task, CacheException.UNEXPECTED_SYSTEM_EXCEPTION, e.toString());
}
}
@Override
public void failure(int rc, Object error) {
switch (rc) {
case CacheException.POOL_DISABLED:
/* Pool manager had outdated
* information about the pool. Give
* it a chance to be updated and
* then retry.
*/
retry(task, RETRY_DELAY);
break;
case CacheException.FILE_NOT_IN_REPOSITORY:
/* Pnfs manager had stale location
* information. The pool clears
* this information as a result of
* this error, so we retry in a
* moment.
*/
retry(task, SMALL_DELAY);
break;
default:
fail(task, rc, error.toString());
break;
}
}
@Override
public void noroute(CellPath path) {
/* The pool must have gone down. Give
* pool manager a moment to notice this
* and then retry.
*/
retry(task, RETRY_DELAY);
}
@Override
public void timeout(CellPath path) {
/* No response from pool. Typically this is
* because the pool is overloaded.
*/
fail(task, CacheException.TIMEOUT, "No reply from " + path);
}
});
}
private void askPoolManager(final PinTask task) {
PoolMgrSelectReadPoolMsg msg =
new PoolMgrSelectReadPoolMsg(
task.getFileAttributes(),
task.getProtocolInfo(),
task.getReadPoolSelectionContext(),
checkStaging(task));
msg.setSubject(task.getSubject());
msg.setSkipCostUpdate(true);
_poolManagerStub.send(
msg,
PoolMgrSelectReadPoolMsg.class,
new AbstractMessageCallback<PoolMgrSelectReadPoolMsg>() {
@Override
public void success(PoolMgrSelectReadPoolMsg msg) {
try {
/* Pool manager expects us
* to keep some state
* between retries.
*/
task.setReadPoolSelectionContext(msg.getContext());
/* Store the pool name in
* the DB so we know what to
* clean up if something
* fails.
*/
String poolName = msg.getPoolName();
CellAddressCore poolAddress = msg.getPoolAddress();
task.getFileAttributes().getLocations().add(poolName);
setPool(task, poolName);
setStickyFlag(task, poolName, poolAddress);
} catch (CacheException e) {
fail(task, e.getRc(), e.getMessage());
} catch (RuntimeException e) {
fail(task, CacheException.UNEXPECTED_SYSTEM_EXCEPTION, e.toString());
}
}
@Override
public void failure(int rc, Object error) {
/* Pool manager expects us to
* keep some state between
* retries.
*/
task.setReadPoolSelectionContext(getReply().getContext());
switch (rc) {
case CacheException.OUT_OF_DATE:
/* Pool manager asked for a
* refresh of the request.
* Retry right away.
*/
retry(task, 0);
break;
case CacheException.FILE_NOT_IN_REPOSITORY:
case CacheException.PERMISSION_DENIED:
fail(task, rc, error.toString());
break;
default:
/* Ideally we would delegate the retry to the door,
* but for the time being the retry is dealed with
* by pin manager.
*/
retry(task, RETRY_DELAY);
break;
}
}
@Override
public void noroute(CellPath path) {
/* Pool manager is
* unreachable. We expect this
* to be transient and retry in
* a moment.
*/
retry(task, RETRY_DELAY);
}
@Override
public void timeout(CellPath path) {
/* Pool manager did not
* respond. We expect this to be
* transient and retry in a
* moment.
*/
retry(task, SMALL_DELAY);
}
});
}