@Override
    public void completed(Set<URI> uris) {
      try {
        descriptor.close();

        FileAttributes fileAttributesForNotification = getFileAttributesForNotification(uris);

        infoMsg.setStorageInfo(fileAttributesForNotification.getStorageInfo());

        PnfsId pnfsId = getFileAttributes().getPnfsId();
        notifyNamespace(pnfsId, fileAttributesForNotification);

        try {
          repository.setState(pnfsId, ReplicaState.CACHED);
        } catch (IllegalTransitionException ignored) {
          /* Apparently the file is no longer precious. Most
           * likely it got deleted, which is fine, since the
           * flush already succeeded.
           */
        }
        done(null);

        LOGGER.info("Flushed {} to nearline storage: {}", pnfsId, Joiner.on(' ').join(uris));
      } catch (Exception e) {
        done(e);
      }
    }
Exemple #2
0
 @Override
 public String call() throws IOException, CacheException, InterruptedException {
   FileAttributes fileAttributes = new FileAttributes();
   fileAttributes.setPnfsId(pnfsId);
   List<StickyRecord> stickyRecords = Collections.emptyList();
   newCompanion(pool, fileAttributes, ReplicaState.CACHED, stickyRecords, null, false, null);
   return "Transfer Initiated";
 }
Exemple #3
0
 public synchronized String ac_pp_get_file_$_2(Args args)
     throws CacheException, IOException, InterruptedException {
   FileAttributes fileAttributes = new FileAttributes();
   fileAttributes.setPnfsId(new PnfsId(args.argv(0)));
   String pool = args.argv(1);
   List<StickyRecord> stickyRecords = Collections.emptyList();
   newCompanion(pool, fileAttributes, EntryState.CACHED, stickyRecords, null, false, null);
   return "Transfer Initiated";
 }
 @Test
 public void shouldSelectAWritableRemoveLocation() throws Exception {
   givenAFileWithSeveralLocations();
   givenThisManyLocationsAreOffline(attributes.getLocations().size() - 1);
   whenRemoveTargetIsSelected();
   assertNotNull(selected);
   assertTrue(attributes.getLocations().contains(selected));
   assertTrue(poolInfoMap.getPoolState(selected).mode.isEnabled());
 }
 /**
  * Injects the StorageInfo into the FileAttributes.
  *
  * <p>Legacy fields that used to be stored in StorageInfo, but are now stored in FileAttributes,
  * are injected into the FileAttributes too.
  *
  * <p>Should only be used when backwards compatibility must be maintained.
  */
 public static FileAttributes injectInto(StorageInfo info, FileAttributes attributes) {
   attributes.setStorageInfo(info);
   attributes.setSize(info.getLegacySize());
   attributes.setAccessLatency(info.getLegacyAccessLatency());
   attributes.setRetentionPolicy(info.getLegacyRetentionPolicy());
   String cFlag = info.getKey("flag-c");
   if (cFlag != null) {
     attributes.setChecksums(
         Sets.newHashSet(
             transform(
                 Splitter.on(',').trimResults().omitEmptyStrings().split(cFlag),
                 new Function<String, Checksum>() {
                   @Override
                   public Checksum apply(String digest) {
                     return Checksum.parseChecksum(digest);
                   }
                 })));
   }
   String uid = info.getKey("uid");
   if (uid != null) {
     attributes.setOwner(Integer.parseInt(uid));
   }
   String gid = info.getKey("gid");
   if (gid != null) {
     attributes.setGroup(Integer.parseInt(gid));
   }
   attributes.setFlags(info.getMap());
   return attributes;
 }
 public StageRequestImpl(NearlineStorage storage, FileAttributes fileAttributes)
     throws CacheException {
   super(storage);
   PnfsId pnfsId = fileAttributes.getPnfsId();
   infoMsg = new StorageInfoMessage(cellAddress.toString(), pnfsId, true);
   infoMsg.setStorageInfo(fileAttributes.getStorageInfo());
   infoMsg.setFileSize(fileAttributes.getSize());
   descriptor =
       repository.createEntry(
           fileAttributes,
           ReplicaState.FROM_STORE,
           ReplicaState.CACHED,
           Collections.emptyList(),
           EnumSet.noneOf(Repository.OpenFlags.class));
   LOGGER.debug("Stage request created for {}.", pnfsId);
 }
 @Test
 public void shouldFailToSelectAWritableRemoveLocation() throws Exception {
   givenAFileWithSeveralLocations();
   givenThisManyLocationsAreOffline(attributes.getLocations().size());
   whenRemoveTargetIsSelected();
   assertNull(selected);
 }
 @Override
 public PnfsId createDirectory(Subject subject, String path, FileAttributes attributes)
     throws CacheException {
   PnfsHandler pnfs = new PnfsHandler(_pnfs, subject, Restrictions.none());
   attributes.setFileType(DIR);
   PnfsCreateEntryMessage returnMsg = pnfs.createPnfsDirectory(path, attributes);
   return returnMsg.getPnfsId();
 }
 private String getFlushCommand(File file, FileAttributes fileAttributes) {
   StorageInfo storageInfo = StorageInfos.extractFrom(fileAttributes);
   StringBuilder sb = new StringBuilder(command);
   sb.append(" put ").append(fileAttributes.getPnfsId()).append(' ').append(file.getPath());
   sb.append(" -si=").append(storageInfo.toString());
   sb.append(options);
   LOGGER.debug("COMMAND: {}", sb);
   return sb.toString();
 }
 private void whenCopyTargetIsSelected() {
   try {
     selected =
         locationSelector.selectCopyTarget(
             group, attributes.getLocations(), Collections.EMPTY_SET, Collections.EMPTY_SET);
   } catch (Exception e) {
     LOGGER.error("{}", new ExceptionMessage(e));
   }
 }
  public PoolIoFileMessage(String pool, ProtocolInfo protocolInfo, FileAttributes fileAttributes) {
    super(pool);

    checkNotNull(fileAttributes);
    checkArgument(fileAttributes.isDefined(EnumSet.of(STORAGEINFO, PNFSID)));

    _fileAttributes = fileAttributes;
    _protocolInfo = protocolInfo;
  }
 private FileAttributes getFileAttributesForNotification(Set<URI> uris) throws CacheException {
   FileAttributes fileAttributes = descriptor.getFileAttributes();
   StorageInfo storageInfo = fileAttributes.getStorageInfo().clone();
   for (URI uri : uris) {
     try {
       HsmLocationExtractorFactory.validate(uri);
       storageInfo.addLocation(uri);
       storageInfo.isSetAddLocation(true);
     } catch (IllegalArgumentException e) {
       throw new CacheException(2, e.getMessage(), e);
     }
   }
   FileAttributes fileAttributesForNotification = new FileAttributes();
   fileAttributesForNotification.setAccessLatency(fileAttributes.getAccessLatency());
   fileAttributesForNotification.setRetentionPolicy(fileAttributes.getRetentionPolicy());
   fileAttributesForNotification.setStorageInfo(storageInfo);
   fileAttributesForNotification.setSize(fileAttributes.getSize());
   return fileAttributesForNotification;
 }
 @Test
 public void shouldSelectAWritableCopyLocation() throws Exception {
   givenAFileWithSeveralLocations();
   givenTheSetOfOnlinePoolsForTheGroup();
   givenThisManyOtherPoolsInTheGroupAreDown(11);
   whenCopyTargetIsSelected();
   assertNotNull(selected);
   assertFalse(attributes.getLocations().contains(selected));
   assertTrue(available.contains(selected));
 }
 /**
  * Stages a file from nearline storage.
  *
  * <p>TODO: Should eventually accept multiple files at once, but the rest of the pool doesn't
  * support that yet.
  *
  * @param file attributes of file to stage
  * @param callback callback notified when file is staged
  */
 public void stage(
     String hsmInstance, FileAttributes file, CompletionHandler<Void, PnfsId> callback) {
   try {
     NearlineStorage nearlineStorage = hsmSet.getNearlineStorageByName(hsmInstance);
     checkArgument(nearlineStorage != null, "No such nearline storage: " + hsmInstance);
     stageRequests.addAll(nearlineStorage, Collections.singleton(file), callback);
   } catch (RuntimeException e) {
     callback.failed(e, file.getPnfsId());
   }
 }
Exemple #15
0
  public synchronized int newCompanion(
      String sourcePoolName,
      FileAttributes fileAttributes,
      EntryState targetState,
      List<StickyRecord> stickyRecords,
      CacheFileAvailable callback,
      boolean forceSourceMode,
      Long atime)
      throws IOException, CacheException, InterruptedException {
    if (getCellEndpoint() == null) {
      throw new IllegalStateException("Endpoint not initialized");
    }
    if (_pool == null) {
      throw new IllegalStateException("Pool stub not initialized");
    }
    if (_executor == null) {
      throw new IllegalStateException("Executor not initialized");
    }
    if (_repository == null) {
      throw new IllegalStateException("Repository not initialized");
    }
    if (_checksumModule == null) {
      throw new IllegalStateException("Checksum module not initialized");
    }
    if (_pnfs == null) {
      throw new IllegalStateException("PNFS stub not initialized");
    }
    if (_repository.getState(fileAttributes.getPnfsId()) != EntryState.NEW) {
      throw new IllegalStateException("Replica already exists");
    }

    Callback cb = new Callback(callback);

    Companion companion =
        new Companion(
            _executor,
            getInterface(),
            _repository,
            _checksumModule,
            _pnfs,
            _pool,
            fileAttributes,
            sourcePoolName,
            getCellName(),
            getCellDomainName(),
            targetState,
            stickyRecords,
            cb,
            forceSourceMode,
            atime);

    int id = addCompanion(companion);
    cb.setId(id);
    return id;
  }
  private void givenThisManyLocationsAreOffline(int offline) throws CacheException {
    collection = attributes.getLocations();
    int count = 0;

    for (String location : collection) {
      givenLocationGoesOffline(location);
      if (++count >= offline) {
        break;
      }
    }
  }
 private void givenThisManyOtherPoolsInTheGroupAreDown(int down) throws CacheException {
   available = Sets.difference(readable, ImmutableSet.copyOf(attributes.getLocations()));
   down = Math.min(available.size(), down);
   int count = 0;
   for (String pool : available) {
     givenLocationGoesOffline(pool);
     if (++count >= down) {
       break;
     }
   }
 }
  public boolean isPinned(FileAttributes fileAttributes, CellStub cellStub)
      throws CacheException, InterruptedException, URISyntaxException {
    boolean isPinned = false;

    PinManagerCountPinsMessage message = new PinManagerCountPinsMessage(fileAttributes.getPnfsId());

    message = cellStub.sendAndWait(message);
    if (message.getCount() != 0) {
      isPinned = true;
    }
    return isPinned;
  }
 @Override
 public FileAttributes createFile(
     Subject subject,
     String path,
     FileAttributes assignAttributes,
     Set<FileAttribute> requestedAttributes)
     throws CacheException {
   PnfsHandler pnfs = new PnfsHandler(_pnfs, subject, Restrictions.none());
   assignAttributes.setFileType(REGULAR);
   PnfsCreateEntryMessage message =
       new PnfsCreateEntryMessage(path, assignAttributes, requestedAttributes);
   return pnfs.request(message).getFileAttributes();
 }
 private String getFetchCommand(File file, FileAttributes attributes) {
   StorageInfo storageInfo = StorageInfos.extractFrom(attributes);
   StringBuilder sb = new StringBuilder(command);
   sb.append(" get ").append(attributes.getPnfsId()).append(' ').append(file.getPath());
   sb.append(" -si=").append(storageInfo.toString());
   for (URI location : getLocations(attributes)) {
     if (location.getScheme().equals(type) && location.getAuthority().equals(name)) {
       sb.append(" -uri=").append(location.toString());
     }
   }
   sb.append(options);
   LOGGER.debug("COMMAND: {}", sb);
   return sb.toString();
 }
 @VisibleForTesting
 String[] getFlushCommand(URI dataFile, FileAttributes fileAttributes) {
   StorageInfo storageInfo = StorageInfos.extractFrom(fileAttributes);
   String[] argsArray =
       Stream.concat(
               Stream.of(
                   command,
                   "put",
                   fileAttributes.getPnfsId().toString(),
                   getFileString(dataFile),
                   "-si=" + storageInfo),
               options.stream())
           .toArray(String[]::new);
   LOGGER.debug("COMMAND: {}", Arrays.deepToString(argsArray));
   return argsArray;
 }
 @VisibleForTesting
 String[] getFetchCommand(URI dataFile, FileAttributes fileAttributes) {
   StorageInfo storageInfo = StorageInfos.extractFrom(fileAttributes);
   String[] argsArray =
       Stream.of(
               Stream.of(
                   command,
                   "get",
                   fileAttributes.getPnfsId().toString(),
                   getFileString(dataFile),
                   "-si=" + storageInfo),
               getLocations(fileAttributes).stream().map(uri -> "-uri=" + uri),
               options.stream())
           .flatMap(s -> s)
           .toArray(String[]::new);
   LOGGER.debug("COMMAND: {}", Arrays.deepToString(argsArray));
   return argsArray;
 }
  public void unpin(FileAttributes fileAttributes, CellStub cellStub) {

    PinManagerUnpinMessage message = new PinManagerUnpinMessage(fileAttributes.getPnfsId());
    message.setRequestId(requestId);
    cellStub.notify(message);
  }
 private static String buildDigest(NettyTransferService<HttpProtocolInfo>.NettyMoverChannel file) {
   FileAttributes attributes = file.getFileAttributes();
   return attributes.getChecksumsIfPresent().transform(TO_RFC3230).or("");
 }
 /**
  * Extracts the StorageInfo stored in the FileAttributes.
  *
  * <p>Initializes legacy fields that used to be stored in StorageInfo, but are now stored in
  * FileAttributes.
  *
  * <p>Should only be used when backwards compatibility must be maintained.
  */
 public static StorageInfo extractFrom(FileAttributes attributes) {
   StorageInfo info = attributes.getStorageInfo();
   if (attributes.isDefined(FileAttribute.SIZE)) {
     info.setLegacySize(attributes.getSize());
   }
   if (attributes.isDefined(FileAttribute.ACCESS_LATENCY)) {
     info.setLegacyAccessLatency(attributes.getAccessLatency());
   }
   if (attributes.isDefined(FileAttribute.RETENTION_POLICY)) {
     info.setLegacyRetentionPolicy(attributes.getRetentionPolicy());
   }
   if (attributes.isDefined(FileAttribute.FLAGS)) {
     for (Map.Entry<String, String> entry : attributes.getFlags().entrySet()) {
       info.setKey(entry.getKey(), entry.getValue());
     }
   }
   if (attributes.isDefined(FileAttribute.CHECKSUM)) {
     info.setKey("flag-c", Joiner.on(',').join(attributes.getChecksums()));
   }
   if (attributes.isDefined(FileAttribute.OWNER)) {
     info.setKey("uid", Integer.toString(attributes.getOwner()));
   }
   if (attributes.isDefined(FileAttribute.OWNER_GROUP)) {
     info.setKey("gid", Integer.toString(attributes.getGroup()));
   }
   return info;
 }
 private void whenReadableMemberLocationsAreRequested() {
   collection = locationSelector.getReadableMemberLocations(group, attributes.getLocations());
 }
 public PoolIoFileMessage(String pool, PnfsId pnfsId, ProtocolInfo protocolInfo) {
   super(pool);
   _protocolInfo = protocolInfo;
   _fileAttributes = new FileAttributes();
   _fileAttributes.setPnfsId(pnfsId);
 }
 private void setInfoFromAttributes() {
   pool = attributes.getLocations().iterator().next();
   group = poolInfoMap.getResilientPoolGroup(poolInfoMap.getPoolIndex(pool));
 }
 public PnfsId getPnfsId() {
   return _fileAttributes.getPnfsId();
 }
 @Override
 protected PnfsId extractKey(FileAttributes file) {
   return file.getPnfsId();
 }