@Override public void completed(Set<URI> uris) { try { descriptor.close(); FileAttributes fileAttributesForNotification = getFileAttributesForNotification(uris); infoMsg.setStorageInfo(fileAttributesForNotification.getStorageInfo()); PnfsId pnfsId = getFileAttributes().getPnfsId(); notifyNamespace(pnfsId, fileAttributesForNotification); try { repository.setState(pnfsId, ReplicaState.CACHED); } catch (IllegalTransitionException ignored) { /* Apparently the file is no longer precious. Most * likely it got deleted, which is fine, since the * flush already succeeded. */ } done(null); LOGGER.info("Flushed {} to nearline storage: {}", pnfsId, Joiner.on(' ').join(uris)); } catch (Exception e) { done(e); } }
@Override public String call() throws IOException, CacheException, InterruptedException { FileAttributes fileAttributes = new FileAttributes(); fileAttributes.setPnfsId(pnfsId); List<StickyRecord> stickyRecords = Collections.emptyList(); newCompanion(pool, fileAttributes, ReplicaState.CACHED, stickyRecords, null, false, null); return "Transfer Initiated"; }
public synchronized String ac_pp_get_file_$_2(Args args) throws CacheException, IOException, InterruptedException { FileAttributes fileAttributes = new FileAttributes(); fileAttributes.setPnfsId(new PnfsId(args.argv(0))); String pool = args.argv(1); List<StickyRecord> stickyRecords = Collections.emptyList(); newCompanion(pool, fileAttributes, EntryState.CACHED, stickyRecords, null, false, null); return "Transfer Initiated"; }
@Test public void shouldSelectAWritableRemoveLocation() throws Exception { givenAFileWithSeveralLocations(); givenThisManyLocationsAreOffline(attributes.getLocations().size() - 1); whenRemoveTargetIsSelected(); assertNotNull(selected); assertTrue(attributes.getLocations().contains(selected)); assertTrue(poolInfoMap.getPoolState(selected).mode.isEnabled()); }
/** * Injects the StorageInfo into the FileAttributes. * * <p>Legacy fields that used to be stored in StorageInfo, but are now stored in FileAttributes, * are injected into the FileAttributes too. * * <p>Should only be used when backwards compatibility must be maintained. */ public static FileAttributes injectInto(StorageInfo info, FileAttributes attributes) { attributes.setStorageInfo(info); attributes.setSize(info.getLegacySize()); attributes.setAccessLatency(info.getLegacyAccessLatency()); attributes.setRetentionPolicy(info.getLegacyRetentionPolicy()); String cFlag = info.getKey("flag-c"); if (cFlag != null) { attributes.setChecksums( Sets.newHashSet( transform( Splitter.on(',').trimResults().omitEmptyStrings().split(cFlag), new Function<String, Checksum>() { @Override public Checksum apply(String digest) { return Checksum.parseChecksum(digest); } }))); } String uid = info.getKey("uid"); if (uid != null) { attributes.setOwner(Integer.parseInt(uid)); } String gid = info.getKey("gid"); if (gid != null) { attributes.setGroup(Integer.parseInt(gid)); } attributes.setFlags(info.getMap()); return attributes; }
public StageRequestImpl(NearlineStorage storage, FileAttributes fileAttributes) throws CacheException { super(storage); PnfsId pnfsId = fileAttributes.getPnfsId(); infoMsg = new StorageInfoMessage(cellAddress.toString(), pnfsId, true); infoMsg.setStorageInfo(fileAttributes.getStorageInfo()); infoMsg.setFileSize(fileAttributes.getSize()); descriptor = repository.createEntry( fileAttributes, ReplicaState.FROM_STORE, ReplicaState.CACHED, Collections.emptyList(), EnumSet.noneOf(Repository.OpenFlags.class)); LOGGER.debug("Stage request created for {}.", pnfsId); }
@Test public void shouldFailToSelectAWritableRemoveLocation() throws Exception { givenAFileWithSeveralLocations(); givenThisManyLocationsAreOffline(attributes.getLocations().size()); whenRemoveTargetIsSelected(); assertNull(selected); }
@Override public PnfsId createDirectory(Subject subject, String path, FileAttributes attributes) throws CacheException { PnfsHandler pnfs = new PnfsHandler(_pnfs, subject, Restrictions.none()); attributes.setFileType(DIR); PnfsCreateEntryMessage returnMsg = pnfs.createPnfsDirectory(path, attributes); return returnMsg.getPnfsId(); }
private String getFlushCommand(File file, FileAttributes fileAttributes) { StorageInfo storageInfo = StorageInfos.extractFrom(fileAttributes); StringBuilder sb = new StringBuilder(command); sb.append(" put ").append(fileAttributes.getPnfsId()).append(' ').append(file.getPath()); sb.append(" -si=").append(storageInfo.toString()); sb.append(options); LOGGER.debug("COMMAND: {}", sb); return sb.toString(); }
private void whenCopyTargetIsSelected() { try { selected = locationSelector.selectCopyTarget( group, attributes.getLocations(), Collections.EMPTY_SET, Collections.EMPTY_SET); } catch (Exception e) { LOGGER.error("{}", new ExceptionMessage(e)); } }
public PoolIoFileMessage(String pool, ProtocolInfo protocolInfo, FileAttributes fileAttributes) { super(pool); checkNotNull(fileAttributes); checkArgument(fileAttributes.isDefined(EnumSet.of(STORAGEINFO, PNFSID))); _fileAttributes = fileAttributes; _protocolInfo = protocolInfo; }
private FileAttributes getFileAttributesForNotification(Set<URI> uris) throws CacheException { FileAttributes fileAttributes = descriptor.getFileAttributes(); StorageInfo storageInfo = fileAttributes.getStorageInfo().clone(); for (URI uri : uris) { try { HsmLocationExtractorFactory.validate(uri); storageInfo.addLocation(uri); storageInfo.isSetAddLocation(true); } catch (IllegalArgumentException e) { throw new CacheException(2, e.getMessage(), e); } } FileAttributes fileAttributesForNotification = new FileAttributes(); fileAttributesForNotification.setAccessLatency(fileAttributes.getAccessLatency()); fileAttributesForNotification.setRetentionPolicy(fileAttributes.getRetentionPolicy()); fileAttributesForNotification.setStorageInfo(storageInfo); fileAttributesForNotification.setSize(fileAttributes.getSize()); return fileAttributesForNotification; }
@Test public void shouldSelectAWritableCopyLocation() throws Exception { givenAFileWithSeveralLocations(); givenTheSetOfOnlinePoolsForTheGroup(); givenThisManyOtherPoolsInTheGroupAreDown(11); whenCopyTargetIsSelected(); assertNotNull(selected); assertFalse(attributes.getLocations().contains(selected)); assertTrue(available.contains(selected)); }
/** * Stages a file from nearline storage. * * <p>TODO: Should eventually accept multiple files at once, but the rest of the pool doesn't * support that yet. * * @param file attributes of file to stage * @param callback callback notified when file is staged */ public void stage( String hsmInstance, FileAttributes file, CompletionHandler<Void, PnfsId> callback) { try { NearlineStorage nearlineStorage = hsmSet.getNearlineStorageByName(hsmInstance); checkArgument(nearlineStorage != null, "No such nearline storage: " + hsmInstance); stageRequests.addAll(nearlineStorage, Collections.singleton(file), callback); } catch (RuntimeException e) { callback.failed(e, file.getPnfsId()); } }
public synchronized int newCompanion( String sourcePoolName, FileAttributes fileAttributes, EntryState targetState, List<StickyRecord> stickyRecords, CacheFileAvailable callback, boolean forceSourceMode, Long atime) throws IOException, CacheException, InterruptedException { if (getCellEndpoint() == null) { throw new IllegalStateException("Endpoint not initialized"); } if (_pool == null) { throw new IllegalStateException("Pool stub not initialized"); } if (_executor == null) { throw new IllegalStateException("Executor not initialized"); } if (_repository == null) { throw new IllegalStateException("Repository not initialized"); } if (_checksumModule == null) { throw new IllegalStateException("Checksum module not initialized"); } if (_pnfs == null) { throw new IllegalStateException("PNFS stub not initialized"); } if (_repository.getState(fileAttributes.getPnfsId()) != EntryState.NEW) { throw new IllegalStateException("Replica already exists"); } Callback cb = new Callback(callback); Companion companion = new Companion( _executor, getInterface(), _repository, _checksumModule, _pnfs, _pool, fileAttributes, sourcePoolName, getCellName(), getCellDomainName(), targetState, stickyRecords, cb, forceSourceMode, atime); int id = addCompanion(companion); cb.setId(id); return id; }
private void givenThisManyLocationsAreOffline(int offline) throws CacheException { collection = attributes.getLocations(); int count = 0; for (String location : collection) { givenLocationGoesOffline(location); if (++count >= offline) { break; } } }
private void givenThisManyOtherPoolsInTheGroupAreDown(int down) throws CacheException { available = Sets.difference(readable, ImmutableSet.copyOf(attributes.getLocations())); down = Math.min(available.size(), down); int count = 0; for (String pool : available) { givenLocationGoesOffline(pool); if (++count >= down) { break; } } }
public boolean isPinned(FileAttributes fileAttributes, CellStub cellStub) throws CacheException, InterruptedException, URISyntaxException { boolean isPinned = false; PinManagerCountPinsMessage message = new PinManagerCountPinsMessage(fileAttributes.getPnfsId()); message = cellStub.sendAndWait(message); if (message.getCount() != 0) { isPinned = true; } return isPinned; }
@Override public FileAttributes createFile( Subject subject, String path, FileAttributes assignAttributes, Set<FileAttribute> requestedAttributes) throws CacheException { PnfsHandler pnfs = new PnfsHandler(_pnfs, subject, Restrictions.none()); assignAttributes.setFileType(REGULAR); PnfsCreateEntryMessage message = new PnfsCreateEntryMessage(path, assignAttributes, requestedAttributes); return pnfs.request(message).getFileAttributes(); }
private String getFetchCommand(File file, FileAttributes attributes) { StorageInfo storageInfo = StorageInfos.extractFrom(attributes); StringBuilder sb = new StringBuilder(command); sb.append(" get ").append(attributes.getPnfsId()).append(' ').append(file.getPath()); sb.append(" -si=").append(storageInfo.toString()); for (URI location : getLocations(attributes)) { if (location.getScheme().equals(type) && location.getAuthority().equals(name)) { sb.append(" -uri=").append(location.toString()); } } sb.append(options); LOGGER.debug("COMMAND: {}", sb); return sb.toString(); }
@VisibleForTesting String[] getFlushCommand(URI dataFile, FileAttributes fileAttributes) { StorageInfo storageInfo = StorageInfos.extractFrom(fileAttributes); String[] argsArray = Stream.concat( Stream.of( command, "put", fileAttributes.getPnfsId().toString(), getFileString(dataFile), "-si=" + storageInfo), options.stream()) .toArray(String[]::new); LOGGER.debug("COMMAND: {}", Arrays.deepToString(argsArray)); return argsArray; }
@VisibleForTesting String[] getFetchCommand(URI dataFile, FileAttributes fileAttributes) { StorageInfo storageInfo = StorageInfos.extractFrom(fileAttributes); String[] argsArray = Stream.of( Stream.of( command, "get", fileAttributes.getPnfsId().toString(), getFileString(dataFile), "-si=" + storageInfo), getLocations(fileAttributes).stream().map(uri -> "-uri=" + uri), options.stream()) .flatMap(s -> s) .toArray(String[]::new); LOGGER.debug("COMMAND: {}", Arrays.deepToString(argsArray)); return argsArray; }
public void unpin(FileAttributes fileAttributes, CellStub cellStub) { PinManagerUnpinMessage message = new PinManagerUnpinMessage(fileAttributes.getPnfsId()); message.setRequestId(requestId); cellStub.notify(message); }
private static String buildDigest(NettyTransferService<HttpProtocolInfo>.NettyMoverChannel file) { FileAttributes attributes = file.getFileAttributes(); return attributes.getChecksumsIfPresent().transform(TO_RFC3230).or(""); }
/** * Extracts the StorageInfo stored in the FileAttributes. * * <p>Initializes legacy fields that used to be stored in StorageInfo, but are now stored in * FileAttributes. * * <p>Should only be used when backwards compatibility must be maintained. */ public static StorageInfo extractFrom(FileAttributes attributes) { StorageInfo info = attributes.getStorageInfo(); if (attributes.isDefined(FileAttribute.SIZE)) { info.setLegacySize(attributes.getSize()); } if (attributes.isDefined(FileAttribute.ACCESS_LATENCY)) { info.setLegacyAccessLatency(attributes.getAccessLatency()); } if (attributes.isDefined(FileAttribute.RETENTION_POLICY)) { info.setLegacyRetentionPolicy(attributes.getRetentionPolicy()); } if (attributes.isDefined(FileAttribute.FLAGS)) { for (Map.Entry<String, String> entry : attributes.getFlags().entrySet()) { info.setKey(entry.getKey(), entry.getValue()); } } if (attributes.isDefined(FileAttribute.CHECKSUM)) { info.setKey("flag-c", Joiner.on(',').join(attributes.getChecksums())); } if (attributes.isDefined(FileAttribute.OWNER)) { info.setKey("uid", Integer.toString(attributes.getOwner())); } if (attributes.isDefined(FileAttribute.OWNER_GROUP)) { info.setKey("gid", Integer.toString(attributes.getGroup())); } return info; }
private void whenReadableMemberLocationsAreRequested() { collection = locationSelector.getReadableMemberLocations(group, attributes.getLocations()); }
public PoolIoFileMessage(String pool, PnfsId pnfsId, ProtocolInfo protocolInfo) { super(pool); _protocolInfo = protocolInfo; _fileAttributes = new FileAttributes(); _fileAttributes.setPnfsId(pnfsId); }
private void setInfoFromAttributes() { pool = attributes.getLocations().iterator().next(); group = poolInfoMap.getResilientPoolGroup(poolInfoMap.getPoolIndex(pool)); }
public PnfsId getPnfsId() { return _fileAttributes.getPnfsId(); }
@Override protected PnfsId extractKey(FileAttributes file) { return file.getPnfsId(); }