private String getFlushCommand(File file, FileAttributes fileAttributes) {
   StorageInfo storageInfo = StorageInfos.extractFrom(fileAttributes);
   StringBuilder sb = new StringBuilder(command);
   sb.append(" put ").append(fileAttributes.getPnfsId()).append(' ').append(file.getPath());
   sb.append(" -si=").append(storageInfo.toString());
   sb.append(options);
   LOGGER.debug("COMMAND: {}", sb);
   return sb.toString();
 }
예제 #2
0
 /**
  * Stages a file from nearline storage.
  *
  * <p>TODO: Should eventually accept multiple files at once, but the rest of the pool doesn't
  * support that yet.
  *
  * @param file attributes of file to stage
  * @param callback callback notified when file is staged
  */
 public void stage(
     String hsmInstance, FileAttributes file, CompletionHandler<Void, PnfsId> callback) {
   try {
     NearlineStorage nearlineStorage = hsmSet.getNearlineStorageByName(hsmInstance);
     checkArgument(nearlineStorage != null, "No such nearline storage: " + hsmInstance);
     stageRequests.addAll(nearlineStorage, Collections.singleton(file), callback);
   } catch (RuntimeException e) {
     callback.failed(e, file.getPnfsId());
   }
 }
예제 #3
0
  public synchronized int newCompanion(
      String sourcePoolName,
      FileAttributes fileAttributes,
      EntryState targetState,
      List<StickyRecord> stickyRecords,
      CacheFileAvailable callback,
      boolean forceSourceMode,
      Long atime)
      throws IOException, CacheException, InterruptedException {
    if (getCellEndpoint() == null) {
      throw new IllegalStateException("Endpoint not initialized");
    }
    if (_pool == null) {
      throw new IllegalStateException("Pool stub not initialized");
    }
    if (_executor == null) {
      throw new IllegalStateException("Executor not initialized");
    }
    if (_repository == null) {
      throw new IllegalStateException("Repository not initialized");
    }
    if (_checksumModule == null) {
      throw new IllegalStateException("Checksum module not initialized");
    }
    if (_pnfs == null) {
      throw new IllegalStateException("PNFS stub not initialized");
    }
    if (_repository.getState(fileAttributes.getPnfsId()) != EntryState.NEW) {
      throw new IllegalStateException("Replica already exists");
    }

    Callback cb = new Callback(callback);

    Companion companion =
        new Companion(
            _executor,
            getInterface(),
            _repository,
            _checksumModule,
            _pnfs,
            _pool,
            fileAttributes,
            sourcePoolName,
            getCellName(),
            getCellDomainName(),
            targetState,
            stickyRecords,
            cb,
            forceSourceMode,
            atime);

    int id = addCompanion(companion);
    cb.setId(id);
    return id;
  }
예제 #4
0
  public boolean isPinned(FileAttributes fileAttributes, CellStub cellStub)
      throws CacheException, InterruptedException, URISyntaxException {
    boolean isPinned = false;

    PinManagerCountPinsMessage message = new PinManagerCountPinsMessage(fileAttributes.getPnfsId());

    message = cellStub.sendAndWait(message);
    if (message.getCount() != 0) {
      isPinned = true;
    }
    return isPinned;
  }
 private String getFetchCommand(File file, FileAttributes attributes) {
   StorageInfo storageInfo = StorageInfos.extractFrom(attributes);
   StringBuilder sb = new StringBuilder(command);
   sb.append(" get ").append(attributes.getPnfsId()).append(' ').append(file.getPath());
   sb.append(" -si=").append(storageInfo.toString());
   for (URI location : getLocations(attributes)) {
     if (location.getScheme().equals(type) && location.getAuthority().equals(name)) {
       sb.append(" -uri=").append(location.toString());
     }
   }
   sb.append(options);
   LOGGER.debug("COMMAND: {}", sb);
   return sb.toString();
 }
예제 #6
0
 public StageRequestImpl(NearlineStorage storage, FileAttributes fileAttributes)
     throws CacheException {
   super(storage);
   PnfsId pnfsId = fileAttributes.getPnfsId();
   infoMsg = new StorageInfoMessage(cellAddress.toString(), pnfsId, true);
   infoMsg.setStorageInfo(fileAttributes.getStorageInfo());
   infoMsg.setFileSize(fileAttributes.getSize());
   descriptor =
       repository.createEntry(
           fileAttributes,
           ReplicaState.FROM_STORE,
           ReplicaState.CACHED,
           Collections.emptyList(),
           EnumSet.noneOf(Repository.OpenFlags.class));
   LOGGER.debug("Stage request created for {}.", pnfsId);
 }
예제 #7
0
 @VisibleForTesting
 String[] getFlushCommand(URI dataFile, FileAttributes fileAttributes) {
   StorageInfo storageInfo = StorageInfos.extractFrom(fileAttributes);
   String[] argsArray =
       Stream.concat(
               Stream.of(
                   command,
                   "put",
                   fileAttributes.getPnfsId().toString(),
                   getFileString(dataFile),
                   "-si=" + storageInfo),
               options.stream())
           .toArray(String[]::new);
   LOGGER.debug("COMMAND: {}", Arrays.deepToString(argsArray));
   return argsArray;
 }
예제 #8
0
 @VisibleForTesting
 String[] getFetchCommand(URI dataFile, FileAttributes fileAttributes) {
   StorageInfo storageInfo = StorageInfos.extractFrom(fileAttributes);
   String[] argsArray =
       Stream.of(
               Stream.of(
                   command,
                   "get",
                   fileAttributes.getPnfsId().toString(),
                   getFileString(dataFile),
                   "-si=" + storageInfo),
               getLocations(fileAttributes).stream().map(uri -> "-uri=" + uri),
               options.stream())
           .flatMap(s -> s)
           .toArray(String[]::new);
   LOGGER.debug("COMMAND: {}", Arrays.deepToString(argsArray));
   return argsArray;
 }
예제 #9
0
  public void unpin(FileAttributes fileAttributes, CellStub cellStub) {

    PinManagerUnpinMessage message = new PinManagerUnpinMessage(fileAttributes.getPnfsId());
    message.setRequestId(requestId);
    cellStub.notify(message);
  }
예제 #10
0
 @Override
 protected PnfsId extractKey(FileAttributes file) {
   return file.getPnfsId();
 }
예제 #11
0
 public PnfsId getPnfsId() {
   return _fileAttributes.getPnfsId();
 }
예제 #12
0
  /** Part of the MoverProtocol interface. */
  @Override
  public void runIO(
      FileAttributes fileAttributes,
      RepositoryChannel fileChannel,
      ProtocolInfo protocol,
      Allocator allocator,
      IoMode access)
      throws Exception {
    if (!(protocol instanceof GFtpProtocolInfo)) {
      throw new CacheException(44, "Protocol info not of type GFtpProtocolInfo");
    }
    GFtpProtocolInfo gftpProtocolInfo = (GFtpProtocolInfo) protocol;

    Role role = access == IoMode.WRITE ? Role.Receiver : Role.Sender;
    int version = gftpProtocolInfo.getMajorVersion();
    String host = gftpProtocolInfo.getSocketAddress().getAddress().getHostAddress();
    int port = gftpProtocolInfo.getSocketAddress().getPort();
    int bufferSize = gftpProtocolInfo.getBufferSize();
    int parallelism = gftpProtocolInfo.getParallelStart();
    long offset = gftpProtocolInfo.getOffset();
    long size = gftpProtocolInfo.getSize();
    boolean passive = gftpProtocolInfo.getPassive() && _allowPassivePool;

    say(
        MessageFormat.format(
            "version={0}, role={1}, mode={2}, host={3}:{4,number,#}, buffer={5}, passive={6}, parallelism={7}",
            version,
            role,
            gftpProtocolInfo.getMode(),
            host,
            port,
            bufferSize,
            passive,
            parallelism));

    /* Sanity check the parameters.
     */
    if (gftpProtocolInfo.getPassive() && version == 1) {
      /* In passive mode we need to be able to send the port we
       * listen on to the client. With GFtp/1, we cannot send
       * this information back to the door.
       */
      throw new CacheException(
          44, "Internal error: Cannot do passive transfer with mover protocol version 1.");
    }

    /* If on transfer checksum calculation is enabled, check if
     * we have a protocol specific preferred algorithm.
     */
    if (_checksumFactory != null) {
      ChecksumFactory factory = getChecksumFactory(gftpProtocolInfo);
      if (factory != null) {
        _checksumFactory = factory;
      }
      _digest = _checksumFactory.create();
    }

    /* We initialise these things early, as the job timeout
     * manager will not kill the job otherwise.
     */
    _transferStarted = System.currentTimeMillis();
    _lastTransferred = _transferStarted;

    Mode mode = createMode(gftpProtocolInfo.getMode(), role, fileChannel);
    mode.setBufferSize(bufferSize);

    /* For GFtp/2, the FTP door expects a
     * GFtpTransferStartedMessage when the mover is ready to
     * transfer the data.
     */
    if (version == 2) {
      GFtpTransferStartedMessage message;

      if (passive) {
        /* When in passive mode, the door passes us the host
         * from which the control channel was created. It
         * seems like a safe assumption that the data channel
         * will be established from the same network.
         */
        InetAddress clientAddress = InetAddress.getByName(gftpProtocolInfo.getClientAddress());
        InetAddress localAddress = NetworkUtils.getLocalAddress(clientAddress);

        /* When using true passive mode, we open a server
         * socket and send a message containing the port
         * number back to the door.
         */
        ServerSocketChannel channel = ServerSocketChannel.open();
        if (bufferSize > 0) {
          channel.socket().setReceiveBufferSize(bufferSize);
        }
        _portRange.bind(channel.socket(), localAddress, 128);

        message =
            new GFtpTransferStartedMessage(
                fileAttributes.getPnfsId().getId(),
                channel.socket().getInetAddress().getHostAddress(),
                channel.socket().getLocalPort());
        mode.setPassive(channel);
      } else {
        /* If passive mode is disabled, then fall back to
         * active mode.  When notified about this, the door
         * will fall back to proxy mode.
         */
        message = new GFtpTransferStartedMessage(fileAttributes.getPnfsId().getId());
      }
      CellPath path =
          new CellPath(
              gftpProtocolInfo.getDoorCellName(), gftpProtocolInfo.getDoorCellDomainName());
      _cell.sendMessage(new CellMessage(path, message));
    }

    if (!passive) {
      /* We use PROXY or ACTIVE mode. In proxy mode, host and
       * port identify the SocketAdapter running at the door. In
       * Active mode, host and port identify the client. Either
       * way, we do not really care.
       */

      try {
        mode.setActive(new InetSocketAddress(host, port));
      } catch (UnresolvedAddressException e) {
        throw new CacheException("Failed to resolve " + host);
      }
    }

    /* - Parallel transfers in stream mode are not defined.
     *
     * - Receiption in E mode must be passive (incomming). If the
     *   connection is outgoing, it means we use a proxy at the door.
     *   This proxy is limitted to one connection from the mover.
     *
     * In either case, set the parallelism to one.
     */
    switch (Character.toUpperCase(gftpProtocolInfo.getMode().charAt(0))) {
      case 'E':
        if (role == Role.Receiver && !passive) {
          parallelism = 1;
        }
        break;
      case 'S':
        parallelism = 1;
        break;
    }
    mode.setParallelism(parallelism);

    /* Setup partial retrieve parameters. These settings have
     * already been checked by the door, but better safe than
     * sorry...
     */
    if (role == Role.Sender) {
      long fileSize = fileChannel.size();
      if (offset < 0) {
        String err = "prm_offset is " + offset;
        esay(err);
        throw new IllegalArgumentException(err);
      }
      if (size < 0) {
        String err = "prm_offset is " + size;
        esay(err);
        throw new IllegalArgumentException(err);
      }
      if (offset + size > fileSize) {
        String err =
            "invalid prm_offset="
                + offset
                + " and prm_size "
                + size
                + " for file of size "
                + fileSize;
        esay(err);
        throw new IllegalArgumentException(err);
      }
      mode.setPartialRetrieveParameters(offset, size);
    }

    try {
      transfer(fileChannel, role, mode, allocator);
    } finally {
      /* Log some useful information about the transfer. This
       * will be send back to the door by the pool cell.
       */
      gftpProtocolInfo.setBytesTransferred(getBytesTransferred());
      gftpProtocolInfo.setTransferTime(getTransferTime());
    }
  }