@Override
 public void send(CellEndpoint endpoint, CellMessage envelope, PoolManagerMessage msg) {
   checkArgument(envelope.getSourcePath().hops() > 0, "Envelope is missing source address.");
   envelope.getDestinationPath().insert(destination);
   envelope.setMessageObject(msg);
   endpoint.sendMessage(envelope, CellEndpoint.SendFlag.PASS_THROUGH);
 }
 @SuppressWarnings("unchecked")
 protected <T extends Message> ListenableFuture<T> submit(
     CellEndpoint endpoint, CellPath path, T msg, long timeout) {
   FutureCellMessageAnswerable<T> callback =
       new FutureCellMessageAnswerable<>((Class<T>) msg.getClass());
   endpoint.sendMessage(
       new CellMessage(path, msg), callback, MoreExecutors.directExecutor(), timeout);
   return callback;
 }
Esempio n. 3
0
  public GFtpProtocol_2_nio(CellEndpoint cell) {
    _cell = cell;

    String range = System.getProperty("org.globus.tcp.port.range");
    if (range != null) {
      _portRange = PortRange.valueOf(range);
    } else {
      _portRange = new PortRange(0);
    }

    if (_cell != null) {
      Args args = _cell.getArgs();
      if (args.hasOption("ftpProxyPassive")) {
        _allowPassivePool = !Boolean.parseBoolean(args.getOpt("ftpProxyPassive"));
      }

      if (args.hasOption("gsiftpBlockSize")) {
        _blockSize = Integer.valueOf(args.getOpt("gsiftpBlockSize"));
      }
    }
  }
Esempio n. 4
0
  /** Part of the MoverProtocol interface. */
  @Override
  public void runIO(
      FileAttributes fileAttributes,
      RepositoryChannel fileChannel,
      ProtocolInfo protocol,
      Allocator allocator,
      IoMode access)
      throws Exception {
    if (!(protocol instanceof GFtpProtocolInfo)) {
      throw new CacheException(44, "Protocol info not of type GFtpProtocolInfo");
    }
    GFtpProtocolInfo gftpProtocolInfo = (GFtpProtocolInfo) protocol;

    Role role = access == IoMode.WRITE ? Role.Receiver : Role.Sender;
    int version = gftpProtocolInfo.getMajorVersion();
    String host = gftpProtocolInfo.getSocketAddress().getAddress().getHostAddress();
    int port = gftpProtocolInfo.getSocketAddress().getPort();
    int bufferSize = gftpProtocolInfo.getBufferSize();
    int parallelism = gftpProtocolInfo.getParallelStart();
    long offset = gftpProtocolInfo.getOffset();
    long size = gftpProtocolInfo.getSize();
    boolean passive = gftpProtocolInfo.getPassive() && _allowPassivePool;

    say(
        MessageFormat.format(
            "version={0}, role={1}, mode={2}, host={3}:{4,number,#}, buffer={5}, passive={6}, parallelism={7}",
            version,
            role,
            gftpProtocolInfo.getMode(),
            host,
            port,
            bufferSize,
            passive,
            parallelism));

    /* Sanity check the parameters.
     */
    if (gftpProtocolInfo.getPassive() && version == 1) {
      /* In passive mode we need to be able to send the port we
       * listen on to the client. With GFtp/1, we cannot send
       * this information back to the door.
       */
      throw new CacheException(
          44, "Internal error: Cannot do passive transfer with mover protocol version 1.");
    }

    /* If on transfer checksum calculation is enabled, check if
     * we have a protocol specific preferred algorithm.
     */
    if (_checksumFactory != null) {
      ChecksumFactory factory = getChecksumFactory(gftpProtocolInfo);
      if (factory != null) {
        _checksumFactory = factory;
      }
      _digest = _checksumFactory.create();
    }

    /* We initialise these things early, as the job timeout
     * manager will not kill the job otherwise.
     */
    _transferStarted = System.currentTimeMillis();
    _lastTransferred = _transferStarted;

    Mode mode = createMode(gftpProtocolInfo.getMode(), role, fileChannel);
    mode.setBufferSize(bufferSize);

    /* For GFtp/2, the FTP door expects a
     * GFtpTransferStartedMessage when the mover is ready to
     * transfer the data.
     */
    if (version == 2) {
      GFtpTransferStartedMessage message;

      if (passive) {
        /* When in passive mode, the door passes us the host
         * from which the control channel was created. It
         * seems like a safe assumption that the data channel
         * will be established from the same network.
         */
        InetAddress clientAddress = InetAddress.getByName(gftpProtocolInfo.getClientAddress());
        InetAddress localAddress = NetworkUtils.getLocalAddress(clientAddress);

        /* When using true passive mode, we open a server
         * socket and send a message containing the port
         * number back to the door.
         */
        ServerSocketChannel channel = ServerSocketChannel.open();
        if (bufferSize > 0) {
          channel.socket().setReceiveBufferSize(bufferSize);
        }
        _portRange.bind(channel.socket(), localAddress, 128);

        message =
            new GFtpTransferStartedMessage(
                fileAttributes.getPnfsId().getId(),
                channel.socket().getInetAddress().getHostAddress(),
                channel.socket().getLocalPort());
        mode.setPassive(channel);
      } else {
        /* If passive mode is disabled, then fall back to
         * active mode.  When notified about this, the door
         * will fall back to proxy mode.
         */
        message = new GFtpTransferStartedMessage(fileAttributes.getPnfsId().getId());
      }
      CellPath path =
          new CellPath(
              gftpProtocolInfo.getDoorCellName(), gftpProtocolInfo.getDoorCellDomainName());
      _cell.sendMessage(new CellMessage(path, message));
    }

    if (!passive) {
      /* We use PROXY or ACTIVE mode. In proxy mode, host and
       * port identify the SocketAdapter running at the door. In
       * Active mode, host and port identify the client. Either
       * way, we do not really care.
       */

      try {
        mode.setActive(new InetSocketAddress(host, port));
      } catch (UnresolvedAddressException e) {
        throw new CacheException("Failed to resolve " + host);
      }
    }

    /* - Parallel transfers in stream mode are not defined.
     *
     * - Receiption in E mode must be passive (incomming). If the
     *   connection is outgoing, it means we use a proxy at the door.
     *   This proxy is limitted to one connection from the mover.
     *
     * In either case, set the parallelism to one.
     */
    switch (Character.toUpperCase(gftpProtocolInfo.getMode().charAt(0))) {
      case 'E':
        if (role == Role.Receiver && !passive) {
          parallelism = 1;
        }
        break;
      case 'S':
        parallelism = 1;
        break;
    }
    mode.setParallelism(parallelism);

    /* Setup partial retrieve parameters. These settings have
     * already been checked by the door, but better safe than
     * sorry...
     */
    if (role == Role.Sender) {
      long fileSize = fileChannel.size();
      if (offset < 0) {
        String err = "prm_offset is " + offset;
        esay(err);
        throw new IllegalArgumentException(err);
      }
      if (size < 0) {
        String err = "prm_offset is " + size;
        esay(err);
        throw new IllegalArgumentException(err);
      }
      if (offset + size > fileSize) {
        String err =
            "invalid prm_offset="
                + offset
                + " and prm_size "
                + size
                + " for file of size "
                + fileSize;
        esay(err);
        throw new IllegalArgumentException(err);
      }
      mode.setPartialRetrieveParameters(offset, size);
    }

    try {
      transfer(fileChannel, role, mode, allocator);
    } finally {
      /* Log some useful information about the transfer. This
       * will be send back to the door by the pool cell.
       */
      gftpProtocolInfo.setBytesTransferred(getBytesTransferred());
      gftpProtocolInfo.setTransferTime(getTransferTime());
    }
  }
Esempio n. 5
0
  /** Receive a file. */
  public void transfer(RepositoryChannel fileChannel, Role role, Mode mode, Allocator allocator)
      throws Exception {
    /* Initialise transfer parameters.
     */
    _role = role;
    _bytesTransferred = 0;
    _blockLog = new BlockLog(this);
    _fileChannel = fileChannel;
    _allocator = allocator;
    _reservedSpace = 0;
    _spaceUsed = 0;
    _status = "None";
    DigestThread digestThread = null;

    /* Startup the transfer. The transfer is performed on a single
     * thread, no matter the number of streams.
     *
     * Checksum computation is performed on a different
     * thread. The checksum computation thread is not allowed to
     * overtake the transfer thread, but we also ensure that the
     * checksum thread does not fall too far behind. This is to
     * increase the chance that data has not yet been evicted from
     * the cache.
     */
    _multiplexer = new Multiplexer(this);
    try {
      _inProgress = true;

      digestThread = createDigestThread();
      if (digestThread != null) {
        Object o = _cell.getDomainContext().get(READ_AHEAD_KEY);
        if (o != null && ((String) o).length() > 0) {
          try {
            digestThread.setReadAhead(Long.parseLong((String) o));
          } catch (NumberFormatException e) {
            esay("Failed parsing read ahead: " + e.getMessage());
          }
        }

        say("Initiated checksum computation thread");
        digestThread.start();
      }

      _multiplexer.add(mode);

      say("Entering event loop");
      _multiplexer.loop();
    } catch (ClosedByInterruptException e) {
      /* Many NIO operations throw a ClosedByInterruptException
       * rather than InterruptedException. We rethrow this as an
       * InterruptedException and clear the interrupt flag on
       * the thread.
       */
      Thread.interrupted();
      throw new InterruptedException();
    } catch (InterruptedException | FTPException e) {
      throw e;
    } catch (Exception e) {
      esay(e);
      throw e;
    } finally {
      _inProgress = false;

      /* It is important that this is done before joining the
       * digest thread, since otherwise the digest thread would
       * not terminate.
       */
      _blockLog.setEof();

      /* Close all open channels.
       */
      say("Left event loop and closing channels");
      _multiplexer.close();

      /* Wait for checksum computation to finish before
       * returning. Otherwise getActualChecksum() could
       * possibly return an incomplete checksum.
       *
       * REVISIT: If the mover gets killed here, we break out
       * with an InterruptedException. This is as such not a
       * major problem, since everything after this point is not
       * essential for clean up. It is however unfortunate that
       * the job gets killed because we wait for checksum
       * computation (in particular because the checksum
       * computation may be the cause of the timeout if it is
       * very slow).
       */
      if (digestThread != null) {
        digestThread.join();
      }

      /* Log some useful information about the transfer.
       */
      long amount = getBytesTransferred();
      long time = getTransferTime();
      if (time > 0) {
        say(
            String.format(
                "Transfer finished: %d bytes transferred in %.2f seconds = %.3f MB/s",
                amount, time / 1000.0, (1000.0 * amount) / (time * 1024 * 1024)));
      } else {
        say(String.format("Transfer finished: %d bytes transferred in less than 1 ms", amount));
      }
    }

    /* REVISIT: Error reporting from the digest thread is not
     * optimal. In case of errors, they are not detected until
     * here. It would be better if digestThread could shutdown the
     * multiplexer. Maybe we should simply embed the DigestThread
     * class into the Mover.
     */
    if (digestThread != null && digestThread.getLastError() != null) {
      esay(digestThread.getLastError());
      throw digestThread.getLastError();
    }

    /* Check that we receive the whole file.
     */
    if (!_blockLog.isComplete()) {
      throw new CacheException(44, "Incomplete file detected");
    }
  }
 @Override
 public void start(CellEndpoint endpoint, CellMessage envelope, PoolIoFileMessage msg) {
   checkArgument(envelope.getSourcePath().hops() > 0, "Envelope is missing source address.");
   envelope.setMessageObject(msg);
   endpoint.sendMessage(envelope, CellEndpoint.SendFlag.PASS_THROUGH);
 }
Esempio n. 7
0
 protected synchronized void send() {
   _envelope.revertDirection();
   _envelope.setMessageObject(_msg);
   _endpoint.sendMessage(_envelope);
 }