Exemple #1
0
 private void makeQosReservation(int fileIndex) throws MalformedURLException, SRMException {
   try {
     CopyFileRequest cfr = getFileRequests().get(fileIndex);
     RequestCredential credential = RequestCredential.getRequestCredential(credentialId);
     QOSTicket qosTicket =
         getQosPlugin()
             .createTicket(
                 credential.getCredentialName(),
                 getStorage().getFileMetaData(getUser(), cfr.getSourceSurl(), false).size,
                 cfr.getSourceSurl().toASCIIString(),
                 cfr.getSourceSurl().getPort(),
                 cfr.getSourceSurl().getPort(),
                 cfr.getSourceSurl().getScheme(),
                 cfr.getDestinationSurl().toASCIIString(),
                 cfr.getDestinationSurl().getPort(),
                 cfr.getDestinationSurl().getPort(),
                 cfr.getDestinationSurl().getScheme());
     getQosPlugin().addTicket(qosTicket);
     if (getQosPlugin().submit()) {
       cfr.setQOSTicket(qosTicket);
       LOG.debug("QOS Ticket Received {}", getQosPlugin());
     }
   } catch (Exception e) {
     LOG.error("Could not create QOS reservation: {}", e.getMessage());
   }
 }
Exemple #2
0
 private CopyFileRequest getFileRequestBySurls(String source, String destination)
     throws SRMInvalidRequestException, SRMInvalidPathException {
   for (CopyFileRequest request : getFileRequests()) {
     if (request.getSourceSurl().toString().equals(source)
         && request.getDestinationSurl().toString().equals(destination)) {
       return request;
     }
   }
   throw new SRMInvalidPathException("request not found");
 }
Exemple #3
0
 @Nonnull
 @Override
 public final CopyFileRequest getFileRequestBySurl(URI surl)
     throws SRMFileRequestNotFoundException {
   for (CopyFileRequest request : getFileRequests()) {
     if (request.getSourceSurl().equals(surl) || request.getDestinationSurl().equals(surl)) {
       return request;
     }
   }
   throw new SRMFileRequestNotFoundException("file request for url=" + surl + " is not found");
 }
Exemple #4
0
  private void getTURLs()
      throws SRMException, IOException, InterruptedException, IllegalStateTransition,
          DataAccessException {
    if (isSourceSrm() && !isSourceLocal()) { // implying destination is local
      if (getStorageType() != null && !storageType.equals(TFileStorageType.PERMANENT)) {
        throw new SRMNotSupportedException(
            "TargetFileStorageType " + getStorageType() + " is not supported");
      }
      RequestCredential credential = RequestCredential.getRequestCredential(credentialId);
      LOG.debug("obtained credential={} id={}", credential, credential.getId());

      for (int i = 0; i < getNumOfFileRequest(); ++i) {
        CopyFileRequest cfr = getFileRequests().get(i);
        if (cfr.getState() == State.UNSCHEDULED && cfr.getSchedulerId() == null) {
          if (cfr.getSourceTurl() != null) {
            cfr.scheduleWith(Scheduler.getScheduler(schedulerId));
          } else {
            // Since source SURLs are local, we can just set the path
            remoteSurlToFileReqIds.put(cfr.getSourceSurl().toASCIIString(), cfr.getId());
            String path = localPathFromSurl(cfr.getDestinationSurl());
            LOG.debug("setting destination path to {}", path);
            cfr.setLocalDestinationPath(path);
            cfr.saveJob();
          }
        }
      }
      String[] remoteSurlsUniqueArray =
          remoteSurlToFileReqIds.keySet().toArray(new String[remoteSurlToFileReqIds.size()]);
      if (LOG.isDebugEnabled()) {
        for (int i = 0; i < remoteSurlsUniqueArray.length; ++i) {
          LOG.debug("remoteSurlsUniqueArray[{}]={}", i, remoteSurlsUniqueArray[i]);
        }
      }
      // need to fetch files from remote SRM system
      setRemoteTurlClient(
          new RemoteTurlGetterV2(
              getStorage(),
              credential,
              remoteSurlsUniqueArray,
              getProtocols(),
              this,
              getConfiguration().getCopyMaxPollPeriod(),
              2,
              this.getRemainingLifetime(),
              getConfiguration().getCaCertificatePath(),
              clientTransport));
      getRemoteTurlClient().getInitialRequest();
      getRemoteTurlClient().run();
      return;
    }

    if (isSourceSrm()) { // source is this storage system
      for (int i = 0; i < getNumOfFileRequest(); ++i) {
        CopyFileRequest cfr = getFileRequests().get(i);

        if (cfr.getState() == State.UNSCHEDULED
            && cfr.getSchedulerId() == null
            && cfr.getLocalSourcePath() == null) {
          String path = localPathFromSurl(cfr.getSourceSurl());
          LOG.debug("setting source path to {}", path);
          cfr.setLocalSourcePath(path);
          cfr.saveJob();
        }
      }
    } else { // source is not SRM, so supplied values are the TURL(s)
      for (int i = 0; i < getNumOfFileRequest(); ++i) {
        CopyFileRequest cfr = getFileRequests().get(i);

        if (cfr.getState() == State.UNSCHEDULED
            && cfr.getSchedulerId() == null
            && cfr.getSourceTurl() == null) {
          LOG.debug("getTurlsArrived, setting \"from\" turl to {}", cfr.getSourceSurl());
          cfr.setSourceTurl(cfr.getSourceSurl());
          cfr.saveJob();
        }
      }
    }

    // now source is known, either as a TURL or a local path.

    if (isDestinationSrm() && isDestinationLocal()) { // destination is this system
      // As we have a source (either a path or a TURL) for all files and
      // the destination is local, we have all the information needed.
      for (int i = 0; i < getNumOfFileRequest(); ++i) {
        CopyFileRequest cfr = getFileRequests().get(i);
        if (cfr.getState() == State.UNSCHEDULED && cfr.getSchedulerId() == null) {
          String path = localPathFromSurl(cfr.getDestinationSurl());
          LOG.debug("setting local destination path to {}", path);
          cfr.setLocalDestinationPath(path);
          cfr.scheduleWith(Scheduler.getScheduler(schedulerId));
        }
      }
      return;
    }

    if (!isDestinationSrm()) { // destination is some TURL
      // As we have a source (either a path or a TURL) for all files and
      // the destination is a TURL, we have all the information needed.
      for (int i = 0; i < getNumOfFileRequest(); ++i) {
        CopyFileRequest cfr = getFileRequests().get(i);
        if (cfr.getState() == State.UNSCHEDULED && cfr.getSchedulerId() == null) {
          LOG.debug("setting destination to {}", cfr.getDestinationSurl());
          cfr.setDestinationTurl(cfr.getDestinationSurl());
          cfr.scheduleWith(Scheduler.getScheduler(schedulerId));
        }
      }
      return;
    }

    // The only possibility left is a remote destination SURLs, for which
    // we need to discover a TURLs.

    for (int i = 0; i < getNumOfFileRequest(); ++i) {
      CopyFileRequest cfr = getFileRequests().get(i);
      if (cfr.getState() != State.UNSCHEDULED || cfr.getSchedulerId() != null) {
        // copy file request has being canceled,failed or scheduled before
      } else if (cfr.getDestinationTurl() != null) {
        // destination TURL has arrived, but request has not been scheduled
        cfr.scheduleWith(Scheduler.getScheduler(schedulerId));
      } else {
        remoteSurlToFileReqIds.put(cfr.getDestinationSurl().toASCIIString(), cfr.getId());
      }
    }

    int uniqueCount = remoteSurlToFileReqIds.size();
    String[] uniqueSurls = remoteSurlToFileReqIds.keySet().toArray(new String[uniqueCount]);
    String[] destinationSurls = new String[uniqueCount];
    long[] sizes = new long[uniqueCount];
    for (int i = 0; i < uniqueCount; ++i) {
      long id = Iterables.get(remoteSurlToFileReqIds.get(uniqueSurls[i]), 0);
      CopyFileRequest cfr = getFileRequest(id);
      sizes[i] = getStorage().getFileMetaData(getUser(), cfr.getSourceSurl(), false).size;
      LOG.debug("local size is {}", sizes[i]);
      cfr.setSize(sizes[i]);
      destinationSurls[i] = cfr.getDestinationSurl().toString();
      if (getQosPlugin() != null) {
        makeQosReservation(i);
      }
    }

    // Now create an SRM client to fetch a TURL for each SURL.

    RequestCredential credential = RequestCredential.getRequestCredential(credentialId);
    setRemoteTurlClient(
        new RemoteTurlPutterV2(
            getStorage(),
            credential,
            destinationSurls,
            sizes,
            getProtocols(),
            this,
            getConfiguration().getCopyMaxPollPeriod(),
            2,
            this.getRemainingLifetime(),
            getStorageType(),
            getTargetRetentionPolicy(),
            getTargetAccessLatency(),
            getOverwriteMode(),
            getTargetSpaceToken(),
            getConfiguration().getCaCertificatePath(),
            clientTransport));
    getRemoteTurlClient().getInitialRequest();

    getRemoteTurlClient().run();
  }
Exemple #5
0
  private void identify() throws IOException, SRMException {
    wlock();
    try {
      URI source = getFileRequests().get(0).getSourceSurl();
      String sourceProtocol = source.getScheme();
      String sourceHost = source.getHost();
      int sourcePort = source.getPort();

      URI destination = getFileRequests().get(0).getDestinationSurl();
      String destinationProtocol = destination.getScheme();
      String destinationHost = destination.getHost();
      int destinationPort = destination.getPort();

      for (CopyFileRequest cfr : getFileRequests().subList(1, getNumOfFileRequest())) {
        URI sourceSurl = cfr.getSourceSurl();
        URI destinationSurl = cfr.getDestinationSurl();
        if (!sourceSurl.getScheme().equals(sourceProtocol)
            || !sourceSurl.getHost().equals(sourceHost)
            || sourceSurl.getPort() != sourcePort) {
          String err = "Source URL " + sourceSurl + " is inconsistent with first source URL";
          LOG.error(err);
          throw new IOException(err);
        }

        if (!destinationSurl.getScheme().equals(destinationProtocol)
            || !destinationSurl.getHost().equals(destinationHost)
            || destinationSurl.getPort() != destinationPort) {
          String err =
              "Destination URL " + destinationSurl + " is inconsistent with first destination URL";
          LOG.error(err);
          throw new IOException(err);
        }
      }

      isSourceSrm = sourceProtocol.equals("srm");
      isDestinationSrm = destinationProtocol.equals("srm");

      Set<String> srmHosts = getConfiguration().getSrmHosts();
      isSourceLocal =
          isSourceSrm
              && (sourcePort == -1 || sourcePort == getConfiguration().getPort())
              && (sourceHost == null || srmHosts.stream().anyMatch(sourceHost::equalsIgnoreCase));
      isDestinationLocal =
          isDestinationSrm
              && (destinationPort == -1 || destinationPort == getConfiguration().getPort())
              && (destinationHost == null
                  || srmHosts.stream().anyMatch(destinationHost::equalsIgnoreCase));

      LOG.debug(
          "src (srm={}, local={}), dest (srm={}, local={})",
          isSourceSrm,
          isSourceLocal,
          isDestinationSrm,
          isDestinationLocal);

      if (!isSourceLocal && !isDestinationLocal) {
        LOG.error("Both source ({}) and destination ({}) URLs are remote.", source, destination);
        throw new SRMInvalidRequestException("Both source and destination URLs are remote.");
      }
    } finally {
      wunlock();
    }
  }