/** Provision a new slave for an EC2 spot instance to call back to Jenkins */
  private EC2AbstractSlave provisionSpot(TaskListener listener)
      throws AmazonClientException, IOException {
    PrintStream logger = listener.getLogger();
    AmazonEC2 ec2 = getParent().connect();

    try {
      logger.println("Launching " + ami + " for template " + description);
      LOGGER.info("Launching " + ami + " for template " + description);

      KeyPair keyPair = getKeyPair(ec2);

      RequestSpotInstancesRequest spotRequest = new RequestSpotInstancesRequest();

      // Validate spot bid before making the request
      if (getSpotMaxBidPrice() == null) {
        // throw new FormException("Invalid Spot price specified: " +
        // getSpotMaxBidPrice(), "spotMaxBidPrice");
        throw new AmazonClientException("Invalid Spot price specified: " + getSpotMaxBidPrice());
      }

      spotRequest.setSpotPrice(getSpotMaxBidPrice());
      spotRequest.setInstanceCount(1);

      LaunchSpecification launchSpecification = new LaunchSpecification();
      InstanceNetworkInterfaceSpecification net = new InstanceNetworkInterfaceSpecification();

      launchSpecification.setImageId(ami);
      launchSpecification.setInstanceType(type);

      if (StringUtils.isNotBlank(getZone())) {
        SpotPlacement placement = new SpotPlacement(getZone());
        launchSpecification.setPlacement(placement);
      }

      if (StringUtils.isNotBlank(getSubnetId())) {
        if (getAssociatePublicIp()) {
          net.setSubnetId(getSubnetId());
        } else {
          launchSpecification.setSubnetId(getSubnetId());
        }

        /*
         * If we have a subnet ID then we can only use VPC security groups
         */
        if (!securityGroupSet.isEmpty()) {
          List<String> groupIds = getEc2SecurityGroups(ec2);
          if (!groupIds.isEmpty()) {
            if (getAssociatePublicIp()) {
              net.setGroups(groupIds);
            } else {
              ArrayList<GroupIdentifier> groups = new ArrayList<GroupIdentifier>();

              for (String group_id : groupIds) {
                GroupIdentifier group = new GroupIdentifier();
                group.setGroupId(group_id);
                groups.add(group);
              }
              if (!groups.isEmpty()) launchSpecification.setAllSecurityGroups(groups);
            }
          }
        }
      } else {
        /* No subnet: we can use standard security groups by name */
        if (!securityGroupSet.isEmpty()) {
          launchSpecification.setSecurityGroups(securityGroupSet);
        }
      }

      String userDataString = Base64.encodeBase64String(userData.getBytes(StandardCharsets.UTF_8));

      launchSpecification.setUserData(userDataString);
      launchSpecification.setKeyName(keyPair.getKeyName());
      launchSpecification.setInstanceType(type.toString());

      if (getAssociatePublicIp()) {
        net.setAssociatePublicIpAddress(true);
        net.setDeviceIndex(0);
        launchSpecification.withNetworkInterfaces(net);
      }

      boolean hasCustomTypeTag = false;
      HashSet<Tag> instTags = null;
      if (tags != null && !tags.isEmpty()) {
        instTags = new HashSet<Tag>();
        for (EC2Tag t : tags) {
          instTags.add(new Tag(t.getName(), t.getValue()));
          if (StringUtils.equals(t.getName(), EC2Tag.TAG_NAME_JENKINS_SLAVE_TYPE)) {
            hasCustomTypeTag = true;
          }
        }
      }
      if (!hasCustomTypeTag) {
        if (instTags != null)
          instTags.add(
              new Tag(
                  EC2Tag.TAG_NAME_JENKINS_SLAVE_TYPE,
                  EC2Cloud.getSlaveTypeTagValue(EC2Cloud.EC2_SLAVE_TYPE_SPOT, description)));
      }

      if (StringUtils.isNotBlank(getIamInstanceProfile())) {
        launchSpecification.setIamInstanceProfile(
            new IamInstanceProfileSpecification().withArn(getIamInstanceProfile()));
      }

      if (useEphemeralDevices) {
        setupEphemeralDeviceMapping(launchSpecification);
      } else {
        setupCustomDeviceMapping(launchSpecification);
      }

      spotRequest.setLaunchSpecification(launchSpecification);

      // Make the request for a new Spot instance
      RequestSpotInstancesResult reqResult = ec2.requestSpotInstances(spotRequest);

      List<SpotInstanceRequest> reqInstances = reqResult.getSpotInstanceRequests();
      if (reqInstances.isEmpty()) {
        throw new AmazonClientException("No spot instances found");
      }

      SpotInstanceRequest spotInstReq = reqInstances.get(0);
      if (spotInstReq == null) {
        throw new AmazonClientException("Spot instance request is null");
      }
      String slaveName = spotInstReq.getSpotInstanceRequestId();

      /* Now that we have our Spot request, we can set tags on it */
      if (instTags != null) {
        updateRemoteTags(
            ec2,
            instTags,
            "InvalidSpotInstanceRequestID.NotFound",
            spotInstReq.getSpotInstanceRequestId());

        // That was a remote request - we should also update our local
        // instance data.
        spotInstReq.setTags(instTags);
      }

      logger.println("Spot instance id in provision: " + spotInstReq.getSpotInstanceRequestId());
      LOGGER.info("Spot instance id in provision: " + spotInstReq.getSpotInstanceRequestId());

      return newSpotSlave(spotInstReq, slaveName);

    } catch (FormException e) {
      throw new AssertionError(); // we should have discovered all
      // configuration issues upfront
    } catch (InterruptedException e) {
      throw new RuntimeException(e);
    }
  }
  @Override
  public List<DiscoveryNode> buildDynamicNodes() {
    List<DiscoveryNode> discoNodes = new ArrayList<>();

    DescribeInstancesResult descInstances;
    try {
      // Query EC2 API based on AZ, instance state, and tag.

      // NOTE: we don't filter by security group during the describe instances request for two
      // reasons:
      // 1. differences in VPCs require different parameters during query (ID vs Name)
      // 2. We want to use two different strategies: (all security groups vs. any security groups)
      descInstances = client.describeInstances(buildDescribeInstancesRequest());
    } catch (AmazonClientException e) {
      logger.info("Exception while retrieving instance list from AWS API: {}", e.getMessage());
      logger.debug("Full exception:", e);
      return discoNodes;
    }

    logger.trace("building dynamic unicast discovery nodes...");
    for (Reservation reservation : descInstances.getReservations()) {
      for (Instance instance : reservation.getInstances()) {
        // lets see if we can filter based on groups
        if (!groups.isEmpty()) {
          List<GroupIdentifier> instanceSecurityGroups = instance.getSecurityGroups();
          ArrayList<String> securityGroupNames = new ArrayList<String>();
          ArrayList<String> securityGroupIds = new ArrayList<String>();
          for (GroupIdentifier sg : instanceSecurityGroups) {
            securityGroupNames.add(sg.getGroupName());
            securityGroupIds.add(sg.getGroupId());
          }
          if (bindAnyGroup) {
            // We check if we can find at least one group name or one group id in groups.
            if (Collections.disjoint(securityGroupNames, groups)
                && Collections.disjoint(securityGroupIds, groups)) {
              logger.trace(
                  "filtering out instance {} based on groups {}, not part of {}",
                  instance.getInstanceId(),
                  instanceSecurityGroups,
                  groups);
              // continue to the next instance
              continue;
            }
          } else {
            // We need tp match all group names or group ids, otherwise we ignore this instance
            if (!(securityGroupNames.containsAll(groups) || securityGroupIds.containsAll(groups))) {
              logger.trace(
                  "filtering out instance {} based on groups {}, does not include all of {}",
                  instance.getInstanceId(),
                  instanceSecurityGroups,
                  groups);
              // continue to the next instance
              continue;
            }
          }
        }

        String address = null;
        switch (hostType) {
          case PRIVATE_DNS:
            address = instance.getPrivateDnsName();
            break;
          case PRIVATE_IP:
            address = instance.getPrivateIpAddress();
            break;
          case PUBLIC_DNS:
            address = instance.getPublicDnsName();
            break;
          case PUBLIC_IP:
            address = instance.getPublicIpAddress();
            break;
        }
        if (address != null) {
          try {
            TransportAddress[] addresses = transportService.addressesFromString(address);
            // we only limit to 1 addresses, makes no sense to ping 100 ports
            for (int i = 0; (i < addresses.length && i < UnicastZenPing.LIMIT_PORTS_COUNT); i++) {
              logger.trace(
                  "adding {}, address {}, transport_address {}",
                  instance.getInstanceId(),
                  address,
                  addresses[i]);
              discoNodes.add(
                  new DiscoveryNode(
                      "#cloud-" + instance.getInstanceId() + "-" + i,
                      addresses[i],
                      version.minimumCompatibilityVersion()));
            }
          } catch (Exception e) {
            logger.warn("failed ot add {}, address {}", e, instance.getInstanceId(), address);
          }
        } else {
          logger.trace(
              "not adding {}, address is null, host_type {}", instance.getInstanceId(), hostType);
        }
      }
    }

    logger.debug("using dynamic discovery nodes {}", discoNodes);

    return discoNodes;
  }