コード例 #1
0
  private Resource parseJsonElementToVolumeResource(String region, JsonNode jsonNode) {
    Validate.notNull(jsonNode);
    long createTime = jsonNode.get("createTime").asLong();

    Resource resource =
        new AWSResource()
            .withId(jsonNode.get("volumeId").getTextValue())
            .withRegion(region)
            .withResourceType(AWSResourceType.EBS_VOLUME)
            .withLaunchTime(new Date(createTime));

    JsonNode tags = jsonNode.get("tags");
    StringBuilder description = new StringBuilder();
    JsonNode size = jsonNode.get("size");
    description.append(String.format("size=%s", size == null ? "unknown" : size.getIntValue()));

    if (tags == null || !tags.isArray() || tags.size() == 0) {
      LOGGER.debug(String.format("No tags is found for %s", resource.getId()));
    } else {
      for (Iterator<JsonNode> it = tags.getElements(); it.hasNext(); ) {
        JsonNode tag = it.next();
        String key = tag.get("key").getTextValue();
        String value = tag.get("value").getTextValue();
        description.append(String.format("; %s=%s", key, value));
        resource.setTag(key, value);
        if (key.equals(PURPOSE)) {
          resource.setAdditionalField(PURPOSE, value);
        }
      }
      resource.setDescription(description.toString());
    }
    ((AWSResource) resource).setAWSResourceState(jsonNode.get("state").getTextValue());
    return resource;
  }
コード例 #2
0
 private void markResource(Resource resource, DateTime now) {
   if (resource.getExpectedTerminationTime() == null) {
     Date terminationTime = calendar.getBusinessDay(new Date(now.getMillis()), retentionDays);
     resource.setExpectedTerminationTime(terminationTime);
     resource.setTerminationReason(
         String.format(
             "Launch config older than %d days. Not in Discovery. No ELB.",
             launchConfigAgeThreshold + retentionDays));
   } else {
     LOGGER.info(
         String.format("Resource %s is already marked as cleanup candidate.", resource.getId()));
   }
 }
コード例 #3
0
 private String getBatchUrl(String region, List<Resource> batch) {
   StringBuilder batchUrl = new StringBuilder(eddaClient.getBaseUrl(region) + "/aws/volumes/");
   boolean isFirst = true;
   for (Resource resource : batch) {
     if (!isFirst) {
       batchUrl.append(',');
     } else {
       isFirst = false;
     }
     batchUrl.append(resource.getId());
   }
   batchUrl.append(
       ";data.state=in-use;_since=0;_expand;_meta:"
           + "(ltime,data:(volumeId,attachments:(deleteOnTermination,instanceId)))");
   return batchUrl.toString();
 }
コード例 #4
0
  /** {@inheritDoc} */
  @Override
  public boolean isValid(Resource resource) {
    Validate.notNull(resource);
    if (!"ASG".equals(resource.getResourceType().name())) {
      return true;
    }

    if (StringUtils.isNotEmpty(resource.getAdditionalField(ASGJanitorCrawler.ASG_FIELD_ELBS))) {
      LOGGER.info(String.format("ASG %s has ELBs.", resource.getId()));
      return true;
    }

    if (instanceValidator.hasActiveInstance(resource)) {
      LOGGER.info(String.format("ASG %s has active instance.", resource.getId()));
      return true;
    }

    String lcName = resource.getAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_NAME);
    DateTime now = new DateTime(calendar.now().getTimeInMillis());
    if (StringUtils.isEmpty(lcName)) {
      LOGGER.error(
          String.format("Failed to find launch configuration for ASG %s", resource.getId()));
      markResource(resource, now);
      return false;
    }

    String lcCreationTime =
        resource.getAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_CREATION_TIME);
    if (StringUtils.isEmpty(lcCreationTime)) {
      LOGGER.error(
          String.format("Failed to find creation time for launch configuration %s", lcName));
      return true;
    }

    DateTime createTime = new DateTime(Long.parseLong(lcCreationTime));
    if (now.isBefore(createTime.plusDays(launchConfigAgeThreshold))) {
      LOGGER.info(
          String.format(
              "The launch configuation %s has not been created for more than %d days",
              lcName, launchConfigAgeThreshold));
      return true;
    }
    LOGGER.info(
        String.format(
            "The launch configuation %s has been created for more than %d days",
            lcName, launchConfigAgeThreshold));

    if (lastChangeDaysThreshold != null) {
      String lastChangeTimeField =
          resource.getAdditionalField(EddaASGJanitorCrawler.ASG_FIELD_LAST_CHANGE_TIME);
      if (StringUtils.isNotBlank(lastChangeTimeField)) {
        DateTime lastChangeTime = new DateTime(Long.parseLong(lastChangeTimeField));
        if (lastChangeTime.plusDays(lastChangeDaysThreshold).isAfter(now)) {
          LOGGER.info(
              String.format(
                  "ASG %s had change during the last %d days",
                  resource.getId(), lastChangeDaysThreshold));
          return true;
        }
      }
    }

    markResource(resource, now);
    return false;
  }
コード例 #5
0
  /**
   * Adds information of last attachment to the resources. To be compatible with the AWS
   * implementation of the same crawler, add the information to the JANITOR_META tag. It always uses
   * the latest information to update the tag in this resource (not writing back to AWS) no matter
   * if the tag exists.
   *
   * @param resources the volume resources
   */
  private void addLastAttachmentInfo(List<Resource> resources) {
    Validate.notNull(resources);
    LOGGER.info(
        String.format("Updating the latest attachment info for %d resources", resources.size()));
    Map<String, List<Resource>> regionToResources = Maps.newHashMap();
    for (Resource resource : resources) {
      List<Resource> regionalList = regionToResources.get(resource.getRegion());
      if (regionalList == null) {
        regionalList = Lists.newArrayList();
        regionToResources.put(resource.getRegion(), regionalList);
      }
      regionalList.add(resource);
    }
    for (Map.Entry<String, List<Resource>> entry : regionToResources.entrySet()) {
      LOGGER.info(
          String.format(
              "Updating the latest attachment info for %d resources in region %s",
              resources.size(), entry.getKey()));
      for (List<Resource> batch : Lists.partition(entry.getValue(), BATCH_SIZE)) {
        LOGGER.info(String.format("Processing batch of size %d", batch.size()));
        String batchUrl = getBatchUrl(entry.getKey(), batch);
        JsonNode batchResult = null;
        try {
          batchResult = eddaClient.getJsonNodeFromUrl(batchUrl);
        } catch (IOException e) {
          LOGGER.error("Failed to get response for the batch.", e);
        }
        Map<String, Resource> idToResource = Maps.newHashMap();
        for (Resource resource : batch) {
          idToResource.put(resource.getId(), resource);
        }
        if (batchResult == null || !batchResult.isArray()) {
          throw new RuntimeException(
              String.format(
                  "Failed to get valid document from %s, got: %s", batchUrl, batchResult));
        }

        Set<String> processedIds = Sets.newHashSet();
        for (Iterator<JsonNode> it = batchResult.getElements(); it.hasNext(); ) {
          JsonNode elem = it.next();
          JsonNode data = elem.get("data");
          String volumeId = data.get("volumeId").getTextValue();
          Resource resource = idToResource.get(volumeId);
          JsonNode attachments = data.get("attachments");

          Validate.isTrue(attachments.isArray() && attachments.size() > 0);
          JsonNode attachment = attachments.get(0);

          JsonNode ltime = elem.get("ltime");
          if (ltime == null || ltime.isNull()) {
            continue;
          }
          DateTime detachTime = new DateTime(ltime.asLong());
          processedIds.add(volumeId);
          setAttachmentInfo(volumeId, attachment, detachTime, resource);
        }

        for (Map.Entry<String, Resource> volumeEntry : idToResource.entrySet()) {
          String id = volumeEntry.getKey();
          if (!processedIds.contains(id)) {
            Resource resource = volumeEntry.getValue();
            LOGGER.info(
                String.format(
                    "Volume %s never was attached, use createTime %s as the detachTime",
                    id, resource.getLaunchTime()));
            setAttachmentInfo(id, null, new DateTime(resource.getLaunchTime().getTime()), resource);
          }
        }
      }
    }
  }