private Resource parseJsonElementToVolumeResource(String region, JsonNode jsonNode) {
Validate.notNull(jsonNode);
long createTime = jsonNode.get("createTime").asLong();
Resource resource =
new AWSResource()
.withId(jsonNode.get("volumeId").getTextValue())
.withRegion(region)
.withResourceType(AWSResourceType.EBS_VOLUME)
.withLaunchTime(new Date(createTime));
JsonNode tags = jsonNode.get("tags");
StringBuilder description = new StringBuilder();
JsonNode size = jsonNode.get("size");
description.append(String.format("size=%s", size == null ? "unknown" : size.getIntValue()));
if (tags == null || !tags.isArray() || tags.size() == 0) {
LOGGER.debug(String.format("No tags is found for %s", resource.getId()));
} else {
for (Iterator<JsonNode> it = tags.getElements(); it.hasNext(); ) {
JsonNode tag = it.next();
String key = tag.get("key").getTextValue();
String value = tag.get("value").getTextValue();
description.append(String.format("; %s=%s", key, value));
resource.setTag(key, value);
if (key.equals(PURPOSE)) {
resource.setAdditionalField(PURPOSE, value);
}
}
resource.setDescription(description.toString());
}
((AWSResource) resource).setAWSResourceState(jsonNode.get("state").getTextValue());
return resource;
}
private void markResource(Resource resource, DateTime now) {
if (resource.getExpectedTerminationTime() == null) {
Date terminationTime = calendar.getBusinessDay(new Date(now.getMillis()), retentionDays);
resource.setExpectedTerminationTime(terminationTime);
resource.setTerminationReason(
String.format(
"Launch config older than %d days. Not in Discovery. No ELB.",
launchConfigAgeThreshold + retentionDays));
} else {
LOGGER.info(
String.format("Resource %s is already marked as cleanup candidate.", resource.getId()));
}
}
private String getBatchUrl(String region, List<Resource> batch) {
StringBuilder batchUrl = new StringBuilder(eddaClient.getBaseUrl(region) + "/aws/volumes/");
boolean isFirst = true;
for (Resource resource : batch) {
if (!isFirst) {
batchUrl.append(',');
} else {
isFirst = false;
}
batchUrl.append(resource.getId());
}
batchUrl.append(
";data.state=in-use;_since=0;_expand;_meta:"
+ "(ltime,data:(volumeId,attachments:(deleteOnTermination,instanceId)))");
return batchUrl.toString();
}
/** {@inheritDoc} */
@Override
public boolean isValid(Resource resource) {
Validate.notNull(resource);
if (!"ASG".equals(resource.getResourceType().name())) {
return true;
}
if (StringUtils.isNotEmpty(resource.getAdditionalField(ASGJanitorCrawler.ASG_FIELD_ELBS))) {
LOGGER.info(String.format("ASG %s has ELBs.", resource.getId()));
return true;
}
if (instanceValidator.hasActiveInstance(resource)) {
LOGGER.info(String.format("ASG %s has active instance.", resource.getId()));
return true;
}
String lcName = resource.getAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_NAME);
DateTime now = new DateTime(calendar.now().getTimeInMillis());
if (StringUtils.isEmpty(lcName)) {
LOGGER.error(
String.format("Failed to find launch configuration for ASG %s", resource.getId()));
markResource(resource, now);
return false;
}
String lcCreationTime =
resource.getAdditionalField(ASGJanitorCrawler.ASG_FIELD_LC_CREATION_TIME);
if (StringUtils.isEmpty(lcCreationTime)) {
LOGGER.error(
String.format("Failed to find creation time for launch configuration %s", lcName));
return true;
}
DateTime createTime = new DateTime(Long.parseLong(lcCreationTime));
if (now.isBefore(createTime.plusDays(launchConfigAgeThreshold))) {
LOGGER.info(
String.format(
"The launch configuation %s has not been created for more than %d days",
lcName, launchConfigAgeThreshold));
return true;
}
LOGGER.info(
String.format(
"The launch configuation %s has been created for more than %d days",
lcName, launchConfigAgeThreshold));
if (lastChangeDaysThreshold != null) {
String lastChangeTimeField =
resource.getAdditionalField(EddaASGJanitorCrawler.ASG_FIELD_LAST_CHANGE_TIME);
if (StringUtils.isNotBlank(lastChangeTimeField)) {
DateTime lastChangeTime = new DateTime(Long.parseLong(lastChangeTimeField));
if (lastChangeTime.plusDays(lastChangeDaysThreshold).isAfter(now)) {
LOGGER.info(
String.format(
"ASG %s had change during the last %d days",
resource.getId(), lastChangeDaysThreshold));
return true;
}
}
}
markResource(resource, now);
return false;
}
/**
* Adds information of last attachment to the resources. To be compatible with the AWS
* implementation of the same crawler, add the information to the JANITOR_META tag. It always uses
* the latest information to update the tag in this resource (not writing back to AWS) no matter
* if the tag exists.
*
* @param resources the volume resources
*/
private void addLastAttachmentInfo(List<Resource> resources) {
Validate.notNull(resources);
LOGGER.info(
String.format("Updating the latest attachment info for %d resources", resources.size()));
Map<String, List<Resource>> regionToResources = Maps.newHashMap();
for (Resource resource : resources) {
List<Resource> regionalList = regionToResources.get(resource.getRegion());
if (regionalList == null) {
regionalList = Lists.newArrayList();
regionToResources.put(resource.getRegion(), regionalList);
}
regionalList.add(resource);
}
for (Map.Entry<String, List<Resource>> entry : regionToResources.entrySet()) {
LOGGER.info(
String.format(
"Updating the latest attachment info for %d resources in region %s",
resources.size(), entry.getKey()));
for (List<Resource> batch : Lists.partition(entry.getValue(), BATCH_SIZE)) {
LOGGER.info(String.format("Processing batch of size %d", batch.size()));
String batchUrl = getBatchUrl(entry.getKey(), batch);
JsonNode batchResult = null;
try {
batchResult = eddaClient.getJsonNodeFromUrl(batchUrl);
} catch (IOException e) {
LOGGER.error("Failed to get response for the batch.", e);
}
Map<String, Resource> idToResource = Maps.newHashMap();
for (Resource resource : batch) {
idToResource.put(resource.getId(), resource);
}
if (batchResult == null || !batchResult.isArray()) {
throw new RuntimeException(
String.format(
"Failed to get valid document from %s, got: %s", batchUrl, batchResult));
}
Set<String> processedIds = Sets.newHashSet();
for (Iterator<JsonNode> it = batchResult.getElements(); it.hasNext(); ) {
JsonNode elem = it.next();
JsonNode data = elem.get("data");
String volumeId = data.get("volumeId").getTextValue();
Resource resource = idToResource.get(volumeId);
JsonNode attachments = data.get("attachments");
Validate.isTrue(attachments.isArray() && attachments.size() > 0);
JsonNode attachment = attachments.get(0);
JsonNode ltime = elem.get("ltime");
if (ltime == null || ltime.isNull()) {
continue;
}
DateTime detachTime = new DateTime(ltime.asLong());
processedIds.add(volumeId);
setAttachmentInfo(volumeId, attachment, detachTime, resource);
}
for (Map.Entry<String, Resource> volumeEntry : idToResource.entrySet()) {
String id = volumeEntry.getKey();
if (!processedIds.contains(id)) {
Resource resource = volumeEntry.getValue();
LOGGER.info(
String.format(
"Volume %s never was attached, use createTime %s as the detachTime",
id, resource.getLaunchTime()));
setAttachmentInfo(id, null, new DateTime(resource.getLaunchTime().getTime()), resource);
}
}
}
}
}