/**
* Gets all volumes that are not attached to any instance. Janitor Monkey only considers
* unattached volumes as cleanup candidates, so there is no need to get volumes that are in-use.
*
* @param region
* @return
*/
private List<Resource> getUnattachedVolumeResourcesInRegion(String region, String... volumeIds) {
String url = eddaClient.getBaseUrl(region) + "/aws/volumes;";
if (volumeIds != null && volumeIds.length != 0) {
url += StringUtils.join(volumeIds, ',');
LOGGER.info(
String.format("Getting volumes in region %s for %d ids", region, volumeIds.length));
} else {
LOGGER.info(String.format("Getting all unattached volumes in region %s", region));
}
url += ";state=available;_expand:(volumeId,createTime,size,state,tags)";
JsonNode jsonNode = null;
try {
jsonNode = eddaClient.getJsonNodeFromUrl(url);
} catch (Exception e) {
LOGGER.error(
String.format(
"Failed to get Jason node from edda for unattached volumes in region %s.", region),
e);
}
if (jsonNode == null || !jsonNode.isArray()) {
throw new RuntimeException(
String.format("Failed to get valid document from %s, got: %s", url, jsonNode));
}
List<Resource> resources = Lists.newArrayList();
for (Iterator<JsonNode> it = jsonNode.getElements(); it.hasNext(); ) {
resources.add(parseJsonElementToVolumeResource(region, it.next()));
}
return resources;
}
private void updateInstanceToOwner(String region) {
LOGGER.info(String.format("Getting owners for all instances in region %s", region));
long startTime = DateTime.now().minusDays(LOOKBACK_DAYS).getMillis();
String url =
String.format(
"%s/view/instances;_since=%d;state.name=running;tags.key=owner;"
+ "_expand:(instanceId,tags:(key,value))",
eddaClient.getBaseUrl(region), startTime);
JsonNode jsonNode = null;
try {
jsonNode = eddaClient.getJsonNodeFromUrl(url);
} catch (Exception e) {
LOGGER.error(
String.format(
"Failed to get Jason node from edda for instance owners in region %s.", region),
e);
}
if (jsonNode == null || !jsonNode.isArray()) {
throw new RuntimeException(
String.format("Failed to get valid document from %s, got: %s", url, jsonNode));
}
for (Iterator<JsonNode> it = jsonNode.getElements(); it.hasNext(); ) {
JsonNode elem = it.next();
String instanceId = elem.get("instanceId").getTextValue();
JsonNode tags = elem.get("tags");
if (tags == null || !tags.isArray() || tags.size() == 0) {
continue;
}
for (Iterator<JsonNode> tagsIt = tags.getElements(); tagsIt.hasNext(); ) {
JsonNode tag = tagsIt.next();
String tagKey = tag.get("key").getTextValue();
if ("owner".equals(tagKey)) {
instanceToOwner.put(instanceId, tag.get("value").getTextValue());
break;
}
}
}
}
private String getBatchUrl(String region, List<Resource> batch) {
StringBuilder batchUrl = new StringBuilder(eddaClient.getBaseUrl(region) + "/aws/volumes/");
boolean isFirst = true;
for (Resource resource : batch) {
if (!isFirst) {
batchUrl.append(',');
} else {
isFirst = false;
}
batchUrl.append(resource.getId());
}
batchUrl.append(
";data.state=in-use;_since=0;_expand;_meta:"
+ "(ltime,data:(volumeId,attachments:(deleteOnTermination,instanceId)))");
return batchUrl.toString();
}
/**
* Adds information of last attachment to the resources. To be compatible with the AWS
* implementation of the same crawler, add the information to the JANITOR_META tag. It always uses
* the latest information to update the tag in this resource (not writing back to AWS) no matter
* if the tag exists.
*
* @param resources the volume resources
*/
private void addLastAttachmentInfo(List<Resource> resources) {
Validate.notNull(resources);
LOGGER.info(
String.format("Updating the latest attachment info for %d resources", resources.size()));
Map<String, List<Resource>> regionToResources = Maps.newHashMap();
for (Resource resource : resources) {
List<Resource> regionalList = regionToResources.get(resource.getRegion());
if (regionalList == null) {
regionalList = Lists.newArrayList();
regionToResources.put(resource.getRegion(), regionalList);
}
regionalList.add(resource);
}
for (Map.Entry<String, List<Resource>> entry : regionToResources.entrySet()) {
LOGGER.info(
String.format(
"Updating the latest attachment info for %d resources in region %s",
resources.size(), entry.getKey()));
for (List<Resource> batch : Lists.partition(entry.getValue(), BATCH_SIZE)) {
LOGGER.info(String.format("Processing batch of size %d", batch.size()));
String batchUrl = getBatchUrl(entry.getKey(), batch);
JsonNode batchResult = null;
try {
batchResult = eddaClient.getJsonNodeFromUrl(batchUrl);
} catch (IOException e) {
LOGGER.error("Failed to get response for the batch.", e);
}
Map<String, Resource> idToResource = Maps.newHashMap();
for (Resource resource : batch) {
idToResource.put(resource.getId(), resource);
}
if (batchResult == null || !batchResult.isArray()) {
throw new RuntimeException(
String.format(
"Failed to get valid document from %s, got: %s", batchUrl, batchResult));
}
Set<String> processedIds = Sets.newHashSet();
for (Iterator<JsonNode> it = batchResult.getElements(); it.hasNext(); ) {
JsonNode elem = it.next();
JsonNode data = elem.get("data");
String volumeId = data.get("volumeId").getTextValue();
Resource resource = idToResource.get(volumeId);
JsonNode attachments = data.get("attachments");
Validate.isTrue(attachments.isArray() && attachments.size() > 0);
JsonNode attachment = attachments.get(0);
JsonNode ltime = elem.get("ltime");
if (ltime == null || ltime.isNull()) {
continue;
}
DateTime detachTime = new DateTime(ltime.asLong());
processedIds.add(volumeId);
setAttachmentInfo(volumeId, attachment, detachTime, resource);
}
for (Map.Entry<String, Resource> volumeEntry : idToResource.entrySet()) {
String id = volumeEntry.getKey();
if (!processedIds.contains(id)) {
Resource resource = volumeEntry.getValue();
LOGGER.info(
String.format(
"Volume %s never was attached, use createTime %s as the detachTime",
id, resource.getLaunchTime()));
setAttachmentInfo(id, null, new DateTime(resource.getLaunchTime().getTime()), resource);
}
}
}
}
}