public void terminateFailedNodes(
ContainerOrchestrator orchestrator, Stack stack, GatewayConfig gatewayConfig, Set<Node> nodes)
throws CloudbreakOrchestratorFailedException {
List<String> allAvailableNode = orchestrator.getAvailableNodes(gatewayConfig, nodes);
List<Node> missingNodes = selectMissingNodes(nodes, allAvailableNode);
if (missingNodes.size() > 0) {
String message =
String.format(
"Bootstrap failed on %s nodes. These nodes will be terminated.", missingNodes.size());
LOGGER.info(message);
eventService.fireCloudbreakEvent(stack.getId(), Status.UPDATE_IN_PROGRESS.name(), message);
for (Node missingNode : missingNodes) {
InstanceMetaData instanceMetaData =
instanceMetaDataRepository.findNotTerminatedByPrivateAddress(
stack.getId(), missingNode.getPrivateIp());
InstanceGroup ig =
instanceGroupRepository.findOneByGroupNameInStack(
stack.getId(), instanceMetaData.getInstanceGroup().getGroupName());
ig.setNodeCount(ig.getNodeCount() - 1);
if (ig.getNodeCount() < 1) {
throw new CloudbreakOrchestratorFailedException(
String.format(
"%s instancegroup nodecount was lower than 1 cluster creation failed.",
ig.getGroupName()));
}
instanceGroupRepository.save(ig);
message =
String.format(
"Delete '%s' node. and Decrease the nodecount on %s instancegroup",
instanceMetaData.getInstanceId(), ig.getGroupName());
LOGGER.info(message);
eventService.fireCloudbreakEvent(stack.getId(), Status.UPDATE_IN_PROGRESS.name(), message);
deleteResourceAndDependencies(stack, instanceMetaData);
deleteInstanceResourceFromDatabase(stack, instanceMetaData);
instanceMetaData.setInstanceStatus(InstanceStatus.TERMINATED);
instanceMetaDataRepository.save(instanceMetaData);
LOGGER.info(
String.format(
"The status of instanceMetadata with %s id and %s name setted to TERMINATED.",
instanceMetaData.getId(), instanceMetaData.getInstanceId()));
}
}
}
private void updateNodeCount(
long stackId,
List<Group> originalGroups,
List<CloudResourceStatus> statuses,
boolean create) {
for (Group group : originalGroups) {
int nodeCount = group.getInstances().size();
List<CloudResourceStatus> failedResources = removeFailedMetadata(stackId, statuses, group);
if (!failedResources.isEmpty() && create) {
int failedCount = failedResources.size();
InstanceGroup instanceGroup =
instanceGroupRepository.findOneByGroupNameInStack(stackId, group.getName());
instanceGroup.setNodeCount(nodeCount - failedCount);
instanceGroupRepository.save(instanceGroup);
String message =
cloudbreakMessagesService.getMessage(
ROLLBACK_MESSAGE,
asList(failedCount, group.getName(), failedResources.get(0).getStatusReason()));
eventService.fireCloudbreakEvent(stackId, Status.UPDATE_IN_PROGRESS.name(), message);
}
}
}