private List<Node> selectMissingNodes(Set<Node> clusterNodes, List<String> availableNodes) {
List<Node> missingNodes = new ArrayList<>();
for (Node node : clusterNodes) {
boolean contains = false;
for (String nodeAddress : availableNodes) {
if (nodeAddress.equals(node.getPrivateIp())) {
contains = true;
break;
}
}
if (!contains) {
missingNodes.add(node);
}
}
return missingNodes;
}
public void terminateFailedNodes(
ContainerOrchestrator orchestrator, Stack stack, GatewayConfig gatewayConfig, Set<Node> nodes)
throws CloudbreakOrchestratorFailedException {
List<String> allAvailableNode = orchestrator.getAvailableNodes(gatewayConfig, nodes);
List<Node> missingNodes = selectMissingNodes(nodes, allAvailableNode);
if (missingNodes.size() > 0) {
String message =
String.format(
"Bootstrap failed on %s nodes. These nodes will be terminated.", missingNodes.size());
LOGGER.info(message);
eventService.fireCloudbreakEvent(stack.getId(), Status.UPDATE_IN_PROGRESS.name(), message);
for (Node missingNode : missingNodes) {
InstanceMetaData instanceMetaData =
instanceMetaDataRepository.findNotTerminatedByPrivateAddress(
stack.getId(), missingNode.getPrivateIp());
InstanceGroup ig =
instanceGroupRepository.findOneByGroupNameInStack(
stack.getId(), instanceMetaData.getInstanceGroup().getGroupName());
ig.setNodeCount(ig.getNodeCount() - 1);
if (ig.getNodeCount() < 1) {
throw new CloudbreakOrchestratorFailedException(
String.format(
"%s instancegroup nodecount was lower than 1 cluster creation failed.",
ig.getGroupName()));
}
instanceGroupRepository.save(ig);
message =
String.format(
"Delete '%s' node. and Decrease the nodecount on %s instancegroup",
instanceMetaData.getInstanceId(), ig.getGroupName());
LOGGER.info(message);
eventService.fireCloudbreakEvent(stack.getId(), Status.UPDATE_IN_PROGRESS.name(), message);
deleteResourceAndDependencies(stack, instanceMetaData);
deleteInstanceResourceFromDatabase(stack, instanceMetaData);
instanceMetaData.setInstanceStatus(InstanceStatus.TERMINATED);
instanceMetaDataRepository.save(instanceMetaData);
LOGGER.info(
String.format(
"The status of instanceMetadata with %s id and %s name setted to TERMINATED.",
instanceMetaData.getId(), instanceMetaData.getInstanceId()));
}
}
}
