/**
* Generate the topological ordering for the given jobGraph.
*
* @param jobgraph the job graph
* @return a {@link List} of the {@link IndigoJob} in topological order
* @throws IllegalArgumentException if the graph has cycles (hence no topological order exists).
*/
protected List<IndigoJob> getJobsTopologicalOrder(Map<String, IndigoJob> jobgraph) {
DefaultDirectedGraph<IndigoJob, DefaultEdge> graph =
new DefaultDirectedGraph<IndigoJob, DefaultEdge>(DefaultEdge.class);
for (IndigoJob job : jobgraph.values()) {
graph.addVertex(job);
}
for (IndigoJob job : jobgraph.values()) {
for (IndigoJob parent : job.getParents()) {
graph.addEdge(parent, job); // job depends on parent
}
}
LOG.debug("IndigoJob graph: {}", graph.toString());
// Are there cycles in the dependencies.
CycleDetector<IndigoJob, DefaultEdge> cycleDetector =
new CycleDetector<IndigoJob, DefaultEdge>(graph);
if (cycleDetector.detectCycles()) {
LOG.error("Job graph has cycles!");
throw new IllegalArgumentException(
String.format(
"Failed to generate topological order for a graph with cycles: <%s>", graph));
}
TopologicalOrderIterator<IndigoJob, DefaultEdge> orderIterator =
new TopologicalOrderIterator<IndigoJob, DefaultEdge>(graph);
List<IndigoJob> topoOrder = Lists.newArrayList(orderIterator);
LOG.debug("IndigoJob topological order: {}", topoOrder);
return topoOrder;
}
/**
* @param deployment the deployment from which create the jobs
* @param jobgraph the graph of the jobs
* @param templateTopologicalOrderIterator the topological order iterator of the jobs
* @param client the Chronos client to use
* @return <tt>true</tt> if the currently checked node is ready, <tt>false</tt> if still in
* progress.
* @throws DeploymentException if the currently node failed.
*/
protected boolean checkJobsOnChronosIteratively(
Deployment deployment,
Map<String, IndigoJob> jobgraph,
TemplateTopologicalOrderIterator templateTopologicalOrderIterator,
Chronos client)
throws DeploymentException {
// Get current job
Resource currentNode = templateTopologicalOrderIterator.getCurrent();
IndigoJob job = jobgraph.get(currentNode.getToscaNodeName());
String jobName = job.getChronosJob().getName();
Job updatedJob = getJobStatus(client, jobName);
if (updatedJob == null) {
String errorMsg =
String.format(
"Failed to deploy deployment <%s>. Chronos job <%s> (id: <%s>) does not exist",
deployment.getId(), job.getToscaNodeName(), jobName);
LOG.error(errorMsg);
// Update job status
updateResource(deployment, job, NodeStates.ERROR);
throw new DeploymentException(errorMsg);
}
JobState jobState = getLastState(updatedJob);
LOG.debug(
"Status of Chronos job <{}> for deployment <{}> is <{}> ({}/{})",
jobName,
deployment.getId(),
jobState,
templateTopologicalOrderIterator.getPosition() + 1,
templateTopologicalOrderIterator.getNodeSize());
// Go ahead only if the job succeeded
if (jobState != JobState.SUCCESS) {
if (jobState != JobState.FAILURE) {
// Job still in progress
LOG.debug("Polling again job <{}> for deployment <{}>", jobName, deployment.getId());
return false;
} else {
// Job failed -> Deployment failed!
String errorMsg =
String.format(
"Failed to deploy deployment <%s>. Chronos job <%s> (id: <%s>) status is <%s>",
deployment.getId(), job.getToscaNodeName(), jobName, jobState);
LOG.error(errorMsg);
// Update job status
updateResource(deployment, job, NodeStates.ERROR);
currentNode.setState(NodeStates.ERROR);
throw new DeploymentException(errorMsg);
}
} else {
// Job finished -> Update job status
updateResource(deployment, job, NodeStates.STARTED);
currentNode.setState(NodeStates.STARTED);
return true;
}
}
/**
* Deletes all the jobs from Chronos.
*
* @param jobgraph the job graph.
* @param client the {@link Chronos} client.
* @param failAtFirst if <tt>true</tt> throws an exception at the first job deletion error,
* otherwise it tries to delete every other job.
* @return <tt>true</tt> if all jobs have been deleted, <tt>false</tt> otherwise.
*/
protected boolean deleteJobsOnChronosIteratively(
Deployment deployment,
Map<String, IndigoJob> jobgraph,
TemplateTopologicalOrderIterator templateTopologicalOrderIterator,
Chronos client,
boolean failAtFirst) {
Resource currentNode = templateTopologicalOrderIterator.getCurrent();
if (currentNode == null) {
return false;
}
IndigoJob currentJob = jobgraph.get(currentNode.getToscaNodeName());
boolean failed = false;
// Delete current job (all jobs iteratively)
try {
try {
updateResource(deployment, currentJob, NodeStates.DELETING);
currentNode.setState(NodeStates.DELETING);
String jobName = currentJob.getChronosJob().getName();
client.deleteJob(jobName);
LOG.debug(
"Deleted Chronos job <{}> for deployment <{}> ({}/{})",
jobName,
deployment.getId(),
templateTopologicalOrderIterator.getPosition() + 1,
templateTopologicalOrderIterator.getNodeSize());
} catch (ChronosException ce) {
// Chronos API hack (to avoid error 400 if the job to delete does not exist)
if (ce.getStatus() != 400 && ce.getStatus() != 404) {
throw new RuntimeException(String.format("Status Code: <%s>", ce.getStatus()));
}
}
} catch (Exception ex) {
// Just log the error
String errorMsg = String.format("Failed to delete job <%s> on Chronos: %s", ex.getMessage());
LOG.error(errorMsg);
failed = true;
// Update job status
updateResource(deployment, currentJob, NodeStates.ERROR);
currentNode.setState(NodeStates.ERROR);
// Only throw exception if required
if (failAtFirst) {
// TODO use a custom exception ?
throw new RuntimeException(errorMsg);
}
}
return !failed;
}
private void updateResource(Deployment deployment, IndigoJob job, NodeStates state) {
// Find the Resource from DB
Resource resource =
resourceRepository.findByToscaNodeNameAndDeployment_id(
job.getToscaNodeName(), deployment.getId());
resource.setState(state);
// resourceRepository.save(resource);
}
/**
* @param deployment the deployment from which create the jobs
* @param jobgraph the graph of the jobs
* @param templateTopologicalOrderIterator the topological order iterator of the jobs
* @param client the Chronos client to use
* @return <tt>true</tt> if there are more nodes to create, <tt>false</tt> otherwise.
*/
protected boolean createJobsOnChronosIteratively(
Deployment deployment,
Map<String, IndigoJob> jobgraph,
TemplateTopologicalOrderIterator templateTopologicalOrderIterator,
Chronos client) {
// Create Jobs in the required order on Chronos
Resource currentNode = templateTopologicalOrderIterator.getCurrent();
if (currentNode == null) {
return false;
}
IndigoJob currentJob = jobgraph.get(currentNode.getToscaNodeName());
// Create jobs based on the topological order
try {
String nodeTypeMsg;
if (currentJob.getParents().isEmpty()) {
// Scheduled job (not dependent)
nodeTypeMsg = "scheduled";
client.createJob(currentJob.getChronosJob());
} else {
// Dependent job
nodeTypeMsg = String.format("parents <%s>", currentJob.getChronosJob().getParents());
client.createDependentJob(currentJob.getChronosJob());
}
LOG.debug(
"Created job for deployment <{}> on Chronos: name <{}>, {} ({}/{})",
deployment.getId(),
currentJob.getChronosJob().getName(),
nodeTypeMsg,
templateTopologicalOrderIterator.getPosition() + 1,
templateTopologicalOrderIterator.getNodeSize());
// Update job status
updateResource(deployment, currentJob, NodeStates.CREATED);
// The node in the iterator is not actually an entity (serialization issues)
currentNode.setState(NodeStates.CREATED);
} catch (ChronosException exception) { // Chronos job launch error
// Update job status
updateResource(deployment, currentJob, NodeStates.ERROR);
currentNode.setState(NodeStates.ERROR);
// TODO use a custom exception ?
throw new RuntimeException(
String.format(
"Failed to launch job <%s> on Chronos. Status Code: <%s>",
currentJob.getChronosJob().getName(), exception.getStatus()));
}
return templateTopologicalOrderIterator.getNext() != null;
}
/**
* Creates the {@link INDIGOJob} graph based on the given {@link Deployment} (the TOSCA template
* is parsed).
*
* @param deployment the input deployment.
* @return the job graph.
*/
protected Map<String, IndigoJob> generateJobGraph(
Deployment deployment, Map<String, OneData> odParameters) {
String deploymentId = deployment.getId();
Map<String, IndigoJob> jobs = new HashMap<String, ChronosServiceImpl.IndigoJob>();
// Parse TOSCA template
Map<String, NodeTemplate> nodes = null;
try {
String customizedTemplate = deployment.getTemplate();
/*
* FIXME TEMPORARY - Replace hard-coded properties in nodes (WARNING: Cannot be done when
* receiving the template because we still miss OneData settings that are obtained during the
* WF after the site choice, which in turns depends on the template nodes and properties...)
*/
customizedTemplate = replaceHardCodedParams(customizedTemplate, odParameters);
// Re-parse template (TODO: serialize the template in-memory representation?)
ArchiveRoot ar = toscaService.prepareTemplate(customizedTemplate, deployment.getParameters());
nodes = ar.getTopology().getNodeTemplates();
} catch (IOException | ParsingException ex) {
throw new OrchestratorException(ex);
}
// TODO Iterate on Chronos nodes and related dependencies (just ignore others - also if invalid
// - for now)
// Populate resources (nodes) hashmap to speed up job creation (id-name mapping is needed)
Map<String, Resource> resources =
deployment
.getResources()
.stream()
.collect(Collectors.toMap(e -> e.getToscaNodeName(), e -> e));
// Only create Indigo Jobs
for (Map.Entry<String, NodeTemplate> node : nodes.entrySet()) {
NodeTemplate nodeTemplate = node.getValue();
String nodeName = node.getKey();
if (isChronosNode(nodeTemplate)) {
Job chronosJob = createJob(nodes, deploymentId, nodeName, nodeTemplate, resources);
IndigoJob job = new IndigoJob(nodeName, chronosJob);
jobs.put(nodeName, job);
}
}
// Create jobs hierarchy
for (Map.Entry<String, IndigoJob> job : jobs.entrySet()) {
IndigoJob indigoJob = job.getValue();
String nodeName = job.getKey();
NodeTemplate nodeTemplate = nodes.get(nodeName);
// Retrieve Job parents
List<String> parentNames = getJobParents(nodeTemplate, nodeName, nodes);
if (parentNames != null && !parentNames.isEmpty()) {
List<String> chronosParentList = new ArrayList<>();
for (String parentName : parentNames) {
IndigoJob parentJob = jobs.get(parentName);
// Add this job to the parent
parentJob.getChildren().add(indigoJob);
// Add the parent to this job
indigoJob.getParents().add(parentJob);
// Add to the Chronos DSL parent list
chronosParentList.add(parentJob.getChronosJob().getName());
}
// Update Chronos DSL parent list
indigoJob.getChronosJob().setParents(chronosParentList);
}
}
// Validate (no cycles!)
// FIXME Shouldn't just return the topological order ?
getJobsTopologicalOrder(jobs);
return jobs;
}
