/** * Generate the topological ordering for the given jobGraph. * * @param jobgraph the job graph * @return a {@link List} of the {@link IndigoJob} in topological order * @throws IllegalArgumentException if the graph has cycles (hence no topological order exists). */ protected List<IndigoJob> getJobsTopologicalOrder(Map<String, IndigoJob> jobgraph) { DefaultDirectedGraph<IndigoJob, DefaultEdge> graph = new DefaultDirectedGraph<IndigoJob, DefaultEdge>(DefaultEdge.class); for (IndigoJob job : jobgraph.values()) { graph.addVertex(job); } for (IndigoJob job : jobgraph.values()) { for (IndigoJob parent : job.getParents()) { graph.addEdge(parent, job); // job depends on parent } } LOG.debug("IndigoJob graph: {}", graph.toString()); // Are there cycles in the dependencies. CycleDetector<IndigoJob, DefaultEdge> cycleDetector = new CycleDetector<IndigoJob, DefaultEdge>(graph); if (cycleDetector.detectCycles()) { LOG.error("Job graph has cycles!"); throw new IllegalArgumentException( String.format( "Failed to generate topological order for a graph with cycles: <%s>", graph)); } TopologicalOrderIterator<IndigoJob, DefaultEdge> orderIterator = new TopologicalOrderIterator<IndigoJob, DefaultEdge>(graph); List<IndigoJob> topoOrder = Lists.newArrayList(orderIterator); LOG.debug("IndigoJob topological order: {}", topoOrder); return topoOrder; }
/** * @param deployment the deployment from which create the jobs * @param jobgraph the graph of the jobs * @param templateTopologicalOrderIterator the topological order iterator of the jobs * @param client the Chronos client to use * @return <tt>true</tt> if the currently checked node is ready, <tt>false</tt> if still in * progress. * @throws DeploymentException if the currently node failed. */ protected boolean checkJobsOnChronosIteratively( Deployment deployment, Map<String, IndigoJob> jobgraph, TemplateTopologicalOrderIterator templateTopologicalOrderIterator, Chronos client) throws DeploymentException { // Get current job Resource currentNode = templateTopologicalOrderIterator.getCurrent(); IndigoJob job = jobgraph.get(currentNode.getToscaNodeName()); String jobName = job.getChronosJob().getName(); Job updatedJob = getJobStatus(client, jobName); if (updatedJob == null) { String errorMsg = String.format( "Failed to deploy deployment <%s>. Chronos job <%s> (id: <%s>) does not exist", deployment.getId(), job.getToscaNodeName(), jobName); LOG.error(errorMsg); // Update job status updateResource(deployment, job, NodeStates.ERROR); throw new DeploymentException(errorMsg); } JobState jobState = getLastState(updatedJob); LOG.debug( "Status of Chronos job <{}> for deployment <{}> is <{}> ({}/{})", jobName, deployment.getId(), jobState, templateTopologicalOrderIterator.getPosition() + 1, templateTopologicalOrderIterator.getNodeSize()); // Go ahead only if the job succeeded if (jobState != JobState.SUCCESS) { if (jobState != JobState.FAILURE) { // Job still in progress LOG.debug("Polling again job <{}> for deployment <{}>", jobName, deployment.getId()); return false; } else { // Job failed -> Deployment failed! String errorMsg = String.format( "Failed to deploy deployment <%s>. Chronos job <%s> (id: <%s>) status is <%s>", deployment.getId(), job.getToscaNodeName(), jobName, jobState); LOG.error(errorMsg); // Update job status updateResource(deployment, job, NodeStates.ERROR); currentNode.setState(NodeStates.ERROR); throw new DeploymentException(errorMsg); } } else { // Job finished -> Update job status updateResource(deployment, job, NodeStates.STARTED); currentNode.setState(NodeStates.STARTED); return true; } }
/** * Deletes all the jobs from Chronos. * * @param jobgraph the job graph. * @param client the {@link Chronos} client. * @param failAtFirst if <tt>true</tt> throws an exception at the first job deletion error, * otherwise it tries to delete every other job. * @return <tt>true</tt> if all jobs have been deleted, <tt>false</tt> otherwise. */ protected boolean deleteJobsOnChronosIteratively( Deployment deployment, Map<String, IndigoJob> jobgraph, TemplateTopologicalOrderIterator templateTopologicalOrderIterator, Chronos client, boolean failAtFirst) { Resource currentNode = templateTopologicalOrderIterator.getCurrent(); if (currentNode == null) { return false; } IndigoJob currentJob = jobgraph.get(currentNode.getToscaNodeName()); boolean failed = false; // Delete current job (all jobs iteratively) try { try { updateResource(deployment, currentJob, NodeStates.DELETING); currentNode.setState(NodeStates.DELETING); String jobName = currentJob.getChronosJob().getName(); client.deleteJob(jobName); LOG.debug( "Deleted Chronos job <{}> for deployment <{}> ({}/{})", jobName, deployment.getId(), templateTopologicalOrderIterator.getPosition() + 1, templateTopologicalOrderIterator.getNodeSize()); } catch (ChronosException ce) { // Chronos API hack (to avoid error 400 if the job to delete does not exist) if (ce.getStatus() != 400 && ce.getStatus() != 404) { throw new RuntimeException(String.format("Status Code: <%s>", ce.getStatus())); } } } catch (Exception ex) { // Just log the error String errorMsg = String.format("Failed to delete job <%s> on Chronos: %s", ex.getMessage()); LOG.error(errorMsg); failed = true; // Update job status updateResource(deployment, currentJob, NodeStates.ERROR); currentNode.setState(NodeStates.ERROR); // Only throw exception if required if (failAtFirst) { // TODO use a custom exception ? throw new RuntimeException(errorMsg); } } return !failed; }
private void updateResource(Deployment deployment, IndigoJob job, NodeStates state) { // Find the Resource from DB Resource resource = resourceRepository.findByToscaNodeNameAndDeployment_id( job.getToscaNodeName(), deployment.getId()); resource.setState(state); // resourceRepository.save(resource); }
/** * @param deployment the deployment from which create the jobs * @param jobgraph the graph of the jobs * @param templateTopologicalOrderIterator the topological order iterator of the jobs * @param client the Chronos client to use * @return <tt>true</tt> if there are more nodes to create, <tt>false</tt> otherwise. */ protected boolean createJobsOnChronosIteratively( Deployment deployment, Map<String, IndigoJob> jobgraph, TemplateTopologicalOrderIterator templateTopologicalOrderIterator, Chronos client) { // Create Jobs in the required order on Chronos Resource currentNode = templateTopologicalOrderIterator.getCurrent(); if (currentNode == null) { return false; } IndigoJob currentJob = jobgraph.get(currentNode.getToscaNodeName()); // Create jobs based on the topological order try { String nodeTypeMsg; if (currentJob.getParents().isEmpty()) { // Scheduled job (not dependent) nodeTypeMsg = "scheduled"; client.createJob(currentJob.getChronosJob()); } else { // Dependent job nodeTypeMsg = String.format("parents <%s>", currentJob.getChronosJob().getParents()); client.createDependentJob(currentJob.getChronosJob()); } LOG.debug( "Created job for deployment <{}> on Chronos: name <{}>, {} ({}/{})", deployment.getId(), currentJob.getChronosJob().getName(), nodeTypeMsg, templateTopologicalOrderIterator.getPosition() + 1, templateTopologicalOrderIterator.getNodeSize()); // Update job status updateResource(deployment, currentJob, NodeStates.CREATED); // The node in the iterator is not actually an entity (serialization issues) currentNode.setState(NodeStates.CREATED); } catch (ChronosException exception) { // Chronos job launch error // Update job status updateResource(deployment, currentJob, NodeStates.ERROR); currentNode.setState(NodeStates.ERROR); // TODO use a custom exception ? throw new RuntimeException( String.format( "Failed to launch job <%s> on Chronos. Status Code: <%s>", currentJob.getChronosJob().getName(), exception.getStatus())); } return templateTopologicalOrderIterator.getNext() != null; }
/** * Creates the {@link INDIGOJob} graph based on the given {@link Deployment} (the TOSCA template * is parsed). * * @param deployment the input deployment. * @return the job graph. */ protected Map<String, IndigoJob> generateJobGraph( Deployment deployment, Map<String, OneData> odParameters) { String deploymentId = deployment.getId(); Map<String, IndigoJob> jobs = new HashMap<String, ChronosServiceImpl.IndigoJob>(); // Parse TOSCA template Map<String, NodeTemplate> nodes = null; try { String customizedTemplate = deployment.getTemplate(); /* * FIXME TEMPORARY - Replace hard-coded properties in nodes (WARNING: Cannot be done when * receiving the template because we still miss OneData settings that are obtained during the * WF after the site choice, which in turns depends on the template nodes and properties...) */ customizedTemplate = replaceHardCodedParams(customizedTemplate, odParameters); // Re-parse template (TODO: serialize the template in-memory representation?) ArchiveRoot ar = toscaService.prepareTemplate(customizedTemplate, deployment.getParameters()); nodes = ar.getTopology().getNodeTemplates(); } catch (IOException | ParsingException ex) { throw new OrchestratorException(ex); } // TODO Iterate on Chronos nodes and related dependencies (just ignore others - also if invalid // - for now) // Populate resources (nodes) hashmap to speed up job creation (id-name mapping is needed) Map<String, Resource> resources = deployment .getResources() .stream() .collect(Collectors.toMap(e -> e.getToscaNodeName(), e -> e)); // Only create Indigo Jobs for (Map.Entry<String, NodeTemplate> node : nodes.entrySet()) { NodeTemplate nodeTemplate = node.getValue(); String nodeName = node.getKey(); if (isChronosNode(nodeTemplate)) { Job chronosJob = createJob(nodes, deploymentId, nodeName, nodeTemplate, resources); IndigoJob job = new IndigoJob(nodeName, chronosJob); jobs.put(nodeName, job); } } // Create jobs hierarchy for (Map.Entry<String, IndigoJob> job : jobs.entrySet()) { IndigoJob indigoJob = job.getValue(); String nodeName = job.getKey(); NodeTemplate nodeTemplate = nodes.get(nodeName); // Retrieve Job parents List<String> parentNames = getJobParents(nodeTemplate, nodeName, nodes); if (parentNames != null && !parentNames.isEmpty()) { List<String> chronosParentList = new ArrayList<>(); for (String parentName : parentNames) { IndigoJob parentJob = jobs.get(parentName); // Add this job to the parent parentJob.getChildren().add(indigoJob); // Add the parent to this job indigoJob.getParents().add(parentJob); // Add to the Chronos DSL parent list chronosParentList.add(parentJob.getChronosJob().getName()); } // Update Chronos DSL parent list indigoJob.getChronosJob().setParents(chronosParentList); } } // Validate (no cycles!) // FIXME Shouldn't just return the topological order ? getJobsTopologicalOrder(jobs); return jobs; }