/** * Generate the topological ordering for the given jobGraph. * * @param jobgraph the job graph * @return a {@link List} of the {@link IndigoJob} in topological order * @throws IllegalArgumentException if the graph has cycles (hence no topological order exists). */ protected List<IndigoJob> getJobsTopologicalOrder(Map<String, IndigoJob> jobgraph) { DefaultDirectedGraph<IndigoJob, DefaultEdge> graph = new DefaultDirectedGraph<IndigoJob, DefaultEdge>(DefaultEdge.class); for (IndigoJob job : jobgraph.values()) { graph.addVertex(job); } for (IndigoJob job : jobgraph.values()) { for (IndigoJob parent : job.getParents()) { graph.addEdge(parent, job); // job depends on parent } } LOG.debug("IndigoJob graph: {}", graph.toString()); // Are there cycles in the dependencies. CycleDetector<IndigoJob, DefaultEdge> cycleDetector = new CycleDetector<IndigoJob, DefaultEdge>(graph); if (cycleDetector.detectCycles()) { LOG.error("Job graph has cycles!"); throw new IllegalArgumentException( String.format( "Failed to generate topological order for a graph with cycles: <%s>", graph)); } TopologicalOrderIterator<IndigoJob, DefaultEdge> orderIterator = new TopologicalOrderIterator<IndigoJob, DefaultEdge>(graph); List<IndigoJob> topoOrder = Lists.newArrayList(orderIterator); LOG.debug("IndigoJob topological order: {}", topoOrder); return topoOrder; }
/** * @param deployment the deployment from which create the jobs * @param jobgraph the graph of the jobs * @param templateTopologicalOrderIterator the topological order iterator of the jobs * @param client the Chronos client to use * @return <tt>true</tt> if there are more nodes to create, <tt>false</tt> otherwise. */ protected boolean createJobsOnChronosIteratively( Deployment deployment, Map<String, IndigoJob> jobgraph, TemplateTopologicalOrderIterator templateTopologicalOrderIterator, Chronos client) { // Create Jobs in the required order on Chronos Resource currentNode = templateTopologicalOrderIterator.getCurrent(); if (currentNode == null) { return false; } IndigoJob currentJob = jobgraph.get(currentNode.getToscaNodeName()); // Create jobs based on the topological order try { String nodeTypeMsg; if (currentJob.getParents().isEmpty()) { // Scheduled job (not dependent) nodeTypeMsg = "scheduled"; client.createJob(currentJob.getChronosJob()); } else { // Dependent job nodeTypeMsg = String.format("parents <%s>", currentJob.getChronosJob().getParents()); client.createDependentJob(currentJob.getChronosJob()); } LOG.debug( "Created job for deployment <{}> on Chronos: name <{}>, {} ({}/{})", deployment.getId(), currentJob.getChronosJob().getName(), nodeTypeMsg, templateTopologicalOrderIterator.getPosition() + 1, templateTopologicalOrderIterator.getNodeSize()); // Update job status updateResource(deployment, currentJob, NodeStates.CREATED); // The node in the iterator is not actually an entity (serialization issues) currentNode.setState(NodeStates.CREATED); } catch (ChronosException exception) { // Chronos job launch error // Update job status updateResource(deployment, currentJob, NodeStates.ERROR); currentNode.setState(NodeStates.ERROR); // TODO use a custom exception ? throw new RuntimeException( String.format( "Failed to launch job <%s> on Chronos. Status Code: <%s>", currentJob.getChronosJob().getName(), exception.getStatus())); } return templateTopologicalOrderIterator.getNext() != null; }
/** * Creates the {@link INDIGOJob} graph based on the given {@link Deployment} (the TOSCA template * is parsed). * * @param deployment the input deployment. * @return the job graph. */ protected Map<String, IndigoJob> generateJobGraph( Deployment deployment, Map<String, OneData> odParameters) { String deploymentId = deployment.getId(); Map<String, IndigoJob> jobs = new HashMap<String, ChronosServiceImpl.IndigoJob>(); // Parse TOSCA template Map<String, NodeTemplate> nodes = null; try { String customizedTemplate = deployment.getTemplate(); /* * FIXME TEMPORARY - Replace hard-coded properties in nodes (WARNING: Cannot be done when * receiving the template because we still miss OneData settings that are obtained during the * WF after the site choice, which in turns depends on the template nodes and properties...) */ customizedTemplate = replaceHardCodedParams(customizedTemplate, odParameters); // Re-parse template (TODO: serialize the template in-memory representation?) ArchiveRoot ar = toscaService.prepareTemplate(customizedTemplate, deployment.getParameters()); nodes = ar.getTopology().getNodeTemplates(); } catch (IOException | ParsingException ex) { throw new OrchestratorException(ex); } // TODO Iterate on Chronos nodes and related dependencies (just ignore others - also if invalid // - for now) // Populate resources (nodes) hashmap to speed up job creation (id-name mapping is needed) Map<String, Resource> resources = deployment .getResources() .stream() .collect(Collectors.toMap(e -> e.getToscaNodeName(), e -> e)); // Only create Indigo Jobs for (Map.Entry<String, NodeTemplate> node : nodes.entrySet()) { NodeTemplate nodeTemplate = node.getValue(); String nodeName = node.getKey(); if (isChronosNode(nodeTemplate)) { Job chronosJob = createJob(nodes, deploymentId, nodeName, nodeTemplate, resources); IndigoJob job = new IndigoJob(nodeName, chronosJob); jobs.put(nodeName, job); } } // Create jobs hierarchy for (Map.Entry<String, IndigoJob> job : jobs.entrySet()) { IndigoJob indigoJob = job.getValue(); String nodeName = job.getKey(); NodeTemplate nodeTemplate = nodes.get(nodeName); // Retrieve Job parents List<String> parentNames = getJobParents(nodeTemplate, nodeName, nodes); if (parentNames != null && !parentNames.isEmpty()) { List<String> chronosParentList = new ArrayList<>(); for (String parentName : parentNames) { IndigoJob parentJob = jobs.get(parentName); // Add this job to the parent parentJob.getChildren().add(indigoJob); // Add the parent to this job indigoJob.getParents().add(parentJob); // Add to the Chronos DSL parent list chronosParentList.add(parentJob.getChronosJob().getName()); } // Update Chronos DSL parent list indigoJob.getChronosJob().setParents(chronosParentList); } } // Validate (no cycles!) // FIXME Shouldn't just return the topological order ? getJobsTopologicalOrder(jobs); return jobs; }