/** * Creates all recipe tasks for cluster and groups them by recipe-name. In other words, by having * a recipe-name such as hadoop::dn you fetch all the tasks in the cluster that are running * hadoop::dn. recipeName -> taskid(recipe + machineid) -> task * * @param cluster * @param clusterEntity * @param clusterStats * @param chefJsons * @param submitter * @param allRecipeTasks * @param dag * @return * @throws KaramelException */ public static Map<String, Map<String, Task>> recipeLevelTasks( JsonCluster cluster, ClusterRuntime clusterEntity, ClusterStats clusterStats, Map<String, JsonObject> chefJsons, TaskSubmitter submitter, Map<String, RunRecipeTask> allRecipeTasks, Dag dag) throws KaramelException { Map<String, Map<String, Task>> map = new HashMap<>(); for (GroupRuntime ge : clusterEntity.getGroups()) { JsonGroup jg = UserClusterDataExtractor.findGroup(cluster, ge.getName()); for (MachineRuntime me : ge.getMachines()) { for (JsonCookbook jc : jg.getCookbooks()) { CookbookUrls urls = jc.getKaramelizedCookbook().getUrls(); for (JsonRecipe rec : jc.getRecipes()) { JsonObject json1 = chefJsons.get(me.getId() + rec.getCanonicalName()); addRecipeTaskForMachineIntoRecipesMap( rec.getCanonicalName(), me, clusterStats, map, json1, submitter, urls.id, jc.getName(), allRecipeTasks, dag); } } } } return map; }
/** * machine -> taskid -> task * * @param cluster * @param clusterEntity * @param clusterStats * @param chefJsons * @param submitter * @param allRecipeTasks * @param dag * @return * @throws KaramelException */ public static Map<String, Map<String, Task>> cookbookLevelTasks( JsonCluster cluster, ClusterRuntime clusterEntity, ClusterStats clusterStats, Map<String, JsonObject> chefJsons, TaskSubmitter submitter, Map<String, RunRecipeTask> allRecipeTasks, Dag dag) throws KaramelException { Map<String, Map<String, Task>> map = new HashMap<>(); for (GroupRuntime ge : clusterEntity.getGroups()) { JsonGroup jg = UserClusterDataExtractor.findGroup(cluster, ge.getName()); for (MachineRuntime me : ge.getMachines()) { Map<String, Task> map1 = new HashMap<>(); for (JsonCookbook jc : jg.getCookbooks()) { CookbookUrls urls = jc.getKaramelizedCookbook().getUrls(); VendorCookbookTask t1 = new VendorCookbookTask( me, clusterStats, submitter, urls.id, Settings.REMOTE_CB_VENDOR_PATH, urls.repoUrl, urls.repoName, urls.cookbookRelPath, urls.branch); dag.addTask(t1); map1.put(t1.uniqueId(), t1); String recipeName = jc.getName() + Settings.COOKBOOK_DELIMITER + Settings.INSTALL_RECIPE; JsonObject json = chefJsons.get(me.getId() + recipeName); RunRecipeTask t2 = makeRecipeTaskIfNotExist( recipeName, me, clusterStats, json, submitter, urls.id, jc.getName(), allRecipeTasks, dag); map1.put(t2.uniqueId(), t2); } logger.debug( String.format( "Cookbook-level tasks for the machine '%s' in the group '%s' are: %s", me.getPublicIp(), ge.getName(), map1.keySet())); if (map.get(me.getId()) != null) { map.get(me.getId()).putAll(map1); } else { map.put(me.getId(), map1); } } } return map; }