/**
* Creates all recipe tasks for cluster and groups them by recipe-name. In other words, by having
* a recipe-name such as hadoop::dn you fetch all the tasks in the cluster that are running
* hadoop::dn. recipeName -> taskid(recipe + machineid) -> task
*
* @param cluster
* @param clusterEntity
* @param clusterStats
* @param chefJsons
* @param submitter
* @param allRecipeTasks
* @param dag
* @return
* @throws KaramelException
*/
public static Map<String, Map<String, Task>> recipeLevelTasks(
JsonCluster cluster,
ClusterRuntime clusterEntity,
ClusterStats clusterStats,
Map<String, JsonObject> chefJsons,
TaskSubmitter submitter,
Map<String, RunRecipeTask> allRecipeTasks,
Dag dag)
throws KaramelException {
Map<String, Map<String, Task>> map = new HashMap<>();
for (GroupRuntime ge : clusterEntity.getGroups()) {
JsonGroup jg = UserClusterDataExtractor.findGroup(cluster, ge.getName());
for (MachineRuntime me : ge.getMachines()) {
for (JsonCookbook jc : jg.getCookbooks()) {
CookbookUrls urls = jc.getKaramelizedCookbook().getUrls();
for (JsonRecipe rec : jc.getRecipes()) {
JsonObject json1 = chefJsons.get(me.getId() + rec.getCanonicalName());
addRecipeTaskForMachineIntoRecipesMap(
rec.getCanonicalName(),
me,
clusterStats,
map,
json1,
submitter,
urls.id,
jc.getName(),
allRecipeTasks,
dag);
}
}
}
}
return map;
}
/**
* machine -> taskid -> task
*
* @param cluster
* @param clusterEntity
* @param clusterStats
* @param chefJsons
* @param submitter
* @param allRecipeTasks
* @param dag
* @return
* @throws KaramelException
*/
public static Map<String, Map<String, Task>> cookbookLevelTasks(
JsonCluster cluster,
ClusterRuntime clusterEntity,
ClusterStats clusterStats,
Map<String, JsonObject> chefJsons,
TaskSubmitter submitter,
Map<String, RunRecipeTask> allRecipeTasks,
Dag dag)
throws KaramelException {
Map<String, Map<String, Task>> map = new HashMap<>();
for (GroupRuntime ge : clusterEntity.getGroups()) {
JsonGroup jg = UserClusterDataExtractor.findGroup(cluster, ge.getName());
for (MachineRuntime me : ge.getMachines()) {
Map<String, Task> map1 = new HashMap<>();
for (JsonCookbook jc : jg.getCookbooks()) {
CookbookUrls urls = jc.getKaramelizedCookbook().getUrls();
VendorCookbookTask t1 =
new VendorCookbookTask(
me,
clusterStats,
submitter,
urls.id,
Settings.REMOTE_CB_VENDOR_PATH,
urls.repoUrl,
urls.repoName,
urls.cookbookRelPath,
urls.branch);
dag.addTask(t1);
map1.put(t1.uniqueId(), t1);
String recipeName = jc.getName() + Settings.COOKBOOK_DELIMITER + Settings.INSTALL_RECIPE;
JsonObject json = chefJsons.get(me.getId() + recipeName);
RunRecipeTask t2 =
makeRecipeTaskIfNotExist(
recipeName,
me,
clusterStats,
json,
submitter,
urls.id,
jc.getName(),
allRecipeTasks,
dag);
map1.put(t2.uniqueId(), t2);
}
logger.debug(
String.format(
"Cookbook-level tasks for the machine '%s' in the group '%s' are: %s",
me.getPublicIp(), ge.getName(), map1.keySet()));
if (map.get(me.getId()) != null) {
map.get(me.getId()).putAll(map1);
} else {
map.put(me.getId(), map1);
}
}
}
return map;
}