public static void machineLevelTasks(
JsonCluster cluster,
ClusterRuntime clusterEntity,
ClusterStats clusterStats,
TaskSubmitter submitter,
Dag dag)
throws KaramelException {
Confs confs = Confs.loadKaramelConfs();
String prepStoragesConf = confs.getProperty(Settings.PREPARE_STORAGES_KEY);
String vendorPath = UserClusterDataExtractor.makeVendorPath(cluster);
for (GroupRuntime ge : clusterEntity.getGroups()) {
for (MachineRuntime me : ge.getMachines()) {
Provider provider = UserClusterDataExtractor.getGroupProvider(cluster, ge.getName());
boolean storagePreparation =
(prepStoragesConf != null
&& prepStoragesConf.equalsIgnoreCase("true")
&& (provider instanceof Ec2));
if (storagePreparation) {
String model = ((Ec2) provider).getType();
InstanceType instanceType = InstanceType.valueByModel(model);
PrepareStoragesTask st =
new PrepareStoragesTask(
me, clusterStats, submitter, instanceType.getStorageDevices());
dag.addTask(st);
}
AptGetEssentialsTask t1 =
new AptGetEssentialsTask(me, clusterStats, submitter, storagePreparation);
InstallBerkshelfTask t2 = new InstallBerkshelfTask(me, clusterStats, submitter);
MakeSoloRbTask t3 = new MakeSoloRbTask(me, vendorPath, clusterStats, submitter);
dag.addTask(t1);
dag.addTask(t2);
dag.addTask(t3);
}
}
}
/**
* Creates all recipe tasks for cluster and groups them by recipe-name. In other words, by having
* a recipe-name such as hadoop::dn you fetch all the tasks in the cluster that are running
* hadoop::dn. recipeName -> taskid(recipe + machineid) -> task
*
* @param cluster
* @param clusterEntity
* @param clusterStats
* @param chefJsons
* @param submitter
* @param allRecipeTasks
* @param dag
* @return
* @throws KaramelException
*/
public static Map<String, Map<String, Task>> recipeLevelTasks(
JsonCluster cluster,
ClusterRuntime clusterEntity,
ClusterStats clusterStats,
Map<String, JsonObject> chefJsons,
TaskSubmitter submitter,
Map<String, RunRecipeTask> allRecipeTasks,
Dag dag)
throws KaramelException {
Map<String, Map<String, Task>> map = new HashMap<>();
for (GroupRuntime ge : clusterEntity.getGroups()) {
JsonGroup jg = UserClusterDataExtractor.findGroup(cluster, ge.getName());
for (MachineRuntime me : ge.getMachines()) {
for (JsonCookbook jc : jg.getCookbooks()) {
CookbookUrls urls = jc.getKaramelizedCookbook().getUrls();
for (JsonRecipe rec : jc.getRecipes()) {
JsonObject json1 = chefJsons.get(me.getId() + rec.getCanonicalName());
addRecipeTaskForMachineIntoRecipesMap(
rec.getCanonicalName(),
me,
clusterStats,
map,
json1,
submitter,
urls.id,
jc.getName(),
allRecipeTasks,
dag);
}
}
}
}
return map;
}
/**
* machine -> taskid -> task
*
* @param cluster
* @param clusterEntity
* @param clusterStats
* @param chefJsons
* @param submitter
* @param allRecipeTasks
* @param dag
* @return
* @throws KaramelException
*/
public static Map<String, Map<String, Task>> cookbookLevelTasks(
JsonCluster cluster,
ClusterRuntime clusterEntity,
ClusterStats clusterStats,
Map<String, JsonObject> chefJsons,
TaskSubmitter submitter,
Map<String, RunRecipeTask> allRecipeTasks,
Dag dag)
throws KaramelException {
Map<String, Map<String, Task>> map = new HashMap<>();
for (GroupRuntime ge : clusterEntity.getGroups()) {
JsonGroup jg = UserClusterDataExtractor.findGroup(cluster, ge.getName());
for (MachineRuntime me : ge.getMachines()) {
Map<String, Task> map1 = new HashMap<>();
for (JsonCookbook jc : jg.getCookbooks()) {
CookbookUrls urls = jc.getKaramelizedCookbook().getUrls();
VendorCookbookTask t1 =
new VendorCookbookTask(
me,
clusterStats,
submitter,
urls.id,
Settings.REMOTE_CB_VENDOR_PATH,
urls.repoUrl,
urls.repoName,
urls.cookbookRelPath,
urls.branch);
dag.addTask(t1);
map1.put(t1.uniqueId(), t1);
String recipeName = jc.getName() + Settings.COOKBOOK_DELIMITER + Settings.INSTALL_RECIPE;
JsonObject json = chefJsons.get(me.getId() + recipeName);
RunRecipeTask t2 =
makeRecipeTaskIfNotExist(
recipeName,
me,
clusterStats,
json,
submitter,
urls.id,
jc.getName(),
allRecipeTasks,
dag);
map1.put(t2.uniqueId(), t2);
}
logger.debug(
String.format(
"Cookbook-level tasks for the machine '%s' in the group '%s' are: %s",
me.getPublicIp(), ge.getName(), map1.keySet()));
if (map.get(me.getId()) != null) {
map.get(me.getId()).putAll(map1);
} else {
map.put(me.getId(), map1);
}
}
}
return map;
}