Beispiel #1
0
 public static void machineLevelTasks(
     JsonCluster cluster,
     ClusterRuntime clusterEntity,
     ClusterStats clusterStats,
     TaskSubmitter submitter,
     Dag dag)
     throws KaramelException {
   Confs confs = Confs.loadKaramelConfs();
   String prepStoragesConf = confs.getProperty(Settings.PREPARE_STORAGES_KEY);
   String vendorPath = UserClusterDataExtractor.makeVendorPath(cluster);
   for (GroupRuntime ge : clusterEntity.getGroups()) {
     for (MachineRuntime me : ge.getMachines()) {
       Provider provider = UserClusterDataExtractor.getGroupProvider(cluster, ge.getName());
       boolean storagePreparation =
           (prepStoragesConf != null
               && prepStoragesConf.equalsIgnoreCase("true")
               && (provider instanceof Ec2));
       if (storagePreparation) {
         String model = ((Ec2) provider).getType();
         InstanceType instanceType = InstanceType.valueByModel(model);
         PrepareStoragesTask st =
             new PrepareStoragesTask(
                 me, clusterStats, submitter, instanceType.getStorageDevices());
         dag.addTask(st);
       }
       AptGetEssentialsTask t1 =
           new AptGetEssentialsTask(me, clusterStats, submitter, storagePreparation);
       InstallBerkshelfTask t2 = new InstallBerkshelfTask(me, clusterStats, submitter);
       MakeSoloRbTask t3 = new MakeSoloRbTask(me, vendorPath, clusterStats, submitter);
       dag.addTask(t1);
       dag.addTask(t2);
       dag.addTask(t3);
     }
   }
 }
Beispiel #2
0
 /**
  * Creates all recipe tasks for cluster and groups them by recipe-name. In other words, by having
  * a recipe-name such as hadoop::dn you fetch all the tasks in the cluster that are running
  * hadoop::dn. recipeName -> taskid(recipe + machineid) -> task
  *
  * @param cluster
  * @param clusterEntity
  * @param clusterStats
  * @param chefJsons
  * @param submitter
  * @param allRecipeTasks
  * @param dag
  * @return
  * @throws KaramelException
  */
 public static Map<String, Map<String, Task>> recipeLevelTasks(
     JsonCluster cluster,
     ClusterRuntime clusterEntity,
     ClusterStats clusterStats,
     Map<String, JsonObject> chefJsons,
     TaskSubmitter submitter,
     Map<String, RunRecipeTask> allRecipeTasks,
     Dag dag)
     throws KaramelException {
   Map<String, Map<String, Task>> map = new HashMap<>();
   for (GroupRuntime ge : clusterEntity.getGroups()) {
     JsonGroup jg = UserClusterDataExtractor.findGroup(cluster, ge.getName());
     for (MachineRuntime me : ge.getMachines()) {
       for (JsonCookbook jc : jg.getCookbooks()) {
         CookbookUrls urls = jc.getKaramelizedCookbook().getUrls();
         for (JsonRecipe rec : jc.getRecipes()) {
           JsonObject json1 = chefJsons.get(me.getId() + rec.getCanonicalName());
           addRecipeTaskForMachineIntoRecipesMap(
               rec.getCanonicalName(),
               me,
               clusterStats,
               map,
               json1,
               submitter,
               urls.id,
               jc.getName(),
               allRecipeTasks,
               dag);
         }
       }
     }
   }
   return map;
 }
Beispiel #3
0
 /**
  * machine -> taskid -> task
  *
  * @param cluster
  * @param clusterEntity
  * @param clusterStats
  * @param chefJsons
  * @param submitter
  * @param allRecipeTasks
  * @param dag
  * @return
  * @throws KaramelException
  */
 public static Map<String, Map<String, Task>> cookbookLevelTasks(
     JsonCluster cluster,
     ClusterRuntime clusterEntity,
     ClusterStats clusterStats,
     Map<String, JsonObject> chefJsons,
     TaskSubmitter submitter,
     Map<String, RunRecipeTask> allRecipeTasks,
     Dag dag)
     throws KaramelException {
   Map<String, Map<String, Task>> map = new HashMap<>();
   for (GroupRuntime ge : clusterEntity.getGroups()) {
     JsonGroup jg = UserClusterDataExtractor.findGroup(cluster, ge.getName());
     for (MachineRuntime me : ge.getMachines()) {
       Map<String, Task> map1 = new HashMap<>();
       for (JsonCookbook jc : jg.getCookbooks()) {
         CookbookUrls urls = jc.getKaramelizedCookbook().getUrls();
         VendorCookbookTask t1 =
             new VendorCookbookTask(
                 me,
                 clusterStats,
                 submitter,
                 urls.id,
                 Settings.REMOTE_CB_VENDOR_PATH,
                 urls.repoUrl,
                 urls.repoName,
                 urls.cookbookRelPath,
                 urls.branch);
         dag.addTask(t1);
         map1.put(t1.uniqueId(), t1);
         String recipeName = jc.getName() + Settings.COOKBOOK_DELIMITER + Settings.INSTALL_RECIPE;
         JsonObject json = chefJsons.get(me.getId() + recipeName);
         RunRecipeTask t2 =
             makeRecipeTaskIfNotExist(
                 recipeName,
                 me,
                 clusterStats,
                 json,
                 submitter,
                 urls.id,
                 jc.getName(),
                 allRecipeTasks,
                 dag);
         map1.put(t2.uniqueId(), t2);
       }
       logger.debug(
           String.format(
               "Cookbook-level tasks for the machine '%s' in the group '%s' are: %s",
               me.getPublicIp(), ge.getName(), map1.keySet()));
       if (map.get(me.getId()) != null) {
         map.get(me.getId()).putAll(map1);
       } else {
         map.put(me.getId(), map1);
       }
     }
   }
   return map;
 }