Esempio n. 1
0
 /*
  * Makes sure recipe-task for machine exists both in the DAG and in the grouping map of recipes
  */
 private static RunRecipeTask addRecipeTaskForMachineIntoRecipesMap(
     String recipeName,
     MachineRuntime machine,
     ClusterStats clusterStats,
     Map<String, Map<String, Task>> map,
     JsonObject chefJson,
     TaskSubmitter submitter,
     String cookbookId,
     String cookbookName,
     Map<String, RunRecipeTask> allRecipeTasks,
     Dag dag)
     throws DagConstructionException {
   RunRecipeTask t1 =
       makeRecipeTaskIfNotExist(
           recipeName,
           machine,
           clusterStats,
           chefJson,
           submitter,
           cookbookId,
           cookbookName,
           allRecipeTasks,
           dag);
   Map<String, Task> map1 = map.get(recipeName);
   if (map1 == null) {
     map1 = new HashMap<>();
     map.put(recipeName, map1);
   }
   map1.put(t1.uniqueId(), t1);
   return t1;
 }
Esempio n. 2
0
 /**
  * machine -> taskid -> task
  *
  * @param cluster
  * @param clusterEntity
  * @param clusterStats
  * @param chefJsons
  * @param submitter
  * @param allRecipeTasks
  * @param dag
  * @return
  * @throws KaramelException
  */
 public static Map<String, Map<String, Task>> cookbookLevelTasks(
     JsonCluster cluster,
     ClusterRuntime clusterEntity,
     ClusterStats clusterStats,
     Map<String, JsonObject> chefJsons,
     TaskSubmitter submitter,
     Map<String, RunRecipeTask> allRecipeTasks,
     Dag dag)
     throws KaramelException {
   Map<String, Map<String, Task>> map = new HashMap<>();
   for (GroupRuntime ge : clusterEntity.getGroups()) {
     JsonGroup jg = UserClusterDataExtractor.findGroup(cluster, ge.getName());
     for (MachineRuntime me : ge.getMachines()) {
       Map<String, Task> map1 = new HashMap<>();
       for (JsonCookbook jc : jg.getCookbooks()) {
         CookbookUrls urls = jc.getKaramelizedCookbook().getUrls();
         VendorCookbookTask t1 =
             new VendorCookbookTask(
                 me,
                 clusterStats,
                 submitter,
                 urls.id,
                 Settings.REMOTE_CB_VENDOR_PATH,
                 urls.repoUrl,
                 urls.repoName,
                 urls.cookbookRelPath,
                 urls.branch);
         dag.addTask(t1);
         map1.put(t1.uniqueId(), t1);
         String recipeName = jc.getName() + Settings.COOKBOOK_DELIMITER + Settings.INSTALL_RECIPE;
         JsonObject json = chefJsons.get(me.getId() + recipeName);
         RunRecipeTask t2 =
             makeRecipeTaskIfNotExist(
                 recipeName,
                 me,
                 clusterStats,
                 json,
                 submitter,
                 urls.id,
                 jc.getName(),
                 allRecipeTasks,
                 dag);
         map1.put(t2.uniqueId(), t2);
       }
       logger.debug(
           String.format(
               "Cookbook-level tasks for the machine '%s' in the group '%s' are: %s",
               me.getPublicIp(), ge.getName(), map1.keySet()));
       if (map.get(me.getId()) != null) {
         map.get(me.getId()).putAll(map1);
       } else {
         map.put(me.getId(), map1);
       }
     }
   }
   return map;
 }
Esempio n. 3
0
 /*
  * Finds recipe task for machine if it has been already created otherwise makes a new one and adds it into the DAG
  */
 private static RunRecipeTask makeRecipeTaskIfNotExist(
     String recipeName,
     MachineRuntime machine,
     ClusterStats clusterStats,
     JsonObject chefJson,
     TaskSubmitter submitter,
     String cookbookId,
     String cookbookName,
     Map<String, RunRecipeTask> allRecipeTasks,
     Dag dag)
     throws DagConstructionException {
   String recId = RunRecipeTask.makeUniqueId(machine.getId(), recipeName);
   RunRecipeTask runRecipeTask = allRecipeTasks.get(recId);
   if (!allRecipeTasks.containsKey(recId)) {
     ChefJsonGenerator.addRunListForRecipe(chefJson, recipeName);
     GsonBuilder builder = new GsonBuilder();
     builder.disableHtmlEscaping();
     Gson gson = builder.setPrettyPrinting().create();
     String jsonString = gson.toJson(chefJson);
     runRecipeTask =
         new RunRecipeTask(
             machine, clusterStats, recipeName, jsonString, submitter, cookbookId, cookbookName);
     dag.addTask(runRecipeTask);
   }
   allRecipeTasks.put(recId, runRecipeTask);
   return runRecipeTask;
 }
Esempio n. 4
0
  private static boolean updateKaramelDependencies(
      Map<String, RunRecipeTask> allRecipeTasks, Dag dag, Map<String, Map<String, Task>> rlts)
      throws KaramelException {
    boolean newDepFound = false;
    for (RunRecipeTask task : allRecipeTasks.values()) {
      String tid = task.uniqueId();
      KaramelizedCookbook kcb = CookbookCache.get(task.getCookbookId());
      KaramelFileYamlDeps dependency =
          kcb.getKaramelFile().getDependency(task.getRecipeCanonicalName());
      if (dependency != null) {
        for (String depRec : dependency.getLocal()) {
          String depId = RunRecipeTask.makeUniqueId(task.getMachineId(), depRec);
          newDepFound |= dag.addDependency(depId, tid);
        }

        for (String depRec : dependency.getGlobal()) {
          Map<String, Task> rlt2 = rlts.get(depRec);
          if (rlt2 != null) {
            for (Map.Entry<String, Task> entry : rlt2.entrySet()) {
              Task t7 = entry.getValue();
              newDepFound |= dag.addDependency(t7.uniqueId(), tid);
            }
          }
        }
      }
    }
    return newDepFound;
  }