Exemple #1
0
 public void initialize(DragonConfiguration conf, JobId id)
     throws IOException, ClassNotFoundException, InterruptedException {
   Class<? extends ResourceCalculatorPlugin> clazz =
       conf.getClass(
           DragonConfig.RESOURCE_CALCULATOR_PLUGIN, null, ResourceCalculatorPlugin.class);
   resourceCalculator = ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
   LOG.info(" Using ResourceCalculatorPlugin : " + resourceCalculator);
   if (resourceCalculator != null) {
     initCpuCumulativeTime = resourceCalculator.getProcResourceValues().getCumulativeCpuTime();
   }
 }
  @VisibleForTesting
  void init(LinuxContainerExecutor lce, ResourceCalculatorPlugin plugin) throws IOException {
    initConfig();

    // mount cgroups if requested
    if (cgroupMount && cgroupMountPath != null) {
      ArrayList<String> cgroupKVs = new ArrayList<String>();
      cgroupKVs.add(CONTROLLER_CPU + "=" + cgroupMountPath + "/" + CONTROLLER_CPU);
      lce.mountCgroups(cgroupKVs, cgroupPrefix);
    }

    initializeControllerPaths();

    // cap overall usage to the number of cores allocated to YARN
    yarnProcessors = NodeManagerHardwareUtils.getContainersCores(plugin, conf);
    int systemProcessors = plugin.getNumProcessors();
    if (systemProcessors != (int) yarnProcessors) {
      LOG.info("YARN containers restricted to " + yarnProcessors + " cores");
      int[] limits = getOverallLimits(yarnProcessors);
      updateCgroup(CONTROLLER_CPU, "", CPU_PERIOD_US, String.valueOf(limits[0]));
      updateCgroup(CONTROLLER_CPU, "", CPU_QUOTA_US, String.valueOf(limits[1]));
    } else if (cpuLimitsExist()) {
      LOG.info("Removing CPU constraints for YARN containers.");
      updateCgroup(CONTROLLER_CPU, "", CPU_QUOTA_US, String.valueOf(-1));
    }
  }
Exemple #3
0
  /** Update resource information counters */
  void updateResourceCounters() {
    // Update generic resource counters
    updateHeapUsageCounter();

    // Updating resources specified in ResourceCalculatorPlugin
    if (resourceCalculator == null) {
      return;
    }
    ProcResourceValues res = resourceCalculator.getProcResourceValues();
    long cpuTime = res.getCumulativeCpuTime();
    long pMem = res.getPhysicalMemorySize();
    long vMem = res.getVirtualMemorySize();
    // Remove the CPU time consumed previously by JVM reuse
    cpuTime -= initCpuCumulativeTime;
    counters.getCounter(TaskCounter.CPU_MILLISECONDS).setValue(cpuTime);
    counters.getCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).setValue(pMem);
    counters.getCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).setValue(vMem);
  }
 public void init(LinuxContainerExecutor lce) throws IOException {
   this.init(lce, ResourceCalculatorPlugin.getResourceCalculatorPlugin(null, conf));
 }