/** 同computeAvailableContainers方法一样,只是返回的结果是向上取整 */
 public static int divideAndCeilContainers(
     Resource required, Resource factor, EnumSet<SchedulerResourceTypes> resourceTypes) {
   if (resourceTypes.contains(SchedulerResourceTypes.CPU)) {
     return Math.max(
         divideAndCeil(required.getMemory(), factor.getMemory()),
         divideAndCeil(required.getVirtualCores(), factor.getVirtualCores()));
   }
   return divideAndCeil(required.getMemory(), factor.getMemory());
 }
 /**
  * 如果包含CPU资源,因此按照cpu资源能满足多少个请求资源,内存资源能满足多少个请求资源,取最小值 否则仅仅判断内存资源能满足多少个请求即可
  *
  * @param available 可用的资源
  * @param required 请求的资源
  * @param resourceTypes 资源类型 return 计算可用资源可以满足多少个请求资源需求,即可以有多少个container
  */
 public static int computeAvailableContainers(
     Resource available, Resource required, EnumSet<SchedulerResourceTypes> resourceTypes) {
   if (resourceTypes.contains(
       SchedulerResourceTypes.CPU)) { // 因为包含CPU资源,因此按照cpu资源能满足多少个请求资源,内存资源能满足多少个请求资源,取最小值
     return Math.min(
         available.getMemory() / required.getMemory(),
         available.getVirtualCores() / required.getVirtualCores());
   }
   return available.getMemory() / required.getMemory();
 }
  static boolean canFit(Resource arg0, Resource arg1) {
    int mem0 = arg0.getMemory();
    int mem1 = arg1.getMemory();
    int cpu0 = arg0.getVirtualCores();
    int cpu1 = arg1.getVirtualCores();

    if (mem0 <= mem1 && cpu0 <= cpu1) {
      return true;
    }
    return false;
  }
예제 #4
0
 @Override
 public int compareTo(Resource other) {
   int diff = this.getMemory() - other.getMemory();
   if (diff == 0) {
     diff = this.getVirtualCores() - other.getVirtualCores();
     if (diff == 0) {
       diff = this.getOutboundNetworkBandwidthMbits() - other.getOutboundNetworkBandwidthMbits();
     }
   }
   return diff;
 }
 @Override
 public int compare(Resource arg0, Resource arg1) {
   int mem0 = arg0.getMemory();
   int mem1 = arg1.getMemory();
   int cpu0 = arg0.getVirtualCores();
   int cpu1 = arg1.getVirtualCores();
   if (mem0 == mem1) {
     if (cpu0 == cpu1) {
       return 0;
     }
     if (cpu0 < cpu1) {
       return 1;
     }
     return -1;
   }
   if (mem0 < mem1) {
     return 1;
   }
   return -1;
 }
  private void updateQueueWithNodeUpdate(NodeUpdateSchedulerEventWrapper eventWrapper) {
    RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode();
    List<UpdatedContainerInfo> containerList = node.getContainerUpdates();
    for (UpdatedContainerInfo info : containerList) {
      for (ContainerStatus status : info.getCompletedContainers()) {
        ContainerId containerId = status.getContainerId();
        SchedulerAppReport app =
            scheduler.getSchedulerAppInfo(containerId.getApplicationAttemptId());

        if (app == null) {
          // this happens for the AM container
          // The app have already removed when the NM sends the release
          // information.
          continue;
        }

        String queue = appQueueMap.get(containerId.getApplicationAttemptId().getApplicationId());
        int releasedMemory = 0, releasedVCores = 0;
        if (status.getExitStatus() == ContainerExitStatus.SUCCESS) {
          for (RMContainer rmc : app.getLiveContainers()) {
            if (rmc.getContainerId() == containerId) {
              releasedMemory += rmc.getContainer().getResource().getMemory();
              releasedVCores += rmc.getContainer().getResource().getVirtualCores();
              break;
            }
          }
        } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) {
          if (preemptionContainerMap.containsKey(containerId)) {
            Resource preResource = preemptionContainerMap.get(containerId);
            releasedMemory += preResource.getMemory();
            releasedVCores += preResource.getVirtualCores();
            preemptionContainerMap.remove(containerId);
          }
        }
        // update queue counters
        updateQueueMetrics(queue, releasedMemory, releasedVCores);
      }
    }
  }
  // package private for testing purposes
  public void setupLimits(ContainerId containerId, Resource containerResource) throws IOException {
    String containerName = containerId.toString();

    if (isCpuWeightEnabled()) {
      int containerVCores = containerResource.getVirtualCores();
      createCgroup(CONTROLLER_CPU, containerName);

      int cpuShares = CPU_DEFAULT_WEIGHT * containerVCores;
      // absolute minimum of 10 shares for zero CPU containers
      cpuShares = Math.max(cpuShares, 10);

      updateCgroup(CONTROLLER_CPU, containerName, "shares", String.valueOf(cpuShares));
      if (strictResourceUsageMode) {
        int nodeVCores =
            conf.getInt(YarnConfiguration.NM_VCORES, YarnConfiguration.DEFAULT_NM_VCORES);
        if (nodeVCores != containerVCores) {
          float containerCPU = (containerVCores * yarnProcessors) / (float) nodeVCores;
          int[] limits = getOverallLimits(containerCPU);
          updateCgroup(CONTROLLER_CPU, containerName, CPU_PERIOD_US, String.valueOf(limits[0]));
          updateCgroup(CONTROLLER_CPU, containerName, CPU_QUOTA_US, String.valueOf(limits[1]));
        }
      }
    }
  }
예제 #8
0
  /**
   * Return a command to execute the given command in OS shell. On Windows, the passed in groupId
   * can be used to launch and associate the given groupId in a process group. On non-Windows,
   * groupId is ignored.
   */
  protected String[] getRunCommand(
      String command,
      String groupId,
      String userName,
      Path pidFile,
      Configuration conf,
      Resource resource) {
    boolean containerSchedPriorityIsSet = false;
    int containerSchedPriorityAdjustment =
        YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY;

    if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) != null) {
      containerSchedPriorityIsSet = true;
      containerSchedPriorityAdjustment =
          conf.getInt(
              YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY,
              YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY);
    }

    if (Shell.WINDOWS) {
      int cpuRate = -1;
      int memory = -1;
      if (resource != null) {
        if (conf.getBoolean(
            YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED,
            YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) {
          memory = resource.getMemory();
        }

        if (conf.getBoolean(
            YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED,
            YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED)) {
          int containerVCores = resource.getVirtualCores();
          int nodeVCores = NodeManagerHardwareUtils.getVCores(conf);
          int nodeCpuPercentage = NodeManagerHardwareUtils.getNodeCpuPercentage(conf);

          float containerCpuPercentage = (float) (nodeCpuPercentage * containerVCores) / nodeVCores;

          // CPU should be set to a percentage * 100, e.g. 20% cpu rate limit
          // should be set as 20 * 100.
          cpuRate = Math.min(10000, (int) (containerCpuPercentage * 100));
        }
      }
      return new String[] {
        Shell.WINUTILS,
        "task",
        "create",
        "-m",
        String.valueOf(memory),
        "-c",
        String.valueOf(cpuRate),
        groupId,
        "cmd /c " + command
      };
    } else {
      List<String> retCommand = new ArrayList<String>();
      if (containerSchedPriorityIsSet) {
        retCommand.addAll(
            Arrays.asList("nice", "-n", Integer.toString(containerSchedPriorityAdjustment)));
      }
      retCommand.addAll(Arrays.asList("bash", command));
      return retCommand.toArray(new String[retCommand.size()]);
    }
  }
 public void subtract(Resource r) {
   memory -= r.getMemory();
   vcores -= r.getVirtualCores();
 }
 public void add(Resource r) {
   memory += r.getMemory();
   vcores += r.getVirtualCores();
 }
 public IntegralResource(Resource resource) {
   this.memory = resource.getMemory();
   this.vcores = resource.getVirtualCores();
 }
  private void updateQueueWithAllocateRequest(
      Allocation allocation,
      ApplicationAttemptId attemptId,
      List<ResourceRequest> resourceRequests,
      List<ContainerId> containerIds)
      throws IOException {
    // update queue information
    Resource pendingResource = Resources.createResource(0, 0);
    Resource allocatedResource = Resources.createResource(0, 0);
    String queueName = appQueueMap.get(attemptId.getApplicationId());
    // container requested
    for (ResourceRequest request : resourceRequests) {
      if (request.getResourceName().equals(ResourceRequest.ANY)) {
        Resources.addTo(
            pendingResource,
            Resources.multiply(request.getCapability(), request.getNumContainers()));
      }
    }
    // container allocated
    for (Container container : allocation.getContainers()) {
      Resources.addTo(allocatedResource, container.getResource());
      Resources.subtractFrom(pendingResource, container.getResource());
    }
    // container released from AM
    SchedulerAppReport report = scheduler.getSchedulerAppInfo(attemptId);
    for (ContainerId containerId : containerIds) {
      Container container = null;
      for (RMContainer c : report.getLiveContainers()) {
        if (c.getContainerId().equals(containerId)) {
          container = c.getContainer();
          break;
        }
      }
      if (container != null) {
        // released allocated containers
        Resources.subtractFrom(allocatedResource, container.getResource());
      } else {
        for (RMContainer c : report.getReservedContainers()) {
          if (c.getContainerId().equals(containerId)) {
            container = c.getContainer();
            break;
          }
        }
        if (container != null) {
          // released reserved containers
          Resources.subtractFrom(pendingResource, container.getResource());
        }
      }
    }
    // containers released/preemption from scheduler
    Set<ContainerId> preemptionContainers = new HashSet<ContainerId>();
    if (allocation.getContainerPreemptions() != null) {
      preemptionContainers.addAll(allocation.getContainerPreemptions());
    }
    if (allocation.getStrictContainerPreemptions() != null) {
      preemptionContainers.addAll(allocation.getStrictContainerPreemptions());
    }
    if (!preemptionContainers.isEmpty()) {
      for (ContainerId containerId : preemptionContainers) {
        if (!preemptionContainerMap.containsKey(containerId)) {
          Container container = null;
          for (RMContainer c : report.getLiveContainers()) {
            if (c.getContainerId().equals(containerId)) {
              container = c.getContainer();
              break;
            }
          }
          if (container != null) {
            preemptionContainerMap.put(containerId, container.getResource());
          }
        }
      }
    }

    // update metrics
    SortedMap<String, Counter> counterMap = metrics.getCounters();
    String names[] =
        new String[] {
          "counter.queue." + queueName + ".pending.memory",
          "counter.queue." + queueName + ".pending.cores",
          "counter.queue." + queueName + ".allocated.memory",
          "counter.queue." + queueName + ".allocated.cores"
        };
    int values[] =
        new int[] {
          pendingResource.getMemory(),
          pendingResource.getVirtualCores(),
          allocatedResource.getMemory(),
          allocatedResource.getVirtualCores()
        };
    for (int i = names.length - 1; i >= 0; i--) {
      if (!counterMap.containsKey(names[i])) {
        metrics.counter(names[i]);
        counterMap = metrics.getCounters();
      }
      counterMap.get(names[i]).inc(values[i]);
    }

    queueLock.lock();
    try {
      if (!schedulerMetrics.isTracked(queueName)) {
        schedulerMetrics.trackQueue(queueName);
      }
    } finally {
      queueLock.unlock();
    }
  }
예제 #13
0
 public void setMaxShare(Resource resource) {
   maxShareMB.set(resource.getMemory());
   maxShareVCores.set(resource.getVirtualCores());
 }
예제 #14
0
 public void setFairShare(Resource resource) {
   fairShareMB.set(resource.getMemory());
   fairShareVCores.set(resource.getVirtualCores());
 }