public static Protos.Resource mem(double mem) {
   return Protos.Resource.newBuilder()
       .setName("mem")
       .setType(Protos.Value.Type.SCALAR)
       .setScalar(Protos.Value.Scalar.newBuilder().setValue(mem).build())
       .build();
 }
 public static Protos.Resource disk(double disk) {
   return Protos.Resource.newBuilder()
       .setName("disk")
       .setType(Protos.Value.Type.SCALAR)
       .setScalar(Protos.Value.Scalar.newBuilder().setValue(disk).build())
       .build();
 }
Beispiel #3
0
  protected Protos.Resource rangeResource(String name, long begin, long end) {
    // For a given named resource and value,
    // find and return the role that matches the name and exceeds the value.
    // Give preference to reserved offers first (those whose roles do not match "*")
    List<Protos.Resource> reservedResources = new LinkedList<>();
    for (Protos.Resource resource : offer.getResourcesList()) {
      if (resource.hasRole() && !resource.getRole().equals("*")) {
        reservedResources.add(resource);
      }
    }

    String role = "*";
    for (Protos.Resource resource : reservedResources) {
      if (resource.getName() == name) {
        Protos.Value.Ranges ranges = resource.getRanges();
        for (Protos.Value.Range range : ranges.getRangeList()) {
          if (range.getBegin() <= begin && range.getEnd() >= end) {
            role = resource.getRole();
            break;
          }
        }
      }
    }

    return Protos.Resource.newBuilder()
        .setType(Protos.Value.Type.RANGES)
        .setName(name)
        .setRanges(
            Protos.Value.Ranges.newBuilder()
                .addRange(Protos.Value.Range.newBuilder().setBegin(begin).setEnd(end))
                .build())
        .setRole(role)
        .build();
  }
 public static Protos.Resource cpus(double cpus) {
   return Protos.Resource.newBuilder()
       .setName("cpus")
       .setType(Protos.Value.Type.SCALAR)
       .setScalar(Protos.Value.Scalar.newBuilder().setValue(cpus).build())
       .build();
 }
Beispiel #5
0
  protected Protos.Resource scalarResource(String name, double value) {
    // For a given named resource and value,
    // find and return the role that matches the name and exceeds the value.
    // Give preference to reserved offers first (those whose roles do not match "*")
    List<Protos.Resource> reservedResources = new LinkedList<>();
    for (Protos.Resource resource : offer.getResourcesList()) {
      if (resource.hasRole() && !resource.getRole().equals("*")) {
        reservedResources.add(resource);
      }
    }

    String role = "*";
    for (Protos.Resource resource : reservedResources) {
      if (resource.getName() == name && resource.getScalar().getValue() >= value) {
        role = resource.getRole();
        break;
      }
    }

    return Protos.Resource.newBuilder()
        .setName(name)
        .setType(Protos.Value.Type.SCALAR)
        .setScalar(Protos.Value.Scalar.newBuilder().setValue(value))
        .setRole(role)
        .build();
  }
 private List<Protos.Resource> getWorkerRequiredResources(long workerCpus, long workerMem) {
   List<Protos.Resource> resources = getCoreRequiredResouces(workerCpus, workerMem);
   // Set worker rcp port, web ui port, data port as range resources for this task.
   // By default, it would require 29998, 29999, 30000 ports for worker process.
   resources.add(
       Protos.Resource.newBuilder()
           .setName(Constants.MESOS_RESOURCE_PORTS)
           .setType(Protos.Value.Type.RANGES)
           .setRanges(
               Protos.Value.Ranges.newBuilder()
                   .addRange(
                       Protos.Value.Range.newBuilder()
                           .setBegin(Configuration.getLong(PropertyKey.WORKER_RPC_PORT))
                           .setEnd(Configuration.getLong(PropertyKey.WORKER_RPC_PORT)))
                   .addRange(
                       Protos.Value.Range.newBuilder()
                           .setBegin(Configuration.getLong(PropertyKey.WORKER_DATA_PORT))
                           .setEnd(Configuration.getLong(PropertyKey.WORKER_DATA_PORT)))
                   .addRange(
                       (Protos.Value.Range.newBuilder()
                           .setBegin(Configuration.getLong(PropertyKey.WORKER_WEB_PORT))
                           .setEnd(Configuration.getLong(PropertyKey.WORKER_WEB_PORT)))))
           .build());
   return resources;
 }
 private List<Protos.Resource> getCoreRequiredResouces(long cpus, long mem) {
   // Build cpu/mem resource for task.
   List<Protos.Resource> resources = new ArrayList<Protos.Resource>();
   resources.add(
       Protos.Resource.newBuilder()
           .setName(Constants.MESOS_RESOURCE_CPUS)
           .setType(Protos.Value.Type.SCALAR)
           .setScalar(Protos.Value.Scalar.newBuilder().setValue(cpus))
           .build());
   resources.add(
       Protos.Resource.newBuilder()
           .setName(Constants.MESOS_RESOURCE_MEM)
           .setType(Protos.Value.Type.SCALAR)
           .setScalar(Protos.Value.Scalar.newBuilder().setValue(mem))
           .build());
   return resources;
 }
 public static Protos.Resource portRange(long beginPort, long endPort) {
   Protos.Value.Range singlePortRange =
       Protos.Value.Range.newBuilder().setBegin(beginPort).setEnd(endPort).build();
   return Protos.Resource.newBuilder()
       .setName("ports")
       .setType(Protos.Value.Type.RANGES)
       .setRanges(Protos.Value.Ranges.newBuilder().addRange(singlePortRange))
       .build();
 }
 private List<Protos.Resource> getExecutorResources() {
   // JIRA: https://issues.apache.org/jira/browse/MESOS-1807
   // From Mesos 0.22.0, executors must set CPU resources to at least 0.01 and
   // memory resources to at least 32MB.
   List<Protos.Resource> resources = new ArrayList<Protos.Resource>(2);
   // Both cpus/mem are "scalar" type, which means a double value should be used.
   // The resource name is "cpus", type is scalar and the value is 0.1 to tell Mesos
   // this executor would allocate 0.1 cpu for itself.
   resources.add(
       Protos.Resource.newBuilder()
           .setName(Constants.MESOS_RESOURCE_CPUS)
           .setType(Protos.Value.Type.SCALAR)
           .setScalar(Protos.Value.Scalar.newBuilder().setValue(0.1d))
           .build());
   // The resource name is "mem", type is scalar and the value is 32.0MB to tell Mesos
   // this executor would allocate 32.0MB mem for itself.
   resources.add(
       Protos.Resource.newBuilder()
           .setName(Constants.MESOS_RESOURCE_MEM)
           .setType(Protos.Value.Type.SCALAR)
           .setScalar(Protos.Value.Scalar.newBuilder().setValue(32.0d))
           .build());
   return resources;
 }
Beispiel #10
0
  @Override
  public void resourceOffers(SchedulerDriver schedulerDriver, List<Protos.Offer> list) {
    double CPUS_PER_TASK = 1;
    double MEM_PER_TASK = 128;

    for (Protos.Offer offer : list) {
      List<Protos.TaskInfo> taskInfoList = new ArrayList<Protos.TaskInfo>();
      double offerCpus = 0;
      double offerMem = 0;
      for (Protos.Resource resource : offer.getResourcesList()) {
        if (resource.getName().equals("cpus")) {
          offerCpus += resource.getScalar().getValue();
        } else if (resource.getName().equals("mem")) {
          offerMem += resource.getScalar().getValue();
        }
      }
      LOGGER.info(
          "Received Offer : "
              + offer.getId().getValue()
              + " with cpus = "
              + offerCpus
              + " and mem ="
              + offerMem);

      double remainingCpus = offerCpus;
      double remainingMem = offerMem;

      if (launchedTasks < totalTasks
          && remainingCpus >= CPUS_PER_TASK
          && remainingMem >= MEM_PER_TASK) {
        Protos.TaskID taskID =
            Protos.TaskID.newBuilder().setValue(Integer.toString(launchedTasks++)).build();
        LOGGER.info(
            "Launching task :"
                + taskID.getValue()
                + " using the offer : "
                + offer.getId().getValue());

        Protos.TaskInfo piTaskInfo =
            Protos.TaskInfo.newBuilder()
                .setName("task " + taskID.getValue())
                .setTaskId(taskID)
                .setSlaveId(offer.getSlaveId())
                .addResources(
                    Protos.Resource.newBuilder()
                        .setName("cpus")
                        .setType(Protos.Value.Type.SCALAR)
                        .setScalar(Protos.Value.Scalar.newBuilder().setValue(CPUS_PER_TASK)))
                .addResources(
                    Protos.Resource.newBuilder()
                        .setName("mem")
                        .setType(Protos.Value.Type.SCALAR)
                        .setScalar(Protos.Value.Scalar.newBuilder().setValue(MEM_PER_TASK)))
                .setExecutor(Protos.ExecutorInfo.newBuilder(piExecutor))
                .build();

        taskID = Protos.TaskID.newBuilder().setValue(Integer.toString(launchedTasks++)).build();
        LOGGER.info(
            "Launching task :"
                + taskID.getValue()
                + " using the offer : "
                + offer.getId().getValue());

        taskInfoList.add(piTaskInfo);
      }
      schedulerDriver.launchTasks(offer.getId(), taskInfoList);
    }
  }