public static Protos.Resource mem(double mem) {
return Protos.Resource.newBuilder()
.setName("mem")
.setType(Protos.Value.Type.SCALAR)
.setScalar(Protos.Value.Scalar.newBuilder().setValue(mem).build())
.build();
}
public static Protos.Resource disk(double disk) {
return Protos.Resource.newBuilder()
.setName("disk")
.setType(Protos.Value.Type.SCALAR)
.setScalar(Protos.Value.Scalar.newBuilder().setValue(disk).build())
.build();
}
protected Protos.Resource rangeResource(String name, long begin, long end) {
// For a given named resource and value,
// find and return the role that matches the name and exceeds the value.
// Give preference to reserved offers first (those whose roles do not match "*")
List<Protos.Resource> reservedResources = new LinkedList<>();
for (Protos.Resource resource : offer.getResourcesList()) {
if (resource.hasRole() && !resource.getRole().equals("*")) {
reservedResources.add(resource);
}
}
String role = "*";
for (Protos.Resource resource : reservedResources) {
if (resource.getName() == name) {
Protos.Value.Ranges ranges = resource.getRanges();
for (Protos.Value.Range range : ranges.getRangeList()) {
if (range.getBegin() <= begin && range.getEnd() >= end) {
role = resource.getRole();
break;
}
}
}
}
return Protos.Resource.newBuilder()
.setType(Protos.Value.Type.RANGES)
.setName(name)
.setRanges(
Protos.Value.Ranges.newBuilder()
.addRange(Protos.Value.Range.newBuilder().setBegin(begin).setEnd(end))
.build())
.setRole(role)
.build();
}
public static Protos.Resource cpus(double cpus) {
return Protos.Resource.newBuilder()
.setName("cpus")
.setType(Protos.Value.Type.SCALAR)
.setScalar(Protos.Value.Scalar.newBuilder().setValue(cpus).build())
.build();
}
protected Protos.Resource scalarResource(String name, double value) {
// For a given named resource and value,
// find and return the role that matches the name and exceeds the value.
// Give preference to reserved offers first (those whose roles do not match "*")
List<Protos.Resource> reservedResources = new LinkedList<>();
for (Protos.Resource resource : offer.getResourcesList()) {
if (resource.hasRole() && !resource.getRole().equals("*")) {
reservedResources.add(resource);
}
}
String role = "*";
for (Protos.Resource resource : reservedResources) {
if (resource.getName() == name && resource.getScalar().getValue() >= value) {
role = resource.getRole();
break;
}
}
return Protos.Resource.newBuilder()
.setName(name)
.setType(Protos.Value.Type.SCALAR)
.setScalar(Protos.Value.Scalar.newBuilder().setValue(value))
.setRole(role)
.build();
}
private List<Protos.Resource> getWorkerRequiredResources(long workerCpus, long workerMem) {
List<Protos.Resource> resources = getCoreRequiredResouces(workerCpus, workerMem);
// Set worker rcp port, web ui port, data port as range resources for this task.
// By default, it would require 29998, 29999, 30000 ports for worker process.
resources.add(
Protos.Resource.newBuilder()
.setName(Constants.MESOS_RESOURCE_PORTS)
.setType(Protos.Value.Type.RANGES)
.setRanges(
Protos.Value.Ranges.newBuilder()
.addRange(
Protos.Value.Range.newBuilder()
.setBegin(Configuration.getLong(PropertyKey.WORKER_RPC_PORT))
.setEnd(Configuration.getLong(PropertyKey.WORKER_RPC_PORT)))
.addRange(
Protos.Value.Range.newBuilder()
.setBegin(Configuration.getLong(PropertyKey.WORKER_DATA_PORT))
.setEnd(Configuration.getLong(PropertyKey.WORKER_DATA_PORT)))
.addRange(
(Protos.Value.Range.newBuilder()
.setBegin(Configuration.getLong(PropertyKey.WORKER_WEB_PORT))
.setEnd(Configuration.getLong(PropertyKey.WORKER_WEB_PORT)))))
.build());
return resources;
}
private List<Protos.Resource> getCoreRequiredResouces(long cpus, long mem) {
// Build cpu/mem resource for task.
List<Protos.Resource> resources = new ArrayList<Protos.Resource>();
resources.add(
Protos.Resource.newBuilder()
.setName(Constants.MESOS_RESOURCE_CPUS)
.setType(Protos.Value.Type.SCALAR)
.setScalar(Protos.Value.Scalar.newBuilder().setValue(cpus))
.build());
resources.add(
Protos.Resource.newBuilder()
.setName(Constants.MESOS_RESOURCE_MEM)
.setType(Protos.Value.Type.SCALAR)
.setScalar(Protos.Value.Scalar.newBuilder().setValue(mem))
.build());
return resources;
}
public static Protos.Resource portRange(long beginPort, long endPort) {
Protos.Value.Range singlePortRange =
Protos.Value.Range.newBuilder().setBegin(beginPort).setEnd(endPort).build();
return Protos.Resource.newBuilder()
.setName("ports")
.setType(Protos.Value.Type.RANGES)
.setRanges(Protos.Value.Ranges.newBuilder().addRange(singlePortRange))
.build();
}
private List<Protos.Resource> getExecutorResources() {
// JIRA: https://issues.apache.org/jira/browse/MESOS-1807
// From Mesos 0.22.0, executors must set CPU resources to at least 0.01 and
// memory resources to at least 32MB.
List<Protos.Resource> resources = new ArrayList<Protos.Resource>(2);
// Both cpus/mem are "scalar" type, which means a double value should be used.
// The resource name is "cpus", type is scalar and the value is 0.1 to tell Mesos
// this executor would allocate 0.1 cpu for itself.
resources.add(
Protos.Resource.newBuilder()
.setName(Constants.MESOS_RESOURCE_CPUS)
.setType(Protos.Value.Type.SCALAR)
.setScalar(Protos.Value.Scalar.newBuilder().setValue(0.1d))
.build());
// The resource name is "mem", type is scalar and the value is 32.0MB to tell Mesos
// this executor would allocate 32.0MB mem for itself.
resources.add(
Protos.Resource.newBuilder()
.setName(Constants.MESOS_RESOURCE_MEM)
.setType(Protos.Value.Type.SCALAR)
.setScalar(Protos.Value.Scalar.newBuilder().setValue(32.0d))
.build());
return resources;
}
@Override
public void resourceOffers(SchedulerDriver schedulerDriver, List<Protos.Offer> list) {
double CPUS_PER_TASK = 1;
double MEM_PER_TASK = 128;
for (Protos.Offer offer : list) {
List<Protos.TaskInfo> taskInfoList = new ArrayList<Protos.TaskInfo>();
double offerCpus = 0;
double offerMem = 0;
for (Protos.Resource resource : offer.getResourcesList()) {
if (resource.getName().equals("cpus")) {
offerCpus += resource.getScalar().getValue();
} else if (resource.getName().equals("mem")) {
offerMem += resource.getScalar().getValue();
}
}
LOGGER.info(
"Received Offer : "
+ offer.getId().getValue()
+ " with cpus = "
+ offerCpus
+ " and mem ="
+ offerMem);
double remainingCpus = offerCpus;
double remainingMem = offerMem;
if (launchedTasks < totalTasks
&& remainingCpus >= CPUS_PER_TASK
&& remainingMem >= MEM_PER_TASK) {
Protos.TaskID taskID =
Protos.TaskID.newBuilder().setValue(Integer.toString(launchedTasks++)).build();
LOGGER.info(
"Launching task :"
+ taskID.getValue()
+ " using the offer : "
+ offer.getId().getValue());
Protos.TaskInfo piTaskInfo =
Protos.TaskInfo.newBuilder()
.setName("task " + taskID.getValue())
.setTaskId(taskID)
.setSlaveId(offer.getSlaveId())
.addResources(
Protos.Resource.newBuilder()
.setName("cpus")
.setType(Protos.Value.Type.SCALAR)
.setScalar(Protos.Value.Scalar.newBuilder().setValue(CPUS_PER_TASK)))
.addResources(
Protos.Resource.newBuilder()
.setName("mem")
.setType(Protos.Value.Type.SCALAR)
.setScalar(Protos.Value.Scalar.newBuilder().setValue(MEM_PER_TASK)))
.setExecutor(Protos.ExecutorInfo.newBuilder(piExecutor))
.build();
taskID = Protos.TaskID.newBuilder().setValue(Integer.toString(launchedTasks++)).build();
LOGGER.info(
"Launching task :"
+ taskID.getValue()
+ " using the offer : "
+ offer.getId().getValue());
taskInfoList.add(piTaskInfo);
}
schedulerDriver.launchTasks(offer.getId(), taskInfoList);
}
}