/** * Construct the Mesos TaskInfo in Protos to launch basing on the LaunchableTask * * @param heronConfig the heron config * @param heronRuntime the heron runtime * @return Mesos TaskInfo in Protos to launch */ public Protos.TaskInfo constructMesosTaskInfo(Config heronConfig, Config heronRuntime) { // String taskIdStr, BaseContainer task, Offer offer String taskIdStr = this.taskId; Protos.TaskID mesosTaskID = Protos.TaskID.newBuilder().setValue(taskIdStr).build(); Protos.TaskInfo.Builder taskInfo = Protos.TaskInfo.newBuilder().setName(baseContainer.name).setTaskId(mesosTaskID); Protos.Environment.Builder environment = Protos.Environment.newBuilder(); // If the job defines custom environment variables, add them to the builder // Don't add them if they already exist to prevent overwriting the defaults Set<String> builtinEnvNames = new HashSet<>(); for (Protos.Environment.Variable variable : environment.getVariablesList()) { builtinEnvNames.add(variable.getName()); } for (BaseContainer.EnvironmentVariable ev : baseContainer.environmentVariables) { environment.addVariables( Protos.Environment.Variable.newBuilder().setName(ev.name).setValue(ev.value)); } taskInfo .addResources(scalarResource(TaskResources.CPUS_RESOURCE_NAME, baseContainer.cpu)) .addResources(scalarResource(TaskResources.MEM_RESOURCE_NAME, baseContainer.memInMB)) .addResources(scalarResource(TaskResources.DISK_RESOURCE_NAME, baseContainer.diskInMB)) .addResources( rangeResource( TaskResources.PORT_RESOURCE_NAME, this.freePorts.get(0), this.freePorts.get(this.freePorts.size() - 1))) .setSlaveId(this.offer.getSlaveId()); int containerIndex = TaskUtils.getContainerIndexForTaskId(taskIdStr); String commandStr = executorCommand(heronConfig, heronRuntime, containerIndex); Protos.CommandInfo.Builder command = Protos.CommandInfo.newBuilder(); List<Protos.CommandInfo.URI> uriProtos = new ArrayList<>(); for (String uri : baseContainer.dependencies) { uriProtos.add(Protos.CommandInfo.URI.newBuilder().setValue(uri).setExtract(true).build()); } command .setValue(commandStr) .setShell(baseContainer.shell) .setEnvironment(environment) .addAllUris(uriProtos); if (!baseContainer.runAsUser.isEmpty()) { command.setUser(baseContainer.runAsUser); } taskInfo.setCommand(command); return taskInfo.build(); }
@Override public void resourceOffers(SchedulerDriver driver, List<Protos.Offer> offers) { long masterCpu = Configuration.getInt(PropertyKey.INTEGRATION_MASTER_RESOURCE_CPU); long masterMem = Configuration.getBytes(PropertyKey.INTEGRATION_MASTER_RESOURCE_MEM) / Constants.MB; long workerCpu = Configuration.getInt(PropertyKey.INTEGRATION_WORKER_RESOURCE_CPU); long workerMem = Configuration.getBytes(PropertyKey.INTEGRATION_WORKER_RESOURCE_MEM) / Constants.MB; LOG.info( "Master launched {}, master count {}, " + "requested master cpu {} mem {} MB and required master hostname {}", mMasterLaunched, mMasterCount, masterCpu, masterMem, mRequiredMasterHostname); for (Protos.Offer offer : offers) { Protos.Offer.Operation.Launch.Builder launch = Protos.Offer.Operation.Launch.newBuilder(); double offerCpu = 0; double offerMem = 0; for (Protos.Resource resource : offer.getResourcesList()) { if (resource.getName().equals(Constants.MESOS_RESOURCE_CPUS)) { offerCpu += resource.getScalar().getValue(); } else if (resource.getName().equals(Constants.MESOS_RESOURCE_MEM)) { offerMem += resource.getScalar().getValue(); } else { // Other resources are currently ignored. } } LOG.info( "Received offer {} on host {} with cpus {} and mem {} MB and hasMasterPorts {}", offer.getId().getValue(), offer.getHostname(), offerCpu, offerMem, OfferUtils.hasAvailableMasterPorts(offer)); Protos.ExecutorInfo.Builder executorBuilder = Protos.ExecutorInfo.newBuilder(); List<Protos.Resource> resources; if (!mMasterLaunched && offerCpu >= masterCpu && offerMem >= masterMem && mMasterCount < Configuration.getInt(PropertyKey.INTEGRATION_MESOS_ALLUXIO_MASTER_NODE_COUNT) && OfferUtils.hasAvailableMasterPorts(offer) && (mRequiredMasterHostname == null || mRequiredMasterHostname.equals(offer.getHostname()))) { LOG.debug("Creating Alluxio Master executor"); executorBuilder .setName("Alluxio Master Executor") .setSource("master") .setExecutorId(Protos.ExecutorID.newBuilder().setValue("master")) .addAllResources(getExecutorResources()) .setCommand( Protos.CommandInfo.newBuilder() .setValue(createStartAlluxioCommand("alluxio-master-mesos.sh")) .addAllUris(getExecutorDependencyURIList()) .setEnvironment( Protos.Environment.newBuilder() .addVariables( Protos.Environment.Variable.newBuilder() .setName("ALLUXIO_UNDERFS_ADDRESS") .setValue(Configuration.get(PropertyKey.UNDERFS_ADDRESS)) .build()) .build())); // pre-build resource list here, then use it to build Protos.Task later. resources = getMasterRequiredResources(masterCpu, masterMem); mMasterHostname = offer.getHostname(); mTaskName = Configuration.get(PropertyKey.INTEGRATION_MESOS_ALLUXIO_MASTER_NAME); mMasterCount++; mMasterTaskId = mLaunchedTasks; } else if (mMasterLaunched && !mWorkers.contains(offer.getHostname()) && offerCpu >= workerCpu && offerMem >= workerMem && OfferUtils.hasAvailableWorkerPorts(offer)) { LOG.debug("Creating Alluxio Worker executor"); final String memSize = FormatUtils.getSizeFromBytes((long) workerMem * Constants.MB); executorBuilder .setName("Alluxio Worker Executor") .setSource("worker") .setExecutorId(Protos.ExecutorID.newBuilder().setValue("worker")) .addAllResources(getExecutorResources()) .setCommand( Protos.CommandInfo.newBuilder() .setValue(createStartAlluxioCommand("alluxio-worker-mesos.sh")) .addAllUris(getExecutorDependencyURIList()) .setEnvironment( Protos.Environment.newBuilder() .addVariables( Protos.Environment.Variable.newBuilder() .setName("ALLUXIO_MASTER_HOSTNAME") .setValue(mMasterHostname) .build()) .addVariables( Protos.Environment.Variable.newBuilder() .setName("ALLUXIO_WORKER_MEMORY_SIZE") .setValue(memSize) .build()) .addVariables( Protos.Environment.Variable.newBuilder() .setName("ALLUXIO_UNDERFS_ADDRESS") .setValue(Configuration.get(PropertyKey.UNDERFS_ADDRESS)) .build()) .build())); // pre-build resource list here, then use it to build Protos.Task later. resources = getWorkerRequiredResources(workerCpu, workerMem); mWorkers.add(offer.getHostname()); mTaskName = Configuration.get(PropertyKey.INTEGRATION_MESOS_ALLUXIO_WORKER_NAME); } else { // The resource offer cannot be used to start either master or a worker. LOG.info("Declining offer {}", offer.getId().getValue()); driver.declineOffer(offer.getId()); continue; } Protos.TaskID taskId = Protos.TaskID.newBuilder().setValue(String.valueOf(mLaunchedTasks)).build(); LOG.info("Launching task {} using offer {}", taskId.getValue(), offer.getId().getValue()); Protos.TaskInfo task = Protos.TaskInfo.newBuilder() .setName(mTaskName) .setTaskId(taskId) .setSlaveId(offer.getSlaveId()) .addAllResources(resources) .setExecutor(executorBuilder) .build(); launch.addTaskInfos(Protos.TaskInfo.newBuilder(task)); mLaunchedTasks++; // NOTE: We use the new API `acceptOffers` here to launch tasks. // The 'launchTasks' API will be deprecated. List<Protos.OfferID> offerIds = new ArrayList<Protos.OfferID>(); offerIds.add(offer.getId()); List<Protos.Offer.Operation> operations = new ArrayList<Protos.Offer.Operation>(); Protos.Offer.Operation operation = Protos.Offer.Operation.newBuilder() .setType(Protos.Offer.Operation.Type.LAUNCH) .setLaunch(launch) .build(); operations.add(operation); Protos.Filters filters = Protos.Filters.newBuilder().setRefuseSeconds(1).build(); driver.acceptOffers(offerIds, operations, filters); } }