@Override public Resource assignContainer(FSSchedulerNode node) { Resource assigned = Resources.none(); if (LOG.isDebugEnabled()) { LOG.debug("Node " + node.getNodeName() + " offered to queue: " + getName()); } if (!assignContainerPreCheck(node)) { return assigned; } Comparator<Schedulable> comparator = policy.getComparator(); Collections.sort(runnableApps, comparator); for (FSAppAttempt sched : runnableApps) { if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) { continue; } assigned = sched.assignContainer(node); if (!assigned.equals(Resources.none())) { break; } } return assigned; }
public static Resource newResource(int memory, int vCores, int gCores) { Resource resource = recordFactory.newRecordInstance(Resource.class); resource.setMemory(memory); resource.setVirtualCores(vCores); resource.setGPUCores(gCores); return resource; }
public void submitApplication( ApplicationId appId, String appName, Map<String, String> env, Map<String, LocalResource> localResources, List<String> commands, int memory) throws URISyntaxException, IOException { if (clientResourceManager == null) throw new IllegalStateException( "Cannot submit an application without connecting to resource manager!"); ApplicationSubmissionContext appCtx = Records.newRecord(ApplicationSubmissionContext.class); appCtx.setApplicationId(appId); appCtx.setApplicationName(appName); appCtx.setQueue("default"); appCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); // System.out.println( "Based on my current user I am: " + // UserGroupInformation.getCurrentUser().getShortUserName() ); Priority prio = Records.newRecord(Priority.class); prio.setPriority(0); appCtx.setPriority(prio); /* for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) { System.out.println("IR:RM: " + entry.getKey() + "/" + entry.getValue().getResource()); } */ /* for (Map.Entry<String, String> entry : env.entrySet()) { System.out.println("IR:ResourceManager -> Env vars: " + entry.getKey() + "/" + entry.getValue() ); } */ // Launch ctx ContainerLaunchContext containerCtx = Records.newRecord(ContainerLaunchContext.class); containerCtx.setLocalResources(localResources); containerCtx.setCommands(commands); containerCtx.setEnvironment(env); containerCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()); Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); containerCtx.setResource(capability); appCtx.setAMContainerSpec(containerCtx); SubmitApplicationRequest submitReq = Records.newRecord(SubmitApplicationRequest.class); submitReq.setApplicationSubmissionContext(appCtx); LOG.info("Submitting application to ASM"); clientResourceManager.submitApplication(submitReq); // Don't return anything, ASM#submit returns an empty response }
static boolean canFit(Resource arg0, Resource arg1) { int mem0 = arg0.getMemory(); int mem1 = arg1.getMemory(); int cpu0 = arg0.getVirtualCores(); int cpu1 = arg1.getVirtualCores(); if (mem0 <= mem1 && cpu0 <= cpu1) { return true; } return false; }
@Override public int compareTo(Resource other) { int diff = this.getMemory() - other.getMemory(); if (diff == 0) { diff = this.getVirtualCores() - other.getVirtualCores(); if (diff == 0) { diff = this.getOutboundNetworkBandwidthMbits() - other.getOutboundNetworkBandwidthMbits(); } } return diff; }
@Test public void testMove() { final String user = "******"; Queue parentQueue = createQueue("parent", null); Queue oldQueue = createQueue("old", parentQueue); Queue newQueue = createQueue("new", parentQueue); QueueMetrics parentMetrics = parentQueue.getMetrics(); QueueMetrics oldMetrics = oldQueue.getMetrics(); QueueMetrics newMetrics = newQueue.getMetrics(); ApplicationAttemptId appAttId = createAppAttemptId(0, 0); SchedulerApplicationAttempt app = new SchedulerApplicationAttempt( appAttId, user, oldQueue, oldQueue.getActiveUsersManager(), null); oldMetrics.submitApp(user); // Resource request Resource requestedResource = Resource.newInstance(1536, 2); Priority requestedPriority = Priority.newInstance(2); ResourceRequest request = ResourceRequest.newInstance(requestedPriority, ResourceRequest.ANY, requestedResource, 3); app.updateResourceRequests(Arrays.asList(request)); // Allocated container RMContainer container1 = createRMContainer(appAttId, 1, requestedResource); app.liveContainers.put(container1.getContainerId(), container1); SchedulerNode node = createNode(); app.appSchedulingInfo.allocate( NodeType.OFF_SWITCH, node, requestedPriority, request, container1.getContainer()); // Reserved container Priority prio1 = Priority.newInstance(1); Resource reservedResource = Resource.newInstance(2048, 3); RMContainer container2 = createReservedRMContainer(appAttId, 1, reservedResource, node.getNodeID(), prio1); Map<NodeId, RMContainer> reservations = new HashMap<NodeId, RMContainer>(); reservations.put(node.getNodeID(), container2); app.reservedContainers.put(prio1, reservations); oldMetrics.reserveResource(user, reservedResource); checkQueueMetrics(oldMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); checkQueueMetrics(newMetrics, 0, 0, 0, 0, 0, 0, 0, 0); checkQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); app.move(newQueue); checkQueueMetrics(oldMetrics, 0, 0, 0, 0, 0, 0, 0, 0); checkQueueMetrics(newMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); checkQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); }
/** * Setup a container request on specified node * * @param node the specified node * @return ResourceRequest sent to RM */ private ResourceRequest setupAContainerAskForRM(String node) { ResourceRequest request = Records.newRecord(ResourceRequest.class); request.setResourceName(node); request.setNumContainers(1); // important Priority priority = Records.newRecord(Priority.class); priority.setPriority(requestPriority); request.setPriority(priority); Resource capability = Records.newRecord(Resource.class); capability.setMemory(containerMemory); request.setCapability(capability); return request; }
/** * Utility method creating a {@link ResourceRequest}. * * @param numContainers number of containers to request * @return request to be sent to resource manager */ private ResourceRequest getContainerResourceRequest( int numContainers, String hostName, boolean relaxLocality) { ResourceRequest request = Records.newRecord(ResourceRequest.class); request.setRelaxLocality(relaxLocality); request.setResourceName(hostName); request.setNumContainers(numContainers); Priority pri = Records.newRecord(Priority.class); pri.setPriority(priority); request.setPriority(pri); Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); ResourceCompat.setVirtualCores(capability, virtualcores); request.setCapability(capability); return request; }
public void runMainLoop() throws Exception { rmClient.registerApplicationMaster("", 0, ""); initRegistry(); startMasterNode(); Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); Resource capability = Records.newRecord(Resource.class); capability.setMemory(ClusterConf.get().getWorkerHeapSize()); capability.setVirtualCores(ClusterConf.get().getNumWorkerThreads()); List<ContainerRequest> containerReq = new ArrayList<ContainerRequest>(); for (int i = 0; i < ClusterConf.get().getNumWorkers(); ++i) { ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority); rmClient.addContainerRequest(containerAsk); containerReq.add(containerAsk); } while (alive) { Thread.sleep(1000); } finish(); /* for (ContainerRequest req : containerReq) { rmClient.removeContainerRequest(req); } int containersToAdd = 2; numContainersToWaitFor = containersToAdd; System.out.println("[Am] finished all containers. Asking for more containers, total=" + numContainersToWaitFor); for (int i = 0; i < containersToAdd; ++i) { ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority); System.out.println("[AM] Making res-req " + i); rmClient.addContainerRequest(containerAsk); } System.out.println("[AM] waiting for containers to finish once more!!!"); while (!doneWithContainers()) { Thread.sleep(100); } */ }
protected void register() { // Register InetSocketAddress serviceAddr = null; if (clientService != null) { serviceAddr = clientService.getBindAddress(); } try { RegisterApplicationMasterRequest request = recordFactory.newRecordInstance(RegisterApplicationMasterRequest.class); if (serviceAddr != null) { request.setHost(serviceAddr.getHostName()); request.setRpcPort(serviceAddr.getPort()); request.setTrackingUrl(serviceAddr.getHostName() + ":" + clientService.getHttpPort()); } RegisterApplicationMasterResponse response = scheduler.registerApplicationMaster(request); maxContainerCapability = response.getMaximumResourceCapability(); this.context.getClusterInfo().setMaxContainerCapability(maxContainerCapability); if (UserGroupInformation.isSecurityEnabled()) { setClientToAMToken(response.getClientToAMTokenMasterKey()); } this.applicationACLs = response.getApplicationACLs(); LOG.info("maxContainerCapability: " + maxContainerCapability.getMemory()); } catch (Exception are) { LOG.error("Exception while registering", are); throw new YarnRuntimeException(are); } }
private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) throws Exception { YarnClientApplication newApp = rmClient.createApplication(); ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId(); // Create launch context for app master ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId); // set the application name appContext.setApplicationName("test"); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(1); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); appContext.setResource(Resource.newInstance(1024, 1)); appContext.setUnmanagedAM(unmanaged); // Submit the application to the applications manager rmClient.submitApplication(appContext); return appId; }
@Test(timeout = 20000) public void testAppSubmissionWithInvalidDelegationToken() throws Exception { Configuration conf = new Configuration(); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); UserGroupInformation.setConfiguration(conf); MockRM rm = new MockRM(conf); ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes()); ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance( new HashMap<String, LocalResource>(), new HashMap<String, String>(), new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens, new HashMap<ApplicationAccessType, String>()); ApplicationSubmissionContext appSubContext = ApplicationSubmissionContext.newInstance( ApplicationId.newInstance(1234121, 0), "BOGUS", "default", Priority.UNDEFINED, amContainer, false, true, 1, Resource.newInstance(1024, 1), "BOGUS"); SubmitApplicationRequest request = SubmitApplicationRequest.newInstance(appSubContext); try { rm.getClientRMService().submitApplication(request); fail("Error was excepted."); } catch (YarnException e) { Assert.assertTrue(e.getMessage().contains("Bad header found in token storage")); } }
public FSLeafQueue(String name, FairScheduler scheduler, FSParentQueue parent) { super(name, scheduler, parent); this.lastTimeAtMinShare = scheduler.getClock().getTime(); this.lastTimeAtHalfFairShare = scheduler.getClock().getTime(); activeUsersManager = new ActiveUsersManager(getMetrics()); amResourceUsage = Resource.newInstance(0, 0); }
private RMNodeImpl getRunningNode() throws IOException { NodeId nodeId = BuilderUtils.newNodeId("localhost", 0); Resource capability = Resource.newInstance(4096, 4); int rpcID = HopYarnAPIUtilities.getRPCID(); byte[] allNMRequestData = new byte[10]; try { RMUtilities.persistAppMasterRPC(rpcID, RPC.Type.RegisterNM, allNMRequestData); } catch (IOException ex) { Logger.getLogger(TestRMNodeTransitions.class.getName()).log(Level.SEVERE, null, ex); } TransactionState ts = new TransactionStateImpl(TransactionType.RM); // TransactionStateRM.newInstance(rpcID); RMNodeImpl node = new RMNodeImpl( nodeId, rmContext, nodeId.getHost(), 0, 0, null, ResourceOption.newInstance(capability, RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT), null); ((TransactionStateImpl) ts).getRMContextInfo().toAddActiveRMNode(nodeId, node, 1); node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.STARTED, ts)); Assert.assertEquals(NodeState.RUNNING, node.getState()); return node; }
/** * Setup the request that will be sent to the RM for the container ask. * * @return the setup ResourceRequest to be sent to RM */ private ContainerRequest setupContainerAskForRM() { // setup requirements for hosts // using * as any host will do for the distributed shell app // set the priority for the request Priority pri = Records.newRecord(Priority.class); // TODO - what is the range for priority? how to decide? pri.setPriority(requestPriority); // Set up resource type requirements // For now, memory and CPU are supported so we set memory and cpu requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(containerMemory); capability.setVirtualCores(containerVirtualCores); ContainerRequest request = new ContainerRequest(capability, null, null, pri); LOG.info("Requested container ask: " + request.toString()); return request; }
/* * Helper to determine the size of the container requested * from yarn. Falls back to Map-reduce's map size if tez * container size isn't set. */ private Resource getContainerResource(Configuration conf) { Resource containerResource; int memory = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0 ? HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) : conf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB); int cpus = conf.getInt(MRJobConfig.MAP_CPU_VCORES, MRJobConfig.DEFAULT_MAP_CPU_VCORES); return Resource.newInstance(memory, cpus); }
@Override public int compare(Resource arg0, Resource arg1) { int mem0 = arg0.getMemory(); int mem1 = arg1.getMemory(); int cpu0 = arg0.getVirtualCores(); int cpu1 = arg1.getVirtualCores(); if (mem0 == mem1) { if (cpu0 == cpu1) { return 0; } if (cpu0 < cpu1) { return 1; } return -1; } if (mem0 < mem1) { return 1; } return -1; }
private void requestContainer(Optional<String> preferredNode) { Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); Resource capability = Records.newRecord(Resource.class); int maxMemoryCapacity = this.maxResourceCapacity.get().getMemory(); capability.setMemory( this.requestedContainerMemoryMbs <= maxMemoryCapacity ? this.requestedContainerMemoryMbs : maxMemoryCapacity); int maxCoreCapacity = this.maxResourceCapacity.get().getVirtualCores(); capability.setVirtualCores( this.requestedContainerCores <= maxCoreCapacity ? this.requestedContainerCores : maxCoreCapacity); String[] preferredNodes = preferredNode.isPresent() ? new String[] {preferredNode.get()} : null; this.amrmClientAsync.addContainerRequest( new AMRMClient.ContainerRequest(capability, preferredNodes, null, priority)); }
private DAGPlan createDAG() { // Create a plan with 3 vertices: A, B, C. Group(A,B)->C Configuration conf = new Configuration(false); int dummyTaskCount = 1; Resource dummyTaskResource = Resource.newInstance(1, 1); org.apache.tez.dag.api.Vertex v1 = new org.apache.tez.dag.api.Vertex( "vertex1", new ProcessorDescriptor("Processor").setHistoryText("vertex1 Processor HistoryText"), dummyTaskCount, dummyTaskResource); v1.addInput( "input1", new InputDescriptor("input.class").setHistoryText("input HistoryText"), null); org.apache.tez.dag.api.Vertex v2 = new org.apache.tez.dag.api.Vertex( "vertex2", new ProcessorDescriptor("Processor").setHistoryText("vertex2 Processor HistoryText"), dummyTaskCount, dummyTaskResource); org.apache.tez.dag.api.Vertex v3 = new org.apache.tez.dag.api.Vertex( "vertex3", new ProcessorDescriptor("Processor").setHistoryText("vertex3 Processor HistoryText"), dummyTaskCount, dummyTaskResource); DAG dag = new DAG("testDag"); String groupName1 = "uv12"; org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2); OutputDescriptor outDesc = new OutputDescriptor("output.class").setHistoryText("uvOut HistoryText"); uv12.addOutput("uvOut", outDesc, OutputCommitter.class); v3.addOutput("uvOut", outDesc, OutputCommitter.class); GroupInputEdge e1 = new GroupInputEdge( uv12, v3, new EdgeProperty( DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, new OutputDescriptor("dummy output class").setHistoryText("Dummy History Text"), new InputDescriptor("dummy input class").setHistoryText("Dummy History Text")), new InputDescriptor("merge.class").setHistoryText("Merge HistoryText")); dag.addVertex(v1); dag.addVertex(v2); dag.addVertex(v3); dag.addEdge(e1); return dag.createDag(conf); }
private void updateQueueWithNodeUpdate(NodeUpdateSchedulerEventWrapper eventWrapper) { RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode(); List<UpdatedContainerInfo> containerList = node.getContainerUpdates(); for (UpdatedContainerInfo info : containerList) { for (ContainerStatus status : info.getCompletedContainers()) { ContainerId containerId = status.getContainerId(); SchedulerAppReport app = scheduler.getSchedulerAppInfo(containerId.getApplicationAttemptId()); if (app == null) { // this happens for the AM container // The app have already removed when the NM sends the release // information. continue; } String queue = appQueueMap.get(containerId.getApplicationAttemptId().getApplicationId()); int releasedMemory = 0, releasedVCores = 0; if (status.getExitStatus() == ContainerExitStatus.SUCCESS) { for (RMContainer rmc : app.getLiveContainers()) { if (rmc.getContainerId() == containerId) { releasedMemory += rmc.getContainer().getResource().getMemory(); releasedVCores += rmc.getContainer().getResource().getVirtualCores(); break; } } } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) { if (preemptionContainerMap.containsKey(containerId)) { Resource preResource = preemptionContainerMap.get(containerId); releasedMemory += preResource.getMemory(); releasedVCores += preResource.getVirtualCores(); preemptionContainerMap.remove(containerId); } } // update queue counters updateQueueMetrics(queue, releasedMemory, releasedVCores); } } }
public FiCaSchedulerNode(RMNode node, boolean usePortForNodeName) { this.rmNode = node; this.availableResource.setMemory(node.getTotalCapability().getMemory()); this.availableResource.setVirtualCores(node.getTotalCapability().getVirtualCores()); totalResourceCapability = Resource.newInstance( node.getTotalCapability().getMemory(), node.getTotalCapability().getVirtualCores()); if (usePortForNodeName) { nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort(); } else { nodeName = rmNode.getHostName(); } }
private ReservationSubmissionRequest createSimpleReservationRequest( int numContainers, long arrival, long deadline, long duration) { // create a request with a single atomic ask ReservationRequest r = ReservationRequest.newInstance(Resource.newInstance(1024, 1), numContainers, 1, duration); ReservationRequests reqs = ReservationRequests.newInstance( Collections.singletonList(r), ReservationRequestInterpreter.R_ALL); ReservationDefinition rDef = ReservationDefinition.newInstance(arrival, deadline, reqs, "testYarnClient#reservation"); ReservationSubmissionRequest request = ReservationSubmissionRequest.newInstance(rDef, ReservationSystemTestUtil.reservationQ); return request; }
private Token createNewContainerToken(ContainerId contId, String containerManagerAddr) { long currentTime = System.currentTimeMillis(); return MRApp.newContainerToken( NodeId.newInstance("127.0.0.1", 1234), "password".getBytes(), new ContainerTokenIdentifier( contId, containerManagerAddr, "user", Resource.newInstance(1024, 1), currentTime + 10000L, 123, currentTime)); }
/** 同computeAvailableContainers方法一样,只是返回的结果是向上取整 */ public static int divideAndCeilContainers( Resource required, Resource factor, EnumSet<SchedulerResourceTypes> resourceTypes) { if (resourceTypes.contains(SchedulerResourceTypes.CPU)) { return Math.max( divideAndCeil(required.getMemory(), factor.getMemory()), divideAndCeil(required.getVirtualCores(), factor.getVirtualCores())); } return divideAndCeil(required.getMemory(), factor.getMemory()); }
/** * 如果包含CPU资源,因此按照cpu资源能满足多少个请求资源,内存资源能满足多少个请求资源,取最小值 否则仅仅判断内存资源能满足多少个请求即可 * * @param available 可用的资源 * @param required 请求的资源 * @param resourceTypes 资源类型 return 计算可用资源可以满足多少个请求资源需求,即可以有多少个container */ public static int computeAvailableContainers( Resource available, Resource required, EnumSet<SchedulerResourceTypes> resourceTypes) { if (resourceTypes.contains( SchedulerResourceTypes.CPU)) { // 因为包含CPU资源,因此按照cpu资源能满足多少个请求资源,内存资源能满足多少个请求资源,取最小值 return Math.min( available.getMemory() / required.getMemory(), available.getVirtualCores() / required.getVirtualCores()); } return available.getMemory() / required.getMemory(); }
public WrappedContainer(boolean shouldProfile, String profileString) { applicationID = ApplicationId.newInstance(rmIdentifier, 1); appAttemptID = ApplicationAttemptId.newInstance(applicationID, 1); containerID = ContainerId.newInstance(appAttemptID, 1); nodeID = NodeId.newInstance("host", 12500); nodeHttpAddress = "host:12501"; resource = Resource.newInstance(1024, 1); priority = Priority.newInstance(1); container = Container.newInstance(containerID, nodeID, nodeHttpAddress, resource, priority, null); chh = mock(ContainerHeartbeatHandler.class); InetSocketAddress addr = new InetSocketAddress("localhost", 0); tal = mock(TaskAttemptListener.class); doReturn(addr).when(tal).getAddress(); dagID = TezDAGID.getInstance(applicationID, 1); vertexID = TezVertexID.getInstance(dagID, 1); taskID = TezTaskID.getInstance(vertexID, 1); taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1); eventHandler = mock(EventHandler.class); historyEventHandler = mock(HistoryEventHandler.class); Configuration conf = new Configuration(false); appContext = mock(AppContext.class); doReturn(new HashMap<ApplicationAccessType, String>()).when(appContext).getApplicationACLs(); doReturn(eventHandler).when(appContext).getEventHandler(); doReturn(appAttemptID).when(appContext).getApplicationAttemptId(); doReturn(applicationID).when(appContext).getApplicationID(); doReturn(new SystemClock()).when(appContext).getClock(); doReturn(historyEventHandler).when(appContext).getHistoryHandler(); doReturn(conf).when(appContext).getAMConf(); mockDAGID(); taskSpec = mock(TaskSpec.class); doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID(); amContainer = new AMContainerImpl(container, chh, tal, new ContainerContextMatcher(), appContext); }
@Before public void setUp() { dispatcher = new DrainDispatcher(); dispatcher.register(DAGEventType.class, mock(EventHandler.class)); dispatcher.register(VertexEventType.class, mock(EventHandler.class)); dispatcher.register(TaskEventType.class, new TaskEventHandler()); dispatcher.register(TaskAttemptEventType.class, taEventHandler); dispatcher.init(new Configuration()); dispatcher.start(); vertex = mock(Vertex.class, RETURNS_DEEP_STUBS); when(vertex.getProcessorDescriptor().getClassName()).thenReturn(""); mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS); when(mockAppContext.getCurrentDAG().getVertex(any(TezVertexID.class))).thenReturn(vertex); mockHistoryEventHandler = new MockHistoryEventHandler(mockAppContext); when(mockAppContext.getHistoryHandler()).thenReturn(mockHistoryEventHandler); task = new TaskImpl( vertexId, 0, dispatcher.getEventHandler(), new Configuration(), mock(TaskCommunicatorManagerInterface.class), new SystemClock(), mock(TaskHeartbeatHandler.class), mockAppContext, false, Resource.newInstance(1, 1), mock(ContainerContext.class), mock(StateChangeNotifier.class), vertex); Map<String, OutputCommitter> committers = new HashMap<String, OutputCommitter>(); committers.put( "out1", new TestOutputCommitter(mock(OutputCommitterContext.class), true, false)); when(task.getVertex().getOutputCommitters()).thenReturn(committers); }
@Test public void testAllcoateRequestWithIncrease() { List<ContainerResourceIncreaseRequest> incRequests = new ArrayList<ContainerResourceIncreaseRequest>(); for (int i = 0; i < 3; i++) { incRequests.add( ContainerResourceIncreaseRequest.newInstance(null, Resource.newInstance(0, i))); } AllocateRequest r = AllocateRequest.newInstance(123, 0f, null, null, null, incRequests); // serde AllocateRequestProto p = ((AllocateRequestPBImpl) r).getProto(); r = new AllocateRequestPBImpl(p); // check value Assert.assertEquals(123, r.getResponseId()); Assert.assertEquals(incRequests.size(), r.getIncreaseRequests().size()); for (int i = 0; i < incRequests.size(); i++) { Assert.assertEquals( r.getIncreaseRequests().get(i).getCapability().getVirtualCores(), incRequests.get(i).getCapability().getVirtualCores()); } }
// package private for testing purposes public void setupLimits(ContainerId containerId, Resource containerResource) throws IOException { String containerName = containerId.toString(); if (isCpuWeightEnabled()) { int containerVCores = containerResource.getVirtualCores(); createCgroup(CONTROLLER_CPU, containerName); int cpuShares = CPU_DEFAULT_WEIGHT * containerVCores; // absolute minimum of 10 shares for zero CPU containers cpuShares = Math.max(cpuShares, 10); updateCgroup(CONTROLLER_CPU, containerName, "shares", String.valueOf(cpuShares)); if (strictResourceUsageMode) { int nodeVCores = conf.getInt(YarnConfiguration.NM_VCORES, YarnConfiguration.DEFAULT_NM_VCORES); if (nodeVCores != containerVCores) { float containerCPU = (containerVCores * yarnProcessors) / (float) nodeVCores; int[] limits = getOverallLimits(containerCPU); updateCgroup(CONTROLLER_CPU, containerName, CPU_PERIOD_US, String.valueOf(limits[0])); updateCgroup(CONTROLLER_CPU, containerName, CPU_QUOTA_US, String.valueOf(limits[1])); } } } }
@Test public void testNodeRegistrationWithMinimumAllocations() throws Exception { Configuration conf = new Configuration(); conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048"); conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4"); rm = new MockRM(conf); rm.start(); ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService(); RegisterNodeManagerRequest req = Records.newRecord(RegisterNodeManagerRequest.class); NodeId nodeId = BuilderUtils.newNodeId("host", 1234); req.setNodeId(nodeId); Resource capability = BuilderUtils.newResource(1024, 1); req.setResource(capability); RegisterNodeManagerResponse response1 = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN, response1.getNodeAction()); capability.setMemory(2048); capability.setVirtualCores(1); req.setResource(capability); RegisterNodeManagerResponse response2 = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN, response2.getNodeAction()); capability.setMemory(1024); capability.setVirtualCores(4); req.setResource(capability); RegisterNodeManagerResponse response3 = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.SHUTDOWN, response3.getNodeAction()); capability.setMemory(2048); capability.setVirtualCores(4); req.setResource(capability); RegisterNodeManagerResponse response4 = resourceTrackerService.registerNodeManager(req); Assert.assertEquals(NodeAction.NORMAL, response4.getNodeAction()); }