@Override
  public Resource assignContainer(FSSchedulerNode node) {
    Resource assigned = Resources.none();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Node " + node.getNodeName() + " offered to queue: " + getName());
    }

    if (!assignContainerPreCheck(node)) {
      return assigned;
    }

    Comparator<Schedulable> comparator = policy.getComparator();
    Collections.sort(runnableApps, comparator);
    for (FSAppAttempt sched : runnableApps) {
      if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) {
        continue;
      }

      assigned = sched.assignContainer(node);
      if (!assigned.equals(Resources.none())) {
        break;
      }
    }
    return assigned;
  }
Example #2
0
 public static Resource newResource(int memory, int vCores, int gCores) {
   Resource resource = recordFactory.newRecordInstance(Resource.class);
   resource.setMemory(memory);
   resource.setVirtualCores(vCores);
   resource.setGPUCores(gCores);
   return resource;
 }
  public void submitApplication(
      ApplicationId appId,
      String appName,
      Map<String, String> env,
      Map<String, LocalResource> localResources,
      List<String> commands,
      int memory)
      throws URISyntaxException, IOException {

    if (clientResourceManager == null)
      throw new IllegalStateException(
          "Cannot submit an application without connecting to resource manager!");

    ApplicationSubmissionContext appCtx = Records.newRecord(ApplicationSubmissionContext.class);
    appCtx.setApplicationId(appId);
    appCtx.setApplicationName(appName);
    appCtx.setQueue("default");
    appCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());

    // System.out.println( "Based on my current user I am: " +
    // UserGroupInformation.getCurrentUser().getShortUserName() );

    Priority prio = Records.newRecord(Priority.class);
    prio.setPriority(0);
    appCtx.setPriority(prio);
    /*
       for (Map.Entry<String, LocalResource> entry : localResources.entrySet())
       {
           System.out.println("IR:RM: " + entry.getKey() + "/" + entry.getValue().getResource());
       }
    */
    /*
        for (Map.Entry<String, String> entry : env.entrySet())
        {
            System.out.println("IR:ResourceManager -> Env vars: " + entry.getKey() + "/" + entry.getValue() );
        }
    */

    // Launch ctx
    ContainerLaunchContext containerCtx = Records.newRecord(ContainerLaunchContext.class);
    containerCtx.setLocalResources(localResources);
    containerCtx.setCommands(commands);
    containerCtx.setEnvironment(env);
    containerCtx.setUser(UserGroupInformation.getCurrentUser().getShortUserName());

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(memory);
    containerCtx.setResource(capability);

    appCtx.setAMContainerSpec(containerCtx);

    SubmitApplicationRequest submitReq = Records.newRecord(SubmitApplicationRequest.class);
    submitReq.setApplicationSubmissionContext(appCtx);

    LOG.info("Submitting application to ASM");
    clientResourceManager.submitApplication(submitReq);

    // Don't return anything, ASM#submit returns an empty response
  }
  static boolean canFit(Resource arg0, Resource arg1) {
    int mem0 = arg0.getMemory();
    int mem1 = arg1.getMemory();
    int cpu0 = arg0.getVirtualCores();
    int cpu1 = arg1.getVirtualCores();

    if (mem0 <= mem1 && cpu0 <= cpu1) {
      return true;
    }
    return false;
  }
Example #5
0
 @Override
 public int compareTo(Resource other) {
   int diff = this.getMemory() - other.getMemory();
   if (diff == 0) {
     diff = this.getVirtualCores() - other.getVirtualCores();
     if (diff == 0) {
       diff = this.getOutboundNetworkBandwidthMbits() - other.getOutboundNetworkBandwidthMbits();
     }
   }
   return diff;
 }
  @Test
  public void testMove() {
    final String user = "******";
    Queue parentQueue = createQueue("parent", null);
    Queue oldQueue = createQueue("old", parentQueue);
    Queue newQueue = createQueue("new", parentQueue);
    QueueMetrics parentMetrics = parentQueue.getMetrics();
    QueueMetrics oldMetrics = oldQueue.getMetrics();
    QueueMetrics newMetrics = newQueue.getMetrics();

    ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
    SchedulerApplicationAttempt app =
        new SchedulerApplicationAttempt(
            appAttId, user, oldQueue, oldQueue.getActiveUsersManager(), null);
    oldMetrics.submitApp(user);

    // Resource request
    Resource requestedResource = Resource.newInstance(1536, 2);
    Priority requestedPriority = Priority.newInstance(2);
    ResourceRequest request =
        ResourceRequest.newInstance(requestedPriority, ResourceRequest.ANY, requestedResource, 3);
    app.updateResourceRequests(Arrays.asList(request));

    // Allocated container
    RMContainer container1 = createRMContainer(appAttId, 1, requestedResource);
    app.liveContainers.put(container1.getContainerId(), container1);
    SchedulerNode node = createNode();
    app.appSchedulingInfo.allocate(
        NodeType.OFF_SWITCH, node, requestedPriority, request, container1.getContainer());

    // Reserved container
    Priority prio1 = Priority.newInstance(1);
    Resource reservedResource = Resource.newInstance(2048, 3);
    RMContainer container2 =
        createReservedRMContainer(appAttId, 1, reservedResource, node.getNodeID(), prio1);
    Map<NodeId, RMContainer> reservations = new HashMap<NodeId, RMContainer>();
    reservations.put(node.getNodeID(), container2);
    app.reservedContainers.put(prio1, reservations);
    oldMetrics.reserveResource(user, reservedResource);

    checkQueueMetrics(oldMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
    checkQueueMetrics(newMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
    checkQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);

    app.move(newQueue);

    checkQueueMetrics(oldMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
    checkQueueMetrics(newMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
    checkQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
  }
  /**
   * Setup a container request on specified node
   *
   * @param node the specified node
   * @return ResourceRequest sent to RM
   */
  private ResourceRequest setupAContainerAskForRM(String node) {
    ResourceRequest request = Records.newRecord(ResourceRequest.class);
    request.setResourceName(node);
    request.setNumContainers(1); // important

    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(requestPriority);
    request.setPriority(priority);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(containerMemory);
    request.setCapability(capability);

    return request;
  }
 /**
  * Utility method creating a {@link ResourceRequest}.
  *
  * @param numContainers number of containers to request
  * @return request to be sent to resource manager
  */
 private ResourceRequest getContainerResourceRequest(
     int numContainers, String hostName, boolean relaxLocality) {
   ResourceRequest request = Records.newRecord(ResourceRequest.class);
   request.setRelaxLocality(relaxLocality);
   request.setResourceName(hostName);
   request.setNumContainers(numContainers);
   Priority pri = Records.newRecord(Priority.class);
   pri.setPriority(priority);
   request.setPriority(pri);
   Resource capability = Records.newRecord(Resource.class);
   capability.setMemory(memory);
   ResourceCompat.setVirtualCores(capability, virtualcores);
   request.setCapability(capability);
   return request;
 }
  public void runMainLoop() throws Exception {

    rmClient.registerApplicationMaster("", 0, "");

    initRegistry();

    startMasterNode();

    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(ClusterConf.get().getWorkerHeapSize());
    capability.setVirtualCores(ClusterConf.get().getNumWorkerThreads());

    List<ContainerRequest> containerReq = new ArrayList<ContainerRequest>();
    for (int i = 0; i < ClusterConf.get().getNumWorkers(); ++i) {
      ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
      rmClient.addContainerRequest(containerAsk);
      containerReq.add(containerAsk);
    }

    while (alive) {
      Thread.sleep(1000);
    }
    finish();

    /*
    for (ContainerRequest req : containerReq) {
        rmClient.removeContainerRequest(req);
    }
    int containersToAdd = 2;
    numContainersToWaitFor = containersToAdd;

    System.out.println("[Am] finished all containers. Asking for more containers, total=" + numContainersToWaitFor);
    for (int i = 0; i < containersToAdd; ++i) {
        ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
        System.out.println("[AM] Making res-req " + i);
        rmClient.addContainerRequest(containerAsk);
    }

    System.out.println("[AM] waiting for containers to finish once more!!!");
    while (!doneWithContainers()) {
        Thread.sleep(100);
    }
    */

  }
Example #10
0
 protected void register() {
   // Register
   InetSocketAddress serviceAddr = null;
   if (clientService != null) {
     serviceAddr = clientService.getBindAddress();
   }
   try {
     RegisterApplicationMasterRequest request =
         recordFactory.newRecordInstance(RegisterApplicationMasterRequest.class);
     if (serviceAddr != null) {
       request.setHost(serviceAddr.getHostName());
       request.setRpcPort(serviceAddr.getPort());
       request.setTrackingUrl(serviceAddr.getHostName() + ":" + clientService.getHttpPort());
     }
     RegisterApplicationMasterResponse response = scheduler.registerApplicationMaster(request);
     maxContainerCapability = response.getMaximumResourceCapability();
     this.context.getClusterInfo().setMaxContainerCapability(maxContainerCapability);
     if (UserGroupInformation.isSecurityEnabled()) {
       setClientToAMToken(response.getClientToAMTokenMasterKey());
     }
     this.applicationACLs = response.getApplicationACLs();
     LOG.info("maxContainerCapability: " + maxContainerCapability.getMemory());
   } catch (Exception are) {
     LOG.error("Exception while registering", are);
     throw new YarnRuntimeException(are);
   }
 }
  private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) throws Exception {
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(1);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("default");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);

    return appId;
  }
 @Test(timeout = 20000)
 public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
   Configuration conf = new Configuration();
   conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
   UserGroupInformation.setConfiguration(conf);
   MockRM rm = new MockRM(conf);
   ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes());
   ContainerLaunchContext amContainer =
       ContainerLaunchContext.newInstance(
           new HashMap<String, LocalResource>(),
           new HashMap<String, String>(),
           new ArrayList<String>(),
           new HashMap<String, ByteBuffer>(),
           tokens,
           new HashMap<ApplicationAccessType, String>());
   ApplicationSubmissionContext appSubContext =
       ApplicationSubmissionContext.newInstance(
           ApplicationId.newInstance(1234121, 0),
           "BOGUS",
           "default",
           Priority.UNDEFINED,
           amContainer,
           false,
           true,
           1,
           Resource.newInstance(1024, 1),
           "BOGUS");
   SubmitApplicationRequest request = SubmitApplicationRequest.newInstance(appSubContext);
   try {
     rm.getClientRMService().submitApplication(request);
     fail("Error was excepted.");
   } catch (YarnException e) {
     Assert.assertTrue(e.getMessage().contains("Bad header found in token storage"));
   }
 }
 public FSLeafQueue(String name, FairScheduler scheduler, FSParentQueue parent) {
   super(name, scheduler, parent);
   this.lastTimeAtMinShare = scheduler.getClock().getTime();
   this.lastTimeAtHalfFairShare = scheduler.getClock().getTime();
   activeUsersManager = new ActiveUsersManager(getMetrics());
   amResourceUsage = Resource.newInstance(0, 0);
 }
  private RMNodeImpl getRunningNode() throws IOException {
    NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
    Resource capability = Resource.newInstance(4096, 4);

    int rpcID = HopYarnAPIUtilities.getRPCID();
    byte[] allNMRequestData = new byte[10];
    try {
      RMUtilities.persistAppMasterRPC(rpcID, RPC.Type.RegisterNM, allNMRequestData);
    } catch (IOException ex) {
      Logger.getLogger(TestRMNodeTransitions.class.getName()).log(Level.SEVERE, null, ex);
    }
    TransactionState ts =
        new TransactionStateImpl(TransactionType.RM); // TransactionStateRM.newInstance(rpcID);

    RMNodeImpl node =
        new RMNodeImpl(
            nodeId,
            rmContext,
            nodeId.getHost(),
            0,
            0,
            null,
            ResourceOption.newInstance(capability, RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT),
            null);
    ((TransactionStateImpl) ts).getRMContextInfo().toAddActiveRMNode(nodeId, node, 1);
    node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.STARTED, ts));
    Assert.assertEquals(NodeState.RUNNING, node.getState());
    return node;
  }
  /**
   * Setup the request that will be sent to the RM for the container ask.
   *
   * @return the setup ResourceRequest to be sent to RM
   */
  private ContainerRequest setupContainerAskForRM() {
    // setup requirements for hosts
    // using * as any host will do for the distributed shell app
    // set the priority for the request
    Priority pri = Records.newRecord(Priority.class);
    // TODO - what is the range for priority? how to decide?
    pri.setPriority(requestPriority);

    // Set up resource type requirements
    // For now, memory and CPU are supported so we set memory and cpu requirements
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(containerMemory);
    capability.setVirtualCores(containerVirtualCores);

    ContainerRequest request = new ContainerRequest(capability, null, null, pri);
    LOG.info("Requested container ask: " + request.toString());
    return request;
  }
Example #16
0
 /*
  * Helper to determine the size of the container requested
  * from yarn. Falls back to Map-reduce's map size if tez
  * container size isn't set.
  */
 private Resource getContainerResource(Configuration conf) {
   Resource containerResource;
   int memory =
       HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE) > 0
           ? HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVETEZCONTAINERSIZE)
           : conf.getInt(MRJobConfig.MAP_MEMORY_MB, MRJobConfig.DEFAULT_MAP_MEMORY_MB);
   int cpus = conf.getInt(MRJobConfig.MAP_CPU_VCORES, MRJobConfig.DEFAULT_MAP_CPU_VCORES);
   return Resource.newInstance(memory, cpus);
 }
 @Override
 public int compare(Resource arg0, Resource arg1) {
   int mem0 = arg0.getMemory();
   int mem1 = arg1.getMemory();
   int cpu0 = arg0.getVirtualCores();
   int cpu1 = arg1.getVirtualCores();
   if (mem0 == mem1) {
     if (cpu0 == cpu1) {
       return 0;
     }
     if (cpu0 < cpu1) {
       return 1;
     }
     return -1;
   }
   if (mem0 < mem1) {
     return 1;
   }
   return -1;
 }
Example #18
0
  private void requestContainer(Optional<String> preferredNode) {
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    Resource capability = Records.newRecord(Resource.class);
    int maxMemoryCapacity = this.maxResourceCapacity.get().getMemory();
    capability.setMemory(
        this.requestedContainerMemoryMbs <= maxMemoryCapacity
            ? this.requestedContainerMemoryMbs
            : maxMemoryCapacity);
    int maxCoreCapacity = this.maxResourceCapacity.get().getVirtualCores();
    capability.setVirtualCores(
        this.requestedContainerCores <= maxCoreCapacity
            ? this.requestedContainerCores
            : maxCoreCapacity);

    String[] preferredNodes = preferredNode.isPresent() ? new String[] {preferredNode.get()} : null;
    this.amrmClientAsync.addContainerRequest(
        new AMRMClient.ContainerRequest(capability, preferredNodes, null, priority));
  }
Example #19
0
  private DAGPlan createDAG() {
    // Create a plan with 3 vertices: A, B, C. Group(A,B)->C
    Configuration conf = new Configuration(false);
    int dummyTaskCount = 1;
    Resource dummyTaskResource = Resource.newInstance(1, 1);
    org.apache.tez.dag.api.Vertex v1 =
        new org.apache.tez.dag.api.Vertex(
            "vertex1",
            new ProcessorDescriptor("Processor").setHistoryText("vertex1 Processor HistoryText"),
            dummyTaskCount,
            dummyTaskResource);
    v1.addInput(
        "input1", new InputDescriptor("input.class").setHistoryText("input HistoryText"), null);
    org.apache.tez.dag.api.Vertex v2 =
        new org.apache.tez.dag.api.Vertex(
            "vertex2",
            new ProcessorDescriptor("Processor").setHistoryText("vertex2 Processor HistoryText"),
            dummyTaskCount,
            dummyTaskResource);
    org.apache.tez.dag.api.Vertex v3 =
        new org.apache.tez.dag.api.Vertex(
            "vertex3",
            new ProcessorDescriptor("Processor").setHistoryText("vertex3 Processor HistoryText"),
            dummyTaskCount,
            dummyTaskResource);

    DAG dag = new DAG("testDag");
    String groupName1 = "uv12";
    org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
    OutputDescriptor outDesc =
        new OutputDescriptor("output.class").setHistoryText("uvOut HistoryText");
    uv12.addOutput("uvOut", outDesc, OutputCommitter.class);
    v3.addOutput("uvOut", outDesc, OutputCommitter.class);

    GroupInputEdge e1 =
        new GroupInputEdge(
            uv12,
            v3,
            new EdgeProperty(
                DataMovementType.SCATTER_GATHER,
                DataSourceType.PERSISTED,
                SchedulingType.SEQUENTIAL,
                new OutputDescriptor("dummy output class").setHistoryText("Dummy History Text"),
                new InputDescriptor("dummy input class").setHistoryText("Dummy History Text")),
            new InputDescriptor("merge.class").setHistoryText("Merge HistoryText"));

    dag.addVertex(v1);
    dag.addVertex(v2);
    dag.addVertex(v3);
    dag.addEdge(e1);
    return dag.createDag(conf);
  }
  private void updateQueueWithNodeUpdate(NodeUpdateSchedulerEventWrapper eventWrapper) {
    RMNodeWrapper node = (RMNodeWrapper) eventWrapper.getRMNode();
    List<UpdatedContainerInfo> containerList = node.getContainerUpdates();
    for (UpdatedContainerInfo info : containerList) {
      for (ContainerStatus status : info.getCompletedContainers()) {
        ContainerId containerId = status.getContainerId();
        SchedulerAppReport app =
            scheduler.getSchedulerAppInfo(containerId.getApplicationAttemptId());

        if (app == null) {
          // this happens for the AM container
          // The app have already removed when the NM sends the release
          // information.
          continue;
        }

        String queue = appQueueMap.get(containerId.getApplicationAttemptId().getApplicationId());
        int releasedMemory = 0, releasedVCores = 0;
        if (status.getExitStatus() == ContainerExitStatus.SUCCESS) {
          for (RMContainer rmc : app.getLiveContainers()) {
            if (rmc.getContainerId() == containerId) {
              releasedMemory += rmc.getContainer().getResource().getMemory();
              releasedVCores += rmc.getContainer().getResource().getVirtualCores();
              break;
            }
          }
        } else if (status.getExitStatus() == ContainerExitStatus.ABORTED) {
          if (preemptionContainerMap.containsKey(containerId)) {
            Resource preResource = preemptionContainerMap.get(containerId);
            releasedMemory += preResource.getMemory();
            releasedVCores += preResource.getVirtualCores();
            preemptionContainerMap.remove(containerId);
          }
        }
        // update queue counters
        updateQueueMetrics(queue, releasedMemory, releasedVCores);
      }
    }
  }
 public FiCaSchedulerNode(RMNode node, boolean usePortForNodeName) {
   this.rmNode = node;
   this.availableResource.setMemory(node.getTotalCapability().getMemory());
   this.availableResource.setVirtualCores(node.getTotalCapability().getVirtualCores());
   totalResourceCapability =
       Resource.newInstance(
           node.getTotalCapability().getMemory(), node.getTotalCapability().getVirtualCores());
   if (usePortForNodeName) {
     nodeName = rmNode.getHostName() + ":" + node.getNodeID().getPort();
   } else {
     nodeName = rmNode.getHostName();
   }
 }
 private ReservationSubmissionRequest createSimpleReservationRequest(
     int numContainers, long arrival, long deadline, long duration) {
   // create a request with a single atomic ask
   ReservationRequest r =
       ReservationRequest.newInstance(Resource.newInstance(1024, 1), numContainers, 1, duration);
   ReservationRequests reqs =
       ReservationRequests.newInstance(
           Collections.singletonList(r), ReservationRequestInterpreter.R_ALL);
   ReservationDefinition rDef =
       ReservationDefinition.newInstance(arrival, deadline, reqs, "testYarnClient#reservation");
   ReservationSubmissionRequest request =
       ReservationSubmissionRequest.newInstance(rDef, ReservationSystemTestUtil.reservationQ);
   return request;
 }
 private Token createNewContainerToken(ContainerId contId, String containerManagerAddr) {
   long currentTime = System.currentTimeMillis();
   return MRApp.newContainerToken(
       NodeId.newInstance("127.0.0.1", 1234),
       "password".getBytes(),
       new ContainerTokenIdentifier(
           contId,
           containerManagerAddr,
           "user",
           Resource.newInstance(1024, 1),
           currentTime + 10000L,
           123,
           currentTime));
 }
 /** 同computeAvailableContainers方法一样,只是返回的结果是向上取整 */
 public static int divideAndCeilContainers(
     Resource required, Resource factor, EnumSet<SchedulerResourceTypes> resourceTypes) {
   if (resourceTypes.contains(SchedulerResourceTypes.CPU)) {
     return Math.max(
         divideAndCeil(required.getMemory(), factor.getMemory()),
         divideAndCeil(required.getVirtualCores(), factor.getVirtualCores()));
   }
   return divideAndCeil(required.getMemory(), factor.getMemory());
 }
 /**
  * 如果包含CPU资源,因此按照cpu资源能满足多少个请求资源,内存资源能满足多少个请求资源,取最小值 否则仅仅判断内存资源能满足多少个请求即可
  *
  * @param available 可用的资源
  * @param required 请求的资源
  * @param resourceTypes 资源类型 return 计算可用资源可以满足多少个请求资源需求,即可以有多少个container
  */
 public static int computeAvailableContainers(
     Resource available, Resource required, EnumSet<SchedulerResourceTypes> resourceTypes) {
   if (resourceTypes.contains(
       SchedulerResourceTypes.CPU)) { // 因为包含CPU资源,因此按照cpu资源能满足多少个请求资源,内存资源能满足多少个请求资源,取最小值
     return Math.min(
         available.getMemory() / required.getMemory(),
         available.getVirtualCores() / required.getVirtualCores());
   }
   return available.getMemory() / required.getMemory();
 }
Example #26
0
    public WrappedContainer(boolean shouldProfile, String profileString) {
      applicationID = ApplicationId.newInstance(rmIdentifier, 1);
      appAttemptID = ApplicationAttemptId.newInstance(applicationID, 1);
      containerID = ContainerId.newInstance(appAttemptID, 1);
      nodeID = NodeId.newInstance("host", 12500);
      nodeHttpAddress = "host:12501";
      resource = Resource.newInstance(1024, 1);
      priority = Priority.newInstance(1);
      container =
          Container.newInstance(containerID, nodeID, nodeHttpAddress, resource, priority, null);

      chh = mock(ContainerHeartbeatHandler.class);

      InetSocketAddress addr = new InetSocketAddress("localhost", 0);
      tal = mock(TaskAttemptListener.class);
      doReturn(addr).when(tal).getAddress();

      dagID = TezDAGID.getInstance(applicationID, 1);
      vertexID = TezVertexID.getInstance(dagID, 1);
      taskID = TezTaskID.getInstance(vertexID, 1);
      taskAttemptID = TezTaskAttemptID.getInstance(taskID, 1);

      eventHandler = mock(EventHandler.class);
      historyEventHandler = mock(HistoryEventHandler.class);

      Configuration conf = new Configuration(false);
      appContext = mock(AppContext.class);
      doReturn(new HashMap<ApplicationAccessType, String>()).when(appContext).getApplicationACLs();
      doReturn(eventHandler).when(appContext).getEventHandler();
      doReturn(appAttemptID).when(appContext).getApplicationAttemptId();
      doReturn(applicationID).when(appContext).getApplicationID();
      doReturn(new SystemClock()).when(appContext).getClock();
      doReturn(historyEventHandler).when(appContext).getHistoryHandler();
      doReturn(conf).when(appContext).getAMConf();
      mockDAGID();

      taskSpec = mock(TaskSpec.class);
      doReturn(taskAttemptID).when(taskSpec).getTaskAttemptID();

      amContainer =
          new AMContainerImpl(container, chh, tal, new ContainerContextMatcher(), appContext);
    }
Example #27
0
  @Before
  public void setUp() {
    dispatcher = new DrainDispatcher();
    dispatcher.register(DAGEventType.class, mock(EventHandler.class));
    dispatcher.register(VertexEventType.class, mock(EventHandler.class));
    dispatcher.register(TaskEventType.class, new TaskEventHandler());
    dispatcher.register(TaskAttemptEventType.class, taEventHandler);
    dispatcher.init(new Configuration());
    dispatcher.start();

    vertex = mock(Vertex.class, RETURNS_DEEP_STUBS);
    when(vertex.getProcessorDescriptor().getClassName()).thenReturn("");

    mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    when(mockAppContext.getCurrentDAG().getVertex(any(TezVertexID.class))).thenReturn(vertex);
    mockHistoryEventHandler = new MockHistoryEventHandler(mockAppContext);
    when(mockAppContext.getHistoryHandler()).thenReturn(mockHistoryEventHandler);
    task =
        new TaskImpl(
            vertexId,
            0,
            dispatcher.getEventHandler(),
            new Configuration(),
            mock(TaskCommunicatorManagerInterface.class),
            new SystemClock(),
            mock(TaskHeartbeatHandler.class),
            mockAppContext,
            false,
            Resource.newInstance(1, 1),
            mock(ContainerContext.class),
            mock(StateChangeNotifier.class),
            vertex);

    Map<String, OutputCommitter> committers = new HashMap<String, OutputCommitter>();
    committers.put(
        "out1", new TestOutputCommitter(mock(OutputCommitterContext.class), true, false));
    when(task.getVertex().getOutputCommitters()).thenReturn(committers);
  }
Example #28
0
  @Test
  public void testAllcoateRequestWithIncrease() {
    List<ContainerResourceIncreaseRequest> incRequests =
        new ArrayList<ContainerResourceIncreaseRequest>();
    for (int i = 0; i < 3; i++) {
      incRequests.add(
          ContainerResourceIncreaseRequest.newInstance(null, Resource.newInstance(0, i)));
    }
    AllocateRequest r = AllocateRequest.newInstance(123, 0f, null, null, null, incRequests);

    // serde
    AllocateRequestProto p = ((AllocateRequestPBImpl) r).getProto();
    r = new AllocateRequestPBImpl(p);

    // check value
    Assert.assertEquals(123, r.getResponseId());
    Assert.assertEquals(incRequests.size(), r.getIncreaseRequests().size());

    for (int i = 0; i < incRequests.size(); i++) {
      Assert.assertEquals(
          r.getIncreaseRequests().get(i).getCapability().getVirtualCores(),
          incRequests.get(i).getCapability().getVirtualCores());
    }
  }
  // package private for testing purposes
  public void setupLimits(ContainerId containerId, Resource containerResource) throws IOException {
    String containerName = containerId.toString();

    if (isCpuWeightEnabled()) {
      int containerVCores = containerResource.getVirtualCores();
      createCgroup(CONTROLLER_CPU, containerName);

      int cpuShares = CPU_DEFAULT_WEIGHT * containerVCores;
      // absolute minimum of 10 shares for zero CPU containers
      cpuShares = Math.max(cpuShares, 10);

      updateCgroup(CONTROLLER_CPU, containerName, "shares", String.valueOf(cpuShares));
      if (strictResourceUsageMode) {
        int nodeVCores =
            conf.getInt(YarnConfiguration.NM_VCORES, YarnConfiguration.DEFAULT_NM_VCORES);
        if (nodeVCores != containerVCores) {
          float containerCPU = (containerVCores * yarnProcessors) / (float) nodeVCores;
          int[] limits = getOverallLimits(containerCPU);
          updateCgroup(CONTROLLER_CPU, containerName, CPU_PERIOD_US, String.valueOf(limits[0]));
          updateCgroup(CONTROLLER_CPU, containerName, CPU_QUOTA_US, String.valueOf(limits[1]));
        }
      }
    }
  }
  @Test
  public void testNodeRegistrationWithMinimumAllocations() throws Exception {
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048");
    conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4");
    rm = new MockRM(conf);
    rm.start();

    ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
    RegisterNodeManagerRequest req = Records.newRecord(RegisterNodeManagerRequest.class);
    NodeId nodeId = BuilderUtils.newNodeId("host", 1234);
    req.setNodeId(nodeId);

    Resource capability = BuilderUtils.newResource(1024, 1);
    req.setResource(capability);
    RegisterNodeManagerResponse response1 = resourceTrackerService.registerNodeManager(req);
    Assert.assertEquals(NodeAction.SHUTDOWN, response1.getNodeAction());

    capability.setMemory(2048);
    capability.setVirtualCores(1);
    req.setResource(capability);
    RegisterNodeManagerResponse response2 = resourceTrackerService.registerNodeManager(req);
    Assert.assertEquals(NodeAction.SHUTDOWN, response2.getNodeAction());

    capability.setMemory(1024);
    capability.setVirtualCores(4);
    req.setResource(capability);
    RegisterNodeManagerResponse response3 = resourceTrackerService.registerNodeManager(req);
    Assert.assertEquals(NodeAction.SHUTDOWN, response3.getNodeAction());

    capability.setMemory(2048);
    capability.setVirtualCores(4);
    req.setResource(capability);
    RegisterNodeManagerResponse response4 = resourceTrackerService.registerNodeManager(req);
    Assert.assertEquals(NodeAction.NORMAL, response4.getNodeAction());
  }