예제 #1
0
 public static ContainerId newContainerId(
     RecordFactory recordFactory,
     ApplicationId appId,
     ApplicationAttemptId appAttemptId,
     int containerId) {
   return ContainerId.newContainerId(appAttemptId, containerId);
 }
예제 #2
0
  @Test(timeout = 10000)
  public void testGetContainers() throws YarnException, IOException {
    Configuration conf = new Configuration();
    final YarnClient client = new MockYarnClient();
    client.init(conf);
    client.start();

    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
    List<ContainerReport> reports = client.getContainers(appAttemptId);
    Assert.assertNotNull(reports);
    Assert.assertEquals(
        reports.get(0).getContainerId(), (ContainerId.newContainerId(appAttemptId, 1)));
    Assert.assertEquals(
        reports.get(1).getContainerId(), (ContainerId.newContainerId(appAttemptId, 2)));
    client.stop();
  }
예제 #3
0
  @Test(timeout = 10000)
  public void testGetContainerReport() throws YarnException, IOException {
    Configuration conf = new Configuration();
    final YarnClient client = new MockYarnClient();
    client.init(conf);
    client.start();

    List<ApplicationReport> expectedReports = ((MockYarnClient) client).getReports();

    ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(applicationId, 1);
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
    ContainerReport report = client.getContainerReport(containerId);
    Assert.assertNotNull(report);
    Assert.assertEquals(
        report.getContainerId().toString(),
        (ContainerId.newContainerId(expectedReports.get(0).getCurrentApplicationAttemptId(), 1))
            .toString());
    client.stop();
  }
예제 #4
0
  @Test
  public void testSuccessfulContainerLaunch()
      throws InterruptedException, IOException, YarnException {

    FileContext localFS = FileContext.getLocalFSFileContext();

    localFS.delete(new Path(localDir.getAbsolutePath()), true);
    localFS.delete(new Path(localLogDir.getAbsolutePath()), true);
    localFS.delete(new Path(remoteLogDir.getAbsolutePath()), true);
    localDir.mkdir();
    localLogDir.mkdir();
    remoteLogDir.mkdir();

    YarnConfiguration conf = new YarnConfiguration();

    Context context =
        new NMContext(
            new NMContainerTokenSecretManager(conf),
            new NMTokenSecretManagerInNM(),
            null,
            null,
            new NMNullStateStoreService()) {
          @Override
          public int getHttpPort() {
            return 1234;
          }
        };

    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath());

    ContainerExecutor exec = new DefaultContainerExecutor();
    exec.setConf(conf);

    DeletionService del = new DeletionService(exec);
    Dispatcher dispatcher = new AsyncDispatcher();
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    NodeHealthCheckerService healthChecker =
        new NodeHealthCheckerService(NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
    healthChecker.init(conf);
    NodeManagerMetrics metrics = NodeManagerMetrics.create();
    NodeStatusUpdater nodeStatusUpdater =
        new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics) {
          @Override
          protected ResourceTracker getRMClient() {
            return new LocalRMInterface();
          };

          @Override
          protected void stopRMProxy() {
            return;
          }

          @Override
          protected void startStatusUpdater() {
            return; // Don't start any updating thread.
          }

          @Override
          public long getRMIdentifier() {
            return SIMULATED_RM_IDENTIFIER;
          }
        };

    DummyContainerManager containerManager =
        new DummyContainerManager(
            context,
            exec,
            del,
            nodeStatusUpdater,
            metrics,
            new ApplicationACLsManager(conf),
            dirsHandler);
    nodeStatusUpdater.init(conf);
    ((NMContext) context).setContainerManager(containerManager);
    nodeStatusUpdater.start();
    containerManager.init(conf);
    containerManager.start();

    ContainerLaunchContext launchContext =
        recordFactory.newRecordInstance(ContainerLaunchContext.class);
    ApplicationId applicationId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
    ContainerId cID = ContainerId.newContainerId(applicationAttemptId, 0);

    String user = "******";
    StartContainerRequest scRequest =
        StartContainerRequest.newInstance(
            launchContext,
            TestContainerManager.createContainerToken(
                cID,
                SIMULATED_RM_IDENTIFIER,
                context.getNodeId(),
                user,
                context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    containerManager.startContainers(allRequests);

    BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.RUNNING);

    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(cID);
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(containerIds);
    containerManager.stopContainers(stopRequest);
    BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.COMPLETE);

    containerManager.stop();
  }
  @Test(timeout = 30000)
  public void testNodeUpdate() throws Exception {
    // set node -> label
    mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z"));

    // set mapping:
    // h1 -> x
    // h2 -> y
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));

    // inject node label manager
    MockRM rm =
        new MockRM(getConfigurationWithQueueLabels(conf)) {
          @Override
          public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
          }
        };

    rm.getRMContext().setNodeLabelManager(mgr);
    rm.start();
    MockNM nm1 = rm.registerNode("h1:1234", 8000);
    MockNM nm2 = rm.registerNode("h2:1234", 8000);
    MockNM nm3 = rm.registerNode("h3:1234", 8000);

    ContainerId containerId;

    // launch an app to queue a1 (label = x), and check all container will
    // be allocated in h1
    RMApp app1 = rm.submitApp(GB, "app", "user", null, "a");
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);

    // request a container.
    am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
    containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    Assert.assertTrue(rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED, 10 * 1000));

    // check used resource:
    // queue-a used x=1G, ""=1G
    checkUsedResource(rm, "a", 1024, "x");
    checkUsedResource(rm, "a", 1024);

    // change h1's label to z, container should be killed
    mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("z")));
    Assert.assertTrue(rm.waitForState(nm1, containerId, RMContainerState.KILLED, 10 * 1000));

    // check used resource:
    // queue-a used x=0G, ""=1G ("" not changed)
    checkUsedResource(rm, "a", 0, "x");
    checkUsedResource(rm, "a", 1024);

    // request a container with label = y
    am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "y");
    containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
    Assert.assertTrue(rm.waitForState(nm2, containerId, RMContainerState.ALLOCATED, 10 * 1000));

    // check used resource:
    // queue-a used y=1G, ""=1G
    checkUsedResource(rm, "a", 1024, "y");
    checkUsedResource(rm, "a", 1024);

    // change h2's label to no label, container should be killed
    mgr.replaceLabelsOnNode(
        ImmutableMap.of(NodeId.newInstance("h2", 0), CommonNodeLabelsManager.EMPTY_STRING_SET));
    Assert.assertTrue(rm.waitForState(nm1, containerId, RMContainerState.KILLED, 10 * 1000));

    // check used resource:
    // queue-a used x=0G, y=0G, ""=1G ("" not changed)
    checkUsedResource(rm, "a", 0, "x");
    checkUsedResource(rm, "a", 0, "y");
    checkUsedResource(rm, "a", 1024);

    containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);

    // change h3's label to z, AM container should be killed
    mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("h3", 0), toSet("z")));
    Assert.assertTrue(rm.waitForState(nm1, containerId, RMContainerState.KILLED, 10 * 1000));

    // check used resource:
    // queue-a used x=0G, y=0G, ""=1G ("" not changed)
    checkUsedResource(rm, "a", 0, "x");
    checkUsedResource(rm, "a", 0, "y");
    checkUsedResource(rm, "a", 0);

    rm.close();
  }
  @SuppressWarnings({"unchecked", "rawtypes"})
  @Test
  public void testHandleContainerStatusInvalidCompletions() throws Exception {
    rm = new MockRM(new YarnConfiguration());
    rm.start();

    EventHandler handler = spy(rm.getRMContext().getDispatcher().getEventHandler());

    // Case 1: Unmanaged AM
    RMApp app = rm.submitApp(1024, true);

    // Case 1.1: AppAttemptId is null
    NMContainerStatus report =
        NMContainerStatus.newInstance(
            ContainerId.newContainerId(
                ApplicationAttemptId.newInstance(app.getApplicationId(), 2), 1),
            ContainerState.COMPLETE,
            Resource.newInstance(1024, 1),
            "Dummy Completed",
            0,
            Priority.newInstance(10),
            1234);
    rm.getResourceTrackerService().handleNMContainerStatus(report, null);
    verify(handler, never()).handle((Event) any());

    // Case 1.2: Master container is null
    RMAppAttemptImpl currentAttempt = (RMAppAttemptImpl) app.getCurrentAppAttempt();
    currentAttempt.setMasterContainer(null);
    report =
        NMContainerStatus.newInstance(
            ContainerId.newContainerId(currentAttempt.getAppAttemptId(), 0),
            ContainerState.COMPLETE,
            Resource.newInstance(1024, 1),
            "Dummy Completed",
            0,
            Priority.newInstance(10),
            1234);
    rm.getResourceTrackerService().handleNMContainerStatus(report, null);
    verify(handler, never()).handle((Event) any());

    // Case 2: Managed AM
    app = rm.submitApp(1024);

    // Case 2.1: AppAttemptId is null
    report =
        NMContainerStatus.newInstance(
            ContainerId.newContainerId(
                ApplicationAttemptId.newInstance(app.getApplicationId(), 2), 1),
            ContainerState.COMPLETE,
            Resource.newInstance(1024, 1),
            "Dummy Completed",
            0,
            Priority.newInstance(10),
            1234);
    try {
      rm.getResourceTrackerService().handleNMContainerStatus(report, null);
    } catch (Exception e) {
      // expected - ignore
    }
    verify(handler, never()).handle((Event) any());

    // Case 2.2: Master container is null
    currentAttempt = (RMAppAttemptImpl) app.getCurrentAppAttempt();
    currentAttempt.setMasterContainer(null);
    report =
        NMContainerStatus.newInstance(
            ContainerId.newContainerId(currentAttempt.getAppAttemptId(), 0),
            ContainerState.COMPLETE,
            Resource.newInstance(1024, 1),
            "Dummy Completed",
            0,
            Priority.newInstance(10),
            1234);
    try {
      rm.getResourceTrackerService().handleNMContainerStatus(report, null);
    } catch (Exception e) {
      // expected - ignore
    }
    verify(handler, never()).handle((Event) any());
  }
예제 #7
0
  public static RMApp newApplication(int i) {
    final ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(newAppID(i), 0);
    final Container masterContainer = Records.newRecord(Container.class);
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, 0);
    masterContainer.setId(containerId);
    masterContainer.setNodeHttpAddress("node:port");
    final String user = newUserName();
    final String name = newAppName();
    final String queue = newQueue();
    final long start = 123456 + i * 1000;
    final long finish = 234567 + i * 1000;
    final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE;
    YarnApplicationState[] allStates = YarnApplicationState.values();
    final YarnApplicationState state = allStates[i % allStates.length];
    final int maxAppAttempts = i % 1000;
    return new ApplicationBase() {
      @Override
      public ApplicationId getApplicationId() {
        return appAttemptId.getApplicationId();
      }

      @Override
      public String getUser() {
        return user;
      }

      @Override
      public String getName() {
        return name;
      }

      @Override
      public String getApplicationType() {
        return type;
      }

      @Override
      public String getQueue() {
        return queue;
      }

      @Override
      public long getStartTime() {
        return start;
      }

      @Override
      public long getFinishTime() {
        return finish;
      }

      @Override
      public String getTrackingUrl() {
        return null;
      }

      @Override
      public YarnApplicationState createApplicationState() {
        return state;
      }

      @Override
      public StringBuilder getDiagnostics() {
        return new StringBuilder();
      }

      @Override
      public float getProgress() {
        return (float) Math.random();
      }

      @Override
      public FinalApplicationStatus getFinalApplicationStatus() {
        return FinalApplicationStatus.UNDEFINED;
      }

      @Override
      public RMAppAttempt getCurrentAppAttempt() {
        return null;
      }

      @Override
      public int getMaxAppAttempts() {
        return maxAppAttempts;
      }

      @Override
      public Set<String> getApplicationTags() {
        return null;
      }

      @Override
      public ApplicationReport createAndGetApplicationReport(
          String clientUserName, boolean allowAccess) {
        ApplicationResourceUsageReport usageReport =
            ApplicationResourceUsageReport.newInstance(0, 0, null, null, null, 0, 0);
        ApplicationReport report =
            ApplicationReport.newInstance(
                getApplicationId(),
                appAttemptId,
                getUser(),
                getQueue(),
                getName(),
                null,
                0,
                null,
                null,
                getDiagnostics().toString(),
                getTrackingUrl(),
                getStartTime(),
                getFinishTime(),
                getFinalApplicationStatus(),
                usageReport,
                null,
                getProgress(),
                type,
                null);
        return report;
      }
    };
  }
예제 #8
0
    private List<ApplicationReport> createAppReports() {
      ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
      ApplicationReport newApplicationReport =
          ApplicationReport.newInstance(
              applicationId,
              ApplicationAttemptId.newInstance(applicationId, 1),
              "user",
              "queue",
              "appname",
              "host",
              124,
              null,
              YarnApplicationState.RUNNING,
              "diagnostics",
              "url",
              0,
              0,
              FinalApplicationStatus.SUCCEEDED,
              null,
              "N/A",
              0.53789f,
              "YARN",
              null);
      List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
      applicationReports.add(newApplicationReport);
      List<ApplicationAttemptReport> appAttempts = new ArrayList<ApplicationAttemptReport>();
      ApplicationAttemptReport attempt =
          ApplicationAttemptReport.newInstance(
              ApplicationAttemptId.newInstance(applicationId, 1),
              "host",
              124,
              "url",
              "oUrl",
              "diagnostics",
              YarnApplicationAttemptState.FINISHED,
              ContainerId.newContainerId(newApplicationReport.getCurrentApplicationAttemptId(), 1));
      appAttempts.add(attempt);
      ApplicationAttemptReport attempt1 =
          ApplicationAttemptReport.newInstance(
              ApplicationAttemptId.newInstance(applicationId, 2),
              "host",
              124,
              "url",
              "oUrl",
              "diagnostics",
              YarnApplicationAttemptState.FINISHED,
              ContainerId.newContainerId(newApplicationReport.getCurrentApplicationAttemptId(), 2));
      appAttempts.add(attempt1);
      attempts.put(applicationId, appAttempts);

      List<ContainerReport> containerReports = new ArrayList<ContainerReport>();
      ContainerReport container =
          ContainerReport.newInstance(
              ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1),
              null,
              NodeId.newInstance("host", 1234),
              Priority.UNDEFINED,
              1234,
              5678,
              "diagnosticInfo",
              "logURL",
              0,
              ContainerState.COMPLETE,
              "http://" + NodeId.newInstance("host", 2345).toString());
      containerReports.add(container);

      ContainerReport container1 =
          ContainerReport.newInstance(
              ContainerId.newContainerId(attempt.getApplicationAttemptId(), 2),
              null,
              NodeId.newInstance("host", 1234),
              Priority.UNDEFINED,
              1234,
              5678,
              "diagnosticInfo",
              "logURL",
              0,
              ContainerState.COMPLETE,
              "http://" + NodeId.newInstance("host", 2345).toString());

      containerReports.add(container1);
      containers.put(attempt.getApplicationAttemptId(), containerReports);

      ApplicationId applicationId2 = ApplicationId.newInstance(1234, 6);
      ApplicationReport newApplicationReport2 =
          ApplicationReport.newInstance(
              applicationId2,
              ApplicationAttemptId.newInstance(applicationId2, 2),
              "user2",
              "queue2",
              "appname2",
              "host2",
              125,
              null,
              YarnApplicationState.FINISHED,
              "diagnostics2",
              "url2",
              2,
              2,
              FinalApplicationStatus.SUCCEEDED,
              null,
              "N/A",
              0.63789f,
              "NON-YARN",
              null);
      applicationReports.add(newApplicationReport2);

      ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
      ApplicationReport newApplicationReport3 =
          ApplicationReport.newInstance(
              applicationId3,
              ApplicationAttemptId.newInstance(applicationId3, 3),
              "user3",
              "queue3",
              "appname3",
              "host3",
              126,
              null,
              YarnApplicationState.RUNNING,
              "diagnostics3",
              "url3",
              3,
              3,
              FinalApplicationStatus.SUCCEEDED,
              null,
              "N/A",
              0.73789f,
              "MAPREDUCE",
              null);
      applicationReports.add(newApplicationReport3);

      ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
      ApplicationReport newApplicationReport4 =
          ApplicationReport.newInstance(
              applicationId4,
              ApplicationAttemptId.newInstance(applicationId4, 4),
              "user4",
              "queue4",
              "appname4",
              "host4",
              127,
              null,
              YarnApplicationState.FAILED,
              "diagnostics4",
              "url4",
              4,
              4,
              FinalApplicationStatus.SUCCEEDED,
              null,
              "N/A",
              0.83789f,
              "NON-MAPREDUCE",
              null);
      applicationReports.add(newApplicationReport4);
      return applicationReports;
    }
예제 #9
0
 public static ContainerId newContainerId(ApplicationAttemptId appAttemptId, long containerId) {
   return ContainerId.newContainerId(appAttemptId, containerId);
 }