@Test
  public void testOutOfOrder() throws Exception {
    LOG.info("STARTING testOutOfOrder");
    YarnRPC mockRpc = mock(YarnRPC.class);
    AppContext mockContext = mock(AppContext.class);
    @SuppressWarnings("rawtypes")
    EventHandler mockEventHandler = mock(EventHandler.class);
    when(mockContext.getEventHandler()).thenReturn(mockEventHandler);

    ContainerManager mockCM = mock(ContainerManager.class);
    when(mockRpc.getProxy(
            eq(ContainerManager.class), any(InetSocketAddress.class), any(Configuration.class)))
        .thenReturn(mockCM);

    ContainerLauncherImplUnderTest ut = new ContainerLauncherImplUnderTest(mockContext, mockRpc);

    Configuration conf = new Configuration();
    ut.init(conf);
    ut.start();
    try {
      ContainerId contId = makeContainerId(0l, 0, 0, 1);
      TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0);
      String cmAddress = "127.0.0.1:8000";
      StartContainerResponse startResp =
          recordFactory.newRecordInstance(StartContainerResponse.class);
      startResp.setServiceResponse(
          ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID, ShuffleHandler.serializeMetaData(80));

      LOG.info("inserting cleanup event");
      ContainerLauncherEvent mockCleanupEvent = mock(ContainerLauncherEvent.class);
      when(mockCleanupEvent.getType()).thenReturn(EventType.CONTAINER_REMOTE_CLEANUP);
      when(mockCleanupEvent.getContainerID()).thenReturn(contId);
      when(mockCleanupEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
      when(mockCleanupEvent.getContainerMgrAddress()).thenReturn(cmAddress);
      ut.handle(mockCleanupEvent);

      ut.waitForPoolToIdle();

      verify(mockCM, never()).stopContainer(any(StopContainerRequest.class));

      LOG.info("inserting launch event");
      ContainerRemoteLaunchEvent mockLaunchEvent = mock(ContainerRemoteLaunchEvent.class);
      when(mockLaunchEvent.getType()).thenReturn(EventType.CONTAINER_REMOTE_LAUNCH);
      when(mockLaunchEvent.getContainerID()).thenReturn(contId);
      when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId);
      when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress);
      when(mockCM.startContainer(any(StartContainerRequest.class))).thenReturn(startResp);
      ut.handle(mockLaunchEvent);

      ut.waitForPoolToIdle();

      verify(mockCM, never()).startContainer(any(StartContainerRequest.class));
    } finally {
      ut.stop();
    }
  }
 protected ResourceTracker getRMClient() {
   Configuration conf = getConfig();
   YarnRPC rpc = YarnRPC.create(conf);
   InetSocketAddress rmAddress =
       NetUtils.createSocketAddr(
           this.rmAddress,
           YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT,
           YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS);
   return (ResourceTracker) rpc.getProxy(ResourceTracker.class, rmAddress, conf);
 }
 @Override
 protected void serviceStart() throws Exception {
   // All the clients to appsManager are supposed to be authenticated via
   // Kerberos if security is enabled, so no secretManager.
   YarnRPC rpc = YarnRPC.create(getConfig());
   Configuration clientServerConf = new Configuration(getConfig());
   this.server =
       rpc.getServer(
           ApplicationClientProtocol.class, this, clientBindAddress, clientServerConf, null, 1);
   this.server.start();
   super.serviceStart();
 }
 /**
  * Delegate responsible for communicating with the Resource Manager's {@link ClientRMProtocol}.
  *
  * @param conf the configuration object.
  */
 public ResourceMgrDelegate(YarnConfiguration conf) {
   this.conf = conf;
   YarnRPC rpc = YarnRPC.create(this.conf);
   InetSocketAddress rmAddress =
       NetUtils.createSocketAddr(
           this.conf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS),
           YarnConfiguration.DEFAULT_RM_PORT,
           YarnConfiguration.RM_ADDRESS);
   this.rmAddress = rmAddress.toString();
   LOG.debug("Connecting to ResourceManager at " + rmAddress);
   applicationsManager =
       (ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, rmAddress, this.conf);
   LOG.debug("Connected to ResourceManager at " + rmAddress);
 }
 public YarnClientRMConnection(YarnConfiguration config) {
   this.config = config;
   InetSocketAddress remoteAddress =
       NetUtils.createSocketAddr(
           config.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));
   Configuration appsManagerServerConf = new Configuration(config);
   appsManagerServerConf.setClass(
       YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_CLIENT_RESOURCEMANAGER,
       ClientRMSecurityInfo.class,
       SecurityInfo.class);
   YarnRPC rpc = YarnRPC.create(appsManagerServerConf);
   crmp =
       ((ClientRMProtocol)
           rpc.getProxy(ClientRMProtocol.class, remoteAddress, appsManagerServerConf));
 }
  public ClientRMProtocol getClientResourceManager() {
    if (clientResourceManager != null) return clientResourceManager;

    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    YarnRPC rpc = YarnRPC.create(yarnConf);
    InetSocketAddress rmAddress =
        NetUtils.createSocketAddr(
            yarnConf.get(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS));

    LOG.info("Connecting to the resource manager (client) at " + rmAddress);

    clientResourceManager =
        (ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, rmAddress, conf);

    return clientResourceManager;
  }
Example #7
0
  public void launchSupervisorOnContainer(Container container) throws IOException {
    LOG.info("Connecting to ContainerManager for containerid=" + container.getId());
    String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
    InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
    LOG.info("Connecting to ContainerManager at " + cmIpPortStr);
    ContainerManager proxy =
        ((ContainerManager) rpc.getProxy(ContainerManager.class, cmAddress, hadoopConf));

    LOG.info("launchSupervisorOnContainer( id:" + container.getId() + " )");
    ContainerLaunchContext launchContext = Records.newRecord(ContainerLaunchContext.class);

    launchContext.setContainerId(container.getId());
    launchContext.setResource(container.getResource());

    try {
      launchContext.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
    } catch (IOException e) {
      LOG.info(
          "Getting current user info failed when trying to launch the container" + e.getMessage());
    }

    Map<String, String> env = new HashMap<String, String>();
    launchContext.setEnvironment(env);

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    String stormVersion = Util.getStormVersion(this.storm_conf);
    Path zip = new Path("/lib/storm/" + stormVersion + "/storm.zip");
    FileSystem fs = FileSystem.get(this.hadoopConf);
    localResources.put(
        "storm",
        Util.newYarnAppResource(
            fs, zip, LocalResourceType.ARCHIVE, LocalResourceVisibility.PUBLIC));

    String appHome = Util.getApplicationHomeForId(this.appAttemptId.toString());

    Path dirDst = Util.createConfigurationFileInFs(fs, appHome, this.storm_conf, this.hadoopConf);

    localResources.put("conf", Util.newYarnAppResource(fs, dirDst));

    launchContext.setLocalResources(localResources);

    List<String> supervisorArgs = Util.buildSupervisorCommands(this.storm_conf);

    launchContext.setCommands(supervisorArgs);

    StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class);
    startRequest.setContainerLaunchContext(launchContext);

    LOG.info(
        "launchSupervisorOnContainer: startRequest prepared, calling startContainer. "
            + startRequest);
    try {
      StartContainerResponse response = proxy.startContainer(startRequest);
      LOG.info("Got a start container response " + response);
    } catch (Exception e) {
      LOG.error("Caught an exception while trying to start a container", e);
      System.exit(-1);
    }
  }
  public AMRMProtocol getAMResourceManager() {
    if (amResourceManager != null) return amResourceManager;

    LOG.debug("Using configuration: " + conf);

    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    YarnRPC rpc = YarnRPC.create(yarnConf);
    InetSocketAddress rmAddress =
        NetUtils.createSocketAddr(
            yarnConf.get(
                YarnConfiguration.RM_SCHEDULER_ADDRESS,
                YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));

    LOG.info("Connecting to the resource manager (scheduling) at " + rmAddress);
    amResourceManager = (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, conf);

    return amResourceManager;
  }
    public void start(Configuration conf) {
      YarnRPC rpc = YarnRPC.create(conf);
      // TODO : use fixed port ??
      InetSocketAddress address = NetUtils.createSocketAddr(hostAddress);
      InetAddress hostNameResolved = null;
      try {
        address.getAddress();
        hostNameResolved = InetAddress.getLocalHost();
      } catch (UnknownHostException e) {
        throw new YarnRuntimeException(e);
      }

      server = rpc.getServer(protocol, this, address, conf, null, 1);
      server.start();
      this.bindAddress = NetUtils.getConnectAddress(server);
      super.start();
      amRunning = true;
    }
 @Test
 public void testLocalizerRPC() throws Exception {
   InetSocketAddress locAddr = new InetSocketAddress("0.0.0.0", 8040);
   LocalizerService server = new LocalizerService(locAddr);
   try {
     server.start();
     Configuration conf = new Configuration();
     YarnRPC rpc = YarnRPC.create(conf);
     LocalizationProtocol client =
         (LocalizationProtocol) rpc.getProxy(LocalizationProtocol.class, locAddr, conf);
     LocalizerStatus status = recordFactory.newRecordInstance(LocalizerStatus.class);
     status.setLocalizerId("localizer0");
     LocalizerHeartbeatResponse response = client.heartbeat(status);
     assertEquals(dieHBResponse(), response);
   } finally {
     server.stop();
   }
   assertTrue(true);
 }
Example #11
0
 public StormAMRMClient(
     ApplicationAttemptId appAttemptId,
     @SuppressWarnings("rawtypes") Map storm_conf,
     YarnConfiguration hadoopConf) {
   super(appAttemptId);
   this.storm_conf = storm_conf;
   this.hadoopConf = hadoopConf;
   this.rpc = YarnRPC.create(hadoopConf);
   Integer pri = Utils.getInt(storm_conf.get(Config.MASTER_CONTAINER_PRIORITY));
   this.DEFAULT_PRIORITY.setPriority(pri);
   this.containers = new TreeSet<Container>();
 }
Example #12
0
  protected void startServer() throws Exception {
    Configuration conf = getConfig();
    YarnRPC rpc = YarnRPC.create(conf);
    this.server =
        (Server)
            rpc.getServer(
                ResourceManagerAdministrationProtocol.class,
                this,
                masterServiceAddress,
                conf,
                null,
                conf.getInt(
                    YarnConfiguration.RM_ADMIN_CLIENT_THREAD_COUNT,
                    YarnConfiguration.DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT));

    // Enable service authorization?
    if (conf.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
      refreshServiceAcls(
          getConfiguration(conf, YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE),
          RMPolicyProvider.getInstance());
    }

    if (rmContext.isHAEnabled()) {
      RPC.setProtocolEngine(conf, HAServiceProtocolPB.class, ProtobufRpcEngine.class);

      HAServiceProtocolServerSideTranslatorPB haServiceProtocolXlator =
          new HAServiceProtocolServerSideTranslatorPB(this);
      BlockingService haPbService =
          HAServiceProtocolProtos.HAServiceProtocolService.newReflectiveBlockingService(
              haServiceProtocolXlator);
      server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, HAServiceProtocol.class, haPbService);
    }

    this.server.start();
    conf.updateConnectAddr(YarnConfiguration.RM_ADMIN_ADDRESS, server.getListenerAddress());
  }
  @Before
  public void setup() throws InterruptedException, IOException {
    conf = createConfiguration();
    rpc = YarnRPC.create(conf);
    rmAddress =
        conf.getSocketAddr(
            YarnConfiguration.RM_ADDRESS,
            YarnConfiguration.DEFAULT_RM_ADDRESS,
            YarnConfiguration.DEFAULT_RM_PORT);

    AccessControlList adminACL = new AccessControlList("");
    conf.set(YarnConfiguration.YARN_ADMIN_ACL, adminACL.getAclString());

    resourceManager =
        new MockRM(conf) {
          protected ClientRMService createClientRMService() {
            return new ClientRMService(
                getRMContext(),
                this.scheduler,
                this.rmAppManager,
                this.applicationACLsManager,
                this.queueACLsManager,
                getRMContext().getRMDelegationTokenSecretManager());
          };

          @Override
          protected void doSecureLogin() throws IOException {}
        };
    new Thread() {
      public void run() {
        resourceManager.start();
      };
    }.start();
    int waitCount = 0;
    while (resourceManager.getServiceState() == STATE.INITED && waitCount++ < 60) {
      LOG.info("Waiting for RM to start...");
      Thread.sleep(1500);
    }
    if (resourceManager.getServiceState() != STATE.STARTED) {
      // RM could have failed.
      throw new IOException(
          "ResourceManager failed to start. Final state is " + resourceManager.getServiceState());
    }
  }
 public void start() {
   Configuration conf = new Configuration();
   YarnRPC rpc = YarnRPC.create(conf);
   server = rpc.getServer(LocalizationProtocol.class, this, locAddr, conf, null, 1);
   server.start();
 }