예제 #1
0
  private static CommitterEventHandler createCommitterEventHandler(
      Dispatcher dispatcher, OutputCommitter committer) {
    final SystemClock clock = new SystemClock();
    AppContext appContext = mock(AppContext.class);
    when(appContext.getEventHandler()).thenReturn(dispatcher.getEventHandler());
    when(appContext.getClock()).thenReturn(clock);
    RMHeartbeatHandler heartbeatHandler =
        new RMHeartbeatHandler() {
          @Override
          public long getLastHeartbeatTime() {
            return clock.getTime();
          }

          @Override
          public void runOnNextHeartbeat(Runnable callback) {
            callback.run();
          }
        };
    ApplicationAttemptId id =
        ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
    when(appContext.getApplicationID()).thenReturn(id.getApplicationId());
    when(appContext.getApplicationAttemptId()).thenReturn(id);
    CommitterEventHandler handler =
        new CommitterEventHandler(appContext, committer, heartbeatHandler);
    dispatcher.register(CommitterEventType.class, handler);
    return handler;
  }
예제 #2
0
 public void fromProto(DAGSubmittedProto proto) {
   this.dagID = TezDAGID.fromString(proto.getDagId());
   this.dagPlan = proto.getDagPlan();
   this.dagName = this.dagPlan.getName();
   this.submitTime = proto.getSubmitTime();
   this.applicationAttemptId =
       ConverterUtils.toApplicationAttemptId(proto.getApplicationAttemptId());
   if (proto.hasCumulativeAdditionalAmResources()) {
     this.cumulativeAdditionalLocalResources =
         DagTypeConverters.convertFromPlanLocalResources(
             proto.getCumulativeAdditionalAmResources());
   }
 }
예제 #3
0
 private ApplicationAttemptStateData createAttemptState(String itemName, byte[] data)
     throws IOException {
   ApplicationAttemptId attemptId = ConverterUtils.toApplicationAttemptId(itemName);
   ApplicationAttemptStateDataPBImpl attemptState =
       new ApplicationAttemptStateDataPBImpl(ApplicationAttemptStateDataProto.parseFrom(data));
   if (!attemptId.equals(attemptState.getAttemptId())) {
     throw new YarnRuntimeException(
         "The database entry for "
             + attemptId
             + " contains data for "
             + attemptState.getAttemptId());
   }
   return attemptState;
 }
 @Override
 public synchronized void storeApplicationAttemptState(
     String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB) throws Exception {
   ApplicationAttemptId appAttemptId = ConverterUtils.toApplicationAttemptId(attemptId);
   Path appDirPath = getAppDir(rmAppRoot, appAttemptId.getApplicationId().toString());
   Path nodeCreatePath = getNodePath(appDirPath, attemptId);
   LOG.info("Storing info for attempt: " + attemptId + " at: " + nodeCreatePath);
   byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
   try {
     // currently throw all exceptions. May need to respond differently for HA
     // based on whether we have lost the right to write to FS
     writeFile(nodeCreatePath, attemptStateData);
   } catch (Exception e) {
     LOG.info("Error storing info for attempt: " + attemptId, e);
     throw e;
   }
 }
예제 #5
0
 public void fromProto(AMStartedProto proto) {
   this.applicationAttemptId =
       ConverterUtils.toApplicationAttemptId(proto.getApplicationAttemptId());
   this.startTime = proto.getStartTime();
 }
  private void loadRMAppState(RMState rmState) throws Exception {
    try {
      List<ApplicationAttemptState> attempts = new ArrayList<ApplicationAttemptState>();

      for (FileStatus appDir : fs.listStatus(rmAppRoot)) {
        for (FileStatus childNodeStatus : fs.listStatus(appDir.getPath())) {
          assert childNodeStatus.isFile();
          String childNodeName = childNodeStatus.getPath().getName();
          byte[] childData = readFile(childNodeStatus.getPath(), childNodeStatus.getLen());
          if (childNodeName.startsWith(ApplicationId.appIdStrPrefix)) {
            // application
            LOG.info("Loading application from node: " + childNodeName);
            ApplicationId appId = ConverterUtils.toApplicationId(childNodeName);
            ApplicationStateDataPBImpl appStateData =
                new ApplicationStateDataPBImpl(ApplicationStateDataProto.parseFrom(childData));
            ApplicationState appState =
                new ApplicationState(
                    appStateData.getSubmitTime(),
                    appStateData.getApplicationSubmissionContext(),
                    appStateData.getUser());
            // assert child node name is same as actual applicationId
            assert appId.equals(appState.context.getApplicationId());
            rmState.appState.put(appId, appState);
          } else if (childNodeName.startsWith(ApplicationAttemptId.appAttemptIdStrPrefix)) {
            // attempt
            LOG.info("Loading application attempt from node: " + childNodeName);
            ApplicationAttemptId attemptId = ConverterUtils.toApplicationAttemptId(childNodeName);
            ApplicationAttemptStateDataPBImpl attemptStateData =
                new ApplicationAttemptStateDataPBImpl(
                    ApplicationAttemptStateDataProto.parseFrom(childData));
            Credentials credentials = null;
            if (attemptStateData.getAppAttemptTokens() != null) {
              credentials = new Credentials();
              DataInputByteBuffer dibb = new DataInputByteBuffer();
              dibb.reset(attemptStateData.getAppAttemptTokens());
              credentials.readTokenStorageStream(dibb);
            }
            ApplicationAttemptState attemptState =
                new ApplicationAttemptState(
                    attemptId, attemptStateData.getMasterContainer(), credentials);

            // assert child node name is same as application attempt id
            assert attemptId.equals(attemptState.getAttemptId());
            attempts.add(attemptState);
          } else {
            LOG.info("Unknown child node with name: " + childNodeName);
          }
        }
      }

      // go through all attempts and add them to their apps, Ideally, each
      // attempt node must have a corresponding app node, because remove
      // directory operation remove both at the same time
      for (ApplicationAttemptState attemptState : attempts) {
        ApplicationId appId = attemptState.getAttemptId().getApplicationId();
        ApplicationState appState = rmState.appState.get(appId);
        assert appState != null;
        appState.attempts.put(attemptState.getAttemptId(), attemptState);
      }
    } catch (Exception e) {
      LOG.error("Failed to load state.", e);
      throw e;
    }
  }
예제 #7
0
  public static void main(String[] args) throws Exception {
    System.err.println("Inside the AM!!!");
    LOG.info("Starting the AM!!!!");

    Options opts = new Options();
    opts.addOption(
        "app_attempt_id", true, "App Attempt ID. Not to be used " + "unless for testing purposes");

    CommandLine cl = new GnuParser().parse(opts, args);

    Map<String, String> envs = System.getenv();
    _appAttemptID = Records.newRecord(ApplicationAttemptId.class);
    if (cl.hasOption("app_attempt_id")) {
      String appIdStr = cl.getOptionValue("app_attempt_id", "");
      _appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
    } else if (envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) {
      ContainerId containerId =
          ConverterUtils.toContainerId(envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV));
      _appAttemptID = containerId.getApplicationAttemptId();
    } else {
      throw new IllegalArgumentException(
          "Container and application attempt IDs not set in the environment");
    }

    @SuppressWarnings("rawtypes")
    Map storm_conf = Config.readStormConfig(null);
    YarnConfiguration hadoopConf = new YarnConfiguration();

    StormAMRMClient client = new StormAMRMClient(_appAttemptID, storm_conf, hadoopConf);
    client.init(hadoopConf);
    client.start();

    BlockingQueue<Container> launcherQueue = new LinkedBlockingQueue<Container>();

    try {
      InetSocketAddress addr =
          NetUtils.createSocketAddr(
              hadoopConf.get(
                  YarnConfiguration.RM_SCHEDULER_ADDRESS,
                  YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS));
      int port = Utils.getInt(storm_conf.get(Config.MASTER_THRIFT_PORT));
      RegisterApplicationMasterResponse resp =
          client.registerApplicationMaster(addr.getHostName(), port, null);
      LOG.info("Got a registration response " + resp);
      LOG.info("Max Capability " + resp.getMaximumResourceCapability());
      client.setMaxResource(resp.getMaximumResourceCapability());
      MasterServer server = new MasterServer(storm_conf, client);
      LOG.info("Starting HB thread");
      initAndStartHeartbeat(
          client, launcherQueue, (Integer) storm_conf.get(Config.MASTER_HEARTBEAT_INTERVAL_MILLIS));
      LOG.info("Starting launcher");
      initAndStartLauncher(client, launcherQueue);
      client.startAllSupervisors();
      server.serve();
      client.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "AllDone", null);
    } finally {
      client.stop();
    }
    System.err.println("See Ya!!!");
    System.exit(0);
  }
  /**
   * Parse command line options
   *
   * @param args Command line args
   * @return Whether init successful and run should be invoked
   * @throws ParseException
   * @throws IOException
   */
  public boolean init(String[] args) throws ParseException, IOException {
    Options opts = new Options();
    opts.addOption(
        "app_attempt_id", true, "App Attempt ID. Not to be used unless for testing purposes");
    opts.addOption(
        "shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs");
    opts.addOption(
        "container_memory",
        true,
        "Amount of memory in MB to be requested to run the shell command");
    opts.addOption(
        "container_vcores",
        true,
        "Amount of virtual cores to be requested to run the shell command");
    opts.addOption(
        "num_containers",
        true,
        "No. of containers on which the shell command needs to be executed");
    opts.addOption("priority", true, "Application Priority. Default 0");
    opts.addOption("debug", false, "Dump out debug information");

    opts.addOption("help", false, "Print usage");
    CommandLine cliParser = new GnuParser().parse(opts, args);

    if (args.length == 0) {
      printUsage(opts);
      throw new IllegalArgumentException("No args specified for application master to initialize");
    }

    // Check whether customer log4j.properties file exists
    if (fileExist(log4jPath)) {
      try {
        Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, log4jPath);
      } catch (Exception e) {
        LOG.warn("Can not set up custom log4j properties. " + e);
      }
    }

    if (cliParser.hasOption("help")) {
      printUsage(opts);
      return false;
    }

    if (cliParser.hasOption("debug")) {
      dumpOutDebugInfo();
    }

    Map<String, String> envs = System.getenv();

    if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
      if (cliParser.hasOption("app_attempt_id")) {
        String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
        appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
      } else {
        throw new IllegalArgumentException("Application Attempt Id not set in the environment");
      }
    } else {
      ContainerId containerId =
          ConverterUtils.toContainerId(envs.get(Environment.CONTAINER_ID.name()));
      appAttemptID = containerId.getApplicationAttemptId();
    }

    if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) {
      throw new RuntimeException(
          ApplicationConstants.APP_SUBMIT_TIME_ENV + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_HOST.name())) {
      throw new RuntimeException(Environment.NM_HOST.name() + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) {
      throw new RuntimeException(Environment.NM_HTTP_PORT + " not set in the environment");
    }
    if (!envs.containsKey(Environment.NM_PORT.name())) {
      throw new RuntimeException(Environment.NM_PORT.name() + " not set in the environment");
    }

    LOG.info(
        "Application master for app"
            + ", appId="
            + appAttemptID.getApplicationId().getId()
            + ", clustertimestamp="
            + appAttemptID.getApplicationId().getClusterTimestamp()
            + ", attemptId="
            + appAttemptID.getAttemptId());

    if (!fileExist(shellCommandPath)
        && envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION).isEmpty()) {
      throw new IllegalArgumentException(
          "No shell command or shell script specified to be executed by application master");
    }

    if (fileExist(shellCommandPath)) {
      shellCommand = readContent(shellCommandPath);
    }

    if (fileExist(shellArgsPath)) {
      shellArgs = readContent(shellArgsPath);
    }

    if (cliParser.hasOption("shell_env")) {
      String shellEnvs[] = cliParser.getOptionValues("shell_env");
      for (String env : shellEnvs) {
        env = env.trim();
        int index = env.indexOf('=');
        if (index == -1) {
          shellEnv.put(env, "");
          continue;
        }
        String key = env.substring(0, index);
        String val = "";
        if (index < (env.length() - 1)) {
          val = env.substring(index + 1);
        }
        shellEnv.put(key, val);
      }
    }

    if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION)) {
      scriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION);

      if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)) {
        shellScriptPathTimestamp =
            Long.valueOf(envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP));
      }
      if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN)) {
        shellScriptPathLen = Long.valueOf(envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN));
      }

      if (!scriptPath.isEmpty() && (shellScriptPathTimestamp <= 0 || shellScriptPathLen <= 0)) {
        LOG.error(
            "Illegal values in env for shell script path"
                + ", path="
                + scriptPath
                + ", len="
                + shellScriptPathLen
                + ", timestamp="
                + shellScriptPathTimestamp);
        throw new IllegalArgumentException("Illegal values in env for shell script path");
      }
    }

    containerMemory = Integer.parseInt(cliParser.getOptionValue("container_memory", "10"));
    containerVirtualCores = Integer.parseInt(cliParser.getOptionValue("container_vcores", "1"));
    numTotalContainers = Integer.parseInt(cliParser.getOptionValue("num_containers", "1"));
    if (numTotalContainers == 0) {
      throw new IllegalArgumentException("Cannot run distributed shell with no containers");
    }
    requestPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0"));

    // Creating the Timeline Client
    timelineClient = TimelineClient.createTimelineClient();
    timelineClient.init(conf);
    timelineClient.start();

    return true;
  }