Beispiel #1
0
 /**
  * Get job corresponding to jobid.
  *
  * @param jobId
  * @return object of {@link Job}
  * @throws IOException
  * @throws InterruptedException
  */
 public Job getJob(JobID jobId) throws IOException, InterruptedException {
   JobStatus status = client.getJobStatus(jobId);
   if (status != null) {
     return Job.getInstance(this, status, new JobConf(status.getJobFile()));
   }
   return null;
 }
  public void getTaskAttempt(String job, String taskid) {
    // -logs
    try {
      JobID jobid = JobID.forName(job);
      TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskid);

      StringBuilder sb = new StringBuilder();

      LogParams logParams = client.getLogFileParams(jobid, taskAttemptID);
      LogCLIHelpers logDumper = new LogCLIHelpers();
      logDumper.setConf(conf);

      sb.append(logParams.getApplicationId());
      sb.append("|" + logParams.getContainerId());
      sb.append("|" + logParams.getNodeId());
      sb.append("|" + logParams.getOwner());
      System.out.println(sb.toString());

      logDumper.dumpAContainersLogs(
          logParams.getApplicationId(),
          logParams.getContainerId(),
          logParams.getNodeId(),
          logParams.getOwner());
    } catch (Exception e) {
      System.out.println(e);
    }
  }
Beispiel #3
0
 /**
  * Renew a delegation token
  *
  * @param token the token to renew
  * @return the new expiration time
  * @throws InvalidToken
  * @throws IOException
  * @deprecated Use {@link Token#renew} instead
  */
 public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
     throws InvalidToken, IOException, InterruptedException {
   try {
     return client.renewDelegationToken(token);
   } catch (RemoteException re) {
     throw re.unwrapRemoteException(InvalidToken.class, AccessControlException.class);
   }
 }
Beispiel #4
0
 /**
  * Cancel a delegation token from the JobTracker
  *
  * @param token the token to cancel
  * @throws IOException
  * @deprecated Use {@link Token#cancel} instead
  */
 public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
     throws IOException, InterruptedException {
   try {
     client.cancelDelegationToken(token);
   } catch (RemoteException re) {
     throw re.unwrapRemoteException(InvalidToken.class, AccessControlException.class);
   }
 }
 public ClusterMetrics getClusterMetrics() {
   ClusterMetrics cluster = null;
   try {
     cluster = client.getClusterMetrics();
   } catch (Exception e) {
     System.out.println(e);
   }
   return cluster;
 }
  public void getTaskDiagnostics(String job) {

    try {
      String errinfo[];
      StringBuilder sb = new StringBuilder();

      JobID jobid = JobID.forName(job);
      JobStatus status = client.getJobStatus(jobid);
      System.out.println(status.getTrackingUrl());
      System.out.println(status.getJobFile());
      System.out.println(status.getHistoryFile());
      System.out.println(status.getQueue());
      System.out.println(status.toString());

      System.out.println("#### QueueAclsInfo ####");
      QueueAclsInfo[] qais = client.getQueueAclsForCurrentUser();
      for (int i = 0; i < qais.length; i++) {
        QueueAclsInfo qai = qais[i];
        System.out.println(qai.getQueueName().toString());
        String[] ops = qai.getOperations();
        for (int j = 0; j < ops.length; j++) {
          System.out.println(ops[j]);
        }
      }

      System.out.println("#### QueueInfo ####");
      QueueInfo[] qis = client.getRootQueues();
      for (int i = 0; i < qis.length; i++) {
        QueueInfo qi = qis[i];
        System.out.println(qi.getQueueName());
        System.out.println(qi.getSchedulingInfo());
        System.out.println(qi.getState());
        System.out.println(qi.getProperties());
      }

      int mapnum = client.getTaskReports(jobid, TaskType.MAP).length;
      int rednum = client.getTaskReports(jobid, TaskType.REDUCE).length;
      float progress = status.getMapProgress();
      sb.append(
          String.format(
              "m %4d %6s",
              mapnum, progress == 1 ? "100%" : String.format("%.2f%%", progress * 100)));
      sb.append("|");
      progress = status.getReduceProgress();
      sb.append(
          String.format(
              "r %4d %6s",
              rednum, progress == 1 ? "100%" : String.format("%.2f%%", progress * 100)));

      System.out.println(sb.toString());
      printTaskAttempt(client.getTaskReports(jobid, TaskType.MAP));
      printTaskAttempt(client.getTaskReports(jobid, TaskType.REDUCE));
      System.out.println(sb.toString());

    } catch (Exception e) {
      System.out.println(e);
    }
  }
Beispiel #7
0
 /**
  * Get a delegation token for the user from the JobTracker.
  *
  * @param renewer the user who can renew the token
  * @return the new token
  * @throws IOException
  */
 public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
     throws IOException, InterruptedException {
   Token<DelegationTokenIdentifier> result = client.getDelegationToken(renewer);
   InetSocketAddress addr = Master.getMasterAddress(conf);
   StringBuilder service = new StringBuilder();
   service.append(NetUtils.normalizeHostName(addr.getAddress().getHostAddress()));
   service.append(':');
   service.append(addr.getPort());
   result.setService(new Text(service.toString()));
   return result;
 }
 /**
  * Get job corresponding to jobid.
  *
  * @param jobId
  * @return object of {@link Job}
  * @throws IOException
  * @throws InterruptedException
  */
 public Job getJob(JobID jobId) throws IOException, InterruptedException {
   JobStatus status = client.getJobStatus(jobId);
   if (status != null) {
     JobConf conf;
     try {
       conf = new JobConf(status.getJobFile());
     } catch (RuntimeException ex) {
       // If job file doesn't exist it means we can't find the job
       if (ex.getCause() instanceof FileNotFoundException) {
         return null;
       } else {
         throw ex;
       }
     }
     return Job.getInstance(this, status, conf);
   }
   return null;
 }
  public void displayAllJobs() throws Exception {
    try {

      JobStatus[] job = client.getAllJobs();

      for (int i = 0; i < job.length; i++) {

        StringBuilder sb = new StringBuilder();
        java.text.SimpleDateFormat sdf = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
        java.util.Date dt = new java.util.Date(job[i].getStartTime());
        String sDateTime = sdf.format(dt);

        sb.append(sDateTime);
        // int mapnum = client.getTaskReports(job[i].getJobID(), TaskType.MAP).length;
        // int rednum = client.getTaskReports(job[i].getJobID(), TaskType.MAP).length;
        // sb.append("|" + mapnum + "-" + rednum);
        sb.append("|" + job[i].getJobID().toString());
        sb.append("|" + String.format("%-10s", job[i].getState()));
        sb.append("|" + String.format("%-8s", job[i].getUsername()));
        sb.append("|" + String.format("%-8s", job[i].getQueue()));
        // JobStatus status=client.getJobStatus( job[i].getJobID() );
        // sb.append(
        // "|"+String.format("m %.2f",status.getMapProgress()) );
        // sb.append(
        // "|"+String.format("r %.2f",status.getReduceProgress()) );
        String jobname = job[i].getJobName();

        Pattern p = Pattern.compile("[\t|\r|\n]*");
        Matcher m = p.matcher(jobname);
        jobname = m.replaceAll("");
        sb.append("|" + jobname);
        String failInfo = job[i].getFailureInfo();
        if (!failInfo.isEmpty()) sb.append("|" + failInfo);

        System.out.println(sb.toString());
        // LOG.info( sb.toString() );
      }

    } catch (Exception e) {
      System.out.println(e);
    }
  }
Beispiel #10
0
  /**
   * Internal method for submitting jobs to the system.
   *
   * <p>The job submission process involves:
   *
   * <ol>
   *   <li>Checking the input and output specifications of the job.
   *   <li>Computing the {@link InputSplit}s for the job.
   *   <li>Setup the requisite accounting information for the {@link DistributedCache} of the job,
   *       if necessary.
   *   <li>Copying the job's jar and configuration to the map-reduce system directory on the
   *       distributed file-system.
   *   <li>Submitting the job to the <code>JobTracker</code> and optionally monitoring it's status.
   * </ol>
   *
   * @param job the configuration to submit
   * @param cluster the handle to the Cluster
   * @throws ClassNotFoundException
   * @throws InterruptedException
   * @throws IOException
   */
  JobStatus submitJobInternal(Job job, Cluster cluster)
      throws ClassNotFoundException, InterruptedException, IOException {

    // validate the jobs output specs
    //// check use new api or not. check output valid or invalid.
    checkSpecs(job);

    //// usually, /tmp/username as staging area.
    Path jobStagingArea = JobSubmissionFiles.getStagingDir(cluster, job.getConfiguration());
    // configure the command line options correctly on the submitting dfs
    Configuration conf = job.getConfiguration();
    InetAddress ip = InetAddress.getLocalHost();
    if (ip != null) {
      submitHostAddress = ip.getHostAddress();
      submitHostName = ip.getHostName();
      conf.set(MRJobConfig.JOB_SUBMITHOST, submitHostName);
      conf.set(MRJobConfig.JOB_SUBMITHOSTADDR, submitHostAddress);
    }
    JobID jobId = submitClient.getNewJobID();
    job.setJobID(jobId);
    Path submitJobDir = new Path(jobStagingArea, jobId.toString());
    JobStatus status = null;
    try {
      conf.set(
          "hadoop.http.filter.initializers",
          "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer");

      //// set mapreduce_job_dir as job staging tmp dir.
      conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, submitJobDir.toString());
      LOG.debug("Configuring job " + jobId + " with " + submitJobDir + " as the submit dir");
      // get delegation token for the dir

      // secret be stored in TokenCache before job be submitted. and get secret during job running.
      //// this is a static method.
      //// secret stored before submission and read during task.
      TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] {submitJobDir}, conf);

      //// get secret keys and store them into TokenCache.
      populateTokenCache(conf, job.getCredentials());

      //// copy lib、files... into submitJobDir
      copyAndConfigureFiles(job, submitJobDir);
      //// change job.xml dir to Path.
      Path submitJobFile = JobSubmissionFiles.getJobConfPath(submitJobDir);

      // Create the splits for the job
      LOG.debug("Creating splits at " + jtFs.makeQualified(submitJobDir));
      //// InputFormat.getSplits()
      int maps = writeSplits(job, submitJobDir);
      conf.setInt(MRJobConfig.NUM_MAPS, maps);
      LOG.info("number of splits:" + maps);

      // write "queue admins of the queue to which job is being submitted"
      // to job file.
      String queue = conf.get(MRJobConfig.QUEUE_NAME, JobConf.DEFAULT_QUEUE_NAME);
      AccessControlList acl = submitClient.getQueueAdmins(queue);
      conf.set(
          toFullPropertyName(queue, QueueACL.ADMINISTER_JOBS.getAclName()), acl.getAclString());

      // removing jobtoken referrals before copying the jobconf to HDFS
      // as the tasks don't need this setting, actually they may break
      // because of it if present as the referral will point to a
      // different job.
      TokenCache.cleanUpTokenReferral(conf);

      // Write job file to submit dir
      writeConf(conf, submitJobFile);

      //
      // Now, actually submit the job (using the submit name)
      //
      printTokens(jobId, job.getCredentials());
      //// find implement usage of submitJob, result: YARNRunner and LocalJobRunner
      status = submitClient.submitJob(jobId, submitJobDir.toString(), job.getCredentials());
      if (status != null) {
        return status;
      } else {
        throw new IOException("Could not launch job");
      }
    } finally {
      if (status == null) {
        LOG.info("Cleaning up the staging area " + submitJobDir);
        if (jtFs != null && submitJobDir != null) jtFs.delete(submitJobDir, true);
      }
    }
  }
Beispiel #11
0
 /**
  * Get the tasktracker expiry interval for the cluster
  *
  * @return the expiry interval in msec
  */
 public long getTaskTrackerExpiryInterval() throws IOException, InterruptedException {
   return client.getTaskTrackerExpiryInterval();
 }
Beispiel #12
0
 /**
  * Get the JobTracker's status.
  *
  * @return {@link JobTrackerStatus} of the JobTracker
  * @throws IOException
  * @throws InterruptedException
  */
 public JobTrackerStatus getJobTrackerStatus() throws IOException, InterruptedException {
   return client.getJobTrackerStatus();
 }
Beispiel #13
0
 /**
  * Returns immediate children of queueName.
  *
  * @param queueName
  * @return array of JobQueueInfo which are children of queueName
  * @throws IOException
  */
 public QueueInfo[] getChildQueues(String queueName) throws IOException, InterruptedException {
   return client.getChildQueues(queueName);
 }
Beispiel #14
0
 /**
  * Gets the root level queues.
  *
  * @return array of JobQueueInfo object.
  * @throws IOException
  */
 public QueueInfo[] getRootQueues() throws IOException, InterruptedException {
   return client.getRootQueues();
 }
Beispiel #15
0
 /**
  * Gets the Queue ACLs for current user
  *
  * @return array of QueueAclsInfo object for current user.
  * @throws IOException
  */
 public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException, InterruptedException {
   return client.getQueueAclsForCurrentUser();
 }
Beispiel #16
0
 /**
  * Get queue information for the specified name.
  *
  * @param name queuename
  * @return object of {@link QueueInfo}
  * @throws IOException
  * @throws InterruptedException
  */
 public QueueInfo getQueue(String name) throws IOException, InterruptedException {
   return client.getQueue(name);
 }
Beispiel #17
0
 /**
  * Grab the jobtracker's view of the staging directory path where job-specific files will be
  * placed.
  *
  * @return the staging directory where job-specific files are to be placed.
  */
 public Path getStagingAreaDir() throws IOException, InterruptedException {
   if (stagingAreaDir == null) {
     stagingAreaDir = new Path(client.getStagingAreaDir());
   }
   return stagingAreaDir;
 }
Beispiel #18
0
 /**
  * Grab the jobtracker system directory path where job-specific files will be placed.
  *
  * @return the system directory where job-specific files are to be placed.
  */
 public Path getSystemDir() throws IOException, InterruptedException {
   if (sysDir == null) {
     sysDir = new Path(client.getSystemDir());
   }
   return sysDir;
 }
Beispiel #19
0
 /**
  * Get job status for all jobs in the cluster.
  *
  * @return job status for all jobs in cluster
  * @throws IOException
  * @throws InterruptedException
  */
 public JobStatus[] getAllJobStatuses() throws IOException, InterruptedException {
   return client.getAllJobs();
 }
 /**
  * Get log parameters for the specified jobID or taskAttemptID
  *
  * @param jobID the job id.
  * @param taskAttemptID the task attempt id. Optional.
  * @return the LogParams
  * @throws IOException
  * @throws InterruptedException
  */
 public LogParams getLogParams(JobID jobID, TaskAttemptID taskAttemptID)
     throws IOException, InterruptedException {
   return client.getLogFileParams(jobID, taskAttemptID);
 }
Beispiel #21
0
 /**
  * Get blacklisted trackers.
  *
  * @return array of {@link TaskTrackerInfo}
  * @throws IOException
  * @throws InterruptedException
  */
 public TaskTrackerInfo[] getBlackListedTaskTrackers() throws IOException, InterruptedException {
   return client.getBlacklistedTrackers();
 }
Beispiel #22
0
 /**
  * Get all active trackers in the cluster.
  *
  * @return array of {@link TaskTrackerInfo}
  * @throws IOException
  * @throws InterruptedException
  */
 public TaskTrackerInfo[] getActiveTaskTrackers() throws IOException, InterruptedException {
   return client.getActiveTrackers();
 }
Beispiel #23
0
 /**
  * Get current cluster status.
  *
  * @return object of {@link ClusterMetrics}
  * @throws IOException
  * @throws InterruptedException
  */
 public ClusterMetrics getClusterStatus() throws IOException, InterruptedException {
   return client.getClusterMetrics();
 }
 /**
  * Get a delegation token for the user from the JobTracker.
  *
  * @param renewer the user who can renew the token
  * @return the new token
  * @throws IOException
  */
 public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
     throws IOException, InterruptedException {
   // client has already set the service
   return client.getDelegationToken(renewer);
 }
Beispiel #25
0
 /**
  * Get all the jobs in cluster.
  *
  * @return array of {@link Job}
  * @throws IOException
  * @throws InterruptedException
  * @deprecated Use {@link #getAllJobStatuses()} instead.
  */
 @Deprecated
 public Job[] getAllJobs() throws IOException, InterruptedException {
   return getJobs(client.getAllJobs());
 }
Beispiel #26
0
 /**
  * Get the job history file path for a given job id. The job history file at this path may or may
  * not be existing depending on the job completion state. The file is present only for the
  * completed jobs.
  *
  * @param jobId the JobID of the job submitted by the current user.
  * @return the file path of the job history file
  * @throws IOException
  * @throws InterruptedException
  */
 public String getJobHistoryUrl(JobID jobId) throws IOException, InterruptedException {
   if (jobHistoryDir == null) {
     jobHistoryDir = new Path(client.getJobHistoryDir());
   }
   return new Path(jobHistoryDir, jobId.toString() + "_" + ugi.getShortUserName()).toString();
 }