/**
* Get job corresponding to jobid.
*
* @param jobId
* @return object of {@link Job}
* @throws IOException
* @throws InterruptedException
*/
public Job getJob(JobID jobId) throws IOException, InterruptedException {
JobStatus status = client.getJobStatus(jobId);
if (status != null) {
return Job.getInstance(this, status, new JobConf(status.getJobFile()));
}
return null;
}
public void getTaskAttempt(String job, String taskid) {
// -logs
try {
JobID jobid = JobID.forName(job);
TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskid);
StringBuilder sb = new StringBuilder();
LogParams logParams = client.getLogFileParams(jobid, taskAttemptID);
LogCLIHelpers logDumper = new LogCLIHelpers();
logDumper.setConf(conf);
sb.append(logParams.getApplicationId());
sb.append("|" + logParams.getContainerId());
sb.append("|" + logParams.getNodeId());
sb.append("|" + logParams.getOwner());
System.out.println(sb.toString());
logDumper.dumpAContainersLogs(
logParams.getApplicationId(),
logParams.getContainerId(),
logParams.getNodeId(),
logParams.getOwner());
} catch (Exception e) {
System.out.println(e);
}
}
/**
* Renew a delegation token
*
* @param token the token to renew
* @return the new expiration time
* @throws InvalidToken
* @throws IOException
* @deprecated Use {@link Token#renew} instead
*/
public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
throws InvalidToken, IOException, InterruptedException {
try {
return client.renewDelegationToken(token);
} catch (RemoteException re) {
throw re.unwrapRemoteException(InvalidToken.class, AccessControlException.class);
}
}
/**
* Cancel a delegation token from the JobTracker
*
* @param token the token to cancel
* @throws IOException
* @deprecated Use {@link Token#cancel} instead
*/
public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
throws IOException, InterruptedException {
try {
client.cancelDelegationToken(token);
} catch (RemoteException re) {
throw re.unwrapRemoteException(InvalidToken.class, AccessControlException.class);
}
}
public ClusterMetrics getClusterMetrics() {
ClusterMetrics cluster = null;
try {
cluster = client.getClusterMetrics();
} catch (Exception e) {
System.out.println(e);
}
return cluster;
}
public void getTaskDiagnostics(String job) {
try {
String errinfo[];
StringBuilder sb = new StringBuilder();
JobID jobid = JobID.forName(job);
JobStatus status = client.getJobStatus(jobid);
System.out.println(status.getTrackingUrl());
System.out.println(status.getJobFile());
System.out.println(status.getHistoryFile());
System.out.println(status.getQueue());
System.out.println(status.toString());
System.out.println("#### QueueAclsInfo ####");
QueueAclsInfo[] qais = client.getQueueAclsForCurrentUser();
for (int i = 0; i < qais.length; i++) {
QueueAclsInfo qai = qais[i];
System.out.println(qai.getQueueName().toString());
String[] ops = qai.getOperations();
for (int j = 0; j < ops.length; j++) {
System.out.println(ops[j]);
}
}
System.out.println("#### QueueInfo ####");
QueueInfo[] qis = client.getRootQueues();
for (int i = 0; i < qis.length; i++) {
QueueInfo qi = qis[i];
System.out.println(qi.getQueueName());
System.out.println(qi.getSchedulingInfo());
System.out.println(qi.getState());
System.out.println(qi.getProperties());
}
int mapnum = client.getTaskReports(jobid, TaskType.MAP).length;
int rednum = client.getTaskReports(jobid, TaskType.REDUCE).length;
float progress = status.getMapProgress();
sb.append(
String.format(
"m %4d %6s",
mapnum, progress == 1 ? "100%" : String.format("%.2f%%", progress * 100)));
sb.append("|");
progress = status.getReduceProgress();
sb.append(
String.format(
"r %4d %6s",
rednum, progress == 1 ? "100%" : String.format("%.2f%%", progress * 100)));
System.out.println(sb.toString());
printTaskAttempt(client.getTaskReports(jobid, TaskType.MAP));
printTaskAttempt(client.getTaskReports(jobid, TaskType.REDUCE));
System.out.println(sb.toString());
} catch (Exception e) {
System.out.println(e);
}
}
/**
* Get a delegation token for the user from the JobTracker.
*
* @param renewer the user who can renew the token
* @return the new token
* @throws IOException
*/
public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
throws IOException, InterruptedException {
Token<DelegationTokenIdentifier> result = client.getDelegationToken(renewer);
InetSocketAddress addr = Master.getMasterAddress(conf);
StringBuilder service = new StringBuilder();
service.append(NetUtils.normalizeHostName(addr.getAddress().getHostAddress()));
service.append(':');
service.append(addr.getPort());
result.setService(new Text(service.toString()));
return result;
}
/**
* Get job corresponding to jobid.
*
* @param jobId
* @return object of {@link Job}
* @throws IOException
* @throws InterruptedException
*/
public Job getJob(JobID jobId) throws IOException, InterruptedException {
JobStatus status = client.getJobStatus(jobId);
if (status != null) {
JobConf conf;
try {
conf = new JobConf(status.getJobFile());
} catch (RuntimeException ex) {
// If job file doesn't exist it means we can't find the job
if (ex.getCause() instanceof FileNotFoundException) {
return null;
} else {
throw ex;
}
}
return Job.getInstance(this, status, conf);
}
return null;
}
public void displayAllJobs() throws Exception {
try {
JobStatus[] job = client.getAllJobs();
for (int i = 0; i < job.length; i++) {
StringBuilder sb = new StringBuilder();
java.text.SimpleDateFormat sdf = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
java.util.Date dt = new java.util.Date(job[i].getStartTime());
String sDateTime = sdf.format(dt);
sb.append(sDateTime);
// int mapnum = client.getTaskReports(job[i].getJobID(), TaskType.MAP).length;
// int rednum = client.getTaskReports(job[i].getJobID(), TaskType.MAP).length;
// sb.append("|" + mapnum + "-" + rednum);
sb.append("|" + job[i].getJobID().toString());
sb.append("|" + String.format("%-10s", job[i].getState()));
sb.append("|" + String.format("%-8s", job[i].getUsername()));
sb.append("|" + String.format("%-8s", job[i].getQueue()));
// JobStatus status=client.getJobStatus( job[i].getJobID() );
// sb.append(
// "|"+String.format("m %.2f",status.getMapProgress()) );
// sb.append(
// "|"+String.format("r %.2f",status.getReduceProgress()) );
String jobname = job[i].getJobName();
Pattern p = Pattern.compile("[\t|\r|\n]*");
Matcher m = p.matcher(jobname);
jobname = m.replaceAll("");
sb.append("|" + jobname);
String failInfo = job[i].getFailureInfo();
if (!failInfo.isEmpty()) sb.append("|" + failInfo);
System.out.println(sb.toString());
// LOG.info( sb.toString() );
}
} catch (Exception e) {
System.out.println(e);
}
}
/**
* Internal method for submitting jobs to the system.
*
* <p>The job submission process involves:
*
* <ol>
* <li>Checking the input and output specifications of the job.
* <li>Computing the {@link InputSplit}s for the job.
* <li>Setup the requisite accounting information for the {@link DistributedCache} of the job,
* if necessary.
* <li>Copying the job's jar and configuration to the map-reduce system directory on the
* distributed file-system.
* <li>Submitting the job to the <code>JobTracker</code> and optionally monitoring it's status.
* </ol>
*
* @param job the configuration to submit
* @param cluster the handle to the Cluster
* @throws ClassNotFoundException
* @throws InterruptedException
* @throws IOException
*/
JobStatus submitJobInternal(Job job, Cluster cluster)
throws ClassNotFoundException, InterruptedException, IOException {
// validate the jobs output specs
//// check use new api or not. check output valid or invalid.
checkSpecs(job);
//// usually, /tmp/username as staging area.
Path jobStagingArea = JobSubmissionFiles.getStagingDir(cluster, job.getConfiguration());
// configure the command line options correctly on the submitting dfs
Configuration conf = job.getConfiguration();
InetAddress ip = InetAddress.getLocalHost();
if (ip != null) {
submitHostAddress = ip.getHostAddress();
submitHostName = ip.getHostName();
conf.set(MRJobConfig.JOB_SUBMITHOST, submitHostName);
conf.set(MRJobConfig.JOB_SUBMITHOSTADDR, submitHostAddress);
}
JobID jobId = submitClient.getNewJobID();
job.setJobID(jobId);
Path submitJobDir = new Path(jobStagingArea, jobId.toString());
JobStatus status = null;
try {
conf.set(
"hadoop.http.filter.initializers",
"org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer");
//// set mapreduce_job_dir as job staging tmp dir.
conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, submitJobDir.toString());
LOG.debug("Configuring job " + jobId + " with " + submitJobDir + " as the submit dir");
// get delegation token for the dir
// secret be stored in TokenCache before job be submitted. and get secret during job running.
//// this is a static method.
//// secret stored before submission and read during task.
TokenCache.obtainTokensForNamenodes(job.getCredentials(), new Path[] {submitJobDir}, conf);
//// get secret keys and store them into TokenCache.
populateTokenCache(conf, job.getCredentials());
//// copy lib、files... into submitJobDir
copyAndConfigureFiles(job, submitJobDir);
//// change job.xml dir to Path.
Path submitJobFile = JobSubmissionFiles.getJobConfPath(submitJobDir);
// Create the splits for the job
LOG.debug("Creating splits at " + jtFs.makeQualified(submitJobDir));
//// InputFormat.getSplits()
int maps = writeSplits(job, submitJobDir);
conf.setInt(MRJobConfig.NUM_MAPS, maps);
LOG.info("number of splits:" + maps);
// write "queue admins of the queue to which job is being submitted"
// to job file.
String queue = conf.get(MRJobConfig.QUEUE_NAME, JobConf.DEFAULT_QUEUE_NAME);
AccessControlList acl = submitClient.getQueueAdmins(queue);
conf.set(
toFullPropertyName(queue, QueueACL.ADMINISTER_JOBS.getAclName()), acl.getAclString());
// removing jobtoken referrals before copying the jobconf to HDFS
// as the tasks don't need this setting, actually they may break
// because of it if present as the referral will point to a
// different job.
TokenCache.cleanUpTokenReferral(conf);
// Write job file to submit dir
writeConf(conf, submitJobFile);
//
// Now, actually submit the job (using the submit name)
//
printTokens(jobId, job.getCredentials());
//// find implement usage of submitJob, result: YARNRunner and LocalJobRunner
status = submitClient.submitJob(jobId, submitJobDir.toString(), job.getCredentials());
if (status != null) {
return status;
} else {
throw new IOException("Could not launch job");
}
} finally {
if (status == null) {
LOG.info("Cleaning up the staging area " + submitJobDir);
if (jtFs != null && submitJobDir != null) jtFs.delete(submitJobDir, true);
}
}
}
/**
* Get the tasktracker expiry interval for the cluster
*
* @return the expiry interval in msec
*/
public long getTaskTrackerExpiryInterval() throws IOException, InterruptedException {
return client.getTaskTrackerExpiryInterval();
}
/**
* Get the JobTracker's status.
*
* @return {@link JobTrackerStatus} of the JobTracker
* @throws IOException
* @throws InterruptedException
*/
public JobTrackerStatus getJobTrackerStatus() throws IOException, InterruptedException {
return client.getJobTrackerStatus();
}
/**
* Returns immediate children of queueName.
*
* @param queueName
* @return array of JobQueueInfo which are children of queueName
* @throws IOException
*/
public QueueInfo[] getChildQueues(String queueName) throws IOException, InterruptedException {
return client.getChildQueues(queueName);
}
/**
* Gets the root level queues.
*
* @return array of JobQueueInfo object.
* @throws IOException
*/
public QueueInfo[] getRootQueues() throws IOException, InterruptedException {
return client.getRootQueues();
}
/**
* Gets the Queue ACLs for current user
*
* @return array of QueueAclsInfo object for current user.
* @throws IOException
*/
public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException, InterruptedException {
return client.getQueueAclsForCurrentUser();
}
/**
* Get queue information for the specified name.
*
* @param name queuename
* @return object of {@link QueueInfo}
* @throws IOException
* @throws InterruptedException
*/
public QueueInfo getQueue(String name) throws IOException, InterruptedException {
return client.getQueue(name);
}
/**
* Grab the jobtracker's view of the staging directory path where job-specific files will be
* placed.
*
* @return the staging directory where job-specific files are to be placed.
*/
public Path getStagingAreaDir() throws IOException, InterruptedException {
if (stagingAreaDir == null) {
stagingAreaDir = new Path(client.getStagingAreaDir());
}
return stagingAreaDir;
}
/**
* Grab the jobtracker system directory path where job-specific files will be placed.
*
* @return the system directory where job-specific files are to be placed.
*/
public Path getSystemDir() throws IOException, InterruptedException {
if (sysDir == null) {
sysDir = new Path(client.getSystemDir());
}
return sysDir;
}
/**
* Get job status for all jobs in the cluster.
*
* @return job status for all jobs in cluster
* @throws IOException
* @throws InterruptedException
*/
public JobStatus[] getAllJobStatuses() throws IOException, InterruptedException {
return client.getAllJobs();
}
/**
* Get log parameters for the specified jobID or taskAttemptID
*
* @param jobID the job id.
* @param taskAttemptID the task attempt id. Optional.
* @return the LogParams
* @throws IOException
* @throws InterruptedException
*/
public LogParams getLogParams(JobID jobID, TaskAttemptID taskAttemptID)
throws IOException, InterruptedException {
return client.getLogFileParams(jobID, taskAttemptID);
}
/**
* Get blacklisted trackers.
*
* @return array of {@link TaskTrackerInfo}
* @throws IOException
* @throws InterruptedException
*/
public TaskTrackerInfo[] getBlackListedTaskTrackers() throws IOException, InterruptedException {
return client.getBlacklistedTrackers();
}
/**
* Get all active trackers in the cluster.
*
* @return array of {@link TaskTrackerInfo}
* @throws IOException
* @throws InterruptedException
*/
public TaskTrackerInfo[] getActiveTaskTrackers() throws IOException, InterruptedException {
return client.getActiveTrackers();
}
/**
* Get current cluster status.
*
* @return object of {@link ClusterMetrics}
* @throws IOException
* @throws InterruptedException
*/
public ClusterMetrics getClusterStatus() throws IOException, InterruptedException {
return client.getClusterMetrics();
}
/**
* Get a delegation token for the user from the JobTracker.
*
* @param renewer the user who can renew the token
* @return the new token
* @throws IOException
*/
public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
throws IOException, InterruptedException {
// client has already set the service
return client.getDelegationToken(renewer);
}
/**
* Get all the jobs in cluster.
*
* @return array of {@link Job}
* @throws IOException
* @throws InterruptedException
* @deprecated Use {@link #getAllJobStatuses()} instead.
*/
@Deprecated
public Job[] getAllJobs() throws IOException, InterruptedException {
return getJobs(client.getAllJobs());
}
/**
* Get the job history file path for a given job id. The job history file at this path may or may
* not be existing depending on the job completion state. The file is present only for the
* completed jobs.
*
* @param jobId the JobID of the job submitted by the current user.
* @return the file path of the job history file
* @throws IOException
* @throws InterruptedException
*/
public String getJobHistoryUrl(JobID jobId) throws IOException, InterruptedException {
if (jobHistoryDir == null) {
jobHistoryDir = new Path(client.getJobHistoryDir());
}
return new Path(jobHistoryDir, jobId.toString() + "_" + ugi.getShortUserName()).toString();
}