@Test
public void testRemoveAclExceedsQuota() throws Exception {
Path filePath = new Path(path, "file1");
Path fileSnapshotPath = new Path(snapshotPath, "file1");
FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0755));
hdfs.allowSnapshot(path);
hdfs.setQuota(path, 3, HdfsConstants.QUOTA_DONT_SET);
FileSystem.create(hdfs, filePath, FsPermission.createImmutable((short) 0600)).close();
hdfs.setPermission(filePath, FsPermission.createImmutable((short) 0600));
List<AclEntry> aclSpec = Lists.newArrayList(aclEntry(ACCESS, USER, "bruce", READ_WRITE));
hdfs.modifyAclEntries(filePath, aclSpec);
hdfs.createSnapshot(path, snapshotName);
AclStatus s = hdfs.getAclStatus(filePath);
AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(
new AclEntry[] {aclEntry(ACCESS, USER, "bruce", READ_WRITE), aclEntry(ACCESS, GROUP, NONE)},
returned);
assertPermission((short) 010660, filePath);
s = hdfs.getAclStatus(fileSnapshotPath);
returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(
new AclEntry[] {aclEntry(ACCESS, USER, "bruce", READ_WRITE), aclEntry(ACCESS, GROUP, NONE)},
returned);
assertPermission((short) 010660, filePath);
aclSpec = Lists.newArrayList(aclEntry(ACCESS, USER, "bruce", READ));
hdfs.removeAcl(filePath);
}
@Test
public void testModifyReadsCurrentState() throws Exception {
FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0700));
SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName);
List<AclEntry> aclSpec = Lists.newArrayList(aclEntry(ACCESS, USER, "bruce", ALL));
hdfs.modifyAclEntries(path, aclSpec);
aclSpec = Lists.newArrayList(aclEntry(ACCESS, USER, "diana", READ_EXECUTE));
hdfs.modifyAclEntries(path, aclSpec);
AclEntry[] expected =
new AclEntry[] {
aclEntry(ACCESS, USER, "bruce", ALL),
aclEntry(ACCESS, USER, "diana", READ_EXECUTE),
aclEntry(ACCESS, GROUP, NONE)
};
AclStatus s = hdfs.getAclStatus(path);
AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(expected, returned);
assertPermission((short) 010770, path);
assertDirPermissionGranted(fsAsBruce, BRUCE, path);
assertDirPermissionGranted(fsAsDiana, DIANA, path);
}
@Before
public void setUp() {
PermissionStatus permStatus =
PermissionStatus.createImmutable(
SUPERUSER, SUPERGROUP, FsPermission.createImmutable((short) 0755));
inodeRoot = new INodeDirectory(INodeId.ROOT_INODE_ID, INodeDirectory.ROOT_NAME, permStatus, 0L);
}
@Test
public void testRemoveAclSnapshotPath() throws Exception {
FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0700));
SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName);
exception.expect(SnapshotAccessControlException.class);
hdfs.removeAcl(snapshotPath);
}
@Test
public void testSetAclSnapshotPath() throws Exception {
FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0700));
SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName);
List<AclEntry> aclSpec = Lists.newArrayList(aclEntry(DEFAULT, USER, "bruce"));
exception.expect(SnapshotAccessControlException.class);
hdfs.setAcl(snapshotPath, aclSpec);
}
private static INodeDirectory createINodeDirectory(
INodeDirectory parent, String name, String owner, String group, short perm)
throws IOException {
PermissionStatus permStatus =
PermissionStatus.createImmutable(owner, group, FsPermission.createImmutable(perm));
INodeDirectory inodeDirectory =
new INodeDirectory(INodeId.GRANDFATHER_INODE_ID, name.getBytes("UTF-8"), permStatus, 0L);
parent.addChild(inodeDirectory);
return inodeDirectory;
}
@Test
public void testOriginalAclEnforcedForSnapshotRootAfterChange() throws Exception {
FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0700));
List<AclEntry> aclSpec =
Lists.newArrayList(
aclEntry(ACCESS, USER, ALL),
aclEntry(ACCESS, USER, "bruce", READ_EXECUTE),
aclEntry(ACCESS, GROUP, NONE),
aclEntry(ACCESS, OTHER, NONE));
hdfs.setAcl(path, aclSpec);
assertDirPermissionGranted(fsAsBruce, BRUCE, path);
assertDirPermissionDenied(fsAsDiana, DIANA, path);
SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName);
// Both original and snapshot still have same ACL.
AclStatus s = hdfs.getAclStatus(path);
AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(
new AclEntry[] {
aclEntry(ACCESS, USER, "bruce", READ_EXECUTE), aclEntry(ACCESS, GROUP, NONE)
},
returned);
assertPermission((short) 010750, path);
s = hdfs.getAclStatus(snapshotPath);
returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(
new AclEntry[] {
aclEntry(ACCESS, USER, "bruce", READ_EXECUTE), aclEntry(ACCESS, GROUP, NONE)
},
returned);
assertPermission((short) 010750, snapshotPath);
assertDirPermissionGranted(fsAsBruce, BRUCE, snapshotPath);
assertDirPermissionDenied(fsAsDiana, DIANA, snapshotPath);
aclSpec =
Lists.newArrayList(
aclEntry(ACCESS, USER, READ_EXECUTE),
aclEntry(ACCESS, USER, "diana", READ_EXECUTE),
aclEntry(ACCESS, GROUP, NONE),
aclEntry(ACCESS, OTHER, NONE));
hdfs.setAcl(path, aclSpec);
// Original has changed, but snapshot still has old ACL.
doSnapshotRootChangeAssertions(path, snapshotPath);
restart(false);
doSnapshotRootChangeAssertions(path, snapshotPath);
restart(true);
doSnapshotRootChangeAssertions(path, snapshotPath);
}
/**
* Test for {@link TFS#setPermission(Path, org.apache.hadoop.fs.permission.FsPermission)}. It will
* test changing the permission of file using TFS.
*/
@Test
public void chmodTest() throws Exception {
Path fileA = new Path("/chmodfileA");
create(sTFS, fileA);
FileStatus fs = sTFS.getFileStatus(fileA);
// Default permission should be 0644
Assert.assertEquals((short) 0644, fs.getPermission().toShort());
sTFS.setPermission(fileA, FsPermission.createImmutable((short) 0755));
Assert.assertEquals((short) 0755, sTFS.getFileStatus(fileA).getPermission().toShort());
}
private static INodeFile createINodeFile(
INodeDirectory parent, String name, String owner, String group, short perm)
throws IOException {
PermissionStatus permStatus =
PermissionStatus.createImmutable(owner, group, FsPermission.createImmutable(perm));
INodeFile inodeFile =
new INodeFile(
INodeId.GRANDFATHER_INODE_ID,
name.getBytes("UTF-8"),
permStatus,
0L,
0L,
null,
REPLICATION,
PREFERRED_BLOCK_SIZE);
parent.addChild(inodeFile);
return inodeFile;
}
@Test
public void testDefaultAclNotCopiedToAccessAclOfNewSnapshot() throws Exception {
FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0700));
List<AclEntry> aclSpec = Lists.newArrayList(aclEntry(DEFAULT, USER, "bruce", READ_EXECUTE));
hdfs.modifyAclEntries(path, aclSpec);
SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName);
AclStatus s = hdfs.getAclStatus(path);
AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(
new AclEntry[] {
aclEntry(DEFAULT, USER, ALL),
aclEntry(DEFAULT, USER, "bruce", READ_EXECUTE),
aclEntry(DEFAULT, GROUP, NONE),
aclEntry(DEFAULT, MASK, READ_EXECUTE),
aclEntry(DEFAULT, OTHER, NONE)
},
returned);
assertPermission((short) 010700, path);
s = hdfs.getAclStatus(snapshotPath);
returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(
new AclEntry[] {
aclEntry(DEFAULT, USER, ALL),
aclEntry(DEFAULT, USER, "bruce", READ_EXECUTE),
aclEntry(DEFAULT, GROUP, NONE),
aclEntry(DEFAULT, MASK, READ_EXECUTE),
aclEntry(DEFAULT, OTHER, NONE)
},
returned);
assertPermission((short) 010700, snapshotPath);
assertDirPermissionDenied(fsAsBruce, BRUCE, snapshotPath);
}
public abstract class ContainerExecutor implements Configurable {
private static final Log LOG = LogFactory.getLog(ContainerExecutor.class);
public static final FsPermission TASK_LAUNCH_SCRIPT_PERMISSION =
FsPermission.createImmutable((short) 0700);
private Configuration conf;
private ConcurrentMap<ContainerId, Path> pidFiles = new ConcurrentHashMap<ContainerId, Path>();
private ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
private final ReadLock readLock = lock.readLock();
private final WriteLock writeLock = lock.writeLock();
@Override
public void setConf(Configuration conf) {
this.conf = conf;
}
@Override
public Configuration getConf() {
return conf;
}
/**
* Run the executor initialization steps. Verify that the necessary configs, permissions are in
* place.
*
* @throws IOException
*/
public abstract void init() throws IOException;
/**
* On Windows the ContainerLaunch creates a temporary special jar manifest of other jars to
* workaround the CLASSPATH length. In a secure cluster this jar must be localized so that the
* container has access to it. This function localizes on-demand the jar.
*
* @param classPathJar
* @param owner
* @throws IOException
*/
public Path localizeClasspathJar(Path classPathJar, Path pwd, String owner) throws IOException {
// Non-secure executor simply use the classpath created
// in the NM fprivate folder
return classPathJar;
}
/**
* Prepare the environment for containers in this application to execute.
*
* <pre>
* For $x in local.dirs
* create $x/$user/$appId
* Copy $nmLocal/appTokens {@literal ->} $N/$user/$appId
* For $rsrc in private resources
* Copy $rsrc {@literal ->} $N/$user/filecache/[idef]
* For $rsrc in job resources
* Copy $rsrc {@literal ->} $N/$user/$appId/filecache/idef
* </pre>
*
* @param ctx LocalizerStartContext that encapsulates necessary information for starting a
* localizer.
* @throws IOException For most application init failures
* @throws InterruptedException If application init thread is halted by NM
*/
public abstract void startLocalizer(LocalizerStartContext ctx)
throws IOException, InterruptedException;
/**
* Launch the container on the node. This is a blocking call and returns only when the container
* exits.
*
* @param ctx Encapsulates information necessary for launching containers.
* @return the return status of the launch
* @throws IOException
*/
public abstract int launchContainer(ContainerStartContext ctx) throws IOException;
/**
* Signal container with the specified signal.
*
* @param ctx Encapsulates information necessary for signaling containers.
* @return returns true if the operation succeeded
* @throws IOException
*/
public abstract boolean signalContainer(ContainerSignalContext ctx) throws IOException;
/**
* Delete specified directories as a given user.
*
* @param ctx Encapsulates information necessary for deletion.
* @throws IOException
* @throws InterruptedException
*/
public abstract void deleteAsUser(DeletionAsUserContext ctx)
throws IOException, InterruptedException;
/**
* Check if a container is alive.
*
* @param ctx Encapsulates information necessary for container liveness check.
* @return true if container is still alive
* @throws IOException
*/
public abstract boolean isContainerAlive(ContainerLivenessContext ctx) throws IOException;
/**
* Recover an already existing container. This is a blocking call and returns only when the
* container exits. Note that the container must have been activated prior to this call.
*
* @param ctx encapsulates information necessary to reacquire container
* @return The exit code of the pre-existing container
* @throws IOException
* @throws InterruptedException
*/
public int reacquireContainer(ContainerReacquisitionContext ctx)
throws IOException, InterruptedException {
Container container = ctx.getContainer();
String user = ctx.getUser();
ContainerId containerId = ctx.getContainerId();
Path pidPath = getPidFilePath(containerId);
if (pidPath == null) {
LOG.warn(containerId + " is not active, returning terminated error");
return ExitCode.TERMINATED.getExitCode();
}
String pid = null;
pid = ProcessIdFileReader.getProcessId(pidPath);
if (pid == null) {
throw new IOException("Unable to determine pid for " + containerId);
}
LOG.info("Reacquiring " + containerId + " with pid " + pid);
ContainerLivenessContext livenessContext =
new ContainerLivenessContext.Builder()
.setContainer(container)
.setUser(user)
.setPid(pid)
.build();
while (isContainerAlive(livenessContext)) {
Thread.sleep(1000);
}
// wait for exit code file to appear
String exitCodeFile = ContainerLaunch.getExitCodeFile(pidPath.toString());
File file = new File(exitCodeFile);
final int sleepMsec = 100;
int msecLeft = 2000;
while (!file.exists() && msecLeft >= 0) {
if (!isContainerActive(containerId)) {
LOG.info(containerId + " was deactivated");
return ExitCode.TERMINATED.getExitCode();
}
Thread.sleep(sleepMsec);
msecLeft -= sleepMsec;
}
if (msecLeft < 0) {
throw new IOException("Timeout while waiting for exit code from " + containerId);
}
try {
return Integer.parseInt(FileUtils.readFileToString(file).trim());
} catch (NumberFormatException e) {
throw new IOException("Error parsing exit code from pid " + pid, e);
}
}
/**
* This method writes out the launch environment of a container. This can be overridden by
* extending ContainerExecutors to provide different behaviors
*
* @param out the output stream to which the environment is written (usually a script file which
* will be executed by the Launcher)
* @param environment The environment variables and their values
* @param resources The resources which have been localized for this container Symlinks will be
* created to these localized resources
* @param command The command that will be run.
* @throws IOException if any errors happened writing to the OutputStream, while creating symlinks
*/
public void writeLaunchEnv(
OutputStream out,
Map<String, String> environment,
Map<Path, List<String>> resources,
List<String> command)
throws IOException {
ContainerLaunch.ShellScriptBuilder sb = ContainerLaunch.ShellScriptBuilder.create();
Set<String> whitelist = new HashSet<String>();
whitelist.add(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME);
whitelist.add(ApplicationConstants.Environment.HADOOP_YARN_HOME.name());
whitelist.add(ApplicationConstants.Environment.HADOOP_COMMON_HOME.name());
whitelist.add(ApplicationConstants.Environment.HADOOP_HDFS_HOME.name());
whitelist.add(ApplicationConstants.Environment.HADOOP_CONF_DIR.name());
whitelist.add(ApplicationConstants.Environment.JAVA_HOME.name());
if (environment != null) {
for (Map.Entry<String, String> env : environment.entrySet()) {
if (!whitelist.contains(env.getKey())) {
sb.env(env.getKey().toString(), env.getValue().toString());
} else {
sb.whitelistedEnv(env.getKey().toString(), env.getValue().toString());
}
}
}
if (resources != null) {
for (Map.Entry<Path, List<String>> entry : resources.entrySet()) {
for (String linkName : entry.getValue()) {
sb.symlink(entry.getKey(), new Path(linkName));
}
}
}
sb.command(command);
PrintStream pout = null;
try {
pout = new PrintStream(out, false, "UTF-8");
sb.write(pout);
} finally {
if (out != null) {
out.close();
}
}
}
public enum ExitCode {
FORCE_KILLED(137),
TERMINATED(143),
LOST(154);
private final int code;
private ExitCode(int exitCode) {
this.code = exitCode;
}
public int getExitCode() {
return code;
}
@Override
public String toString() {
return String.valueOf(code);
}
}
/** The constants for the signals. */
public enum Signal {
NULL(0, "NULL"),
QUIT(3, "SIGQUIT"),
KILL(9, "SIGKILL"),
TERM(15, "SIGTERM");
private final int value;
private final String str;
private Signal(int value, String str) {
this.str = str;
this.value = value;
}
public int getValue() {
return value;
}
@Override
public String toString() {
return str;
}
}
protected void logOutput(String output) {
String shExecOutput = output;
if (shExecOutput != null) {
for (String str : shExecOutput.split("\n")) {
LOG.info(str);
}
}
}
/**
* Get the pidFile of the container.
*
* @param containerId
* @return the path of the pid-file for the given containerId.
*/
protected Path getPidFilePath(ContainerId containerId) {
try {
readLock.lock();
return (this.pidFiles.get(containerId));
} finally {
readLock.unlock();
}
}
protected String[] getRunCommand(
String command, String groupId, String userName, Path pidFile, Configuration conf) {
return getRunCommand(command, groupId, userName, pidFile, conf, null);
}
/**
* Return a command to execute the given command in OS shell. On Windows, the passed in groupId
* can be used to launch and associate the given groupId in a process group. On non-Windows,
* groupId is ignored.
*/
protected String[] getRunCommand(
String command,
String groupId,
String userName,
Path pidFile,
Configuration conf,
Resource resource) {
boolean containerSchedPriorityIsSet = false;
int containerSchedPriorityAdjustment =
YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY;
if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) != null) {
containerSchedPriorityIsSet = true;
containerSchedPriorityAdjustment =
conf.getInt(
YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY,
YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY);
}
if (Shell.WINDOWS) {
int cpuRate = -1;
int memory = -1;
if (resource != null) {
if (conf.getBoolean(
YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED,
YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) {
memory = resource.getMemory();
}
if (conf.getBoolean(
YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED,
YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED)) {
int containerVCores = resource.getVirtualCores();
int nodeVCores = NodeManagerHardwareUtils.getVCores(conf);
int nodeCpuPercentage = NodeManagerHardwareUtils.getNodeCpuPercentage(conf);
float containerCpuPercentage = (float) (nodeCpuPercentage * containerVCores) / nodeVCores;
// CPU should be set to a percentage * 100, e.g. 20% cpu rate limit
// should be set as 20 * 100.
cpuRate = Math.min(10000, (int) (containerCpuPercentage * 100));
}
}
return new String[] {
Shell.WINUTILS,
"task",
"create",
"-m",
String.valueOf(memory),
"-c",
String.valueOf(cpuRate),
groupId,
"cmd /c " + command
};
} else {
List<String> retCommand = new ArrayList<String>();
if (containerSchedPriorityIsSet) {
retCommand.addAll(
Arrays.asList("nice", "-n", Integer.toString(containerSchedPriorityAdjustment)));
}
retCommand.addAll(Arrays.asList("bash", command));
return retCommand.toArray(new String[retCommand.size()]);
}
}
/**
* Is the container still active?
*
* @param containerId
* @return true if the container is active else false.
*/
protected boolean isContainerActive(ContainerId containerId) {
try {
readLock.lock();
return (this.pidFiles.containsKey(containerId));
} finally {
readLock.unlock();
}
}
/**
* Mark the container as active
*
* @param containerId the ContainerId
* @param pidFilePath Path where the executor should write the pid of the launched process
*/
public void activateContainer(ContainerId containerId, Path pidFilePath) {
try {
writeLock.lock();
this.pidFiles.put(containerId, pidFilePath);
} finally {
writeLock.unlock();
}
}
/**
* Mark the container as inactive. Done iff the container is still active. Else treat it as a
* no-op
*/
public void deactivateContainer(ContainerId containerId) {
try {
writeLock.lock();
this.pidFiles.remove(containerId);
} finally {
writeLock.unlock();
}
}
/**
* Get the process-identifier for the container
*
* @param containerID
* @return the processid of the container if it has already launched, otherwise return null
*/
public String getProcessId(ContainerId containerID) {
String pid = null;
Path pidFile = pidFiles.get(containerID);
if (pidFile == null) {
// This container isn't even launched yet.
return pid;
}
try {
pid = ProcessIdFileReader.getProcessId(pidFile);
} catch (IOException e) {
LOG.error("Got exception reading pid from pid-file " + pidFile, e);
}
return pid;
}
public static class DelayedProcessKiller extends Thread {
private Container container;
private final String user;
private final String pid;
private final long delay;
private final Signal signal;
private final ContainerExecutor containerExecutor;
public DelayedProcessKiller(
Container container,
String user,
String pid,
long delay,
Signal signal,
ContainerExecutor containerExecutor) {
this.container = container;
this.user = user;
this.pid = pid;
this.delay = delay;
this.signal = signal;
this.containerExecutor = containerExecutor;
setName("Task killer for " + pid);
setDaemon(false);
}
@Override
public void run() {
try {
Thread.sleep(delay);
containerExecutor.signalContainer(
new ContainerSignalContext.Builder()
.setContainer(container)
.setUser(user)
.setPid(pid)
.setSignal(signal)
.build());
} catch (InterruptedException e) {
return;
} catch (IOException e) {
String message =
"Exception when user "
+ user
+ " killing task "
+ pid
+ " in DelayedProcessKiller: "
+ StringUtils.stringifyException(e);
LOG.warn(message);
container.handle(new ContainerDiagnosticsUpdateEvent(container.getContainerId(), message));
}
}
}
}
public class TajoResourceAllocator extends AbstractResourceAllocator {
private static final Log LOG = LogFactory.getLog(TajoResourceAllocator.class);
static AtomicInteger containerIdSeq = new AtomicInteger(0);
private TajoConf tajoConf;
private QueryMasterTask.QueryMasterTaskContext queryTaskContext;
private final ExecutorService executorService;
private AtomicBoolean stopped = new AtomicBoolean(false);
public TajoResourceAllocator(QueryMasterTask.QueryMasterTaskContext queryTaskContext) {
this.queryTaskContext = queryTaskContext;
executorService =
Executors.newFixedThreadPool(
queryTaskContext
.getConf()
.getIntVar(TajoConf.ConfVars.YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM));
}
@Override
public ContainerId makeContainerId(YarnProtos.ContainerIdProto containerIdProto) {
TajoWorkerContainerId containerId = new TajoWorkerContainerId();
ApplicationAttemptId appAttemptId =
new ApplicationAttemptIdPBImpl(containerIdProto.getAppAttemptId());
containerId.setApplicationAttemptId(appAttemptId);
containerId.setId(containerIdProto.getId());
return containerId;
}
@Override
public void allocateTaskWorker() {}
@Override
public int calculateNumRequestContainers(TajoWorker.WorkerContext workerContext, int numTasks) {
int clusterSlots = workerContext.getNumClusterSlots();
return clusterSlots == 0 ? 1 : Math.min(numTasks, clusterSlots);
}
@Override
public void init(Configuration conf) {
tajoConf = (TajoConf) conf;
queryTaskContext
.getDispatcher()
.register(TaskRunnerGroupEvent.EventType.class, new TajoTaskRunnerLauncher());
//
queryTaskContext
.getDispatcher()
.register(ContainerAllocatorEventType.class, new TajoWorkerAllocationHandler());
super.init(conf);
}
@Override
public synchronized void stop() {
if (stopped.get()) {
return;
}
stopped.set(true);
executorService.shutdownNow();
Map<ContainerId, ContainerProxy> containers =
queryTaskContext.getResourceAllocator().getContainers();
List<ContainerProxy> list = new ArrayList<ContainerProxy>(containers.values());
for (ContainerProxy eachProxy : list) {
try {
eachProxy.stopContainer();
} catch (Exception e) {
}
}
super.stop();
}
@Override
public void start() {
super.start();
}
public static final FsPermission QUERYCONF_FILE_PERMISSION =
FsPermission.createImmutable((short) 0644); // rw-r--r--
private static void writeConf(Configuration conf, Path queryConfFile) throws IOException {
// Write job file to Tajo's fs
FileSystem fs = queryConfFile.getFileSystem(conf);
FSDataOutputStream out =
FileSystem.create(fs, queryConfFile, new FsPermission(QUERYCONF_FILE_PERMISSION));
try {
conf.writeXml(out);
} finally {
out.close();
}
}
class TajoTaskRunnerLauncher implements TaskRunnerLauncher {
@Override
public void handle(TaskRunnerGroupEvent event) {
if (event.getType() == TaskRunnerGroupEvent.EventType.CONTAINER_REMOTE_LAUNCH) {
launchTaskRunners(event.getExecutionBlockId(), event.getContainers());
} else if (event.getType() == TaskRunnerGroupEvent.EventType.CONTAINER_REMOTE_CLEANUP) {
stopContainers(event.getContainers());
}
}
}
private void launchTaskRunners(
ExecutionBlockId executionBlockId, Collection<Container> containers) {
// Query in standby mode doesn't need launch Worker.
// But, Assign ExecutionBlock to assigned tajo worker
for (Container eachContainer : containers) {
TajoContainerProxy containerProxy =
new TajoContainerProxy(queryTaskContext, tajoConf, eachContainer, executionBlockId);
executorService.submit(new LaunchRunner(eachContainer.getId(), containerProxy));
}
}
protected class LaunchRunner implements Runnable {
private final ContainerProxy proxy;
private final ContainerId id;
public LaunchRunner(ContainerId id, ContainerProxy proxy) {
this.proxy = proxy;
this.id = id;
}
@Override
public void run() {
proxy.launch(null);
LOG.info("ContainerProxy started:" + id);
}
}
private void stopContainers(Collection<Container> containers) {
for (Container container : containers) {
final ContainerProxy proxy =
queryTaskContext.getResourceAllocator().getContainer(container.getId());
executorService.submit(new StopContainerRunner(container.getId(), proxy));
}
}
private class StopContainerRunner implements Runnable {
private final ContainerProxy proxy;
private final ContainerId id;
public StopContainerRunner(ContainerId id, ContainerProxy proxy) {
this.id = id;
this.proxy = proxy;
}
@Override
public void run() {
LOG.info("ContainerProxy stopped:" + id + "," + proxy.getId());
proxy.stopContainer();
}
}
class TajoWorkerAllocationHandler implements EventHandler<ContainerAllocationEvent> {
@Override
public void handle(ContainerAllocationEvent event) {
executorService.submit(new TajoWorkerAllocationThread(event));
}
}
class TajoWorkerAllocationThread extends Thread {
ContainerAllocationEvent event;
TajoWorkerAllocationThread(ContainerAllocationEvent event) {
this.event = event;
}
@Override
public void run() {
LOG.info("Start TajoWorkerAllocationThread");
CallFuture<TajoMasterProtocol.WorkerResourceAllocationResponse> callBack =
new CallFuture<TajoMasterProtocol.WorkerResourceAllocationResponse>();
int requiredMemoryMBSlot = 512; // TODO
int requiredDiskSlots = 1; // TODO
TajoMasterProtocol.WorkerResourceAllocationRequest request =
TajoMasterProtocol.WorkerResourceAllocationRequest.newBuilder()
.setMemoryMBSlots(requiredMemoryMBSlot)
.setDiskSlots(requiredDiskSlots)
.setNumWorks(event.getRequiredNum())
.setExecutionBlockId(event.getExecutionBlockId().getProto())
.build();
RpcConnectionPool connPool = RpcConnectionPool.getPool(queryTaskContext.getConf());
NettyClientBase tmClient = null;
try {
tmClient =
connPool.getConnection(
queryTaskContext.getQueryMasterContext().getWorkerContext().getTajoMasterAddress(),
TajoMasterProtocol.class,
true);
TajoMasterProtocol.TajoMasterProtocolService masterClientService = tmClient.getStub();
masterClientService.allocateWorkerResources(null, request, callBack);
} catch (Exception e) {
connPool.closeConnection(tmClient);
tmClient = null;
LOG.error(e.getMessage(), e);
} finally {
connPool.releaseConnection(tmClient);
}
TajoMasterProtocol.WorkerResourceAllocationResponse response = null;
while (!stopped.get()) {
try {
response = callBack.get(3, TimeUnit.SECONDS);
break;
} catch (InterruptedException e) {
if (stopped.get()) {
return;
}
} catch (TimeoutException e) {
LOG.info("No available worker resource for " + event.getExecutionBlockId());
continue;
}
}
int numAllocatedWorkers = 0;
if (response != null) {
List<TajoMasterProtocol.WorkerAllocatedResource> workerHosts =
response.getWorkerAllocatedResourceList();
ExecutionBlockId executionBlockId = event.getExecutionBlockId();
List<Container> containers = new ArrayList<Container>();
for (TajoMasterProtocol.WorkerAllocatedResource eachWorker : workerHosts) {
TajoWorkerContainer container = new TajoWorkerContainer();
NodeIdPBImpl nodeId = new NodeIdPBImpl();
nodeId.setHost(eachWorker.getWorkerHost());
nodeId.setPort(eachWorker.getPeerRpcPort());
TajoWorkerContainerId containerId = new TajoWorkerContainerId();
containerId.setApplicationAttemptId(
ApplicationIdUtils.createApplicationAttemptId(executionBlockId.getQueryId()));
containerId.setId(containerIdSeq.incrementAndGet());
container.setId(containerId);
container.setNodeId(nodeId);
WorkerResource workerResource = new WorkerResource();
workerResource.setAllocatedHost(nodeId.getHost());
workerResource.setPeerRpcPort(nodeId.getPort());
workerResource.setQueryMasterPort(eachWorker.getQueryMasterPort());
workerResource.setPullServerPort(eachWorker.getWorkerPullServerPort());
workerResource.setMemoryMBSlots(requiredMemoryMBSlot);
workerResource.setDiskSlots(requiredDiskSlots);
container.setWorkerResource(workerResource);
containers.add(container);
}
SubQueryState state = queryTaskContext.getSubQuery(executionBlockId).getState();
if (!SubQuery.isRunningState(state)) {
List<WorkerResource> workerResources = new ArrayList<WorkerResource>();
for (Container eachContainer : containers) {
workerResources.add(((TajoWorkerContainer) eachContainer).getWorkerResource());
}
try {
TajoContainerProxy.releaseWorkerResource(
queryTaskContext, executionBlockId, workerResources);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
return;
}
if (workerHosts.size() > 0) {
if (LOG.isDebugEnabled()) {
LOG.debug("SubQueryContainerAllocationEvent fire:" + executionBlockId);
}
queryTaskContext
.getEventHandler()
.handle(new SubQueryContainerAllocationEvent(executionBlockId, containers));
}
numAllocatedWorkers += workerHosts.size();
}
if (event.getRequiredNum() > numAllocatedWorkers) {
ContainerAllocationEvent shortRequestEvent =
new ContainerAllocationEvent(
event.getType(),
event.getExecutionBlockId(),
event.getPriority(),
event.getResource(),
event.getRequiredNum() - numAllocatedWorkers,
event.isLeafQuery(),
event.getProgress());
queryTaskContext.getEventHandler().handle(shortRequestEvent);
}
LOG.info("Stop TajoWorkerAllocationThread");
}
}
}
/** HDFS UnderFilesystem implementation */
public class HdfsUnderFileSystem extends UnderFileSystem {
private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE);
private static final int MAX_TRY = 5;
private FileSystem mFs = null;
private String mUfsPrefix = null;
// TODO add sticky bit and narrow down the permission in hadoop 2
private static final FsPermission PERMISSION =
new FsPermission((short) 0777).applyUMask(FsPermission.createImmutable((short) 0000));
public HdfsUnderFileSystem(String fsDefaultName, TachyonConf tachyonConf, Object conf) {
super(tachyonConf);
mUfsPrefix = fsDefaultName;
Configuration tConf;
if (conf != null && conf instanceof Configuration) {
tConf = (Configuration) conf;
} else {
tConf = new Configuration();
}
prepareConfiguration(fsDefaultName, tachyonConf, tConf);
tConf.addResource(new Path(tConf.get(Constants.UNDERFS_HADOOP_CONFIGURATION)));
HdfsUnderFileSystemUtils.addS3Credentials(tConf);
Path path = new Path(mUfsPrefix);
try {
mFs = path.getFileSystem(tConf);
} catch (IOException e) {
LOG.error("Exception thrown when trying to get FileSystem for " + mUfsPrefix, e);
throw Throwables.propagate(e);
}
}
/**
* Prepares the Hadoop configuration necessary to successfully obtain a {@link FileSystem}
* instance that can access the provided path
*
* <p>Derived implementations that work with specialised Hadoop {@linkplain FileSystem} API
* compatible implementations can override this method to add implementation specific
* configuration necessary for obtaining a usable {@linkplain FileSystem} instance.
*
* @param path File system path
* @param config Hadoop Configuration
*/
protected void prepareConfiguration(String path, TachyonConf tachyonConf, Configuration config) {
// On Hadoop 2.x this is strictly unnecessary since it uses ServiceLoader to automatically
// discover available file system implementations. However this configuration setting is
// required for earlier Hadoop versions plus it is still honoured as an override even in 2.x so
// if present propagate it to the Hadoop configuration
String ufsHdfsImpl = mTachyonConf.get(Constants.UNDERFS_HDFS_IMPL, null);
if (!StringUtils.isEmpty(ufsHdfsImpl)) {
config.set("fs.hdfs.impl", ufsHdfsImpl);
}
// To disable the instance cache for hdfs client, otherwise it causes the
// FileSystem closed exception. Being configurable for unit/integration
// test only, and not expose to the end-user currently.
config.set(
"fs.hdfs.impl.disable.cache", System.getProperty("fs.hdfs.impl.disable.cache", "false"));
HdfsUnderFileSystemUtils.addKey(config, tachyonConf, Constants.UNDERFS_HADOOP_CONFIGURATION);
}
@Override
public void close() throws IOException {
mFs.close();
}
@Override
public FSDataOutputStream create(String path) throws IOException {
IOException te = null;
int cnt = 0;
while (cnt < MAX_TRY) {
try {
LOG.debug("Creating HDFS file at {}", path);
return FileSystem.create(mFs, new Path(path), PERMISSION);
} catch (IOException e) {
cnt++;
LOG.error(cnt + " : " + e.getMessage(), e);
te = e;
}
}
throw te;
}
/** BlockSize should be a multiple of 512 */
@Override
public FSDataOutputStream create(String path, int blockSizeByte) throws IOException {
// TODO Fix this
// return create(path, (short) Math.min(3, mFs.getDefaultReplication()), blockSizeByte);
return create(path);
}
@Override
public FSDataOutputStream create(String path, short replication, int blockSizeByte)
throws IOException {
// TODO Fix this
// return create(path, (short) Math.min(3, mFs.getDefaultReplication()), blockSizeByte);
return create(path);
// LOG.info(path + " " + replication + " " + blockSizeByte);
// IOException te = null;
// int cnt = 0;
// while (cnt < MAX_TRY) {
// try {
// return mFs.create(new Path(path), true, 4096, replication, blockSizeByte);
// } catch (IOException e) {
// cnt ++;
// LOG.error(cnt + " : " + e.getMessage(), e);
// te = e;
// continue;
// }
// }
// throw te;
}
@Override
public boolean delete(String path, boolean recursive) throws IOException {
LOG.debug("deleting {} {}", path, recursive);
IOException te = null;
int cnt = 0;
while (cnt < MAX_TRY) {
try {
return mFs.delete(new Path(path), recursive);
} catch (IOException e) {
cnt++;
LOG.error(cnt + " : " + e.getMessage(), e);
te = e;
}
}
throw te;
}
@Override
public boolean exists(String path) throws IOException {
IOException te = null;
int cnt = 0;
while (cnt < MAX_TRY) {
try {
return mFs.exists(new Path(path));
} catch (IOException e) {
cnt++;
LOG.error(cnt + " try to check if " + path + " exists " + " : " + e.getMessage(), e);
te = e;
}
}
throw te;
}
@Override
public long getBlockSizeByte(String path) throws IOException {
Path tPath = new Path(path);
if (!mFs.exists(tPath)) {
throw new FileNotFoundException(path);
}
FileStatus fs = mFs.getFileStatus(tPath);
return fs.getBlockSize();
}
@Override
public Object getConf() {
return mFs.getConf();
}
@Override
public List<String> getFileLocations(String path) throws IOException {
return getFileLocations(path, 0);
}
@Override
public List<String> getFileLocations(String path, long offset) throws IOException {
List<String> ret = new ArrayList<String>();
try {
FileStatus fStatus = mFs.getFileStatus(new Path(path));
BlockLocation[] bLocations = mFs.getFileBlockLocations(fStatus, offset, 1);
if (bLocations.length > 0) {
String[] names = bLocations[0].getNames();
Collections.addAll(ret, names);
}
} catch (IOException e) {
LOG.error("Unable to get file location for " + path, e);
}
return ret;
}
@Override
public long getFileSize(String path) throws IOException {
int cnt = 0;
Path tPath = new Path(path);
while (cnt < MAX_TRY) {
try {
FileStatus fs = mFs.getFileStatus(tPath);
return fs.getLen();
} catch (IOException e) {
cnt++;
LOG.error(cnt + " try to get file size for " + path + " : " + e.getMessage(), e);
}
}
return -1;
}
@Override
public long getModificationTimeMs(String path) throws IOException {
Path tPath = new Path(path);
if (!mFs.exists(tPath)) {
throw new FileNotFoundException(path);
}
FileStatus fs = mFs.getFileStatus(tPath);
return fs.getModificationTime();
}
@Override
public long getSpace(String path, SpaceType type) throws IOException {
// Ignoring the path given, will give information for entire cluster
// as Tachyon can load/store data out of entire HDFS cluster
if (mFs instanceof DistributedFileSystem) {
switch (type) {
case SPACE_TOTAL:
return ((DistributedFileSystem) mFs).getDiskStatus().getCapacity();
case SPACE_USED:
return ((DistributedFileSystem) mFs).getDiskStatus().getDfsUsed();
case SPACE_FREE:
return ((DistributedFileSystem) mFs).getDiskStatus().getRemaining();
default:
throw new IOException("Unknown getSpace parameter: " + type);
}
}
return -1;
}
@Override
public boolean isFile(String path) throws IOException {
return mFs.isFile(new Path(path));
}
@Override
public String[] list(String path) throws IOException {
FileStatus[] files = mFs.listStatus(new Path(path));
if (files != null) {
String[] rtn = new String[files.length];
int i = 0;
for (FileStatus status : files) {
TachyonURI filePathURI = new TachyonURI(status.getPath().toUri().toString());
String filePath = NetworkUtils.replaceHostName(filePathURI).toString();
// only return the relative path, to keep consistent with java.io.File.list()
rtn[i++] = filePath.substring(path.length()); // mUfsPrefix
}
return rtn;
} else {
return null;
}
}
@Override
public void connectFromMaster(TachyonConf conf, String host) throws IOException {
String masterKeytab = conf.get(Constants.MASTER_KEYTAB_KEY, null);
String masterPrincipal = conf.get(Constants.MASTER_PRINCIPAL_KEY, null);
if (masterKeytab == null || masterPrincipal == null) {
return;
}
login(
Constants.MASTER_KEYTAB_KEY,
masterKeytab,
Constants.MASTER_PRINCIPAL_KEY,
masterPrincipal,
host);
}
@Override
public void connectFromWorker(TachyonConf conf, String host) throws IOException {
String workerKeytab = conf.get(Constants.WORKER_KEYTAB_KEY, null);
String workerPrincipal = conf.get(Constants.WORKER_PRINCIPAL_KEY, null);
if (workerKeytab == null || workerPrincipal == null) {
return;
}
login(
Constants.WORKER_KEYTAB_KEY,
workerKeytab,
Constants.WORKER_PRINCIPAL_KEY,
workerPrincipal,
host);
}
private void login(
String keytabFileKey,
String keytabFile,
String principalKey,
String principal,
String hostname)
throws IOException {
Configuration conf = new Configuration();
conf.set(keytabFileKey, keytabFile);
conf.set(principalKey, principal);
SecurityUtil.login(conf, keytabFileKey, principalKey, hostname);
}
@Override
public boolean mkdirs(String path, boolean createParent) throws IOException {
IOException te = null;
int cnt = 0;
while (cnt < MAX_TRY) {
try {
if (mFs.exists(new Path(path))) {
LOG.debug("Trying to create existing directory at {}", path);
return false;
}
return mFs.mkdirs(new Path(path), PERMISSION);
} catch (IOException e) {
cnt++;
LOG.error(cnt + " try to make directory for " + path + " : " + e.getMessage(), e);
te = e;
}
}
throw te;
}
@Override
public FSDataInputStream open(String path) throws IOException {
IOException te = null;
int cnt = 0;
while (cnt < MAX_TRY) {
try {
return mFs.open(new Path(path));
} catch (IOException e) {
cnt++;
LOG.error(cnt + " try to open " + path + " : " + e.getMessage(), e);
te = e;
}
}
throw te;
}
@Override
public boolean rename(String src, String dst) throws IOException {
LOG.debug("Renaming from {} to {}", src, dst);
if (!exists(src)) {
LOG.error("File " + src + " does not exist. Therefore rename to " + dst + " failed.");
return false;
}
if (exists(dst)) {
LOG.error("File " + dst + " does exist. Therefore rename from " + src + " failed.");
return false;
}
int cnt = 0;
IOException te = null;
while (cnt < MAX_TRY) {
try {
return mFs.rename(new Path(src), new Path(dst));
} catch (IOException e) {
cnt++;
LOG.error(cnt + " try to rename " + src + " to " + dst + " : " + e.getMessage(), e);
te = e;
}
}
throw te;
}
@Override
public void setConf(Object conf) {
mFs.setConf((Configuration) conf);
}
@Override
public void setPermission(String path, String posixPerm) throws IOException {
try {
FileStatus fileStatus = mFs.getFileStatus(new Path(path));
LOG.info(
"Changing file '"
+ fileStatus.getPath()
+ "' permissions from: "
+ fileStatus.getPermission()
+ " to "
+ posixPerm);
FsPermission perm = new FsPermission(Short.parseShort(posixPerm));
mFs.setPermission(fileStatus.getPath(), perm);
} catch (IOException e) {
LOG.error("Fail to set permission for " + path + " with perm " + posixPerm, e);
throw e;
}
}
}
public class CoronaJobHistory {
public static final Log LOG = LogFactory.getLog(CoronaJobHistory.class);
static final FsPermission HISTORY_DIR_PERMISSION =
FsPermission.createImmutable((short) 0755); // rwxr-x---
static final FsPermission HISTORY_FILE_PERMISSION =
FsPermission.createImmutable((short) 0744); // rwxr-----
Path logDir;
FileSystem logDirFs;
Path doneDir;
FileSystem doneDirFs;
Path logFile;
Path doneFile;
JobConf conf;
boolean disableHistory = true;
long jobHistoryBlockSize = 0;
CoronaJobHistoryFilesManager fileManager = null;
JobID jobId;
ArrayList<PrintWriter> writers = null;
public CoronaJobHistory(JobConf conf, JobID jobId, String logPath) {
try {
this.conf = conf;
this.jobId = jobId;
if (logPath == null) {
logPath =
"file:///"
+ new File(System.getProperty("hadoop.log.dir", "/tmp")).getAbsolutePath()
+ File.separator
+ "history";
}
logDir = new Path(logPath);
logDirFs = logDir.getFileSystem(conf);
if (!logDirFs.exists(logDir)) {
LOG.info("Creating history folder at " + logDir);
if (!logDirFs.mkdirs(logDir, new FsPermission(HISTORY_DIR_PERMISSION))) {
throw new IOException("Mkdirs failed to create " + logDir.toString());
}
}
conf.set("hadoop.job.history.location", logDir.toString());
disableHistory = false;
// set the job history block size (default is 3MB)
jobHistoryBlockSize =
conf.getLong("mapred.jobtracker.job.history.block.size", 3 * 1024 * 1024);
doneDir = new Path(logDir, "done");
doneDirFs = logDirFs;
if (!doneDirFs.exists(doneDir)) {
LOG.info("Creating DONE folder at " + doneDir);
if (!doneDirFs.mkdirs(doneDir, new FsPermission(HISTORY_DIR_PERMISSION))) {
throw new IOException("Mkdirs failed to create " + doneDir);
}
}
String logFileName =
encodeJobHistoryFileName(CoronaJobHistoryFilesManager.getHistoryFilename(jobId));
logFile = new Path(logDir, logFileName);
doneFile = new Path(doneDir, logFileName);
// initialize the file manager
conf.setInt("mapred.jobtracker.historythreads.maximum", 1);
fileManager =
new CoronaJobHistoryFilesManager(
conf,
new JobHistoryObserver() {
public void historyFileCopied(JobID jobid, String historyFile) {}
},
logDir);
fileManager.setDoneDir(doneDir);
// sleeping with the past means tolerating two start methods instead of one
fileManager.start();
fileManager.startIOExecutor();
} catch (IOException e) {
LOG.error("Failed to initialize JobHistory log file", e);
disableHistory = true;
}
}
public void shutdown() {
if (fileManager != null) {
fileManager.shutdown();
}
}
public boolean isDisabled() {
return disableHistory;
}
public static String encodeJobHistoryFileName(String logFileName) throws IOException {
return JobHistory.JobInfo.encodeJobHistoryFileName(logFileName);
}
public String getCompletedJobHistoryPath() {
return doneFile.toString();
}
/** Get the history location for completed jobs */
public Path getCompletedJobHistoryLocation() {
return doneDir;
}
public static String encodeJobHistoryFilePath(String logFile) throws IOException {
return JobHistory.JobInfo.encodeJobHistoryFilePath(logFile);
}
private String getJobName() {
String jobName = ((JobConf) conf).getJobName();
if (jobName == null || jobName.length() == 0) {
jobName = "NA";
}
return jobName;
}
public String getUserName() {
String user = ((JobConf) conf).getUser();
if (user == null || user.length() == 0) {
user = "******";
}
return user;
}
private static void closeAndClear(List<PrintWriter> writers) {
for (PrintWriter out : writers) {
out.close();
}
// By clearning the writers and notify the thread waiting on it,
// we will prevent JobHistory.moveToDone() from
// waiting on writer
synchronized (writers) {
writers.clear();
writers.notifyAll();
}
}
/**
* Log job submitted event to history. Creates a new file in history for the job. if history file
* creation fails, it disables history for all other events.
*
* @param jobConfPath path to job conf xml file in HDFS.
* @param submitTime time when job tracker received the job
* @throws IOException
*/
public void logSubmitted(String jobConfPath, long submitTime, String jobTrackerId)
throws IOException {
if (disableHistory) {
return;
}
// create output stream for logging in hadoop.job.history.location
int defaultBufferSize = logDirFs.getConf().getInt("io.file.buffer.size", 4096);
try {
FSDataOutputStream out = null;
PrintWriter writer = null;
// In case the old JT is still running, but we can't connect to it, we
// should ensure that it won't write to our (new JT's) job history file.
if (logDirFs.exists(logFile)) {
LOG.info("Remove the old history file " + logFile);
logDirFs.delete(logFile, true);
}
out =
logDirFs.create(
logFile,
new FsPermission(HISTORY_FILE_PERMISSION),
true,
defaultBufferSize,
logDirFs.getDefaultReplication(),
jobHistoryBlockSize,
null);
writer = new PrintWriter(out);
fileManager.addWriter(jobId, writer);
// cache it ...
fileManager.setHistoryFile(jobId, logFile);
writers = fileManager.getWriters(jobId);
if (null != writers) {
log(
writers,
RecordTypes.Meta,
new Keys[] {Keys.VERSION},
new String[] {String.valueOf(JobHistory.VERSION)});
}
String jobName = getJobName();
String user = getUserName();
// add to writer as well
log(
writers,
RecordTypes.Job,
new Keys[] {
Keys.JOBID, Keys.JOBNAME, Keys.USER, Keys.SUBMIT_TIME, Keys.JOBCONF, Keys.JOBTRACKERID
},
new String[] {
jobId.toString(), jobName, user,
String.valueOf(submitTime), jobConfPath, jobTrackerId
});
} catch (IOException e) {
// Disable history if we have errors other than in the user log.
disableHistory = true;
}
/* Storing the job conf on the log dir */
Path jobFilePath = new Path(logDir, CoronaJobHistoryFilesManager.getConfFilename(jobId));
fileManager.setConfFile(jobId, jobFilePath);
FSDataOutputStream jobFileOut = null;
try {
if (!logDirFs.exists(jobFilePath)) {
jobFileOut =
logDirFs.create(
jobFilePath,
new FsPermission(HISTORY_FILE_PERMISSION),
true,
defaultBufferSize,
logDirFs.getDefaultReplication(),
logDirFs.getDefaultBlockSize(),
null);
conf.writeXml(jobFileOut);
jobFileOut.close();
}
} catch (IOException ioe) {
LOG.error("Failed to store job conf in the log dir", ioe);
} finally {
if (jobFileOut != null) {
try {
jobFileOut.close();
} catch (IOException ie) {
LOG.info(
"Failed to close the job configuration file " + StringUtils.stringifyException(ie));
}
}
}
}
/**
* Logs launch time of job.
*
* @param startTime start time of job.
* @param totalMaps total maps assigned by jobtracker.
* @param totalReduces total reduces.
*/
public void logInited(long startTime, int totalMaps, int totalReduces) {
if (disableHistory) {
return;
}
if (null != writers) {
log(
writers,
RecordTypes.Job,
new Keys[] {
Keys.JOBID, Keys.LAUNCH_TIME, Keys.TOTAL_MAPS, Keys.TOTAL_REDUCES, Keys.JOB_STATUS
},
new String[] {
jobId.toString(),
String.valueOf(startTime),
String.valueOf(totalMaps),
String.valueOf(totalReduces),
Values.PREP.name()
});
}
}
/** Logs job as running */
public void logStarted() {
if (disableHistory) {
return;
}
if (null != writers) {
log(
writers,
RecordTypes.Job,
new Keys[] {Keys.JOBID, Keys.JOB_STATUS},
new String[] {jobId.toString(), Values.RUNNING.name()});
}
}
/**
* Log job finished. closes the job file in history.
*
* @param finishTime finish time of job in ms.
* @param finishedMaps no of maps successfully finished.
* @param finishedReduces no of reduces finished sucessfully.
* @param failedMaps no of failed map tasks. (includes killed)
* @param failedReduces no of failed reduce tasks. (includes killed)
* @param killedMaps no of killed map tasks.
* @param killedReduces no of killed reduce tasks.
* @param counters the counters from the job
*/
public void logFinished(
long finishTime,
int finishedMaps,
int finishedReduces,
int failedMaps,
int failedReduces,
int killedMaps,
int killedReduces,
Counters mapCounters,
Counters reduceCounters,
Counters counters) {
if (disableHistory) {
return;
}
if (null != writers) {
log(
writers,
RecordTypes.Job,
new Keys[] {
Keys.JOBID,
Keys.FINISH_TIME,
Keys.JOB_STATUS,
Keys.FINISHED_MAPS,
Keys.FINISHED_REDUCES,
Keys.FAILED_MAPS,
Keys.FAILED_REDUCES,
Keys.KILLED_MAPS,
Keys.KILLED_REDUCES,
Keys.MAP_COUNTERS,
Keys.REDUCE_COUNTERS,
Keys.COUNTERS
},
new String[] {
jobId.toString(),
Long.toString(finishTime),
Values.SUCCESS.name(),
String.valueOf(finishedMaps),
String.valueOf(finishedReduces),
String.valueOf(failedMaps),
String.valueOf(failedReduces),
String.valueOf(killedMaps),
String.valueOf(killedReduces),
mapCounters.makeEscapedCompactString(),
reduceCounters.makeEscapedCompactString(),
counters.makeEscapedCompactString()
},
true);
closeAndClear(writers);
}
// NOTE: history cleaning stuff deleted from here. We should do that
// somewhere else!
}
/**
* Logs job failed event. Closes the job history log file.
*
* @param timestamp time when job failure was detected in ms.
* @param finishedMaps no finished map tasks.
* @param finishedReduces no of finished reduce tasks.
*/
public void logFailed(long timestamp, int finishedMaps, int finishedReduces, Counters counters) {
if (disableHistory) {
return;
}
if (null != writers) {
log(
writers,
RecordTypes.Job,
new Keys[] {
Keys.JOBID, Keys.FINISH_TIME,
Keys.JOB_STATUS, Keys.FINISHED_MAPS,
Keys.FINISHED_REDUCES, Keys.COUNTERS
},
new String[] {
jobId.toString(),
String.valueOf(timestamp),
Values.FAILED.name(),
String.valueOf(finishedMaps),
String.valueOf(finishedReduces),
counters.makeEscapedCompactString()
},
true);
closeAndClear(writers);
}
}
/**
* Logs job killed event. Closes the job history log file.
*
* @param timestamp time when job killed was issued in ms.
* @param finishedMaps no finished map tasks.
* @param finishedReduces no of finished reduce tasks.
*/
public void logKilled(long timestamp, int finishedMaps, int finishedReduces, Counters counters) {
if (disableHistory) {
return;
}
if (null != writers) {
log(
writers,
RecordTypes.Job,
new Keys[] {
Keys.JOBID,
Keys.FINISH_TIME,
Keys.JOB_STATUS,
Keys.FINISHED_MAPS,
Keys.FINISHED_REDUCES,
Keys.COUNTERS
},
new String[] {
jobId.toString(),
String.valueOf(timestamp),
Values.KILLED.name(),
String.valueOf(finishedMaps),
String.valueOf(finishedReduces),
counters.makeEscapedCompactString()
},
true);
closeAndClear(writers);
}
}
/**
* Log job's priority.
*
* @param priority Jobs priority
*/
public void logJobPriority(JobID jobid, JobPriority priority) {
if (disableHistory) {
return;
}
if (null != writers) {
log(
writers,
RecordTypes.Job,
new Keys[] {Keys.JOBID, Keys.JOB_PRIORITY},
new String[] {jobId.toString(), priority.toString()});
}
}
/** Move the completed job into the completed folder. */
public void markCompleted() throws IOException {
if (disableHistory) {
return;
}
fileManager.moveToDone(jobId, true, CoronaJobTracker.getMainJobID(jobId));
}
/**
* Log start time of task (TIP).
*
* @param taskId task id
* @param taskType MAP or REDUCE
* @param startTime startTime of tip.
*/
public void logTaskStarted(
TaskID taskId, String taskType, long startTime, String splitLocations) {
if (disableHistory) {
return;
}
JobID id = taskId.getJobID();
if (!this.jobId.equals(id)) {
throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId);
}
if (null != writers) {
log(
writers,
RecordTypes.Task,
new Keys[] {
Keys.TASKID, Keys.TASK_TYPE,
Keys.START_TIME, Keys.SPLITS
},
new String[] {taskId.toString(), taskType, String.valueOf(startTime), splitLocations});
}
}
/**
* Log finish time of task.
*
* @param taskId task id
* @param taskType MAP or REDUCE
* @param finishTime finish timeof task in ms
*/
public void logTaskFinished(TaskID taskId, String taskType, long finishTime, Counters counters) {
if (disableHistory) {
return;
}
JobID id = taskId.getJobID();
if (!this.jobId.equals(id)) {
throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId);
}
if (null != writers) {
log(
writers,
RecordTypes.Task,
new Keys[] {
Keys.TASKID, Keys.TASK_TYPE,
Keys.TASK_STATUS, Keys.FINISH_TIME,
Keys.COUNTERS
},
new String[] {
taskId.toString(),
taskType,
Values.SUCCESS.name(),
String.valueOf(finishTime),
counters.makeEscapedCompactString()
});
}
}
/**
* Update the finish time of task.
*
* @param taskId task id
* @param finishTime finish time of task in ms
*/
public void logTaskUpdates(TaskID taskId, long finishTime) {
if (disableHistory) {
return;
}
JobID id = taskId.getJobID();
if (!this.jobId.equals(id)) {
throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId);
}
if (null != writers) {
log(
writers,
RecordTypes.Task,
new Keys[] {Keys.TASKID, Keys.FINISH_TIME},
new String[] {taskId.toString(), String.valueOf(finishTime)});
}
}
/**
* Log job failed event.
*
* @param taskId task id
* @param taskType MAP or REDUCE.
* @param time timestamp when job failed detected.
* @param error error message for failure.
*/
public void logTaskFailed(TaskID taskId, String taskType, long time, String error) {
logTaskFailed(taskId, taskType, time, error, null);
}
/**
* Log the task failure
*
* @param taskId the task that failed
* @param taskType the type of the task
* @param time the time of the failure
* @param error the error the task failed with
* @param failedDueToAttempt The attempt that caused the failure, if any
*/
public void logTaskFailed(
TaskID taskId, String taskType, long time, String error, TaskAttemptID failedDueToAttempt) {
if (disableHistory) {
return;
}
JobID id = taskId.getJobID();
if (!this.jobId.equals(id)) {
throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId);
}
if (null != writers) {
String failedAttempt = failedDueToAttempt == null ? "" : failedDueToAttempt.toString();
log(
writers,
RecordTypes.Task,
new Keys[] {
Keys.TASKID, Keys.TASK_TYPE,
Keys.TASK_STATUS, Keys.FINISH_TIME,
Keys.ERROR, Keys.TASK_ATTEMPT_ID
},
new String[] {
taskId.toString(),
taskType,
Values.FAILED.name(),
String.valueOf(time),
error,
failedAttempt
});
}
}
/**
* Log start time of this map task attempt.
*
* @param taskAttemptId task attempt id
* @param startTime start time of task attempt as reported by task tracker.
* @param trackerName name of the tracker executing the task attempt.
* @param httpPort http port of the task tracker executing the task attempt
* @param taskType Whether the attempt is cleanup or setup or map
*/
public void logMapTaskStarted(
TaskAttemptID taskAttemptId,
long startTime,
String trackerName,
int httpPort,
String taskType) {
if (disableHistory) {
return;
}
JobID id = taskAttemptId.getJobID();
if (!this.jobId.equals(id)) {
throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId);
}
if (null != writers) {
log(
writers,
RecordTypes.MapAttempt,
new Keys[] {
Keys.TASK_TYPE, Keys.TASKID,
Keys.TASK_ATTEMPT_ID, Keys.START_TIME,
Keys.TRACKER_NAME, Keys.HTTP_PORT
},
new String[] {
taskType,
taskAttemptId.getTaskID().toString(),
taskAttemptId.toString(),
String.valueOf(startTime),
trackerName,
httpPort == -1 ? "" : String.valueOf(httpPort)
});
}
}
/**
* Log finish time of map task attempt.
*
* @param taskAttemptId task attempt id
* @param finishTime finish time
* @param hostName host name
* @param taskType Whether the attempt is cleanup or setup or map
* @param stateString state string of the task attempt
* @param counter counters of the task attempt
*/
public void logMapTaskFinished(
TaskAttemptID taskAttemptId,
long finishTime,
String hostName,
String taskType,
String stateString,
Counters counter) {
if (disableHistory) {
return;
}
JobID id = taskAttemptId.getJobID();
if (!this.jobId.equals(id)) {
throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId);
}
if (null != writers) {
log(
writers,
RecordTypes.MapAttempt,
new Keys[] {
Keys.TASK_TYPE, Keys.TASKID,
Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS,
Keys.FINISH_TIME, Keys.HOSTNAME,
Keys.STATE_STRING, Keys.COUNTERS
},
new String[] {
taskType,
taskAttemptId.getTaskID().toString(),
taskAttemptId.toString(),
Values.SUCCESS.name(),
String.valueOf(finishTime),
hostName,
stateString,
counter.makeEscapedCompactString()
});
}
}
/**
* Log task attempt failed event.
*
* @param taskAttemptId task attempt id
* @param timestamp timestamp
* @param hostName hostname of this task attempt.
* @param error error message if any for this task attempt.
* @param taskType Whether the attempt is cleanup or setup or map
*/
public void logMapTaskFailed(
TaskAttemptID taskAttemptId, long timestamp, String hostName, String error, String taskType) {
if (disableHistory) {
return;
}
JobID id = taskAttemptId.getJobID();
if (!this.jobId.equals(id)) {
throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId);
}
if (null != writers) {
log(
writers,
RecordTypes.MapAttempt,
new Keys[] {
Keys.TASK_TYPE,
Keys.TASKID,
Keys.TASK_ATTEMPT_ID,
Keys.TASK_STATUS,
Keys.FINISH_TIME,
Keys.HOSTNAME,
Keys.ERROR
},
new String[] {
taskType,
taskAttemptId.getTaskID().toString(),
taskAttemptId.toString(),
Values.FAILED.name(),
String.valueOf(timestamp),
hostName,
error
});
}
}
/**
* Log task attempt killed event.
*
* @param taskAttemptId task attempt id
* @param timestamp timestamp
* @param hostName hostname of this task attempt.
* @param error error message if any for this task attempt.
* @param taskType Whether the attempt is cleanup or setup or map
*/
public void logMapTaskKilled(
TaskAttemptID taskAttemptId, long timestamp, String hostName, String error, String taskType) {
if (disableHistory) {
return;
}
JobID id = taskAttemptId.getJobID();
if (!this.jobId.equals(id)) {
throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId);
}
if (null != writers) {
log(
writers,
RecordTypes.MapAttempt,
new Keys[] {
Keys.TASK_TYPE, Keys.TASKID,
Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS,
Keys.FINISH_TIME, Keys.HOSTNAME,
Keys.ERROR
},
new String[] {
taskType,
taskAttemptId.getTaskID().toString(),
taskAttemptId.toString(),
Values.KILLED.name(),
String.valueOf(timestamp),
hostName,
error
});
}
}
/**
* Log start time of Reduce task attempt.
*
* @param taskAttemptId task attempt id
* @param startTime start time
* @param trackerName tracker name
* @param httpPort the http port of the tracker executing the task attempt
* @param taskType Whether the attempt is cleanup or setup or reduce
*/
public void logReduceTaskStarted(
TaskAttemptID taskAttemptId,
long startTime,
String trackerName,
int httpPort,
String taskType) {
if (disableHistory) {
return;
}
JobID id = taskAttemptId.getJobID();
if (!this.jobId.equals(id)) {
throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId);
}
if (null != writers) {
log(
writers,
RecordTypes.ReduceAttempt,
new Keys[] {
Keys.TASK_TYPE, Keys.TASKID,
Keys.TASK_ATTEMPT_ID, Keys.START_TIME,
Keys.TRACKER_NAME, Keys.HTTP_PORT
},
new String[] {
taskType,
taskAttemptId.getTaskID().toString(),
taskAttemptId.toString(),
String.valueOf(startTime),
trackerName,
httpPort == -1 ? "" : String.valueOf(httpPort)
});
}
}
/**
* Log finished event of this task.
*
* @param taskAttemptId task attempt id
* @param shuffleFinished shuffle finish time
* @param sortFinished sort finish time
* @param finishTime finish time of task
* @param hostName host name where task attempt executed
* @param taskType Whether the attempt is cleanup or setup or reduce
* @param stateString the state string of the attempt
* @param counter counters of the attempt
*/
public void logReduceTaskFinished(
TaskAttemptID taskAttemptId,
long shuffleFinished,
long sortFinished,
long finishTime,
String hostName,
String taskType,
String stateString,
Counters counter) {
if (disableHistory) {
return;
}
JobID id = taskAttemptId.getJobID();
if (!this.jobId.equals(id)) {
throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId);
}
if (null != writers) {
log(
writers,
RecordTypes.ReduceAttempt,
new Keys[] {
Keys.TASK_TYPE, Keys.TASKID,
Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS,
Keys.SHUFFLE_FINISHED, Keys.SORT_FINISHED,
Keys.FINISH_TIME, Keys.HOSTNAME,
Keys.STATE_STRING, Keys.COUNTERS
},
new String[] {
taskType,
taskAttemptId.getTaskID().toString(),
taskAttemptId.toString(),
Values.SUCCESS.name(),
String.valueOf(shuffleFinished),
String.valueOf(sortFinished),
String.valueOf(finishTime),
hostName,
stateString,
counter.makeEscapedCompactString()
});
}
}
/**
* Log failed reduce task attempt.
*
* @param taskAttemptId task attempt id
* @param timestamp time stamp when task failed
* @param hostName host name of the task attempt.
* @param error error message of the task.
* @param taskType Whether the attempt is cleanup or setup or reduce
*/
public void logReduceTaskFailed(
TaskAttemptID taskAttemptId, long timestamp, String hostName, String error, String taskType) {
if (disableHistory) {
return;
}
JobID id = taskAttemptId.getJobID();
if (!this.jobId.equals(id)) {
throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId);
}
if (null != writers) {
log(
writers,
RecordTypes.ReduceAttempt,
new Keys[] {
Keys.TASK_TYPE, Keys.TASKID,
Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS,
Keys.FINISH_TIME, Keys.HOSTNAME,
Keys.ERROR
},
new String[] {
taskType,
taskAttemptId.getTaskID().toString(),
taskAttemptId.toString(),
Values.FAILED.name(),
String.valueOf(timestamp),
hostName,
error
});
}
}
/**
* Log killed reduce task attempt.
*
* @param taskAttemptId task attempt id
* @param timestamp time stamp when task failed
* @param hostName host name of the task attempt.
* @param error error message of the task.
* @param taskType Whether the attempt is cleanup or setup or reduce
*/
public void logReduceTaskKilled(
TaskAttemptID taskAttemptId, long timestamp, String hostName, String error, String taskType) {
if (disableHistory) {
return;
}
JobID id = taskAttemptId.getJobID();
if (!this.jobId.equals(id)) {
throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId);
}
if (null != writers) {
log(
writers,
RecordTypes.ReduceAttempt,
new Keys[] {
Keys.TASK_TYPE, Keys.TASKID,
Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS,
Keys.FINISH_TIME, Keys.HOSTNAME,
Keys.ERROR
},
new String[] {
taskType,
taskAttemptId.getTaskID().toString(),
taskAttemptId.toString(),
Values.KILLED.name(),
String.valueOf(timestamp),
hostName,
error
});
}
}
/**
* Log a number of keys and values with record. the array length of keys and values should be
* same.
*
* @param writers the writers to send the data to
* @param recordType type of log event
* @param keys type of log event
* @param values type of log event
*/
private void log(
ArrayList<PrintWriter> writers, RecordTypes recordType, Keys[] keys, String[] values) {
log(writers, recordType, keys, values, false);
}
/**
* Log a number of keys and values with the record. This method allows to do it in a synchronous
* fashion
*
* @param writers the writers to send the data to
* @param recordType the type to log
* @param keys keys to log
* @param values values to log
* @param sync if true - will block until the data is written
*/
private void log(
ArrayList<PrintWriter> writers,
RecordTypes recordType,
Keys[] keys,
String[] values,
boolean sync) {
StringBuffer buf = new StringBuffer(recordType.name());
buf.append(JobHistory.DELIMITER);
for (int i = 0; i < keys.length; i++) {
buf.append(keys[i]);
buf.append("=\"");
values[i] = JobHistory.escapeString(values[i]);
buf.append(values[i]);
buf.append("\"");
buf.append(JobHistory.DELIMITER);
}
buf.append(JobHistory.LINE_DELIMITER_CHAR);
for (PrintWriter out : writers) {
LogTask task = new LogTask(out, buf.toString());
if (sync) {
task.run();
} else {
fileManager.addWriteTask(task);
}
}
}
}
public void testCreate() throws Exception {
Configuration conf = new HdfsConfiguration();
conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true);
conf.set(FsPermission.UMASK_LABEL, "000");
MiniDFSCluster cluster = null;
FileSystem fs = null;
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
cluster.waitActive();
fs = FileSystem.get(conf);
FsPermission rootPerm = checkPermission(fs, "/", null);
FsPermission inheritPerm = FsPermission.createImmutable((short) (rootPerm.toShort() | 0300));
FsPermission dirPerm = new FsPermission((short) 0777);
fs.mkdirs(new Path("/a1/a2/a3"), dirPerm);
checkPermission(fs, "/a1", dirPerm);
checkPermission(fs, "/a1/a2", dirPerm);
checkPermission(fs, "/a1/a2/a3", dirPerm);
dirPerm = new FsPermission((short) 0123);
FsPermission permission = FsPermission.createImmutable((short) (dirPerm.toShort() | 0300));
fs.mkdirs(new Path("/aa/1/aa/2/aa/3"), dirPerm);
checkPermission(fs, "/aa/1", permission);
checkPermission(fs, "/aa/1/aa/2", permission);
checkPermission(fs, "/aa/1/aa/2/aa/3", dirPerm);
FsPermission filePerm = new FsPermission((short) 0444);
FSDataOutputStream out =
fs.create(
new Path("/b1/b2/b3.txt"),
filePerm,
true,
conf.getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
fs.getDefaultReplication(),
fs.getDefaultBlockSize(),
null);
out.write(123);
out.close();
checkPermission(fs, "/b1", inheritPerm);
checkPermission(fs, "/b1/b2", inheritPerm);
checkPermission(fs, "/b1/b2/b3.txt", filePerm);
conf.set(FsPermission.UMASK_LABEL, "022");
permission = FsPermission.createImmutable((short) 0666);
FileSystem.mkdirs(fs, new Path("/c1"), new FsPermission(permission));
FileSystem.create(fs, new Path("/c1/c2.txt"), new FsPermission(permission));
checkPermission(fs, "/c1", permission);
checkPermission(fs, "/c1/c2.txt", permission);
} finally {
try {
if (fs != null) fs.close();
} catch (Exception e) {
LOG.error(StringUtils.stringifyException(e));
}
try {
if (cluster != null) cluster.shutdown();
} catch (Exception e) {
LOG.error(StringUtils.stringifyException(e));
}
}
}
/**
* A utility to manage job submission files.<br>
* <b><i>Note that this class is for framework internal usage only and is not to be used by users
* directly.</i></b>
*/
public class JobSubmissionFiles {
private static final Log LOG = LogFactory.getLog(JobSubmissionFiles.class);
// job submission directory is private!
public static final FsPermission JOB_DIR_PERMISSION =
FsPermission.createImmutable((short) 0700); // rwx--------
// job files are world-wide readable and owner writable
public static final FsPermission JOB_FILE_PERMISSION =
FsPermission.createImmutable((short) 0644); // rw-r--r--
public static Path getJobSplitFile(Path jobSubmissionDir) {
return new Path(jobSubmissionDir, "job.split");
}
public static Path getJobSplitMetaFile(Path jobSubmissionDir) {
return new Path(jobSubmissionDir, "job.splitmetainfo");
}
/** Get the job conf path. */
public static Path getJobConfPath(Path jobSubmitDir) {
return new Path(jobSubmitDir, "job.xml");
}
/** Get the job jar path. */
public static Path getJobJar(Path jobSubmitDir) {
return new Path(jobSubmitDir, "job.jar");
}
/**
* Get the job distributed cache files path.
*
* @param jobSubmitDir
*/
public static Path getJobDistCacheFiles(Path jobSubmitDir) {
return new Path(jobSubmitDir, "files");
}
/**
* Get the job distributed cache archives path.
*
* @param jobSubmitDir
*/
public static Path getJobDistCacheArchives(Path jobSubmitDir) {
return new Path(jobSubmitDir, "archives");
}
/**
* Get the job distributed cache libjars path.
*
* @param jobSubmitDir
*/
public static Path getJobDistCacheLibjars(Path jobSubmitDir) {
return new Path(jobSubmitDir, "libjars");
}
/**
* Initializes the staging directory and returns the path. It also keeps track of all necessary
* ownership & permissions
*
* @param client
* @param conf
*/
public static Path getStagingDir(JobClient client, Configuration conf)
throws IOException, InterruptedException {
Path stagingArea = client.getStagingAreaDir();
FileSystem fs = stagingArea.getFileSystem(conf);
String realUser;
String currentUser;
UserGroupInformation ugi = UserGroupInformation.getLoginUser();
realUser = ugi.getShortUserName();
currentUser = UserGroupInformation.getCurrentUser().getShortUserName();
if (fs.exists(stagingArea)) {
FileStatus fsStatus = fs.getFileStatus(stagingArea);
String owner = fsStatus.getOwner();
if (!(owner.equals(currentUser) || owner.equals(realUser))) {
throw new IOException(
"The ownership on the staging directory "
+ stagingArea
+ " is not as expected. "
+ "It is owned by "
+ owner
+ ". The directory must "
+ "be owned by the submitter "
+ currentUser
+ " or "
+ "by "
+ realUser);
}
if (!fsStatus.getPermission().equals(JOB_DIR_PERMISSION)) {
LOG.info(
"Permissions on staging directory "
+ stagingArea
+ " are "
+ "incorrect: "
+ fsStatus.getPermission()
+ ". Fixing permissions "
+ "to correct value "
+ JOB_DIR_PERMISSION);
fs.setPermission(stagingArea, JOB_DIR_PERMISSION);
}
} else {
fs.mkdirs(stagingArea, new FsPermission(JOB_DIR_PERMISSION));
}
return stagingArea;
}
}
@Override
public void truncateLogsAsUser(String user, List<Task> allAttempts) throws IOException {
Task firstTask = allAttempts.get(0);
String taskid = firstTask.getTaskID().toString();
LocalDirAllocator ldirAlloc = new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY);
String taskRanFile = TaskTracker.TT_LOG_TMP_DIR + Path.SEPARATOR + taskid;
Configuration conf = getConf();
// write the serialized task information to a file to pass to the truncater
Path taskRanFilePath = ldirAlloc.getLocalPathForWrite(taskRanFile, conf);
LocalFileSystem lfs = FileSystem.getLocal(conf);
FSDataOutputStream out = lfs.create(taskRanFilePath);
out.writeInt(allAttempts.size());
for (Task t : allAttempts) {
out.writeBoolean(t.isMapTask());
t.write(out);
}
out.close();
lfs.setPermission(taskRanFilePath, FsPermission.createImmutable((short) 0755));
List<String> command = new ArrayList<String>();
File jvm = // use same jvm as parent
new File(new File(System.getProperty("java.home"), "bin"), "java");
command.add(jvm.toString());
command.add("-Djava.library.path=" + System.getProperty("java.library.path"));
command.add("-Dhadoop.log.dir=" + TaskLog.getBaseLogDir());
command.add("-Dhadoop.root.logger=INFO,console");
command.add("-classpath");
command.add(System.getProperty("java.class.path"));
// main of TaskLogsTruncater
command.add(TaskLogsTruncater.class.getName());
command.add(taskRanFilePath.toString());
String[] taskControllerCmd = new String[4 + command.size()];
taskControllerCmd[0] = taskControllerExe;
taskControllerCmd[1] = user;
taskControllerCmd[2] = localStorage.getDirsString();
taskControllerCmd[3] = Integer.toString(Commands.RUN_COMMAND_AS_USER.getValue());
int i = 4;
for (String cmdArg : command) {
taskControllerCmd[i++] = cmdArg;
}
if (LOG.isDebugEnabled()) {
for (String cmd : taskControllerCmd) {
LOG.debug("taskctrl command = " + cmd);
}
}
ShellCommandExecutor shExec = new ShellCommandExecutor(taskControllerCmd);
try {
shExec.execute();
} catch (Exception e) {
LOG.warn(
"Exit code from "
+ taskControllerExe.toString()
+ " is : "
+ shExec.getExitCode()
+ " for truncateLogs");
LOG.warn(
"Exception thrown by "
+ taskControllerExe.toString()
+ " : "
+ StringUtils.stringifyException(e));
LOG.info("Output from LinuxTaskController's " + taskControllerExe.toString() + " follows:");
logOutput(shExec.getOutput());
lfs.delete(taskRanFilePath, false);
throw new IOException(e);
}
lfs.delete(taskRanFilePath, false);
if (LOG.isDebugEnabled()) {
LOG.info("Output from LinuxTaskController's " + taskControllerExe.toString() + " follows:");
logOutput(shExec.getOutput());
}
}
class INodeFile extends INode {
static final FsPermission UMASK = FsPermission.createImmutable((short) 0111);
// Number of bits for Block size
static final short BLOCKBITS = 48;
// Header mask 64-bit representation
// Format: [16 bits for replication][48 bits for PreferredBlockSize]
static final long HEADERMASK = 0xffffL << BLOCKBITS;
protected long header;
protected BlockInfo blocks[] = null;
INodeFile(
PermissionStatus permissions,
int nrBlocks,
short replication,
long modificationTime,
long atime,
long preferredBlockSize) {
this(
permissions,
new BlockInfo[nrBlocks],
replication,
modificationTime,
atime,
preferredBlockSize);
}
protected INodeFile() {
blocks = null;
header = 0;
}
protected INodeFile(
PermissionStatus permissions,
BlockInfo[] blklist,
short replication,
long modificationTime,
long atime,
long preferredBlockSize) {
super(permissions, modificationTime, atime);
this.setReplication(replication);
this.setPreferredBlockSize(preferredBlockSize);
blocks = blklist;
}
/**
* Set the {@link FsPermission} of this {@link INodeFile}. Since this is a file, the {@link
* FsAction#EXECUTE} action, if any, is ignored.
*/
protected void setPermission(FsPermission permission) {
super.setPermission(permission.applyUMask(UMASK));
}
public boolean isDirectory() {
return false;
}
/**
* Get block replication for the file
*
* @return block replication value
*/
public short getReplication() {
return (short) ((header & HEADERMASK) >> BLOCKBITS);
}
public void setReplication(short replication) {
if (replication <= 0)
throw new IllegalArgumentException("Unexpected value for the replication");
header = ((long) replication << BLOCKBITS) | (header & ~HEADERMASK);
}
/**
* Get preferred block size for the file
*
* @return preferred block size in bytes
*/
public long getPreferredBlockSize() {
return header & ~HEADERMASK;
}
public void setPreferredBlockSize(long preferredBlkSize) {
if ((preferredBlkSize < 0) || (preferredBlkSize > ~HEADERMASK))
throw new IllegalArgumentException("Unexpected value for the block size");
header = (header & HEADERMASK) | (preferredBlkSize & ~HEADERMASK);
}
/**
* Get file blocks
*
* @return file blocks
*/
BlockInfo[] getBlocks() {
return this.blocks;
}
/** Return the last block in this file, or null if there are no blocks. */
Block getLastBlock() {
if (this.blocks == null || this.blocks.length == 0) return null;
return this.blocks[this.blocks.length - 1];
}
/** add a block to the block list */
void addBlock(BlockInfo newblock) {
if (this.blocks == null) {
this.blocks = new BlockInfo[1];
this.blocks[0] = newblock;
} else {
int size = this.blocks.length;
BlockInfo[] newlist = new BlockInfo[size + 1];
System.arraycopy(this.blocks, 0, newlist, 0, size);
newlist[size] = newblock;
this.blocks = newlist;
}
}
/** Set file block */
void setBlock(int idx, BlockInfo blk) {
this.blocks[idx] = blk;
}
int collectSubtreeBlocksAndClear(List<Block> v) {
parent = null;
for (Block blk : blocks) {
v.add(blk);
}
blocks = null;
return 1;
}
/** {@inheritDoc} */
long[] computeContentSummary(long[] summary) {
long bytes = 0;
for (Block blk : blocks) {
bytes += blk.getNumBytes();
}
summary[0] += bytes;
summary[1]++;
summary[3] += diskspaceConsumed();
return summary;
}
@Override
DirCounts spaceConsumedInTree(DirCounts counts) {
counts.nsCount += 1;
counts.dsCount += diskspaceConsumed();
return counts;
}
long diskspaceConsumed() {
return diskspaceConsumed(blocks);
}
long diskspaceConsumed(Block[] blkArr) {
long size = 0;
for (Block blk : blkArr) {
if (blk != null) {
size += blk.getNumBytes();
}
}
/* If the last block is being written to, use prefferedBlockSize
* rather than the actual block size.
*/
if (blkArr.length > 0 && blkArr[blkArr.length - 1] != null && isUnderConstruction()) {
size += getPreferredBlockSize() - blocks[blocks.length - 1].getNumBytes();
}
return size * getReplication();
}
/** Return the penultimate allocated block for this file. */
Block getPenultimateBlock() {
if (blocks == null || blocks.length <= 1) {
return null;
}
return blocks[blocks.length - 2];
}
}
@Test
public void testOriginalAclEnforcedForSnapshotContentsAfterRemoval() throws Exception {
Path filePath = new Path(path, "file1");
Path subdirPath = new Path(path, "subdir1");
Path fileSnapshotPath = new Path(snapshotPath, "file1");
Path subdirSnapshotPath = new Path(snapshotPath, "subdir1");
FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0777));
FileSystem.create(hdfs, filePath, FsPermission.createImmutable((short) 0600)).close();
FileSystem.mkdirs(hdfs, subdirPath, FsPermission.createImmutable((short) 0700));
List<AclEntry> aclSpec =
Lists.newArrayList(
aclEntry(ACCESS, USER, READ_EXECUTE),
aclEntry(ACCESS, USER, "bruce", READ_EXECUTE),
aclEntry(ACCESS, GROUP, NONE),
aclEntry(ACCESS, OTHER, NONE));
hdfs.setAcl(filePath, aclSpec);
hdfs.setAcl(subdirPath, aclSpec);
assertFilePermissionGranted(fsAsBruce, BRUCE, filePath);
assertFilePermissionDenied(fsAsDiana, DIANA, filePath);
assertDirPermissionGranted(fsAsBruce, BRUCE, subdirPath);
assertDirPermissionDenied(fsAsDiana, DIANA, subdirPath);
SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName);
// Both original and snapshot still have same ACL.
AclEntry[] expected =
new AclEntry[] {
aclEntry(ACCESS, USER, "bruce", READ_EXECUTE), aclEntry(ACCESS, GROUP, NONE)
};
AclStatus s = hdfs.getAclStatus(filePath);
AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(expected, returned);
assertPermission((short) 010550, filePath);
s = hdfs.getAclStatus(subdirPath);
returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(expected, returned);
assertPermission((short) 010550, subdirPath);
s = hdfs.getAclStatus(fileSnapshotPath);
returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(expected, returned);
assertPermission((short) 010550, fileSnapshotPath);
assertFilePermissionGranted(fsAsBruce, BRUCE, fileSnapshotPath);
assertFilePermissionDenied(fsAsDiana, DIANA, fileSnapshotPath);
s = hdfs.getAclStatus(subdirSnapshotPath);
returned = s.getEntries().toArray(new AclEntry[0]);
assertArrayEquals(expected, returned);
assertPermission((short) 010550, subdirSnapshotPath);
assertDirPermissionGranted(fsAsBruce, BRUCE, subdirSnapshotPath);
assertDirPermissionDenied(fsAsDiana, DIANA, subdirSnapshotPath);
hdfs.removeAcl(filePath);
hdfs.removeAcl(subdirPath);
// Original has changed, but snapshot still has old ACL.
doSnapshotContentsRemovalAssertions(filePath, fileSnapshotPath, subdirPath, subdirSnapshotPath);
restart(false);
doSnapshotContentsRemovalAssertions(filePath, fileSnapshotPath, subdirPath, subdirSnapshotPath);
restart(true);
doSnapshotContentsRemovalAssertions(filePath, fileSnapshotPath, subdirPath, subdirSnapshotPath);
}
@Public
@Evolving
public class AggregatedLogFormat {
private static final Log LOG = LogFactory.getLog(AggregatedLogFormat.class);
private static final LogKey APPLICATION_ACL_KEY = new LogKey("APPLICATION_ACL");
private static final LogKey APPLICATION_OWNER_KEY = new LogKey("APPLICATION_OWNER");
private static final LogKey VERSION_KEY = new LogKey("VERSION");
private static final Map<String, LogKey> RESERVED_KEYS;
// Maybe write out the retention policy.
// Maybe write out a list of containerLogs skipped by the retention policy.
private static final int VERSION = 1;
/** Umask for the log file. */
private static final FsPermission APP_LOG_FILE_UMASK =
FsPermission.createImmutable((short) (0640 ^ 0777));
static {
RESERVED_KEYS = new HashMap<String, AggregatedLogFormat.LogKey>();
RESERVED_KEYS.put(APPLICATION_ACL_KEY.toString(), APPLICATION_ACL_KEY);
RESERVED_KEYS.put(APPLICATION_OWNER_KEY.toString(), APPLICATION_OWNER_KEY);
RESERVED_KEYS.put(VERSION_KEY.toString(), VERSION_KEY);
}
@Public
public static class LogKey implements Writable {
private String keyString;
public LogKey() {}
public LogKey(ContainerId containerId) {
this.keyString = containerId.toString();
}
public LogKey(String keyString) {
this.keyString = keyString;
}
@Override
public int hashCode() {
return keyString == null ? 0 : keyString.hashCode();
}
@Override
public boolean equals(Object obj) {
if (obj instanceof LogKey) {
LogKey other = (LogKey) obj;
if (this.keyString == null) {
return other.keyString == null;
}
return this.keyString.equals(other.keyString);
}
return false;
}
@Private
@Override
public void write(DataOutput out) throws IOException {
out.writeUTF(this.keyString);
}
@Private
@Override
public void readFields(DataInput in) throws IOException {
this.keyString = in.readUTF();
}
@Override
public String toString() {
return this.keyString;
}
}
@Private
public static class LogValue {
private final List<String> rootLogDirs;
private final ContainerId containerId;
private final String user;
// TODO Maybe add a version string here. Instead of changing the version of
// the entire k-v format
public LogValue(List<String> rootLogDirs, ContainerId containerId, String user) {
this.rootLogDirs = new ArrayList<String>(rootLogDirs);
this.containerId = containerId;
this.user = user;
// Ensure logs are processed in lexical order
Collections.sort(this.rootLogDirs);
}
public void write(DataOutputStream out) throws IOException {
for (String rootLogDir : this.rootLogDirs) {
File appLogDir =
new File(
rootLogDir,
ConverterUtils.toString(
this.containerId.getApplicationAttemptId().getApplicationId()));
File containerLogDir = new File(appLogDir, ConverterUtils.toString(this.containerId));
if (!containerLogDir.isDirectory()) {
continue; // ContainerDir may have been deleted by the user.
}
// Write out log files in lexical order
File[] logFiles = containerLogDir.listFiles();
Arrays.sort(logFiles);
for (File logFile : logFiles) {
// Write the logFile Type
out.writeUTF(logFile.getName());
// Write the log length as UTF so that it is printable
out.writeUTF(String.valueOf(logFile.length()));
// Write the log itself
FileInputStream in = null;
try {
in = SecureIOUtils.openForRead(logFile, getUser(), null);
byte[] buf = new byte[65535];
int len = 0;
while ((len = in.read(buf)) != -1) {
out.write(buf, 0, len);
}
} catch (IOException e) {
String message =
"Error aggregating log file. Log file : "
+ logFile.getAbsolutePath()
+ e.getMessage();
LOG.error(message, e);
out.write(message.getBytes());
} finally {
if (in != null) {
in.close();
}
}
}
}
}
// Added for testing purpose.
public String getUser() {
return user;
}
}
/** The writer that writes out the aggregated logs. */
@Private
public static class LogWriter {
private final FSDataOutputStream fsDataOStream;
private final TFile.Writer writer;
public LogWriter(
final Configuration conf, final Path remoteAppLogFile, UserGroupInformation userUgi)
throws IOException {
try {
this.fsDataOStream =
userUgi.doAs(
new PrivilegedExceptionAction<FSDataOutputStream>() {
@Override
public FSDataOutputStream run() throws Exception {
FileContext fc = FileContext.getFileContext(conf);
fc.setUMask(APP_LOG_FILE_UMASK);
return fc.create(
remoteAppLogFile,
EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
new Options.CreateOpts[] {});
}
});
} catch (InterruptedException e) {
throw new IOException(e);
}
// Keys are not sorted: null arg
// 256KB minBlockSize : Expected log size for each container too
this.writer =
new TFile.Writer(
this.fsDataOStream,
256 * 1024,
conf.get(
YarnConfiguration.NM_LOG_AGG_COMPRESSION_TYPE,
YarnConfiguration.DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE),
null,
conf);
// Write the version string
writeVersion();
}
private void writeVersion() throws IOException {
DataOutputStream out = this.writer.prepareAppendKey(-1);
VERSION_KEY.write(out);
out.close();
out = this.writer.prepareAppendValue(-1);
out.writeInt(VERSION);
out.close();
}
public void writeApplicationOwner(String user) throws IOException {
DataOutputStream out = this.writer.prepareAppendKey(-1);
APPLICATION_OWNER_KEY.write(out);
out.close();
out = this.writer.prepareAppendValue(-1);
out.writeUTF(user);
out.close();
}
public void writeApplicationACLs(Map<ApplicationAccessType, String> appAcls)
throws IOException {
DataOutputStream out = this.writer.prepareAppendKey(-1);
APPLICATION_ACL_KEY.write(out);
out.close();
out = this.writer.prepareAppendValue(-1);
for (Entry<ApplicationAccessType, String> entry : appAcls.entrySet()) {
out.writeUTF(entry.getKey().toString());
out.writeUTF(entry.getValue());
}
out.close();
}
public void append(LogKey logKey, LogValue logValue) throws IOException {
DataOutputStream out = this.writer.prepareAppendKey(-1);
logKey.write(out);
out.close();
out = this.writer.prepareAppendValue(-1);
logValue.write(out);
out.close();
}
public void close() {
try {
this.writer.close();
} catch (IOException e) {
LOG.warn("Exception closing writer", e);
}
try {
this.fsDataOStream.close();
} catch (IOException e) {
LOG.warn("Exception closing output-stream", e);
}
}
}
@Public
@Evolving
public static class LogReader {
private final FSDataInputStream fsDataIStream;
private final TFile.Reader.Scanner scanner;
private final TFile.Reader reader;
public LogReader(Configuration conf, Path remoteAppLogFile) throws IOException {
FileContext fileContext = FileContext.getFileContext(conf);
this.fsDataIStream = fileContext.open(remoteAppLogFile);
reader =
new TFile.Reader(
this.fsDataIStream, fileContext.getFileStatus(remoteAppLogFile).getLen(), conf);
this.scanner = reader.createScanner();
}
private boolean atBeginning = true;
/**
* Returns the owner of the application.
*
* @return the application owner.
* @throws IOException
*/
public String getApplicationOwner() throws IOException {
TFile.Reader.Scanner ownerScanner = reader.createScanner();
LogKey key = new LogKey();
while (!ownerScanner.atEnd()) {
TFile.Reader.Scanner.Entry entry = ownerScanner.entry();
key.readFields(entry.getKeyStream());
if (key.toString().equals(APPLICATION_OWNER_KEY.toString())) {
DataInputStream valueStream = entry.getValueStream();
return valueStream.readUTF();
}
ownerScanner.advance();
}
return null;
}
/**
* Returns ACLs for the application. An empty map is returned if no ACLs are found.
*
* @return a map of the Application ACLs.
* @throws IOException
*/
public Map<ApplicationAccessType, String> getApplicationAcls() throws IOException {
// TODO Seek directly to the key once a comparator is specified.
TFile.Reader.Scanner aclScanner = reader.createScanner();
LogKey key = new LogKey();
Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>();
while (!aclScanner.atEnd()) {
TFile.Reader.Scanner.Entry entry = aclScanner.entry();
key.readFields(entry.getKeyStream());
if (key.toString().equals(APPLICATION_ACL_KEY.toString())) {
DataInputStream valueStream = entry.getValueStream();
while (true) {
String appAccessOp = null;
String aclString = null;
try {
appAccessOp = valueStream.readUTF();
} catch (EOFException e) {
// Valid end of stream.
break;
}
try {
aclString = valueStream.readUTF();
} catch (EOFException e) {
throw new YarnRuntimeException("Error reading ACLs", e);
}
acls.put(ApplicationAccessType.valueOf(appAccessOp), aclString);
}
}
aclScanner.advance();
}
return acls;
}
/**
* Read the next key and return the value-stream.
*
* @param key
* @return the valueStream if there are more keys or null otherwise.
* @throws IOException
*/
public DataInputStream next(LogKey key) throws IOException {
if (!this.atBeginning) {
this.scanner.advance();
} else {
this.atBeginning = false;
}
if (this.scanner.atEnd()) {
return null;
}
TFile.Reader.Scanner.Entry entry = this.scanner.entry();
key.readFields(entry.getKeyStream());
// Skip META keys
if (RESERVED_KEYS.containsKey(key.toString())) {
return next(key);
}
DataInputStream valueStream = entry.getValueStream();
return valueStream;
}
/**
* Get a ContainerLogsReader to read the logs for the specified container.
*
* @param containerId
* @return object to read the container's logs or null if the logs could not be found
* @throws IOException
*/
@Private
public ContainerLogsReader getContainerLogsReader(ContainerId containerId) throws IOException {
ContainerLogsReader logReader = null;
final LogKey containerKey = new LogKey(containerId);
LogKey key = new LogKey();
DataInputStream valueStream = next(key);
while (valueStream != null && !key.equals(containerKey)) {
valueStream = next(key);
}
if (valueStream != null) {
logReader = new ContainerLogsReader(valueStream);
}
return logReader;
}
// TODO Change Log format and interfaces to be containerId specific.
// Avoid returning completeValueStreams.
// public List<String> getTypesForContainer(DataInputStream valueStream){}
//
// /**
// * @param valueStream
// * The Log stream for the container.
// * @param fileType
// * the log type required.
// * @return An InputStreamReader for the required log type or null if the
// * type is not found.
// * @throws IOException
// */
// public InputStreamReader getLogStreamForType(DataInputStream valueStream,
// String fileType) throws IOException {
// valueStream.reset();
// try {
// while (true) {
// String ft = valueStream.readUTF();
// String fileLengthStr = valueStream.readUTF();
// long fileLength = Long.parseLong(fileLengthStr);
// if (ft.equals(fileType)) {
// BoundedInputStream bis =
// new BoundedInputStream(valueStream, fileLength);
// return new InputStreamReader(bis);
// } else {
// long totalSkipped = 0;
// long currSkipped = 0;
// while (currSkipped != -1 && totalSkipped < fileLength) {
// currSkipped = valueStream.skip(fileLength - totalSkipped);
// totalSkipped += currSkipped;
// }
// // TODO Verify skip behaviour.
// if (currSkipped == -1) {
// return null;
// }
// }
// }
// } catch (EOFException e) {
// return null;
// }
// }
/**
* Writes all logs for a single container to the provided writer.
*
* @param valueStream
* @param writer
* @throws IOException
*/
public static void readAcontainerLogs(DataInputStream valueStream, Writer writer)
throws IOException {
int bufferSize = 65536;
char[] cbuf = new char[bufferSize];
String fileType;
String fileLengthStr;
long fileLength;
while (true) {
try {
fileType = valueStream.readUTF();
} catch (EOFException e) {
// EndOfFile
return;
}
fileLengthStr = valueStream.readUTF();
fileLength = Long.parseLong(fileLengthStr);
writer.write("\n\nLogType:");
writer.write(fileType);
writer.write("\nLogLength:");
writer.write(fileLengthStr);
writer.write("\nLog Contents:\n");
// ByteLevel
BoundedInputStream bis = new BoundedInputStream(valueStream, fileLength);
InputStreamReader reader = new InputStreamReader(bis);
int currentRead = 0;
int totalRead = 0;
while ((currentRead = reader.read(cbuf, 0, bufferSize)) != -1) {
writer.write(cbuf, 0, currentRead);
totalRead += currentRead;
}
}
}
/**
* Keep calling this till you get a {@link EOFException} for getting logs of all types for a
* single container.
*
* @param valueStream
* @param out
* @throws IOException
*/
public static void readAContainerLogsForALogType(DataInputStream valueStream, PrintStream out)
throws IOException {
byte[] buf = new byte[65535];
String fileType = valueStream.readUTF();
String fileLengthStr = valueStream.readUTF();
long fileLength = Long.parseLong(fileLengthStr);
out.print("LogType: ");
out.println(fileType);
out.print("LogLength: ");
out.println(fileLengthStr);
out.println("Log Contents:");
int curRead = 0;
long pendingRead = fileLength - curRead;
int toRead = pendingRead > buf.length ? buf.length : (int) pendingRead;
int len = valueStream.read(buf, 0, toRead);
while (len != -1 && curRead < fileLength) {
out.write(buf, 0, len);
curRead += len;
pendingRead = fileLength - curRead;
toRead = pendingRead > buf.length ? buf.length : (int) pendingRead;
len = valueStream.read(buf, 0, toRead);
}
out.println("");
}
public void close() {
IOUtils.cleanup(LOG, scanner, reader, fsDataIStream);
}
}
@Private
public static class ContainerLogsReader {
private DataInputStream valueStream;
private String currentLogType = null;
private long currentLogLength = 0;
private BoundedInputStream currentLogData = null;
private InputStreamReader currentLogISR;
public ContainerLogsReader(DataInputStream stream) {
valueStream = stream;
}
public String nextLog() throws IOException {
if (currentLogData != null && currentLogLength > 0) {
// seek to the end of the current log, relying on BoundedInputStream
// to prevent seeking past the end of the current log
do {
if (currentLogData.skip(currentLogLength) < 0) {
break;
}
} while (currentLogData.read() != -1);
}
currentLogType = null;
currentLogLength = 0;
currentLogData = null;
currentLogISR = null;
try {
String logType = valueStream.readUTF();
String logLengthStr = valueStream.readUTF();
currentLogLength = Long.parseLong(logLengthStr);
currentLogData = new BoundedInputStream(valueStream, currentLogLength);
currentLogData.setPropagateClose(false);
currentLogISR = new InputStreamReader(currentLogData);
currentLogType = logType;
} catch (EOFException e) {
}
return currentLogType;
}
public String getCurrentLogType() {
return currentLogType;
}
public long getCurrentLogLength() {
return currentLogLength;
}
public long skip(long n) throws IOException {
return currentLogData.skip(n);
}
public int read(byte[] buf, int off, int len) throws IOException {
return currentLogData.read(buf, off, len);
}
public int read(char[] buf, int off, int len) throws IOException {
return currentLogISR.read(buf, off, len);
}
}
}
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class JobHistoryUtils {
/** Permissions for the history staging dir while JobInProgress. */
public static final FsPermission HISTORY_STAGING_DIR_PERMISSIONS =
FsPermission.createImmutable((short) 0700);
/** Permissions for the user directory under the staging directory. */
public static final FsPermission HISTORY_STAGING_USER_DIR_PERMISSIONS =
FsPermission.createImmutable((short) 0700);
/** Permissions for the history done dir and derivatives. */
public static final FsPermission HISTORY_DONE_DIR_PERMISSION =
FsPermission.createImmutable((short) 0770);
public static final FsPermission HISTORY_DONE_FILE_PERMISSION =
FsPermission.createImmutable((short) 0770); // rwx------
/** Umask for the done dir and derivatives. */
public static final FsPermission HISTORY_DONE_DIR_UMASK =
FsPermission.createImmutable((short) (0770 ^ 0777));
/** Permissions for the intermediate done directory. */
public static final FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS =
FsPermission.createImmutable((short) 01777);
/** Permissions for the user directory under the intermediate done directory. */
public static final FsPermission HISTORY_INTERMEDIATE_USER_DIR_PERMISSIONS =
FsPermission.createImmutable((short) 0770);
public static final FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS =
FsPermission.createImmutable((short) 0770); // rwx------
/** Suffix for configuration files. */
public static final String CONF_FILE_NAME_SUFFIX = "_conf.xml";
/** Suffix for summary files. */
public static final String SUMMARY_FILE_NAME_SUFFIX = ".summary";
/** Job History File extension. */
public static final String JOB_HISTORY_FILE_EXTENSION = ".jhist";
public static final int VERSION = 4;
public static final int SERIAL_NUMBER_DIRECTORY_DIGITS = 6;
public static final String TIMESTAMP_DIR_REGEX =
"\\d{4}" + "\\" + Path.SEPARATOR + "\\d{2}" + "\\" + Path.SEPARATOR + "\\d{2}";
public static final Pattern TIMESTAMP_DIR_PATTERN = Pattern.compile(TIMESTAMP_DIR_REGEX);
private static final String TIMESTAMP_DIR_FORMAT =
"%04d" + File.separator + "%02d" + File.separator + "%02d";
private static final PathFilter CONF_FILTER =
new PathFilter() {
@Override
public boolean accept(Path path) {
return path.getName().endsWith(CONF_FILE_NAME_SUFFIX);
}
};
private static final PathFilter JOB_HISTORY_FILE_FILTER =
new PathFilter() {
@Override
public boolean accept(Path path) {
return path.getName().endsWith(JOB_HISTORY_FILE_EXTENSION);
}
};
/**
* Checks whether the provided path string is a valid job history file.
*
* @param pathString the path to be checked.
* @return true is the path is a valid job history filename else return false
*/
public static boolean isValidJobHistoryFileName(String pathString) {
return pathString.endsWith(JOB_HISTORY_FILE_EXTENSION);
}
/**
* Returns the jobId from a job history file name.
*
* @param pathString the path string.
* @return the JobId
* @throws IOException if the filename format is invalid.
*/
public static JobID getJobIDFromHistoryFilePath(String pathString) throws IOException {
String[] parts = pathString.split(Path.SEPARATOR);
String fileNamePart = parts[parts.length - 1];
JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fileNamePart);
return TypeConverter.fromYarn(jobIndexInfo.getJobId());
}
/**
* Gets a PathFilter which would match configuration files.
*
* @return the patch filter {@link PathFilter} for matching conf files.
*/
public static PathFilter getConfFileFilter() {
return CONF_FILTER;
}
/**
* Gets a PathFilter which would match job history file names.
*
* @return the path filter {@link PathFilter} matching job history files.
*/
public static PathFilter getHistoryFileFilter() {
return JOB_HISTORY_FILE_FILTER;
}
/**
* Gets the configured directory prefix for In Progress history files.
*
* @param conf the configuration for hte job
* @param jobId the id of the job the history file is for.
* @return A string representation of the prefix.
*/
public static String getConfiguredHistoryStagingDirPrefix(Configuration conf, String jobId)
throws IOException {
String user = UserGroupInformation.getCurrentUser().getShortUserName();
Path stagingPath = MRApps.getStagingAreaDir(conf, user);
Path path = new Path(stagingPath, jobId);
String logDir = path.toString();
return logDir;
}
/**
* Gets the configured directory prefix for intermediate done history files.
*
* @param conf
* @return A string representation of the prefix.
*/
public static String getConfiguredHistoryIntermediateDoneDirPrefix(Configuration conf) {
String doneDirPrefix = conf.get(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR);
if (doneDirPrefix == null) {
doneDirPrefix =
conf.get(MRJobConfig.MR_AM_STAGING_DIR, MRJobConfig.DEFAULT_MR_AM_STAGING_DIR)
+ "/history/done_intermediate";
}
return doneDirPrefix;
}
/**
* Gets the configured directory prefix for Done history files.
*
* @param conf the configuration object
* @return the done history directory
*/
public static String getConfiguredHistoryServerDoneDirPrefix(Configuration conf) {
String doneDirPrefix = conf.get(JHAdminConfig.MR_HISTORY_DONE_DIR);
if (doneDirPrefix == null) {
doneDirPrefix =
conf.get(MRJobConfig.MR_AM_STAGING_DIR, MRJobConfig.DEFAULT_MR_AM_STAGING_DIR)
+ "/history/done";
}
return doneDirPrefix;
}
/**
* Gets the user directory for intermediate done history files.
*
* @param conf the configuration object
* @return the intermediate done directory for jobhistory files.
*/
public static String getHistoryIntermediateDoneDirForUser(Configuration conf) throws IOException {
return getConfiguredHistoryIntermediateDoneDirPrefix(conf)
+ File.separator
+ UserGroupInformation.getCurrentUser().getShortUserName();
}
public static boolean shouldCreateNonUserDirectory(Configuration conf) {
// Returning true by default to allow non secure single node clusters to work
// without any configuration change.
return conf.getBoolean(MRJobConfig.MR_AM_CREATE_JH_INTERMEDIATE_BASE_DIR, true);
}
/** Get the job history file path for non Done history files. */
public static Path getStagingJobHistoryFile(Path dir, JobId jobId, int attempt) {
return getStagingJobHistoryFile(dir, TypeConverter.fromYarn(jobId).toString(), attempt);
}
/** Get the job history file path for non Done history files. */
public static Path getStagingJobHistoryFile(Path dir, String jobId, int attempt) {
return new Path(dir, jobId + "_" + attempt + JOB_HISTORY_FILE_EXTENSION);
}
/**
* Get the done configuration file name for a job.
*
* @param jobId the jobId.
* @return the conf file name.
*/
public static String getIntermediateConfFileName(JobId jobId) {
return TypeConverter.fromYarn(jobId).toString() + CONF_FILE_NAME_SUFFIX;
}
/**
* Get the done summary file name for a job.
*
* @param jobId the jobId.
* @return the conf file name.
*/
public static String getIntermediateSummaryFileName(JobId jobId) {
return TypeConverter.fromYarn(jobId).toString() + SUMMARY_FILE_NAME_SUFFIX;
}
/**
* Gets the conf file path for jobs in progress.
*
* @param logDir the log directory prefix.
* @param jobId the jobId.
* @param attempt attempt number for this job.
* @return the conf file path for jobs in progress.
*/
public static Path getStagingConfFile(Path logDir, JobId jobId, int attempt) {
Path jobFilePath = null;
if (logDir != null) {
jobFilePath =
new Path(
logDir,
TypeConverter.fromYarn(jobId).toString() + "_" + attempt + CONF_FILE_NAME_SUFFIX);
}
return jobFilePath;
}
/**
* Gets the serial number part of the path based on the jobId and serialNumber format.
*
* @param id
* @param serialNumberFormat
* @return the serial number part of the patch based on the jobId and serial number format.
*/
public static String serialNumberDirectoryComponent(JobId id, String serialNumberFormat) {
return String.format(serialNumberFormat, Integer.valueOf(jobSerialNumber(id)))
.substring(0, SERIAL_NUMBER_DIRECTORY_DIGITS);
}
/**
* Extracts the timstamp component from the path.
*
* @param path
* @return the timestamp component from the path
*/
public static String getTimestampPartFromPath(String path) {
Matcher matcher = TIMESTAMP_DIR_PATTERN.matcher(path);
if (matcher.find()) {
String matched = matcher.group();
String ret = matched.intern();
return ret;
} else {
return null;
}
}
/**
* Gets the history subdirectory based on the jobId, timestamp and serial number format.
*
* @param id
* @param timestampComponent
* @param serialNumberFormat
* @return the history sub directory based on the jobid, timestamp and serial number format
*/
public static String historyLogSubdirectory(
JobId id, String timestampComponent, String serialNumberFormat) {
// String result = LOG_VERSION_STRING;
String result = "";
String serialNumberDirectory = serialNumberDirectoryComponent(id, serialNumberFormat);
result = result + timestampComponent + File.separator + serialNumberDirectory + File.separator;
return result;
}
/**
* Gets the timestamp component based on millisecond time.
*
* @param millisecondTime
* @return the timestamp component based on millisecond time
*/
public static String timestampDirectoryComponent(long millisecondTime) {
Calendar timestamp = Calendar.getInstance();
timestamp.setTimeInMillis(millisecondTime);
String dateString = null;
dateString =
String.format(
TIMESTAMP_DIR_FORMAT,
timestamp.get(Calendar.YEAR),
// months are 0-based in Calendar, but people will expect January to
// be month #1.
timestamp.get(Calendar.MONTH) + 1,
timestamp.get(Calendar.DAY_OF_MONTH));
dateString = dateString.intern();
return dateString;
}
public static String doneSubdirsBeforeSerialTail() {
// date
String result = "/*/*/*"; // YYYY/MM/DD ;
return result;
}
/**
* Computes a serial number used as part of directory naming for the given jobId.
*
* @param id the jobId.
* @return the serial number used as part of directory naming for the given jobid
*/
public static int jobSerialNumber(JobId id) {
return id.getId();
}
public static List<FileStatus> localGlobber(FileContext fc, Path root, String tail)
throws IOException {
return localGlobber(fc, root, tail, null);
}
public static List<FileStatus> localGlobber(
FileContext fc, Path root, String tail, PathFilter filter) throws IOException {
return localGlobber(fc, root, tail, filter, null);
}
// hasMismatches is just used to return a second value if you want
// one. I would have used MutableBoxedBoolean if such had been provided.
public static List<FileStatus> localGlobber(
FileContext fc, Path root, String tail, PathFilter filter, AtomicBoolean hasFlatFiles)
throws IOException {
if (tail.equals("")) {
return (listFilteredStatus(fc, root, filter));
}
if (tail.startsWith("/*")) {
Path[] subdirs =
filteredStat2Paths(remoteIterToList(fc.listStatus(root)), true, hasFlatFiles);
List<List<FileStatus>> subsubdirs = new LinkedList<List<FileStatus>>();
int subsubdirCount = 0;
if (subdirs.length == 0) {
return new LinkedList<FileStatus>();
}
String newTail = tail.substring(2);
for (int i = 0; i < subdirs.length; ++i) {
subsubdirs.add(localGlobber(fc, subdirs[i], newTail, filter, null));
// subsubdirs.set(i, localGlobber(fc, subdirs[i], newTail, filter,
// null));
subsubdirCount += subsubdirs.get(i).size();
}
List<FileStatus> result = new LinkedList<FileStatus>();
for (int i = 0; i < subsubdirs.size(); ++i) {
result.addAll(subsubdirs.get(i));
}
return result;
}
if (tail.startsWith("/")) {
int split = tail.indexOf('/', 1);
if (split < 0) {
return listFilteredStatus(fc, new Path(root, tail.substring(1)), filter);
} else {
String thisSegment = tail.substring(1, split);
String newTail = tail.substring(split);
return localGlobber(fc, new Path(root, thisSegment), newTail, filter, hasFlatFiles);
}
}
IOException e = new IOException("localGlobber: bad tail");
throw e;
}
private static List<FileStatus> listFilteredStatus(FileContext fc, Path root, PathFilter filter)
throws IOException {
List<FileStatus> fsList = remoteIterToList(fc.listStatus(root));
if (filter == null) {
return fsList;
} else {
List<FileStatus> filteredList = new LinkedList<FileStatus>();
for (FileStatus fs : fsList) {
if (filter.accept(fs.getPath())) {
filteredList.add(fs);
}
}
return filteredList;
}
}
private static List<FileStatus> remoteIterToList(RemoteIterator<FileStatus> rIter)
throws IOException {
List<FileStatus> fsList = new LinkedList<FileStatus>();
if (rIter == null) return fsList;
while (rIter.hasNext()) {
fsList.add(rIter.next());
}
return fsList;
}
// hasMismatches is just used to return a second value if you want
// one. I would have used MutableBoxedBoolean if such had been provided.
private static Path[] filteredStat2Paths(
List<FileStatus> stats, boolean dirs, AtomicBoolean hasMismatches) {
int resultCount = 0;
if (hasMismatches == null) {
hasMismatches = new AtomicBoolean(false);
}
for (int i = 0; i < stats.size(); ++i) {
if (stats.get(i).isDirectory() == dirs) {
stats.set(resultCount++, stats.get(i));
} else {
hasMismatches.set(true);
}
}
Path[] result = new Path[resultCount];
for (int i = 0; i < resultCount; i++) {
result[i] = stats.get(i).getPath();
}
return result;
}
public static Path getPreviousJobHistoryPath(
Configuration conf, ApplicationAttemptId applicationAttemptId) throws IOException {
String jobId = TypeConverter.fromYarn(applicationAttemptId.getApplicationId()).toString();
String jobhistoryDir = JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(conf, jobId);
Path histDirPath = FileContext.getFileContext(conf).makeQualified(new Path(jobhistoryDir));
FileContext fc = FileContext.getFileContext(histDirPath.toUri(), conf);
return fc.makeQualified(
JobHistoryUtils.getStagingJobHistoryFile(
histDirPath, jobId, (applicationAttemptId.getAttemptId() - 1)));
}
/**
* Looks for the dirs to clean. The folder structure is YYYY/MM/DD/Serial so we can use that to
* more efficiently find the directories to clean by comparing the cutoff timestamp with the
* timestamp from the folder structure.
*
* @param fc done dir FileContext
* @param root folder for completed jobs
* @param cutoff The cutoff for the max history age
* @return The list of directories for cleaning
* @throws IOException
*/
public static List<FileStatus> getHistoryDirsForCleaning(FileContext fc, Path root, long cutoff)
throws IOException {
List<FileStatus> fsList = new ArrayList<FileStatus>();
Calendar cCal = Calendar.getInstance();
cCal.setTimeInMillis(cutoff);
int cYear = cCal.get(Calendar.YEAR);
int cMonth = cCal.get(Calendar.MONTH) + 1;
int cDate = cCal.get(Calendar.DATE);
RemoteIterator<FileStatus> yearDirIt = fc.listStatus(root);
while (yearDirIt.hasNext()) {
FileStatus yearDir = yearDirIt.next();
try {
int year = Integer.parseInt(yearDir.getPath().getName());
if (year <= cYear) {
RemoteIterator<FileStatus> monthDirIt = fc.listStatus(yearDir.getPath());
while (monthDirIt.hasNext()) {
FileStatus monthDir = monthDirIt.next();
try {
int month = Integer.parseInt(monthDir.getPath().getName());
// If we only checked the month here, then something like 07/2013
// would incorrectly not pass when the cutoff is 06/2014
if (year < cYear || month <= cMonth) {
RemoteIterator<FileStatus> dateDirIt = fc.listStatus(monthDir.getPath());
while (dateDirIt.hasNext()) {
FileStatus dateDir = dateDirIt.next();
try {
int date = Integer.parseInt(dateDir.getPath().getName());
// If we only checked the date here, then something like
// 07/21/2013 would incorrectly not pass when the cutoff is
// 08/20/2013 or 07/20/2012
if (year < cYear || month < cMonth || date <= cDate) {
fsList.addAll(remoteIterToList(fc.listStatus(dateDir.getPath())));
}
} catch (NumberFormatException nfe) {
// the directory didn't fit the format we're looking for so
// skip the dir
}
}
}
} catch (NumberFormatException nfe) {
// the directory didn't fit the format we're looking for so skip
// the dir
}
}
}
} catch (NumberFormatException nfe) {
// the directory didn't fit the format we're looking for so skip the dir
}
}
return fsList;
}
}
