@Test public void testRemoveAclExceedsQuota() throws Exception { Path filePath = new Path(path, "file1"); Path fileSnapshotPath = new Path(snapshotPath, "file1"); FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0755)); hdfs.allowSnapshot(path); hdfs.setQuota(path, 3, HdfsConstants.QUOTA_DONT_SET); FileSystem.create(hdfs, filePath, FsPermission.createImmutable((short) 0600)).close(); hdfs.setPermission(filePath, FsPermission.createImmutable((short) 0600)); List<AclEntry> aclSpec = Lists.newArrayList(aclEntry(ACCESS, USER, "bruce", READ_WRITE)); hdfs.modifyAclEntries(filePath, aclSpec); hdfs.createSnapshot(path, snapshotName); AclStatus s = hdfs.getAclStatus(filePath); AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals( new AclEntry[] {aclEntry(ACCESS, USER, "bruce", READ_WRITE), aclEntry(ACCESS, GROUP, NONE)}, returned); assertPermission((short) 010660, filePath); s = hdfs.getAclStatus(fileSnapshotPath); returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals( new AclEntry[] {aclEntry(ACCESS, USER, "bruce", READ_WRITE), aclEntry(ACCESS, GROUP, NONE)}, returned); assertPermission((short) 010660, filePath); aclSpec = Lists.newArrayList(aclEntry(ACCESS, USER, "bruce", READ)); hdfs.removeAcl(filePath); }
@Test public void testModifyReadsCurrentState() throws Exception { FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0700)); SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName); List<AclEntry> aclSpec = Lists.newArrayList(aclEntry(ACCESS, USER, "bruce", ALL)); hdfs.modifyAclEntries(path, aclSpec); aclSpec = Lists.newArrayList(aclEntry(ACCESS, USER, "diana", READ_EXECUTE)); hdfs.modifyAclEntries(path, aclSpec); AclEntry[] expected = new AclEntry[] { aclEntry(ACCESS, USER, "bruce", ALL), aclEntry(ACCESS, USER, "diana", READ_EXECUTE), aclEntry(ACCESS, GROUP, NONE) }; AclStatus s = hdfs.getAclStatus(path); AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals(expected, returned); assertPermission((short) 010770, path); assertDirPermissionGranted(fsAsBruce, BRUCE, path); assertDirPermissionGranted(fsAsDiana, DIANA, path); }
@Before public void setUp() { PermissionStatus permStatus = PermissionStatus.createImmutable( SUPERUSER, SUPERGROUP, FsPermission.createImmutable((short) 0755)); inodeRoot = new INodeDirectory(INodeId.ROOT_INODE_ID, INodeDirectory.ROOT_NAME, permStatus, 0L); }
@Test public void testRemoveAclSnapshotPath() throws Exception { FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0700)); SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName); exception.expect(SnapshotAccessControlException.class); hdfs.removeAcl(snapshotPath); }
@Test public void testSetAclSnapshotPath() throws Exception { FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0700)); SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName); List<AclEntry> aclSpec = Lists.newArrayList(aclEntry(DEFAULT, USER, "bruce")); exception.expect(SnapshotAccessControlException.class); hdfs.setAcl(snapshotPath, aclSpec); }
private static INodeDirectory createINodeDirectory( INodeDirectory parent, String name, String owner, String group, short perm) throws IOException { PermissionStatus permStatus = PermissionStatus.createImmutable(owner, group, FsPermission.createImmutable(perm)); INodeDirectory inodeDirectory = new INodeDirectory(INodeId.GRANDFATHER_INODE_ID, name.getBytes("UTF-8"), permStatus, 0L); parent.addChild(inodeDirectory); return inodeDirectory; }
@Test public void testOriginalAclEnforcedForSnapshotRootAfterChange() throws Exception { FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0700)); List<AclEntry> aclSpec = Lists.newArrayList( aclEntry(ACCESS, USER, ALL), aclEntry(ACCESS, USER, "bruce", READ_EXECUTE), aclEntry(ACCESS, GROUP, NONE), aclEntry(ACCESS, OTHER, NONE)); hdfs.setAcl(path, aclSpec); assertDirPermissionGranted(fsAsBruce, BRUCE, path); assertDirPermissionDenied(fsAsDiana, DIANA, path); SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName); // Both original and snapshot still have same ACL. AclStatus s = hdfs.getAclStatus(path); AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals( new AclEntry[] { aclEntry(ACCESS, USER, "bruce", READ_EXECUTE), aclEntry(ACCESS, GROUP, NONE) }, returned); assertPermission((short) 010750, path); s = hdfs.getAclStatus(snapshotPath); returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals( new AclEntry[] { aclEntry(ACCESS, USER, "bruce", READ_EXECUTE), aclEntry(ACCESS, GROUP, NONE) }, returned); assertPermission((short) 010750, snapshotPath); assertDirPermissionGranted(fsAsBruce, BRUCE, snapshotPath); assertDirPermissionDenied(fsAsDiana, DIANA, snapshotPath); aclSpec = Lists.newArrayList( aclEntry(ACCESS, USER, READ_EXECUTE), aclEntry(ACCESS, USER, "diana", READ_EXECUTE), aclEntry(ACCESS, GROUP, NONE), aclEntry(ACCESS, OTHER, NONE)); hdfs.setAcl(path, aclSpec); // Original has changed, but snapshot still has old ACL. doSnapshotRootChangeAssertions(path, snapshotPath); restart(false); doSnapshotRootChangeAssertions(path, snapshotPath); restart(true); doSnapshotRootChangeAssertions(path, snapshotPath); }
/** * Test for {@link TFS#setPermission(Path, org.apache.hadoop.fs.permission.FsPermission)}. It will * test changing the permission of file using TFS. */ @Test public void chmodTest() throws Exception { Path fileA = new Path("/chmodfileA"); create(sTFS, fileA); FileStatus fs = sTFS.getFileStatus(fileA); // Default permission should be 0644 Assert.assertEquals((short) 0644, fs.getPermission().toShort()); sTFS.setPermission(fileA, FsPermission.createImmutable((short) 0755)); Assert.assertEquals((short) 0755, sTFS.getFileStatus(fileA).getPermission().toShort()); }
private static INodeFile createINodeFile( INodeDirectory parent, String name, String owner, String group, short perm) throws IOException { PermissionStatus permStatus = PermissionStatus.createImmutable(owner, group, FsPermission.createImmutable(perm)); INodeFile inodeFile = new INodeFile( INodeId.GRANDFATHER_INODE_ID, name.getBytes("UTF-8"), permStatus, 0L, 0L, null, REPLICATION, PREFERRED_BLOCK_SIZE); parent.addChild(inodeFile); return inodeFile; }
@Test public void testDefaultAclNotCopiedToAccessAclOfNewSnapshot() throws Exception { FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0700)); List<AclEntry> aclSpec = Lists.newArrayList(aclEntry(DEFAULT, USER, "bruce", READ_EXECUTE)); hdfs.modifyAclEntries(path, aclSpec); SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName); AclStatus s = hdfs.getAclStatus(path); AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals( new AclEntry[] { aclEntry(DEFAULT, USER, ALL), aclEntry(DEFAULT, USER, "bruce", READ_EXECUTE), aclEntry(DEFAULT, GROUP, NONE), aclEntry(DEFAULT, MASK, READ_EXECUTE), aclEntry(DEFAULT, OTHER, NONE) }, returned); assertPermission((short) 010700, path); s = hdfs.getAclStatus(snapshotPath); returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals( new AclEntry[] { aclEntry(DEFAULT, USER, ALL), aclEntry(DEFAULT, USER, "bruce", READ_EXECUTE), aclEntry(DEFAULT, GROUP, NONE), aclEntry(DEFAULT, MASK, READ_EXECUTE), aclEntry(DEFAULT, OTHER, NONE) }, returned); assertPermission((short) 010700, snapshotPath); assertDirPermissionDenied(fsAsBruce, BRUCE, snapshotPath); }
public abstract class ContainerExecutor implements Configurable { private static final Log LOG = LogFactory.getLog(ContainerExecutor.class); public static final FsPermission TASK_LAUNCH_SCRIPT_PERMISSION = FsPermission.createImmutable((short) 0700); private Configuration conf; private ConcurrentMap<ContainerId, Path> pidFiles = new ConcurrentHashMap<ContainerId, Path>(); private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); private final ReadLock readLock = lock.readLock(); private final WriteLock writeLock = lock.writeLock(); @Override public void setConf(Configuration conf) { this.conf = conf; } @Override public Configuration getConf() { return conf; } /** * Run the executor initialization steps. Verify that the necessary configs, permissions are in * place. * * @throws IOException */ public abstract void init() throws IOException; /** * On Windows the ContainerLaunch creates a temporary special jar manifest of other jars to * workaround the CLASSPATH length. In a secure cluster this jar must be localized so that the * container has access to it. This function localizes on-demand the jar. * * @param classPathJar * @param owner * @throws IOException */ public Path localizeClasspathJar(Path classPathJar, Path pwd, String owner) throws IOException { // Non-secure executor simply use the classpath created // in the NM fprivate folder return classPathJar; } /** * Prepare the environment for containers in this application to execute. * * <pre> * For $x in local.dirs * create $x/$user/$appId * Copy $nmLocal/appTokens {@literal ->} $N/$user/$appId * For $rsrc in private resources * Copy $rsrc {@literal ->} $N/$user/filecache/[idef] * For $rsrc in job resources * Copy $rsrc {@literal ->} $N/$user/$appId/filecache/idef * </pre> * * @param ctx LocalizerStartContext that encapsulates necessary information for starting a * localizer. * @throws IOException For most application init failures * @throws InterruptedException If application init thread is halted by NM */ public abstract void startLocalizer(LocalizerStartContext ctx) throws IOException, InterruptedException; /** * Launch the container on the node. This is a blocking call and returns only when the container * exits. * * @param ctx Encapsulates information necessary for launching containers. * @return the return status of the launch * @throws IOException */ public abstract int launchContainer(ContainerStartContext ctx) throws IOException; /** * Signal container with the specified signal. * * @param ctx Encapsulates information necessary for signaling containers. * @return returns true if the operation succeeded * @throws IOException */ public abstract boolean signalContainer(ContainerSignalContext ctx) throws IOException; /** * Delete specified directories as a given user. * * @param ctx Encapsulates information necessary for deletion. * @throws IOException * @throws InterruptedException */ public abstract void deleteAsUser(DeletionAsUserContext ctx) throws IOException, InterruptedException; /** * Check if a container is alive. * * @param ctx Encapsulates information necessary for container liveness check. * @return true if container is still alive * @throws IOException */ public abstract boolean isContainerAlive(ContainerLivenessContext ctx) throws IOException; /** * Recover an already existing container. This is a blocking call and returns only when the * container exits. Note that the container must have been activated prior to this call. * * @param ctx encapsulates information necessary to reacquire container * @return The exit code of the pre-existing container * @throws IOException * @throws InterruptedException */ public int reacquireContainer(ContainerReacquisitionContext ctx) throws IOException, InterruptedException { Container container = ctx.getContainer(); String user = ctx.getUser(); ContainerId containerId = ctx.getContainerId(); Path pidPath = getPidFilePath(containerId); if (pidPath == null) { LOG.warn(containerId + " is not active, returning terminated error"); return ExitCode.TERMINATED.getExitCode(); } String pid = null; pid = ProcessIdFileReader.getProcessId(pidPath); if (pid == null) { throw new IOException("Unable to determine pid for " + containerId); } LOG.info("Reacquiring " + containerId + " with pid " + pid); ContainerLivenessContext livenessContext = new ContainerLivenessContext.Builder() .setContainer(container) .setUser(user) .setPid(pid) .build(); while (isContainerAlive(livenessContext)) { Thread.sleep(1000); } // wait for exit code file to appear String exitCodeFile = ContainerLaunch.getExitCodeFile(pidPath.toString()); File file = new File(exitCodeFile); final int sleepMsec = 100; int msecLeft = 2000; while (!file.exists() && msecLeft >= 0) { if (!isContainerActive(containerId)) { LOG.info(containerId + " was deactivated"); return ExitCode.TERMINATED.getExitCode(); } Thread.sleep(sleepMsec); msecLeft -= sleepMsec; } if (msecLeft < 0) { throw new IOException("Timeout while waiting for exit code from " + containerId); } try { return Integer.parseInt(FileUtils.readFileToString(file).trim()); } catch (NumberFormatException e) { throw new IOException("Error parsing exit code from pid " + pid, e); } } /** * This method writes out the launch environment of a container. This can be overridden by * extending ContainerExecutors to provide different behaviors * * @param out the output stream to which the environment is written (usually a script file which * will be executed by the Launcher) * @param environment The environment variables and their values * @param resources The resources which have been localized for this container Symlinks will be * created to these localized resources * @param command The command that will be run. * @throws IOException if any errors happened writing to the OutputStream, while creating symlinks */ public void writeLaunchEnv( OutputStream out, Map<String, String> environment, Map<Path, List<String>> resources, List<String> command) throws IOException { ContainerLaunch.ShellScriptBuilder sb = ContainerLaunch.ShellScriptBuilder.create(); Set<String> whitelist = new HashSet<String>(); whitelist.add(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME); whitelist.add(ApplicationConstants.Environment.HADOOP_YARN_HOME.name()); whitelist.add(ApplicationConstants.Environment.HADOOP_COMMON_HOME.name()); whitelist.add(ApplicationConstants.Environment.HADOOP_HDFS_HOME.name()); whitelist.add(ApplicationConstants.Environment.HADOOP_CONF_DIR.name()); whitelist.add(ApplicationConstants.Environment.JAVA_HOME.name()); if (environment != null) { for (Map.Entry<String, String> env : environment.entrySet()) { if (!whitelist.contains(env.getKey())) { sb.env(env.getKey().toString(), env.getValue().toString()); } else { sb.whitelistedEnv(env.getKey().toString(), env.getValue().toString()); } } } if (resources != null) { for (Map.Entry<Path, List<String>> entry : resources.entrySet()) { for (String linkName : entry.getValue()) { sb.symlink(entry.getKey(), new Path(linkName)); } } } sb.command(command); PrintStream pout = null; try { pout = new PrintStream(out, false, "UTF-8"); sb.write(pout); } finally { if (out != null) { out.close(); } } } public enum ExitCode { FORCE_KILLED(137), TERMINATED(143), LOST(154); private final int code; private ExitCode(int exitCode) { this.code = exitCode; } public int getExitCode() { return code; } @Override public String toString() { return String.valueOf(code); } } /** The constants for the signals. */ public enum Signal { NULL(0, "NULL"), QUIT(3, "SIGQUIT"), KILL(9, "SIGKILL"), TERM(15, "SIGTERM"); private final int value; private final String str; private Signal(int value, String str) { this.str = str; this.value = value; } public int getValue() { return value; } @Override public String toString() { return str; } } protected void logOutput(String output) { String shExecOutput = output; if (shExecOutput != null) { for (String str : shExecOutput.split("\n")) { LOG.info(str); } } } /** * Get the pidFile of the container. * * @param containerId * @return the path of the pid-file for the given containerId. */ protected Path getPidFilePath(ContainerId containerId) { try { readLock.lock(); return (this.pidFiles.get(containerId)); } finally { readLock.unlock(); } } protected String[] getRunCommand( String command, String groupId, String userName, Path pidFile, Configuration conf) { return getRunCommand(command, groupId, userName, pidFile, conf, null); } /** * Return a command to execute the given command in OS shell. On Windows, the passed in groupId * can be used to launch and associate the given groupId in a process group. On non-Windows, * groupId is ignored. */ protected String[] getRunCommand( String command, String groupId, String userName, Path pidFile, Configuration conf, Resource resource) { boolean containerSchedPriorityIsSet = false; int containerSchedPriorityAdjustment = YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY; if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) != null) { containerSchedPriorityIsSet = true; containerSchedPriorityAdjustment = conf.getInt( YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY); } if (Shell.WINDOWS) { int cpuRate = -1; int memory = -1; if (resource != null) { if (conf.getBoolean( YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED, YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) { memory = resource.getMemory(); } if (conf.getBoolean( YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED, YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED)) { int containerVCores = resource.getVirtualCores(); int nodeVCores = NodeManagerHardwareUtils.getVCores(conf); int nodeCpuPercentage = NodeManagerHardwareUtils.getNodeCpuPercentage(conf); float containerCpuPercentage = (float) (nodeCpuPercentage * containerVCores) / nodeVCores; // CPU should be set to a percentage * 100, e.g. 20% cpu rate limit // should be set as 20 * 100. cpuRate = Math.min(10000, (int) (containerCpuPercentage * 100)); } } return new String[] { Shell.WINUTILS, "task", "create", "-m", String.valueOf(memory), "-c", String.valueOf(cpuRate), groupId, "cmd /c " + command }; } else { List<String> retCommand = new ArrayList<String>(); if (containerSchedPriorityIsSet) { retCommand.addAll( Arrays.asList("nice", "-n", Integer.toString(containerSchedPriorityAdjustment))); } retCommand.addAll(Arrays.asList("bash", command)); return retCommand.toArray(new String[retCommand.size()]); } } /** * Is the container still active? * * @param containerId * @return true if the container is active else false. */ protected boolean isContainerActive(ContainerId containerId) { try { readLock.lock(); return (this.pidFiles.containsKey(containerId)); } finally { readLock.unlock(); } } /** * Mark the container as active * * @param containerId the ContainerId * @param pidFilePath Path where the executor should write the pid of the launched process */ public void activateContainer(ContainerId containerId, Path pidFilePath) { try { writeLock.lock(); this.pidFiles.put(containerId, pidFilePath); } finally { writeLock.unlock(); } } /** * Mark the container as inactive. Done iff the container is still active. Else treat it as a * no-op */ public void deactivateContainer(ContainerId containerId) { try { writeLock.lock(); this.pidFiles.remove(containerId); } finally { writeLock.unlock(); } } /** * Get the process-identifier for the container * * @param containerID * @return the processid of the container if it has already launched, otherwise return null */ public String getProcessId(ContainerId containerID) { String pid = null; Path pidFile = pidFiles.get(containerID); if (pidFile == null) { // This container isn't even launched yet. return pid; } try { pid = ProcessIdFileReader.getProcessId(pidFile); } catch (IOException e) { LOG.error("Got exception reading pid from pid-file " + pidFile, e); } return pid; } public static class DelayedProcessKiller extends Thread { private Container container; private final String user; private final String pid; private final long delay; private final Signal signal; private final ContainerExecutor containerExecutor; public DelayedProcessKiller( Container container, String user, String pid, long delay, Signal signal, ContainerExecutor containerExecutor) { this.container = container; this.user = user; this.pid = pid; this.delay = delay; this.signal = signal; this.containerExecutor = containerExecutor; setName("Task killer for " + pid); setDaemon(false); } @Override public void run() { try { Thread.sleep(delay); containerExecutor.signalContainer( new ContainerSignalContext.Builder() .setContainer(container) .setUser(user) .setPid(pid) .setSignal(signal) .build()); } catch (InterruptedException e) { return; } catch (IOException e) { String message = "Exception when user " + user + " killing task " + pid + " in DelayedProcessKiller: " + StringUtils.stringifyException(e); LOG.warn(message); container.handle(new ContainerDiagnosticsUpdateEvent(container.getContainerId(), message)); } } } }
public class TajoResourceAllocator extends AbstractResourceAllocator { private static final Log LOG = LogFactory.getLog(TajoResourceAllocator.class); static AtomicInteger containerIdSeq = new AtomicInteger(0); private TajoConf tajoConf; private QueryMasterTask.QueryMasterTaskContext queryTaskContext; private final ExecutorService executorService; private AtomicBoolean stopped = new AtomicBoolean(false); public TajoResourceAllocator(QueryMasterTask.QueryMasterTaskContext queryTaskContext) { this.queryTaskContext = queryTaskContext; executorService = Executors.newFixedThreadPool( queryTaskContext .getConf() .getIntVar(TajoConf.ConfVars.YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM)); } @Override public ContainerId makeContainerId(YarnProtos.ContainerIdProto containerIdProto) { TajoWorkerContainerId containerId = new TajoWorkerContainerId(); ApplicationAttemptId appAttemptId = new ApplicationAttemptIdPBImpl(containerIdProto.getAppAttemptId()); containerId.setApplicationAttemptId(appAttemptId); containerId.setId(containerIdProto.getId()); return containerId; } @Override public void allocateTaskWorker() {} @Override public int calculateNumRequestContainers(TajoWorker.WorkerContext workerContext, int numTasks) { int clusterSlots = workerContext.getNumClusterSlots(); return clusterSlots == 0 ? 1 : Math.min(numTasks, clusterSlots); } @Override public void init(Configuration conf) { tajoConf = (TajoConf) conf; queryTaskContext .getDispatcher() .register(TaskRunnerGroupEvent.EventType.class, new TajoTaskRunnerLauncher()); // queryTaskContext .getDispatcher() .register(ContainerAllocatorEventType.class, new TajoWorkerAllocationHandler()); super.init(conf); } @Override public synchronized void stop() { if (stopped.get()) { return; } stopped.set(true); executorService.shutdownNow(); Map<ContainerId, ContainerProxy> containers = queryTaskContext.getResourceAllocator().getContainers(); List<ContainerProxy> list = new ArrayList<ContainerProxy>(containers.values()); for (ContainerProxy eachProxy : list) { try { eachProxy.stopContainer(); } catch (Exception e) { } } super.stop(); } @Override public void start() { super.start(); } public static final FsPermission QUERYCONF_FILE_PERMISSION = FsPermission.createImmutable((short) 0644); // rw-r--r-- private static void writeConf(Configuration conf, Path queryConfFile) throws IOException { // Write job file to Tajo's fs FileSystem fs = queryConfFile.getFileSystem(conf); FSDataOutputStream out = FileSystem.create(fs, queryConfFile, new FsPermission(QUERYCONF_FILE_PERMISSION)); try { conf.writeXml(out); } finally { out.close(); } } class TajoTaskRunnerLauncher implements TaskRunnerLauncher { @Override public void handle(TaskRunnerGroupEvent event) { if (event.getType() == TaskRunnerGroupEvent.EventType.CONTAINER_REMOTE_LAUNCH) { launchTaskRunners(event.getExecutionBlockId(), event.getContainers()); } else if (event.getType() == TaskRunnerGroupEvent.EventType.CONTAINER_REMOTE_CLEANUP) { stopContainers(event.getContainers()); } } } private void launchTaskRunners( ExecutionBlockId executionBlockId, Collection<Container> containers) { // Query in standby mode doesn't need launch Worker. // But, Assign ExecutionBlock to assigned tajo worker for (Container eachContainer : containers) { TajoContainerProxy containerProxy = new TajoContainerProxy(queryTaskContext, tajoConf, eachContainer, executionBlockId); executorService.submit(new LaunchRunner(eachContainer.getId(), containerProxy)); } } protected class LaunchRunner implements Runnable { private final ContainerProxy proxy; private final ContainerId id; public LaunchRunner(ContainerId id, ContainerProxy proxy) { this.proxy = proxy; this.id = id; } @Override public void run() { proxy.launch(null); LOG.info("ContainerProxy started:" + id); } } private void stopContainers(Collection<Container> containers) { for (Container container : containers) { final ContainerProxy proxy = queryTaskContext.getResourceAllocator().getContainer(container.getId()); executorService.submit(new StopContainerRunner(container.getId(), proxy)); } } private class StopContainerRunner implements Runnable { private final ContainerProxy proxy; private final ContainerId id; public StopContainerRunner(ContainerId id, ContainerProxy proxy) { this.id = id; this.proxy = proxy; } @Override public void run() { LOG.info("ContainerProxy stopped:" + id + "," + proxy.getId()); proxy.stopContainer(); } } class TajoWorkerAllocationHandler implements EventHandler<ContainerAllocationEvent> { @Override public void handle(ContainerAllocationEvent event) { executorService.submit(new TajoWorkerAllocationThread(event)); } } class TajoWorkerAllocationThread extends Thread { ContainerAllocationEvent event; TajoWorkerAllocationThread(ContainerAllocationEvent event) { this.event = event; } @Override public void run() { LOG.info("Start TajoWorkerAllocationThread"); CallFuture<TajoMasterProtocol.WorkerResourceAllocationResponse> callBack = new CallFuture<TajoMasterProtocol.WorkerResourceAllocationResponse>(); int requiredMemoryMBSlot = 512; // TODO int requiredDiskSlots = 1; // TODO TajoMasterProtocol.WorkerResourceAllocationRequest request = TajoMasterProtocol.WorkerResourceAllocationRequest.newBuilder() .setMemoryMBSlots(requiredMemoryMBSlot) .setDiskSlots(requiredDiskSlots) .setNumWorks(event.getRequiredNum()) .setExecutionBlockId(event.getExecutionBlockId().getProto()) .build(); RpcConnectionPool connPool = RpcConnectionPool.getPool(queryTaskContext.getConf()); NettyClientBase tmClient = null; try { tmClient = connPool.getConnection( queryTaskContext.getQueryMasterContext().getWorkerContext().getTajoMasterAddress(), TajoMasterProtocol.class, true); TajoMasterProtocol.TajoMasterProtocolService masterClientService = tmClient.getStub(); masterClientService.allocateWorkerResources(null, request, callBack); } catch (Exception e) { connPool.closeConnection(tmClient); tmClient = null; LOG.error(e.getMessage(), e); } finally { connPool.releaseConnection(tmClient); } TajoMasterProtocol.WorkerResourceAllocationResponse response = null; while (!stopped.get()) { try { response = callBack.get(3, TimeUnit.SECONDS); break; } catch (InterruptedException e) { if (stopped.get()) { return; } } catch (TimeoutException e) { LOG.info("No available worker resource for " + event.getExecutionBlockId()); continue; } } int numAllocatedWorkers = 0; if (response != null) { List<TajoMasterProtocol.WorkerAllocatedResource> workerHosts = response.getWorkerAllocatedResourceList(); ExecutionBlockId executionBlockId = event.getExecutionBlockId(); List<Container> containers = new ArrayList<Container>(); for (TajoMasterProtocol.WorkerAllocatedResource eachWorker : workerHosts) { TajoWorkerContainer container = new TajoWorkerContainer(); NodeIdPBImpl nodeId = new NodeIdPBImpl(); nodeId.setHost(eachWorker.getWorkerHost()); nodeId.setPort(eachWorker.getPeerRpcPort()); TajoWorkerContainerId containerId = new TajoWorkerContainerId(); containerId.setApplicationAttemptId( ApplicationIdUtils.createApplicationAttemptId(executionBlockId.getQueryId())); containerId.setId(containerIdSeq.incrementAndGet()); container.setId(containerId); container.setNodeId(nodeId); WorkerResource workerResource = new WorkerResource(); workerResource.setAllocatedHost(nodeId.getHost()); workerResource.setPeerRpcPort(nodeId.getPort()); workerResource.setQueryMasterPort(eachWorker.getQueryMasterPort()); workerResource.setPullServerPort(eachWorker.getWorkerPullServerPort()); workerResource.setMemoryMBSlots(requiredMemoryMBSlot); workerResource.setDiskSlots(requiredDiskSlots); container.setWorkerResource(workerResource); containers.add(container); } SubQueryState state = queryTaskContext.getSubQuery(executionBlockId).getState(); if (!SubQuery.isRunningState(state)) { List<WorkerResource> workerResources = new ArrayList<WorkerResource>(); for (Container eachContainer : containers) { workerResources.add(((TajoWorkerContainer) eachContainer).getWorkerResource()); } try { TajoContainerProxy.releaseWorkerResource( queryTaskContext, executionBlockId, workerResources); } catch (Exception e) { LOG.error(e.getMessage(), e); } return; } if (workerHosts.size() > 0) { if (LOG.isDebugEnabled()) { LOG.debug("SubQueryContainerAllocationEvent fire:" + executionBlockId); } queryTaskContext .getEventHandler() .handle(new SubQueryContainerAllocationEvent(executionBlockId, containers)); } numAllocatedWorkers += workerHosts.size(); } if (event.getRequiredNum() > numAllocatedWorkers) { ContainerAllocationEvent shortRequestEvent = new ContainerAllocationEvent( event.getType(), event.getExecutionBlockId(), event.getPriority(), event.getResource(), event.getRequiredNum() - numAllocatedWorkers, event.isLeafQuery(), event.getProgress()); queryTaskContext.getEventHandler().handle(shortRequestEvent); } LOG.info("Stop TajoWorkerAllocationThread"); } } }
/** HDFS UnderFilesystem implementation */ public class HdfsUnderFileSystem extends UnderFileSystem { private static final Logger LOG = LoggerFactory.getLogger(Constants.LOGGER_TYPE); private static final int MAX_TRY = 5; private FileSystem mFs = null; private String mUfsPrefix = null; // TODO add sticky bit and narrow down the permission in hadoop 2 private static final FsPermission PERMISSION = new FsPermission((short) 0777).applyUMask(FsPermission.createImmutable((short) 0000)); public HdfsUnderFileSystem(String fsDefaultName, TachyonConf tachyonConf, Object conf) { super(tachyonConf); mUfsPrefix = fsDefaultName; Configuration tConf; if (conf != null && conf instanceof Configuration) { tConf = (Configuration) conf; } else { tConf = new Configuration(); } prepareConfiguration(fsDefaultName, tachyonConf, tConf); tConf.addResource(new Path(tConf.get(Constants.UNDERFS_HADOOP_CONFIGURATION))); HdfsUnderFileSystemUtils.addS3Credentials(tConf); Path path = new Path(mUfsPrefix); try { mFs = path.getFileSystem(tConf); } catch (IOException e) { LOG.error("Exception thrown when trying to get FileSystem for " + mUfsPrefix, e); throw Throwables.propagate(e); } } /** * Prepares the Hadoop configuration necessary to successfully obtain a {@link FileSystem} * instance that can access the provided path * * <p>Derived implementations that work with specialised Hadoop {@linkplain FileSystem} API * compatible implementations can override this method to add implementation specific * configuration necessary for obtaining a usable {@linkplain FileSystem} instance. * * @param path File system path * @param config Hadoop Configuration */ protected void prepareConfiguration(String path, TachyonConf tachyonConf, Configuration config) { // On Hadoop 2.x this is strictly unnecessary since it uses ServiceLoader to automatically // discover available file system implementations. However this configuration setting is // required for earlier Hadoop versions plus it is still honoured as an override even in 2.x so // if present propagate it to the Hadoop configuration String ufsHdfsImpl = mTachyonConf.get(Constants.UNDERFS_HDFS_IMPL, null); if (!StringUtils.isEmpty(ufsHdfsImpl)) { config.set("fs.hdfs.impl", ufsHdfsImpl); } // To disable the instance cache for hdfs client, otherwise it causes the // FileSystem closed exception. Being configurable for unit/integration // test only, and not expose to the end-user currently. config.set( "fs.hdfs.impl.disable.cache", System.getProperty("fs.hdfs.impl.disable.cache", "false")); HdfsUnderFileSystemUtils.addKey(config, tachyonConf, Constants.UNDERFS_HADOOP_CONFIGURATION); } @Override public void close() throws IOException { mFs.close(); } @Override public FSDataOutputStream create(String path) throws IOException { IOException te = null; int cnt = 0; while (cnt < MAX_TRY) { try { LOG.debug("Creating HDFS file at {}", path); return FileSystem.create(mFs, new Path(path), PERMISSION); } catch (IOException e) { cnt++; LOG.error(cnt + " : " + e.getMessage(), e); te = e; } } throw te; } /** BlockSize should be a multiple of 512 */ @Override public FSDataOutputStream create(String path, int blockSizeByte) throws IOException { // TODO Fix this // return create(path, (short) Math.min(3, mFs.getDefaultReplication()), blockSizeByte); return create(path); } @Override public FSDataOutputStream create(String path, short replication, int blockSizeByte) throws IOException { // TODO Fix this // return create(path, (short) Math.min(3, mFs.getDefaultReplication()), blockSizeByte); return create(path); // LOG.info(path + " " + replication + " " + blockSizeByte); // IOException te = null; // int cnt = 0; // while (cnt < MAX_TRY) { // try { // return mFs.create(new Path(path), true, 4096, replication, blockSizeByte); // } catch (IOException e) { // cnt ++; // LOG.error(cnt + " : " + e.getMessage(), e); // te = e; // continue; // } // } // throw te; } @Override public boolean delete(String path, boolean recursive) throws IOException { LOG.debug("deleting {} {}", path, recursive); IOException te = null; int cnt = 0; while (cnt < MAX_TRY) { try { return mFs.delete(new Path(path), recursive); } catch (IOException e) { cnt++; LOG.error(cnt + " : " + e.getMessage(), e); te = e; } } throw te; } @Override public boolean exists(String path) throws IOException { IOException te = null; int cnt = 0; while (cnt < MAX_TRY) { try { return mFs.exists(new Path(path)); } catch (IOException e) { cnt++; LOG.error(cnt + " try to check if " + path + " exists " + " : " + e.getMessage(), e); te = e; } } throw te; } @Override public long getBlockSizeByte(String path) throws IOException { Path tPath = new Path(path); if (!mFs.exists(tPath)) { throw new FileNotFoundException(path); } FileStatus fs = mFs.getFileStatus(tPath); return fs.getBlockSize(); } @Override public Object getConf() { return mFs.getConf(); } @Override public List<String> getFileLocations(String path) throws IOException { return getFileLocations(path, 0); } @Override public List<String> getFileLocations(String path, long offset) throws IOException { List<String> ret = new ArrayList<String>(); try { FileStatus fStatus = mFs.getFileStatus(new Path(path)); BlockLocation[] bLocations = mFs.getFileBlockLocations(fStatus, offset, 1); if (bLocations.length > 0) { String[] names = bLocations[0].getNames(); Collections.addAll(ret, names); } } catch (IOException e) { LOG.error("Unable to get file location for " + path, e); } return ret; } @Override public long getFileSize(String path) throws IOException { int cnt = 0; Path tPath = new Path(path); while (cnt < MAX_TRY) { try { FileStatus fs = mFs.getFileStatus(tPath); return fs.getLen(); } catch (IOException e) { cnt++; LOG.error(cnt + " try to get file size for " + path + " : " + e.getMessage(), e); } } return -1; } @Override public long getModificationTimeMs(String path) throws IOException { Path tPath = new Path(path); if (!mFs.exists(tPath)) { throw new FileNotFoundException(path); } FileStatus fs = mFs.getFileStatus(tPath); return fs.getModificationTime(); } @Override public long getSpace(String path, SpaceType type) throws IOException { // Ignoring the path given, will give information for entire cluster // as Tachyon can load/store data out of entire HDFS cluster if (mFs instanceof DistributedFileSystem) { switch (type) { case SPACE_TOTAL: return ((DistributedFileSystem) mFs).getDiskStatus().getCapacity(); case SPACE_USED: return ((DistributedFileSystem) mFs).getDiskStatus().getDfsUsed(); case SPACE_FREE: return ((DistributedFileSystem) mFs).getDiskStatus().getRemaining(); default: throw new IOException("Unknown getSpace parameter: " + type); } } return -1; } @Override public boolean isFile(String path) throws IOException { return mFs.isFile(new Path(path)); } @Override public String[] list(String path) throws IOException { FileStatus[] files = mFs.listStatus(new Path(path)); if (files != null) { String[] rtn = new String[files.length]; int i = 0; for (FileStatus status : files) { TachyonURI filePathURI = new TachyonURI(status.getPath().toUri().toString()); String filePath = NetworkUtils.replaceHostName(filePathURI).toString(); // only return the relative path, to keep consistent with java.io.File.list() rtn[i++] = filePath.substring(path.length()); // mUfsPrefix } return rtn; } else { return null; } } @Override public void connectFromMaster(TachyonConf conf, String host) throws IOException { String masterKeytab = conf.get(Constants.MASTER_KEYTAB_KEY, null); String masterPrincipal = conf.get(Constants.MASTER_PRINCIPAL_KEY, null); if (masterKeytab == null || masterPrincipal == null) { return; } login( Constants.MASTER_KEYTAB_KEY, masterKeytab, Constants.MASTER_PRINCIPAL_KEY, masterPrincipal, host); } @Override public void connectFromWorker(TachyonConf conf, String host) throws IOException { String workerKeytab = conf.get(Constants.WORKER_KEYTAB_KEY, null); String workerPrincipal = conf.get(Constants.WORKER_PRINCIPAL_KEY, null); if (workerKeytab == null || workerPrincipal == null) { return; } login( Constants.WORKER_KEYTAB_KEY, workerKeytab, Constants.WORKER_PRINCIPAL_KEY, workerPrincipal, host); } private void login( String keytabFileKey, String keytabFile, String principalKey, String principal, String hostname) throws IOException { Configuration conf = new Configuration(); conf.set(keytabFileKey, keytabFile); conf.set(principalKey, principal); SecurityUtil.login(conf, keytabFileKey, principalKey, hostname); } @Override public boolean mkdirs(String path, boolean createParent) throws IOException { IOException te = null; int cnt = 0; while (cnt < MAX_TRY) { try { if (mFs.exists(new Path(path))) { LOG.debug("Trying to create existing directory at {}", path); return false; } return mFs.mkdirs(new Path(path), PERMISSION); } catch (IOException e) { cnt++; LOG.error(cnt + " try to make directory for " + path + " : " + e.getMessage(), e); te = e; } } throw te; } @Override public FSDataInputStream open(String path) throws IOException { IOException te = null; int cnt = 0; while (cnt < MAX_TRY) { try { return mFs.open(new Path(path)); } catch (IOException e) { cnt++; LOG.error(cnt + " try to open " + path + " : " + e.getMessage(), e); te = e; } } throw te; } @Override public boolean rename(String src, String dst) throws IOException { LOG.debug("Renaming from {} to {}", src, dst); if (!exists(src)) { LOG.error("File " + src + " does not exist. Therefore rename to " + dst + " failed."); return false; } if (exists(dst)) { LOG.error("File " + dst + " does exist. Therefore rename from " + src + " failed."); return false; } int cnt = 0; IOException te = null; while (cnt < MAX_TRY) { try { return mFs.rename(new Path(src), new Path(dst)); } catch (IOException e) { cnt++; LOG.error(cnt + " try to rename " + src + " to " + dst + " : " + e.getMessage(), e); te = e; } } throw te; } @Override public void setConf(Object conf) { mFs.setConf((Configuration) conf); } @Override public void setPermission(String path, String posixPerm) throws IOException { try { FileStatus fileStatus = mFs.getFileStatus(new Path(path)); LOG.info( "Changing file '" + fileStatus.getPath() + "' permissions from: " + fileStatus.getPermission() + " to " + posixPerm); FsPermission perm = new FsPermission(Short.parseShort(posixPerm)); mFs.setPermission(fileStatus.getPath(), perm); } catch (IOException e) { LOG.error("Fail to set permission for " + path + " with perm " + posixPerm, e); throw e; } } }
public class CoronaJobHistory { public static final Log LOG = LogFactory.getLog(CoronaJobHistory.class); static final FsPermission HISTORY_DIR_PERMISSION = FsPermission.createImmutable((short) 0755); // rwxr-x--- static final FsPermission HISTORY_FILE_PERMISSION = FsPermission.createImmutable((short) 0744); // rwxr----- Path logDir; FileSystem logDirFs; Path doneDir; FileSystem doneDirFs; Path logFile; Path doneFile; JobConf conf; boolean disableHistory = true; long jobHistoryBlockSize = 0; CoronaJobHistoryFilesManager fileManager = null; JobID jobId; ArrayList<PrintWriter> writers = null; public CoronaJobHistory(JobConf conf, JobID jobId, String logPath) { try { this.conf = conf; this.jobId = jobId; if (logPath == null) { logPath = "file:///" + new File(System.getProperty("hadoop.log.dir", "/tmp")).getAbsolutePath() + File.separator + "history"; } logDir = new Path(logPath); logDirFs = logDir.getFileSystem(conf); if (!logDirFs.exists(logDir)) { LOG.info("Creating history folder at " + logDir); if (!logDirFs.mkdirs(logDir, new FsPermission(HISTORY_DIR_PERMISSION))) { throw new IOException("Mkdirs failed to create " + logDir.toString()); } } conf.set("hadoop.job.history.location", logDir.toString()); disableHistory = false; // set the job history block size (default is 3MB) jobHistoryBlockSize = conf.getLong("mapred.jobtracker.job.history.block.size", 3 * 1024 * 1024); doneDir = new Path(logDir, "done"); doneDirFs = logDirFs; if (!doneDirFs.exists(doneDir)) { LOG.info("Creating DONE folder at " + doneDir); if (!doneDirFs.mkdirs(doneDir, new FsPermission(HISTORY_DIR_PERMISSION))) { throw new IOException("Mkdirs failed to create " + doneDir); } } String logFileName = encodeJobHistoryFileName(CoronaJobHistoryFilesManager.getHistoryFilename(jobId)); logFile = new Path(logDir, logFileName); doneFile = new Path(doneDir, logFileName); // initialize the file manager conf.setInt("mapred.jobtracker.historythreads.maximum", 1); fileManager = new CoronaJobHistoryFilesManager( conf, new JobHistoryObserver() { public void historyFileCopied(JobID jobid, String historyFile) {} }, logDir); fileManager.setDoneDir(doneDir); // sleeping with the past means tolerating two start methods instead of one fileManager.start(); fileManager.startIOExecutor(); } catch (IOException e) { LOG.error("Failed to initialize JobHistory log file", e); disableHistory = true; } } public void shutdown() { if (fileManager != null) { fileManager.shutdown(); } } public boolean isDisabled() { return disableHistory; } public static String encodeJobHistoryFileName(String logFileName) throws IOException { return JobHistory.JobInfo.encodeJobHistoryFileName(logFileName); } public String getCompletedJobHistoryPath() { return doneFile.toString(); } /** Get the history location for completed jobs */ public Path getCompletedJobHistoryLocation() { return doneDir; } public static String encodeJobHistoryFilePath(String logFile) throws IOException { return JobHistory.JobInfo.encodeJobHistoryFilePath(logFile); } private String getJobName() { String jobName = ((JobConf) conf).getJobName(); if (jobName == null || jobName.length() == 0) { jobName = "NA"; } return jobName; } public String getUserName() { String user = ((JobConf) conf).getUser(); if (user == null || user.length() == 0) { user = "******"; } return user; } private static void closeAndClear(List<PrintWriter> writers) { for (PrintWriter out : writers) { out.close(); } // By clearning the writers and notify the thread waiting on it, // we will prevent JobHistory.moveToDone() from // waiting on writer synchronized (writers) { writers.clear(); writers.notifyAll(); } } /** * Log job submitted event to history. Creates a new file in history for the job. if history file * creation fails, it disables history for all other events. * * @param jobConfPath path to job conf xml file in HDFS. * @param submitTime time when job tracker received the job * @throws IOException */ public void logSubmitted(String jobConfPath, long submitTime, String jobTrackerId) throws IOException { if (disableHistory) { return; } // create output stream for logging in hadoop.job.history.location int defaultBufferSize = logDirFs.getConf().getInt("io.file.buffer.size", 4096); try { FSDataOutputStream out = null; PrintWriter writer = null; // In case the old JT is still running, but we can't connect to it, we // should ensure that it won't write to our (new JT's) job history file. if (logDirFs.exists(logFile)) { LOG.info("Remove the old history file " + logFile); logDirFs.delete(logFile, true); } out = logDirFs.create( logFile, new FsPermission(HISTORY_FILE_PERMISSION), true, defaultBufferSize, logDirFs.getDefaultReplication(), jobHistoryBlockSize, null); writer = new PrintWriter(out); fileManager.addWriter(jobId, writer); // cache it ... fileManager.setHistoryFile(jobId, logFile); writers = fileManager.getWriters(jobId); if (null != writers) { log( writers, RecordTypes.Meta, new Keys[] {Keys.VERSION}, new String[] {String.valueOf(JobHistory.VERSION)}); } String jobName = getJobName(); String user = getUserName(); // add to writer as well log( writers, RecordTypes.Job, new Keys[] { Keys.JOBID, Keys.JOBNAME, Keys.USER, Keys.SUBMIT_TIME, Keys.JOBCONF, Keys.JOBTRACKERID }, new String[] { jobId.toString(), jobName, user, String.valueOf(submitTime), jobConfPath, jobTrackerId }); } catch (IOException e) { // Disable history if we have errors other than in the user log. disableHistory = true; } /* Storing the job conf on the log dir */ Path jobFilePath = new Path(logDir, CoronaJobHistoryFilesManager.getConfFilename(jobId)); fileManager.setConfFile(jobId, jobFilePath); FSDataOutputStream jobFileOut = null; try { if (!logDirFs.exists(jobFilePath)) { jobFileOut = logDirFs.create( jobFilePath, new FsPermission(HISTORY_FILE_PERMISSION), true, defaultBufferSize, logDirFs.getDefaultReplication(), logDirFs.getDefaultBlockSize(), null); conf.writeXml(jobFileOut); jobFileOut.close(); } } catch (IOException ioe) { LOG.error("Failed to store job conf in the log dir", ioe); } finally { if (jobFileOut != null) { try { jobFileOut.close(); } catch (IOException ie) { LOG.info( "Failed to close the job configuration file " + StringUtils.stringifyException(ie)); } } } } /** * Logs launch time of job. * * @param startTime start time of job. * @param totalMaps total maps assigned by jobtracker. * @param totalReduces total reduces. */ public void logInited(long startTime, int totalMaps, int totalReduces) { if (disableHistory) { return; } if (null != writers) { log( writers, RecordTypes.Job, new Keys[] { Keys.JOBID, Keys.LAUNCH_TIME, Keys.TOTAL_MAPS, Keys.TOTAL_REDUCES, Keys.JOB_STATUS }, new String[] { jobId.toString(), String.valueOf(startTime), String.valueOf(totalMaps), String.valueOf(totalReduces), Values.PREP.name() }); } } /** Logs job as running */ public void logStarted() { if (disableHistory) { return; } if (null != writers) { log( writers, RecordTypes.Job, new Keys[] {Keys.JOBID, Keys.JOB_STATUS}, new String[] {jobId.toString(), Values.RUNNING.name()}); } } /** * Log job finished. closes the job file in history. * * @param finishTime finish time of job in ms. * @param finishedMaps no of maps successfully finished. * @param finishedReduces no of reduces finished sucessfully. * @param failedMaps no of failed map tasks. (includes killed) * @param failedReduces no of failed reduce tasks. (includes killed) * @param killedMaps no of killed map tasks. * @param killedReduces no of killed reduce tasks. * @param counters the counters from the job */ public void logFinished( long finishTime, int finishedMaps, int finishedReduces, int failedMaps, int failedReduces, int killedMaps, int killedReduces, Counters mapCounters, Counters reduceCounters, Counters counters) { if (disableHistory) { return; } if (null != writers) { log( writers, RecordTypes.Job, new Keys[] { Keys.JOBID, Keys.FINISH_TIME, Keys.JOB_STATUS, Keys.FINISHED_MAPS, Keys.FINISHED_REDUCES, Keys.FAILED_MAPS, Keys.FAILED_REDUCES, Keys.KILLED_MAPS, Keys.KILLED_REDUCES, Keys.MAP_COUNTERS, Keys.REDUCE_COUNTERS, Keys.COUNTERS }, new String[] { jobId.toString(), Long.toString(finishTime), Values.SUCCESS.name(), String.valueOf(finishedMaps), String.valueOf(finishedReduces), String.valueOf(failedMaps), String.valueOf(failedReduces), String.valueOf(killedMaps), String.valueOf(killedReduces), mapCounters.makeEscapedCompactString(), reduceCounters.makeEscapedCompactString(), counters.makeEscapedCompactString() }, true); closeAndClear(writers); } // NOTE: history cleaning stuff deleted from here. We should do that // somewhere else! } /** * Logs job failed event. Closes the job history log file. * * @param timestamp time when job failure was detected in ms. * @param finishedMaps no finished map tasks. * @param finishedReduces no of finished reduce tasks. */ public void logFailed(long timestamp, int finishedMaps, int finishedReduces, Counters counters) { if (disableHistory) { return; } if (null != writers) { log( writers, RecordTypes.Job, new Keys[] { Keys.JOBID, Keys.FINISH_TIME, Keys.JOB_STATUS, Keys.FINISHED_MAPS, Keys.FINISHED_REDUCES, Keys.COUNTERS }, new String[] { jobId.toString(), String.valueOf(timestamp), Values.FAILED.name(), String.valueOf(finishedMaps), String.valueOf(finishedReduces), counters.makeEscapedCompactString() }, true); closeAndClear(writers); } } /** * Logs job killed event. Closes the job history log file. * * @param timestamp time when job killed was issued in ms. * @param finishedMaps no finished map tasks. * @param finishedReduces no of finished reduce tasks. */ public void logKilled(long timestamp, int finishedMaps, int finishedReduces, Counters counters) { if (disableHistory) { return; } if (null != writers) { log( writers, RecordTypes.Job, new Keys[] { Keys.JOBID, Keys.FINISH_TIME, Keys.JOB_STATUS, Keys.FINISHED_MAPS, Keys.FINISHED_REDUCES, Keys.COUNTERS }, new String[] { jobId.toString(), String.valueOf(timestamp), Values.KILLED.name(), String.valueOf(finishedMaps), String.valueOf(finishedReduces), counters.makeEscapedCompactString() }, true); closeAndClear(writers); } } /** * Log job's priority. * * @param priority Jobs priority */ public void logJobPriority(JobID jobid, JobPriority priority) { if (disableHistory) { return; } if (null != writers) { log( writers, RecordTypes.Job, new Keys[] {Keys.JOBID, Keys.JOB_PRIORITY}, new String[] {jobId.toString(), priority.toString()}); } } /** Move the completed job into the completed folder. */ public void markCompleted() throws IOException { if (disableHistory) { return; } fileManager.moveToDone(jobId, true, CoronaJobTracker.getMainJobID(jobId)); } /** * Log start time of task (TIP). * * @param taskId task id * @param taskType MAP or REDUCE * @param startTime startTime of tip. */ public void logTaskStarted( TaskID taskId, String taskType, long startTime, String splitLocations) { if (disableHistory) { return; } JobID id = taskId.getJobID(); if (!this.jobId.equals(id)) { throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId); } if (null != writers) { log( writers, RecordTypes.Task, new Keys[] { Keys.TASKID, Keys.TASK_TYPE, Keys.START_TIME, Keys.SPLITS }, new String[] {taskId.toString(), taskType, String.valueOf(startTime), splitLocations}); } } /** * Log finish time of task. * * @param taskId task id * @param taskType MAP or REDUCE * @param finishTime finish timeof task in ms */ public void logTaskFinished(TaskID taskId, String taskType, long finishTime, Counters counters) { if (disableHistory) { return; } JobID id = taskId.getJobID(); if (!this.jobId.equals(id)) { throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId); } if (null != writers) { log( writers, RecordTypes.Task, new Keys[] { Keys.TASKID, Keys.TASK_TYPE, Keys.TASK_STATUS, Keys.FINISH_TIME, Keys.COUNTERS }, new String[] { taskId.toString(), taskType, Values.SUCCESS.name(), String.valueOf(finishTime), counters.makeEscapedCompactString() }); } } /** * Update the finish time of task. * * @param taskId task id * @param finishTime finish time of task in ms */ public void logTaskUpdates(TaskID taskId, long finishTime) { if (disableHistory) { return; } JobID id = taskId.getJobID(); if (!this.jobId.equals(id)) { throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId); } if (null != writers) { log( writers, RecordTypes.Task, new Keys[] {Keys.TASKID, Keys.FINISH_TIME}, new String[] {taskId.toString(), String.valueOf(finishTime)}); } } /** * Log job failed event. * * @param taskId task id * @param taskType MAP or REDUCE. * @param time timestamp when job failed detected. * @param error error message for failure. */ public void logTaskFailed(TaskID taskId, String taskType, long time, String error) { logTaskFailed(taskId, taskType, time, error, null); } /** * Log the task failure * * @param taskId the task that failed * @param taskType the type of the task * @param time the time of the failure * @param error the error the task failed with * @param failedDueToAttempt The attempt that caused the failure, if any */ public void logTaskFailed( TaskID taskId, String taskType, long time, String error, TaskAttemptID failedDueToAttempt) { if (disableHistory) { return; } JobID id = taskId.getJobID(); if (!this.jobId.equals(id)) { throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId); } if (null != writers) { String failedAttempt = failedDueToAttempt == null ? "" : failedDueToAttempt.toString(); log( writers, RecordTypes.Task, new Keys[] { Keys.TASKID, Keys.TASK_TYPE, Keys.TASK_STATUS, Keys.FINISH_TIME, Keys.ERROR, Keys.TASK_ATTEMPT_ID }, new String[] { taskId.toString(), taskType, Values.FAILED.name(), String.valueOf(time), error, failedAttempt }); } } /** * Log start time of this map task attempt. * * @param taskAttemptId task attempt id * @param startTime start time of task attempt as reported by task tracker. * @param trackerName name of the tracker executing the task attempt. * @param httpPort http port of the task tracker executing the task attempt * @param taskType Whether the attempt is cleanup or setup or map */ public void logMapTaskStarted( TaskAttemptID taskAttemptId, long startTime, String trackerName, int httpPort, String taskType) { if (disableHistory) { return; } JobID id = taskAttemptId.getJobID(); if (!this.jobId.equals(id)) { throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId); } if (null != writers) { log( writers, RecordTypes.MapAttempt, new Keys[] { Keys.TASK_TYPE, Keys.TASKID, Keys.TASK_ATTEMPT_ID, Keys.START_TIME, Keys.TRACKER_NAME, Keys.HTTP_PORT }, new String[] { taskType, taskAttemptId.getTaskID().toString(), taskAttemptId.toString(), String.valueOf(startTime), trackerName, httpPort == -1 ? "" : String.valueOf(httpPort) }); } } /** * Log finish time of map task attempt. * * @param taskAttemptId task attempt id * @param finishTime finish time * @param hostName host name * @param taskType Whether the attempt is cleanup or setup or map * @param stateString state string of the task attempt * @param counter counters of the task attempt */ public void logMapTaskFinished( TaskAttemptID taskAttemptId, long finishTime, String hostName, String taskType, String stateString, Counters counter) { if (disableHistory) { return; } JobID id = taskAttemptId.getJobID(); if (!this.jobId.equals(id)) { throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId); } if (null != writers) { log( writers, RecordTypes.MapAttempt, new Keys[] { Keys.TASK_TYPE, Keys.TASKID, Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS, Keys.FINISH_TIME, Keys.HOSTNAME, Keys.STATE_STRING, Keys.COUNTERS }, new String[] { taskType, taskAttemptId.getTaskID().toString(), taskAttemptId.toString(), Values.SUCCESS.name(), String.valueOf(finishTime), hostName, stateString, counter.makeEscapedCompactString() }); } } /** * Log task attempt failed event. * * @param taskAttemptId task attempt id * @param timestamp timestamp * @param hostName hostname of this task attempt. * @param error error message if any for this task attempt. * @param taskType Whether the attempt is cleanup or setup or map */ public void logMapTaskFailed( TaskAttemptID taskAttemptId, long timestamp, String hostName, String error, String taskType) { if (disableHistory) { return; } JobID id = taskAttemptId.getJobID(); if (!this.jobId.equals(id)) { throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId); } if (null != writers) { log( writers, RecordTypes.MapAttempt, new Keys[] { Keys.TASK_TYPE, Keys.TASKID, Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS, Keys.FINISH_TIME, Keys.HOSTNAME, Keys.ERROR }, new String[] { taskType, taskAttemptId.getTaskID().toString(), taskAttemptId.toString(), Values.FAILED.name(), String.valueOf(timestamp), hostName, error }); } } /** * Log task attempt killed event. * * @param taskAttemptId task attempt id * @param timestamp timestamp * @param hostName hostname of this task attempt. * @param error error message if any for this task attempt. * @param taskType Whether the attempt is cleanup or setup or map */ public void logMapTaskKilled( TaskAttemptID taskAttemptId, long timestamp, String hostName, String error, String taskType) { if (disableHistory) { return; } JobID id = taskAttemptId.getJobID(); if (!this.jobId.equals(id)) { throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId); } if (null != writers) { log( writers, RecordTypes.MapAttempt, new Keys[] { Keys.TASK_TYPE, Keys.TASKID, Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS, Keys.FINISH_TIME, Keys.HOSTNAME, Keys.ERROR }, new String[] { taskType, taskAttemptId.getTaskID().toString(), taskAttemptId.toString(), Values.KILLED.name(), String.valueOf(timestamp), hostName, error }); } } /** * Log start time of Reduce task attempt. * * @param taskAttemptId task attempt id * @param startTime start time * @param trackerName tracker name * @param httpPort the http port of the tracker executing the task attempt * @param taskType Whether the attempt is cleanup or setup or reduce */ public void logReduceTaskStarted( TaskAttemptID taskAttemptId, long startTime, String trackerName, int httpPort, String taskType) { if (disableHistory) { return; } JobID id = taskAttemptId.getJobID(); if (!this.jobId.equals(id)) { throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId); } if (null != writers) { log( writers, RecordTypes.ReduceAttempt, new Keys[] { Keys.TASK_TYPE, Keys.TASKID, Keys.TASK_ATTEMPT_ID, Keys.START_TIME, Keys.TRACKER_NAME, Keys.HTTP_PORT }, new String[] { taskType, taskAttemptId.getTaskID().toString(), taskAttemptId.toString(), String.valueOf(startTime), trackerName, httpPort == -1 ? "" : String.valueOf(httpPort) }); } } /** * Log finished event of this task. * * @param taskAttemptId task attempt id * @param shuffleFinished shuffle finish time * @param sortFinished sort finish time * @param finishTime finish time of task * @param hostName host name where task attempt executed * @param taskType Whether the attempt is cleanup or setup or reduce * @param stateString the state string of the attempt * @param counter counters of the attempt */ public void logReduceTaskFinished( TaskAttemptID taskAttemptId, long shuffleFinished, long sortFinished, long finishTime, String hostName, String taskType, String stateString, Counters counter) { if (disableHistory) { return; } JobID id = taskAttemptId.getJobID(); if (!this.jobId.equals(id)) { throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId); } if (null != writers) { log( writers, RecordTypes.ReduceAttempt, new Keys[] { Keys.TASK_TYPE, Keys.TASKID, Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS, Keys.SHUFFLE_FINISHED, Keys.SORT_FINISHED, Keys.FINISH_TIME, Keys.HOSTNAME, Keys.STATE_STRING, Keys.COUNTERS }, new String[] { taskType, taskAttemptId.getTaskID().toString(), taskAttemptId.toString(), Values.SUCCESS.name(), String.valueOf(shuffleFinished), String.valueOf(sortFinished), String.valueOf(finishTime), hostName, stateString, counter.makeEscapedCompactString() }); } } /** * Log failed reduce task attempt. * * @param taskAttemptId task attempt id * @param timestamp time stamp when task failed * @param hostName host name of the task attempt. * @param error error message of the task. * @param taskType Whether the attempt is cleanup or setup or reduce */ public void logReduceTaskFailed( TaskAttemptID taskAttemptId, long timestamp, String hostName, String error, String taskType) { if (disableHistory) { return; } JobID id = taskAttemptId.getJobID(); if (!this.jobId.equals(id)) { throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId); } if (null != writers) { log( writers, RecordTypes.ReduceAttempt, new Keys[] { Keys.TASK_TYPE, Keys.TASKID, Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS, Keys.FINISH_TIME, Keys.HOSTNAME, Keys.ERROR }, new String[] { taskType, taskAttemptId.getTaskID().toString(), taskAttemptId.toString(), Values.FAILED.name(), String.valueOf(timestamp), hostName, error }); } } /** * Log killed reduce task attempt. * * @param taskAttemptId task attempt id * @param timestamp time stamp when task failed * @param hostName host name of the task attempt. * @param error error message of the task. * @param taskType Whether the attempt is cleanup or setup or reduce */ public void logReduceTaskKilled( TaskAttemptID taskAttemptId, long timestamp, String hostName, String error, String taskType) { if (disableHistory) { return; } JobID id = taskAttemptId.getJobID(); if (!this.jobId.equals(id)) { throw new RuntimeException("JobId from task: " + id + " does not match expected: " + jobId); } if (null != writers) { log( writers, RecordTypes.ReduceAttempt, new Keys[] { Keys.TASK_TYPE, Keys.TASKID, Keys.TASK_ATTEMPT_ID, Keys.TASK_STATUS, Keys.FINISH_TIME, Keys.HOSTNAME, Keys.ERROR }, new String[] { taskType, taskAttemptId.getTaskID().toString(), taskAttemptId.toString(), Values.KILLED.name(), String.valueOf(timestamp), hostName, error }); } } /** * Log a number of keys and values with record. the array length of keys and values should be * same. * * @param writers the writers to send the data to * @param recordType type of log event * @param keys type of log event * @param values type of log event */ private void log( ArrayList<PrintWriter> writers, RecordTypes recordType, Keys[] keys, String[] values) { log(writers, recordType, keys, values, false); } /** * Log a number of keys and values with the record. This method allows to do it in a synchronous * fashion * * @param writers the writers to send the data to * @param recordType the type to log * @param keys keys to log * @param values values to log * @param sync if true - will block until the data is written */ private void log( ArrayList<PrintWriter> writers, RecordTypes recordType, Keys[] keys, String[] values, boolean sync) { StringBuffer buf = new StringBuffer(recordType.name()); buf.append(JobHistory.DELIMITER); for (int i = 0; i < keys.length; i++) { buf.append(keys[i]); buf.append("=\""); values[i] = JobHistory.escapeString(values[i]); buf.append(values[i]); buf.append("\""); buf.append(JobHistory.DELIMITER); } buf.append(JobHistory.LINE_DELIMITER_CHAR); for (PrintWriter out : writers) { LogTask task = new LogTask(out, buf.toString()); if (sync) { task.run(); } else { fileManager.addWriteTask(task); } } } }
public void testCreate() throws Exception { Configuration conf = new HdfsConfiguration(); conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true); conf.set(FsPermission.UMASK_LABEL, "000"); MiniDFSCluster cluster = null; FileSystem fs = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); cluster.waitActive(); fs = FileSystem.get(conf); FsPermission rootPerm = checkPermission(fs, "/", null); FsPermission inheritPerm = FsPermission.createImmutable((short) (rootPerm.toShort() | 0300)); FsPermission dirPerm = new FsPermission((short) 0777); fs.mkdirs(new Path("/a1/a2/a3"), dirPerm); checkPermission(fs, "/a1", dirPerm); checkPermission(fs, "/a1/a2", dirPerm); checkPermission(fs, "/a1/a2/a3", dirPerm); dirPerm = new FsPermission((short) 0123); FsPermission permission = FsPermission.createImmutable((short) (dirPerm.toShort() | 0300)); fs.mkdirs(new Path("/aa/1/aa/2/aa/3"), dirPerm); checkPermission(fs, "/aa/1", permission); checkPermission(fs, "/aa/1/aa/2", permission); checkPermission(fs, "/aa/1/aa/2/aa/3", dirPerm); FsPermission filePerm = new FsPermission((short) 0444); FSDataOutputStream out = fs.create( new Path("/b1/b2/b3.txt"), filePerm, true, conf.getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), fs.getDefaultReplication(), fs.getDefaultBlockSize(), null); out.write(123); out.close(); checkPermission(fs, "/b1", inheritPerm); checkPermission(fs, "/b1/b2", inheritPerm); checkPermission(fs, "/b1/b2/b3.txt", filePerm); conf.set(FsPermission.UMASK_LABEL, "022"); permission = FsPermission.createImmutable((short) 0666); FileSystem.mkdirs(fs, new Path("/c1"), new FsPermission(permission)); FileSystem.create(fs, new Path("/c1/c2.txt"), new FsPermission(permission)); checkPermission(fs, "/c1", permission); checkPermission(fs, "/c1/c2.txt", permission); } finally { try { if (fs != null) fs.close(); } catch (Exception e) { LOG.error(StringUtils.stringifyException(e)); } try { if (cluster != null) cluster.shutdown(); } catch (Exception e) { LOG.error(StringUtils.stringifyException(e)); } } }
/** * A utility to manage job submission files.<br> * <b><i>Note that this class is for framework internal usage only and is not to be used by users * directly.</i></b> */ public class JobSubmissionFiles { private static final Log LOG = LogFactory.getLog(JobSubmissionFiles.class); // job submission directory is private! public static final FsPermission JOB_DIR_PERMISSION = FsPermission.createImmutable((short) 0700); // rwx-------- // job files are world-wide readable and owner writable public static final FsPermission JOB_FILE_PERMISSION = FsPermission.createImmutable((short) 0644); // rw-r--r-- public static Path getJobSplitFile(Path jobSubmissionDir) { return new Path(jobSubmissionDir, "job.split"); } public static Path getJobSplitMetaFile(Path jobSubmissionDir) { return new Path(jobSubmissionDir, "job.splitmetainfo"); } /** Get the job conf path. */ public static Path getJobConfPath(Path jobSubmitDir) { return new Path(jobSubmitDir, "job.xml"); } /** Get the job jar path. */ public static Path getJobJar(Path jobSubmitDir) { return new Path(jobSubmitDir, "job.jar"); } /** * Get the job distributed cache files path. * * @param jobSubmitDir */ public static Path getJobDistCacheFiles(Path jobSubmitDir) { return new Path(jobSubmitDir, "files"); } /** * Get the job distributed cache archives path. * * @param jobSubmitDir */ public static Path getJobDistCacheArchives(Path jobSubmitDir) { return new Path(jobSubmitDir, "archives"); } /** * Get the job distributed cache libjars path. * * @param jobSubmitDir */ public static Path getJobDistCacheLibjars(Path jobSubmitDir) { return new Path(jobSubmitDir, "libjars"); } /** * Initializes the staging directory and returns the path. It also keeps track of all necessary * ownership & permissions * * @param client * @param conf */ public static Path getStagingDir(JobClient client, Configuration conf) throws IOException, InterruptedException { Path stagingArea = client.getStagingAreaDir(); FileSystem fs = stagingArea.getFileSystem(conf); String realUser; String currentUser; UserGroupInformation ugi = UserGroupInformation.getLoginUser(); realUser = ugi.getShortUserName(); currentUser = UserGroupInformation.getCurrentUser().getShortUserName(); if (fs.exists(stagingArea)) { FileStatus fsStatus = fs.getFileStatus(stagingArea); String owner = fsStatus.getOwner(); if (!(owner.equals(currentUser) || owner.equals(realUser))) { throw new IOException( "The ownership on the staging directory " + stagingArea + " is not as expected. " + "It is owned by " + owner + ". The directory must " + "be owned by the submitter " + currentUser + " or " + "by " + realUser); } if (!fsStatus.getPermission().equals(JOB_DIR_PERMISSION)) { LOG.info( "Permissions on staging directory " + stagingArea + " are " + "incorrect: " + fsStatus.getPermission() + ". Fixing permissions " + "to correct value " + JOB_DIR_PERMISSION); fs.setPermission(stagingArea, JOB_DIR_PERMISSION); } } else { fs.mkdirs(stagingArea, new FsPermission(JOB_DIR_PERMISSION)); } return stagingArea; } }
@Override public void truncateLogsAsUser(String user, List<Task> allAttempts) throws IOException { Task firstTask = allAttempts.get(0); String taskid = firstTask.getTaskID().toString(); LocalDirAllocator ldirAlloc = new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY); String taskRanFile = TaskTracker.TT_LOG_TMP_DIR + Path.SEPARATOR + taskid; Configuration conf = getConf(); // write the serialized task information to a file to pass to the truncater Path taskRanFilePath = ldirAlloc.getLocalPathForWrite(taskRanFile, conf); LocalFileSystem lfs = FileSystem.getLocal(conf); FSDataOutputStream out = lfs.create(taskRanFilePath); out.writeInt(allAttempts.size()); for (Task t : allAttempts) { out.writeBoolean(t.isMapTask()); t.write(out); } out.close(); lfs.setPermission(taskRanFilePath, FsPermission.createImmutable((short) 0755)); List<String> command = new ArrayList<String>(); File jvm = // use same jvm as parent new File(new File(System.getProperty("java.home"), "bin"), "java"); command.add(jvm.toString()); command.add("-Djava.library.path=" + System.getProperty("java.library.path")); command.add("-Dhadoop.log.dir=" + TaskLog.getBaseLogDir()); command.add("-Dhadoop.root.logger=INFO,console"); command.add("-classpath"); command.add(System.getProperty("java.class.path")); // main of TaskLogsTruncater command.add(TaskLogsTruncater.class.getName()); command.add(taskRanFilePath.toString()); String[] taskControllerCmd = new String[4 + command.size()]; taskControllerCmd[0] = taskControllerExe; taskControllerCmd[1] = user; taskControllerCmd[2] = localStorage.getDirsString(); taskControllerCmd[3] = Integer.toString(Commands.RUN_COMMAND_AS_USER.getValue()); int i = 4; for (String cmdArg : command) { taskControllerCmd[i++] = cmdArg; } if (LOG.isDebugEnabled()) { for (String cmd : taskControllerCmd) { LOG.debug("taskctrl command = " + cmd); } } ShellCommandExecutor shExec = new ShellCommandExecutor(taskControllerCmd); try { shExec.execute(); } catch (Exception e) { LOG.warn( "Exit code from " + taskControllerExe.toString() + " is : " + shExec.getExitCode() + " for truncateLogs"); LOG.warn( "Exception thrown by " + taskControllerExe.toString() + " : " + StringUtils.stringifyException(e)); LOG.info("Output from LinuxTaskController's " + taskControllerExe.toString() + " follows:"); logOutput(shExec.getOutput()); lfs.delete(taskRanFilePath, false); throw new IOException(e); } lfs.delete(taskRanFilePath, false); if (LOG.isDebugEnabled()) { LOG.info("Output from LinuxTaskController's " + taskControllerExe.toString() + " follows:"); logOutput(shExec.getOutput()); } }
class INodeFile extends INode { static final FsPermission UMASK = FsPermission.createImmutable((short) 0111); // Number of bits for Block size static final short BLOCKBITS = 48; // Header mask 64-bit representation // Format: [16 bits for replication][48 bits for PreferredBlockSize] static final long HEADERMASK = 0xffffL << BLOCKBITS; protected long header; protected BlockInfo blocks[] = null; INodeFile( PermissionStatus permissions, int nrBlocks, short replication, long modificationTime, long atime, long preferredBlockSize) { this( permissions, new BlockInfo[nrBlocks], replication, modificationTime, atime, preferredBlockSize); } protected INodeFile() { blocks = null; header = 0; } protected INodeFile( PermissionStatus permissions, BlockInfo[] blklist, short replication, long modificationTime, long atime, long preferredBlockSize) { super(permissions, modificationTime, atime); this.setReplication(replication); this.setPreferredBlockSize(preferredBlockSize); blocks = blklist; } /** * Set the {@link FsPermission} of this {@link INodeFile}. Since this is a file, the {@link * FsAction#EXECUTE} action, if any, is ignored. */ protected void setPermission(FsPermission permission) { super.setPermission(permission.applyUMask(UMASK)); } public boolean isDirectory() { return false; } /** * Get block replication for the file * * @return block replication value */ public short getReplication() { return (short) ((header & HEADERMASK) >> BLOCKBITS); } public void setReplication(short replication) { if (replication <= 0) throw new IllegalArgumentException("Unexpected value for the replication"); header = ((long) replication << BLOCKBITS) | (header & ~HEADERMASK); } /** * Get preferred block size for the file * * @return preferred block size in bytes */ public long getPreferredBlockSize() { return header & ~HEADERMASK; } public void setPreferredBlockSize(long preferredBlkSize) { if ((preferredBlkSize < 0) || (preferredBlkSize > ~HEADERMASK)) throw new IllegalArgumentException("Unexpected value for the block size"); header = (header & HEADERMASK) | (preferredBlkSize & ~HEADERMASK); } /** * Get file blocks * * @return file blocks */ BlockInfo[] getBlocks() { return this.blocks; } /** Return the last block in this file, or null if there are no blocks. */ Block getLastBlock() { if (this.blocks == null || this.blocks.length == 0) return null; return this.blocks[this.blocks.length - 1]; } /** add a block to the block list */ void addBlock(BlockInfo newblock) { if (this.blocks == null) { this.blocks = new BlockInfo[1]; this.blocks[0] = newblock; } else { int size = this.blocks.length; BlockInfo[] newlist = new BlockInfo[size + 1]; System.arraycopy(this.blocks, 0, newlist, 0, size); newlist[size] = newblock; this.blocks = newlist; } } /** Set file block */ void setBlock(int idx, BlockInfo blk) { this.blocks[idx] = blk; } int collectSubtreeBlocksAndClear(List<Block> v) { parent = null; for (Block blk : blocks) { v.add(blk); } blocks = null; return 1; } /** {@inheritDoc} */ long[] computeContentSummary(long[] summary) { long bytes = 0; for (Block blk : blocks) { bytes += blk.getNumBytes(); } summary[0] += bytes; summary[1]++; summary[3] += diskspaceConsumed(); return summary; } @Override DirCounts spaceConsumedInTree(DirCounts counts) { counts.nsCount += 1; counts.dsCount += diskspaceConsumed(); return counts; } long diskspaceConsumed() { return diskspaceConsumed(blocks); } long diskspaceConsumed(Block[] blkArr) { long size = 0; for (Block blk : blkArr) { if (blk != null) { size += blk.getNumBytes(); } } /* If the last block is being written to, use prefferedBlockSize * rather than the actual block size. */ if (blkArr.length > 0 && blkArr[blkArr.length - 1] != null && isUnderConstruction()) { size += getPreferredBlockSize() - blocks[blocks.length - 1].getNumBytes(); } return size * getReplication(); } /** Return the penultimate allocated block for this file. */ Block getPenultimateBlock() { if (blocks == null || blocks.length <= 1) { return null; } return blocks[blocks.length - 2]; } }
@Test public void testOriginalAclEnforcedForSnapshotContentsAfterRemoval() throws Exception { Path filePath = new Path(path, "file1"); Path subdirPath = new Path(path, "subdir1"); Path fileSnapshotPath = new Path(snapshotPath, "file1"); Path subdirSnapshotPath = new Path(snapshotPath, "subdir1"); FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0777)); FileSystem.create(hdfs, filePath, FsPermission.createImmutable((short) 0600)).close(); FileSystem.mkdirs(hdfs, subdirPath, FsPermission.createImmutable((short) 0700)); List<AclEntry> aclSpec = Lists.newArrayList( aclEntry(ACCESS, USER, READ_EXECUTE), aclEntry(ACCESS, USER, "bruce", READ_EXECUTE), aclEntry(ACCESS, GROUP, NONE), aclEntry(ACCESS, OTHER, NONE)); hdfs.setAcl(filePath, aclSpec); hdfs.setAcl(subdirPath, aclSpec); assertFilePermissionGranted(fsAsBruce, BRUCE, filePath); assertFilePermissionDenied(fsAsDiana, DIANA, filePath); assertDirPermissionGranted(fsAsBruce, BRUCE, subdirPath); assertDirPermissionDenied(fsAsDiana, DIANA, subdirPath); SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName); // Both original and snapshot still have same ACL. AclEntry[] expected = new AclEntry[] { aclEntry(ACCESS, USER, "bruce", READ_EXECUTE), aclEntry(ACCESS, GROUP, NONE) }; AclStatus s = hdfs.getAclStatus(filePath); AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals(expected, returned); assertPermission((short) 010550, filePath); s = hdfs.getAclStatus(subdirPath); returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals(expected, returned); assertPermission((short) 010550, subdirPath); s = hdfs.getAclStatus(fileSnapshotPath); returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals(expected, returned); assertPermission((short) 010550, fileSnapshotPath); assertFilePermissionGranted(fsAsBruce, BRUCE, fileSnapshotPath); assertFilePermissionDenied(fsAsDiana, DIANA, fileSnapshotPath); s = hdfs.getAclStatus(subdirSnapshotPath); returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals(expected, returned); assertPermission((short) 010550, subdirSnapshotPath); assertDirPermissionGranted(fsAsBruce, BRUCE, subdirSnapshotPath); assertDirPermissionDenied(fsAsDiana, DIANA, subdirSnapshotPath); hdfs.removeAcl(filePath); hdfs.removeAcl(subdirPath); // Original has changed, but snapshot still has old ACL. doSnapshotContentsRemovalAssertions(filePath, fileSnapshotPath, subdirPath, subdirSnapshotPath); restart(false); doSnapshotContentsRemovalAssertions(filePath, fileSnapshotPath, subdirPath, subdirSnapshotPath); restart(true); doSnapshotContentsRemovalAssertions(filePath, fileSnapshotPath, subdirPath, subdirSnapshotPath); }
@Public @Evolving public class AggregatedLogFormat { private static final Log LOG = LogFactory.getLog(AggregatedLogFormat.class); private static final LogKey APPLICATION_ACL_KEY = new LogKey("APPLICATION_ACL"); private static final LogKey APPLICATION_OWNER_KEY = new LogKey("APPLICATION_OWNER"); private static final LogKey VERSION_KEY = new LogKey("VERSION"); private static final Map<String, LogKey> RESERVED_KEYS; // Maybe write out the retention policy. // Maybe write out a list of containerLogs skipped by the retention policy. private static final int VERSION = 1; /** Umask for the log file. */ private static final FsPermission APP_LOG_FILE_UMASK = FsPermission.createImmutable((short) (0640 ^ 0777)); static { RESERVED_KEYS = new HashMap<String, AggregatedLogFormat.LogKey>(); RESERVED_KEYS.put(APPLICATION_ACL_KEY.toString(), APPLICATION_ACL_KEY); RESERVED_KEYS.put(APPLICATION_OWNER_KEY.toString(), APPLICATION_OWNER_KEY); RESERVED_KEYS.put(VERSION_KEY.toString(), VERSION_KEY); } @Public public static class LogKey implements Writable { private String keyString; public LogKey() {} public LogKey(ContainerId containerId) { this.keyString = containerId.toString(); } public LogKey(String keyString) { this.keyString = keyString; } @Override public int hashCode() { return keyString == null ? 0 : keyString.hashCode(); } @Override public boolean equals(Object obj) { if (obj instanceof LogKey) { LogKey other = (LogKey) obj; if (this.keyString == null) { return other.keyString == null; } return this.keyString.equals(other.keyString); } return false; } @Private @Override public void write(DataOutput out) throws IOException { out.writeUTF(this.keyString); } @Private @Override public void readFields(DataInput in) throws IOException { this.keyString = in.readUTF(); } @Override public String toString() { return this.keyString; } } @Private public static class LogValue { private final List<String> rootLogDirs; private final ContainerId containerId; private final String user; // TODO Maybe add a version string here. Instead of changing the version of // the entire k-v format public LogValue(List<String> rootLogDirs, ContainerId containerId, String user) { this.rootLogDirs = new ArrayList<String>(rootLogDirs); this.containerId = containerId; this.user = user; // Ensure logs are processed in lexical order Collections.sort(this.rootLogDirs); } public void write(DataOutputStream out) throws IOException { for (String rootLogDir : this.rootLogDirs) { File appLogDir = new File( rootLogDir, ConverterUtils.toString( this.containerId.getApplicationAttemptId().getApplicationId())); File containerLogDir = new File(appLogDir, ConverterUtils.toString(this.containerId)); if (!containerLogDir.isDirectory()) { continue; // ContainerDir may have been deleted by the user. } // Write out log files in lexical order File[] logFiles = containerLogDir.listFiles(); Arrays.sort(logFiles); for (File logFile : logFiles) { // Write the logFile Type out.writeUTF(logFile.getName()); // Write the log length as UTF so that it is printable out.writeUTF(String.valueOf(logFile.length())); // Write the log itself FileInputStream in = null; try { in = SecureIOUtils.openForRead(logFile, getUser(), null); byte[] buf = new byte[65535]; int len = 0; while ((len = in.read(buf)) != -1) { out.write(buf, 0, len); } } catch (IOException e) { String message = "Error aggregating log file. Log file : " + logFile.getAbsolutePath() + e.getMessage(); LOG.error(message, e); out.write(message.getBytes()); } finally { if (in != null) { in.close(); } } } } } // Added for testing purpose. public String getUser() { return user; } } /** The writer that writes out the aggregated logs. */ @Private public static class LogWriter { private final FSDataOutputStream fsDataOStream; private final TFile.Writer writer; public LogWriter( final Configuration conf, final Path remoteAppLogFile, UserGroupInformation userUgi) throws IOException { try { this.fsDataOStream = userUgi.doAs( new PrivilegedExceptionAction<FSDataOutputStream>() { @Override public FSDataOutputStream run() throws Exception { FileContext fc = FileContext.getFileContext(conf); fc.setUMask(APP_LOG_FILE_UMASK); return fc.create( remoteAppLogFile, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), new Options.CreateOpts[] {}); } }); } catch (InterruptedException e) { throw new IOException(e); } // Keys are not sorted: null arg // 256KB minBlockSize : Expected log size for each container too this.writer = new TFile.Writer( this.fsDataOStream, 256 * 1024, conf.get( YarnConfiguration.NM_LOG_AGG_COMPRESSION_TYPE, YarnConfiguration.DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE), null, conf); // Write the version string writeVersion(); } private void writeVersion() throws IOException { DataOutputStream out = this.writer.prepareAppendKey(-1); VERSION_KEY.write(out); out.close(); out = this.writer.prepareAppendValue(-1); out.writeInt(VERSION); out.close(); } public void writeApplicationOwner(String user) throws IOException { DataOutputStream out = this.writer.prepareAppendKey(-1); APPLICATION_OWNER_KEY.write(out); out.close(); out = this.writer.prepareAppendValue(-1); out.writeUTF(user); out.close(); } public void writeApplicationACLs(Map<ApplicationAccessType, String> appAcls) throws IOException { DataOutputStream out = this.writer.prepareAppendKey(-1); APPLICATION_ACL_KEY.write(out); out.close(); out = this.writer.prepareAppendValue(-1); for (Entry<ApplicationAccessType, String> entry : appAcls.entrySet()) { out.writeUTF(entry.getKey().toString()); out.writeUTF(entry.getValue()); } out.close(); } public void append(LogKey logKey, LogValue logValue) throws IOException { DataOutputStream out = this.writer.prepareAppendKey(-1); logKey.write(out); out.close(); out = this.writer.prepareAppendValue(-1); logValue.write(out); out.close(); } public void close() { try { this.writer.close(); } catch (IOException e) { LOG.warn("Exception closing writer", e); } try { this.fsDataOStream.close(); } catch (IOException e) { LOG.warn("Exception closing output-stream", e); } } } @Public @Evolving public static class LogReader { private final FSDataInputStream fsDataIStream; private final TFile.Reader.Scanner scanner; private final TFile.Reader reader; public LogReader(Configuration conf, Path remoteAppLogFile) throws IOException { FileContext fileContext = FileContext.getFileContext(conf); this.fsDataIStream = fileContext.open(remoteAppLogFile); reader = new TFile.Reader( this.fsDataIStream, fileContext.getFileStatus(remoteAppLogFile).getLen(), conf); this.scanner = reader.createScanner(); } private boolean atBeginning = true; /** * Returns the owner of the application. * * @return the application owner. * @throws IOException */ public String getApplicationOwner() throws IOException { TFile.Reader.Scanner ownerScanner = reader.createScanner(); LogKey key = new LogKey(); while (!ownerScanner.atEnd()) { TFile.Reader.Scanner.Entry entry = ownerScanner.entry(); key.readFields(entry.getKeyStream()); if (key.toString().equals(APPLICATION_OWNER_KEY.toString())) { DataInputStream valueStream = entry.getValueStream(); return valueStream.readUTF(); } ownerScanner.advance(); } return null; } /** * Returns ACLs for the application. An empty map is returned if no ACLs are found. * * @return a map of the Application ACLs. * @throws IOException */ public Map<ApplicationAccessType, String> getApplicationAcls() throws IOException { // TODO Seek directly to the key once a comparator is specified. TFile.Reader.Scanner aclScanner = reader.createScanner(); LogKey key = new LogKey(); Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>(); while (!aclScanner.atEnd()) { TFile.Reader.Scanner.Entry entry = aclScanner.entry(); key.readFields(entry.getKeyStream()); if (key.toString().equals(APPLICATION_ACL_KEY.toString())) { DataInputStream valueStream = entry.getValueStream(); while (true) { String appAccessOp = null; String aclString = null; try { appAccessOp = valueStream.readUTF(); } catch (EOFException e) { // Valid end of stream. break; } try { aclString = valueStream.readUTF(); } catch (EOFException e) { throw new YarnRuntimeException("Error reading ACLs", e); } acls.put(ApplicationAccessType.valueOf(appAccessOp), aclString); } } aclScanner.advance(); } return acls; } /** * Read the next key and return the value-stream. * * @param key * @return the valueStream if there are more keys or null otherwise. * @throws IOException */ public DataInputStream next(LogKey key) throws IOException { if (!this.atBeginning) { this.scanner.advance(); } else { this.atBeginning = false; } if (this.scanner.atEnd()) { return null; } TFile.Reader.Scanner.Entry entry = this.scanner.entry(); key.readFields(entry.getKeyStream()); // Skip META keys if (RESERVED_KEYS.containsKey(key.toString())) { return next(key); } DataInputStream valueStream = entry.getValueStream(); return valueStream; } /** * Get a ContainerLogsReader to read the logs for the specified container. * * @param containerId * @return object to read the container's logs or null if the logs could not be found * @throws IOException */ @Private public ContainerLogsReader getContainerLogsReader(ContainerId containerId) throws IOException { ContainerLogsReader logReader = null; final LogKey containerKey = new LogKey(containerId); LogKey key = new LogKey(); DataInputStream valueStream = next(key); while (valueStream != null && !key.equals(containerKey)) { valueStream = next(key); } if (valueStream != null) { logReader = new ContainerLogsReader(valueStream); } return logReader; } // TODO Change Log format and interfaces to be containerId specific. // Avoid returning completeValueStreams. // public List<String> getTypesForContainer(DataInputStream valueStream){} // // /** // * @param valueStream // * The Log stream for the container. // * @param fileType // * the log type required. // * @return An InputStreamReader for the required log type or null if the // * type is not found. // * @throws IOException // */ // public InputStreamReader getLogStreamForType(DataInputStream valueStream, // String fileType) throws IOException { // valueStream.reset(); // try { // while (true) { // String ft = valueStream.readUTF(); // String fileLengthStr = valueStream.readUTF(); // long fileLength = Long.parseLong(fileLengthStr); // if (ft.equals(fileType)) { // BoundedInputStream bis = // new BoundedInputStream(valueStream, fileLength); // return new InputStreamReader(bis); // } else { // long totalSkipped = 0; // long currSkipped = 0; // while (currSkipped != -1 && totalSkipped < fileLength) { // currSkipped = valueStream.skip(fileLength - totalSkipped); // totalSkipped += currSkipped; // } // // TODO Verify skip behaviour. // if (currSkipped == -1) { // return null; // } // } // } // } catch (EOFException e) { // return null; // } // } /** * Writes all logs for a single container to the provided writer. * * @param valueStream * @param writer * @throws IOException */ public static void readAcontainerLogs(DataInputStream valueStream, Writer writer) throws IOException { int bufferSize = 65536; char[] cbuf = new char[bufferSize]; String fileType; String fileLengthStr; long fileLength; while (true) { try { fileType = valueStream.readUTF(); } catch (EOFException e) { // EndOfFile return; } fileLengthStr = valueStream.readUTF(); fileLength = Long.parseLong(fileLengthStr); writer.write("\n\nLogType:"); writer.write(fileType); writer.write("\nLogLength:"); writer.write(fileLengthStr); writer.write("\nLog Contents:\n"); // ByteLevel BoundedInputStream bis = new BoundedInputStream(valueStream, fileLength); InputStreamReader reader = new InputStreamReader(bis); int currentRead = 0; int totalRead = 0; while ((currentRead = reader.read(cbuf, 0, bufferSize)) != -1) { writer.write(cbuf, 0, currentRead); totalRead += currentRead; } } } /** * Keep calling this till you get a {@link EOFException} for getting logs of all types for a * single container. * * @param valueStream * @param out * @throws IOException */ public static void readAContainerLogsForALogType(DataInputStream valueStream, PrintStream out) throws IOException { byte[] buf = new byte[65535]; String fileType = valueStream.readUTF(); String fileLengthStr = valueStream.readUTF(); long fileLength = Long.parseLong(fileLengthStr); out.print("LogType: "); out.println(fileType); out.print("LogLength: "); out.println(fileLengthStr); out.println("Log Contents:"); int curRead = 0; long pendingRead = fileLength - curRead; int toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; int len = valueStream.read(buf, 0, toRead); while (len != -1 && curRead < fileLength) { out.write(buf, 0, len); curRead += len; pendingRead = fileLength - curRead; toRead = pendingRead > buf.length ? buf.length : (int) pendingRead; len = valueStream.read(buf, 0, toRead); } out.println(""); } public void close() { IOUtils.cleanup(LOG, scanner, reader, fsDataIStream); } } @Private public static class ContainerLogsReader { private DataInputStream valueStream; private String currentLogType = null; private long currentLogLength = 0; private BoundedInputStream currentLogData = null; private InputStreamReader currentLogISR; public ContainerLogsReader(DataInputStream stream) { valueStream = stream; } public String nextLog() throws IOException { if (currentLogData != null && currentLogLength > 0) { // seek to the end of the current log, relying on BoundedInputStream // to prevent seeking past the end of the current log do { if (currentLogData.skip(currentLogLength) < 0) { break; } } while (currentLogData.read() != -1); } currentLogType = null; currentLogLength = 0; currentLogData = null; currentLogISR = null; try { String logType = valueStream.readUTF(); String logLengthStr = valueStream.readUTF(); currentLogLength = Long.parseLong(logLengthStr); currentLogData = new BoundedInputStream(valueStream, currentLogLength); currentLogData.setPropagateClose(false); currentLogISR = new InputStreamReader(currentLogData); currentLogType = logType; } catch (EOFException e) { } return currentLogType; } public String getCurrentLogType() { return currentLogType; } public long getCurrentLogLength() { return currentLogLength; } public long skip(long n) throws IOException { return currentLogData.skip(n); } public int read(byte[] buf, int off, int len) throws IOException { return currentLogData.read(buf, off, len); } public int read(char[] buf, int off, int len) throws IOException { return currentLogISR.read(buf, off, len); } } }
@InterfaceAudience.Private @InterfaceStability.Unstable public class JobHistoryUtils { /** Permissions for the history staging dir while JobInProgress. */ public static final FsPermission HISTORY_STAGING_DIR_PERMISSIONS = FsPermission.createImmutable((short) 0700); /** Permissions for the user directory under the staging directory. */ public static final FsPermission HISTORY_STAGING_USER_DIR_PERMISSIONS = FsPermission.createImmutable((short) 0700); /** Permissions for the history done dir and derivatives. */ public static final FsPermission HISTORY_DONE_DIR_PERMISSION = FsPermission.createImmutable((short) 0770); public static final FsPermission HISTORY_DONE_FILE_PERMISSION = FsPermission.createImmutable((short) 0770); // rwx------ /** Umask for the done dir and derivatives. */ public static final FsPermission HISTORY_DONE_DIR_UMASK = FsPermission.createImmutable((short) (0770 ^ 0777)); /** Permissions for the intermediate done directory. */ public static final FsPermission HISTORY_INTERMEDIATE_DONE_DIR_PERMISSIONS = FsPermission.createImmutable((short) 01777); /** Permissions for the user directory under the intermediate done directory. */ public static final FsPermission HISTORY_INTERMEDIATE_USER_DIR_PERMISSIONS = FsPermission.createImmutable((short) 0770); public static final FsPermission HISTORY_INTERMEDIATE_FILE_PERMISSIONS = FsPermission.createImmutable((short) 0770); // rwx------ /** Suffix for configuration files. */ public static final String CONF_FILE_NAME_SUFFIX = "_conf.xml"; /** Suffix for summary files. */ public static final String SUMMARY_FILE_NAME_SUFFIX = ".summary"; /** Job History File extension. */ public static final String JOB_HISTORY_FILE_EXTENSION = ".jhist"; public static final int VERSION = 4; public static final int SERIAL_NUMBER_DIRECTORY_DIGITS = 6; public static final String TIMESTAMP_DIR_REGEX = "\\d{4}" + "\\" + Path.SEPARATOR + "\\d{2}" + "\\" + Path.SEPARATOR + "\\d{2}"; public static final Pattern TIMESTAMP_DIR_PATTERN = Pattern.compile(TIMESTAMP_DIR_REGEX); private static final String TIMESTAMP_DIR_FORMAT = "%04d" + File.separator + "%02d" + File.separator + "%02d"; private static final PathFilter CONF_FILTER = new PathFilter() { @Override public boolean accept(Path path) { return path.getName().endsWith(CONF_FILE_NAME_SUFFIX); } }; private static final PathFilter JOB_HISTORY_FILE_FILTER = new PathFilter() { @Override public boolean accept(Path path) { return path.getName().endsWith(JOB_HISTORY_FILE_EXTENSION); } }; /** * Checks whether the provided path string is a valid job history file. * * @param pathString the path to be checked. * @return true is the path is a valid job history filename else return false */ public static boolean isValidJobHistoryFileName(String pathString) { return pathString.endsWith(JOB_HISTORY_FILE_EXTENSION); } /** * Returns the jobId from a job history file name. * * @param pathString the path string. * @return the JobId * @throws IOException if the filename format is invalid. */ public static JobID getJobIDFromHistoryFilePath(String pathString) throws IOException { String[] parts = pathString.split(Path.SEPARATOR); String fileNamePart = parts[parts.length - 1]; JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fileNamePart); return TypeConverter.fromYarn(jobIndexInfo.getJobId()); } /** * Gets a PathFilter which would match configuration files. * * @return the patch filter {@link PathFilter} for matching conf files. */ public static PathFilter getConfFileFilter() { return CONF_FILTER; } /** * Gets a PathFilter which would match job history file names. * * @return the path filter {@link PathFilter} matching job history files. */ public static PathFilter getHistoryFileFilter() { return JOB_HISTORY_FILE_FILTER; } /** * Gets the configured directory prefix for In Progress history files. * * @param conf the configuration for hte job * @param jobId the id of the job the history file is for. * @return A string representation of the prefix. */ public static String getConfiguredHistoryStagingDirPrefix(Configuration conf, String jobId) throws IOException { String user = UserGroupInformation.getCurrentUser().getShortUserName(); Path stagingPath = MRApps.getStagingAreaDir(conf, user); Path path = new Path(stagingPath, jobId); String logDir = path.toString(); return logDir; } /** * Gets the configured directory prefix for intermediate done history files. * * @param conf * @return A string representation of the prefix. */ public static String getConfiguredHistoryIntermediateDoneDirPrefix(Configuration conf) { String doneDirPrefix = conf.get(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR); if (doneDirPrefix == null) { doneDirPrefix = conf.get(MRJobConfig.MR_AM_STAGING_DIR, MRJobConfig.DEFAULT_MR_AM_STAGING_DIR) + "/history/done_intermediate"; } return doneDirPrefix; } /** * Gets the configured directory prefix for Done history files. * * @param conf the configuration object * @return the done history directory */ public static String getConfiguredHistoryServerDoneDirPrefix(Configuration conf) { String doneDirPrefix = conf.get(JHAdminConfig.MR_HISTORY_DONE_DIR); if (doneDirPrefix == null) { doneDirPrefix = conf.get(MRJobConfig.MR_AM_STAGING_DIR, MRJobConfig.DEFAULT_MR_AM_STAGING_DIR) + "/history/done"; } return doneDirPrefix; } /** * Gets the user directory for intermediate done history files. * * @param conf the configuration object * @return the intermediate done directory for jobhistory files. */ public static String getHistoryIntermediateDoneDirForUser(Configuration conf) throws IOException { return getConfiguredHistoryIntermediateDoneDirPrefix(conf) + File.separator + UserGroupInformation.getCurrentUser().getShortUserName(); } public static boolean shouldCreateNonUserDirectory(Configuration conf) { // Returning true by default to allow non secure single node clusters to work // without any configuration change. return conf.getBoolean(MRJobConfig.MR_AM_CREATE_JH_INTERMEDIATE_BASE_DIR, true); } /** Get the job history file path for non Done history files. */ public static Path getStagingJobHistoryFile(Path dir, JobId jobId, int attempt) { return getStagingJobHistoryFile(dir, TypeConverter.fromYarn(jobId).toString(), attempt); } /** Get the job history file path for non Done history files. */ public static Path getStagingJobHistoryFile(Path dir, String jobId, int attempt) { return new Path(dir, jobId + "_" + attempt + JOB_HISTORY_FILE_EXTENSION); } /** * Get the done configuration file name for a job. * * @param jobId the jobId. * @return the conf file name. */ public static String getIntermediateConfFileName(JobId jobId) { return TypeConverter.fromYarn(jobId).toString() + CONF_FILE_NAME_SUFFIX; } /** * Get the done summary file name for a job. * * @param jobId the jobId. * @return the conf file name. */ public static String getIntermediateSummaryFileName(JobId jobId) { return TypeConverter.fromYarn(jobId).toString() + SUMMARY_FILE_NAME_SUFFIX; } /** * Gets the conf file path for jobs in progress. * * @param logDir the log directory prefix. * @param jobId the jobId. * @param attempt attempt number for this job. * @return the conf file path for jobs in progress. */ public static Path getStagingConfFile(Path logDir, JobId jobId, int attempt) { Path jobFilePath = null; if (logDir != null) { jobFilePath = new Path( logDir, TypeConverter.fromYarn(jobId).toString() + "_" + attempt + CONF_FILE_NAME_SUFFIX); } return jobFilePath; } /** * Gets the serial number part of the path based on the jobId and serialNumber format. * * @param id * @param serialNumberFormat * @return the serial number part of the patch based on the jobId and serial number format. */ public static String serialNumberDirectoryComponent(JobId id, String serialNumberFormat) { return String.format(serialNumberFormat, Integer.valueOf(jobSerialNumber(id))) .substring(0, SERIAL_NUMBER_DIRECTORY_DIGITS); } /** * Extracts the timstamp component from the path. * * @param path * @return the timestamp component from the path */ public static String getTimestampPartFromPath(String path) { Matcher matcher = TIMESTAMP_DIR_PATTERN.matcher(path); if (matcher.find()) { String matched = matcher.group(); String ret = matched.intern(); return ret; } else { return null; } } /** * Gets the history subdirectory based on the jobId, timestamp and serial number format. * * @param id * @param timestampComponent * @param serialNumberFormat * @return the history sub directory based on the jobid, timestamp and serial number format */ public static String historyLogSubdirectory( JobId id, String timestampComponent, String serialNumberFormat) { // String result = LOG_VERSION_STRING; String result = ""; String serialNumberDirectory = serialNumberDirectoryComponent(id, serialNumberFormat); result = result + timestampComponent + File.separator + serialNumberDirectory + File.separator; return result; } /** * Gets the timestamp component based on millisecond time. * * @param millisecondTime * @return the timestamp component based on millisecond time */ public static String timestampDirectoryComponent(long millisecondTime) { Calendar timestamp = Calendar.getInstance(); timestamp.setTimeInMillis(millisecondTime); String dateString = null; dateString = String.format( TIMESTAMP_DIR_FORMAT, timestamp.get(Calendar.YEAR), // months are 0-based in Calendar, but people will expect January to // be month #1. timestamp.get(Calendar.MONTH) + 1, timestamp.get(Calendar.DAY_OF_MONTH)); dateString = dateString.intern(); return dateString; } public static String doneSubdirsBeforeSerialTail() { // date String result = "/*/*/*"; // YYYY/MM/DD ; return result; } /** * Computes a serial number used as part of directory naming for the given jobId. * * @param id the jobId. * @return the serial number used as part of directory naming for the given jobid */ public static int jobSerialNumber(JobId id) { return id.getId(); } public static List<FileStatus> localGlobber(FileContext fc, Path root, String tail) throws IOException { return localGlobber(fc, root, tail, null); } public static List<FileStatus> localGlobber( FileContext fc, Path root, String tail, PathFilter filter) throws IOException { return localGlobber(fc, root, tail, filter, null); } // hasMismatches is just used to return a second value if you want // one. I would have used MutableBoxedBoolean if such had been provided. public static List<FileStatus> localGlobber( FileContext fc, Path root, String tail, PathFilter filter, AtomicBoolean hasFlatFiles) throws IOException { if (tail.equals("")) { return (listFilteredStatus(fc, root, filter)); } if (tail.startsWith("/*")) { Path[] subdirs = filteredStat2Paths(remoteIterToList(fc.listStatus(root)), true, hasFlatFiles); List<List<FileStatus>> subsubdirs = new LinkedList<List<FileStatus>>(); int subsubdirCount = 0; if (subdirs.length == 0) { return new LinkedList<FileStatus>(); } String newTail = tail.substring(2); for (int i = 0; i < subdirs.length; ++i) { subsubdirs.add(localGlobber(fc, subdirs[i], newTail, filter, null)); // subsubdirs.set(i, localGlobber(fc, subdirs[i], newTail, filter, // null)); subsubdirCount += subsubdirs.get(i).size(); } List<FileStatus> result = new LinkedList<FileStatus>(); for (int i = 0; i < subsubdirs.size(); ++i) { result.addAll(subsubdirs.get(i)); } return result; } if (tail.startsWith("/")) { int split = tail.indexOf('/', 1); if (split < 0) { return listFilteredStatus(fc, new Path(root, tail.substring(1)), filter); } else { String thisSegment = tail.substring(1, split); String newTail = tail.substring(split); return localGlobber(fc, new Path(root, thisSegment), newTail, filter, hasFlatFiles); } } IOException e = new IOException("localGlobber: bad tail"); throw e; } private static List<FileStatus> listFilteredStatus(FileContext fc, Path root, PathFilter filter) throws IOException { List<FileStatus> fsList = remoteIterToList(fc.listStatus(root)); if (filter == null) { return fsList; } else { List<FileStatus> filteredList = new LinkedList<FileStatus>(); for (FileStatus fs : fsList) { if (filter.accept(fs.getPath())) { filteredList.add(fs); } } return filteredList; } } private static List<FileStatus> remoteIterToList(RemoteIterator<FileStatus> rIter) throws IOException { List<FileStatus> fsList = new LinkedList<FileStatus>(); if (rIter == null) return fsList; while (rIter.hasNext()) { fsList.add(rIter.next()); } return fsList; } // hasMismatches is just used to return a second value if you want // one. I would have used MutableBoxedBoolean if such had been provided. private static Path[] filteredStat2Paths( List<FileStatus> stats, boolean dirs, AtomicBoolean hasMismatches) { int resultCount = 0; if (hasMismatches == null) { hasMismatches = new AtomicBoolean(false); } for (int i = 0; i < stats.size(); ++i) { if (stats.get(i).isDirectory() == dirs) { stats.set(resultCount++, stats.get(i)); } else { hasMismatches.set(true); } } Path[] result = new Path[resultCount]; for (int i = 0; i < resultCount; i++) { result[i] = stats.get(i).getPath(); } return result; } public static Path getPreviousJobHistoryPath( Configuration conf, ApplicationAttemptId applicationAttemptId) throws IOException { String jobId = TypeConverter.fromYarn(applicationAttemptId.getApplicationId()).toString(); String jobhistoryDir = JobHistoryUtils.getConfiguredHistoryStagingDirPrefix(conf, jobId); Path histDirPath = FileContext.getFileContext(conf).makeQualified(new Path(jobhistoryDir)); FileContext fc = FileContext.getFileContext(histDirPath.toUri(), conf); return fc.makeQualified( JobHistoryUtils.getStagingJobHistoryFile( histDirPath, jobId, (applicationAttemptId.getAttemptId() - 1))); } /** * Looks for the dirs to clean. The folder structure is YYYY/MM/DD/Serial so we can use that to * more efficiently find the directories to clean by comparing the cutoff timestamp with the * timestamp from the folder structure. * * @param fc done dir FileContext * @param root folder for completed jobs * @param cutoff The cutoff for the max history age * @return The list of directories for cleaning * @throws IOException */ public static List<FileStatus> getHistoryDirsForCleaning(FileContext fc, Path root, long cutoff) throws IOException { List<FileStatus> fsList = new ArrayList<FileStatus>(); Calendar cCal = Calendar.getInstance(); cCal.setTimeInMillis(cutoff); int cYear = cCal.get(Calendar.YEAR); int cMonth = cCal.get(Calendar.MONTH) + 1; int cDate = cCal.get(Calendar.DATE); RemoteIterator<FileStatus> yearDirIt = fc.listStatus(root); while (yearDirIt.hasNext()) { FileStatus yearDir = yearDirIt.next(); try { int year = Integer.parseInt(yearDir.getPath().getName()); if (year <= cYear) { RemoteIterator<FileStatus> monthDirIt = fc.listStatus(yearDir.getPath()); while (monthDirIt.hasNext()) { FileStatus monthDir = monthDirIt.next(); try { int month = Integer.parseInt(monthDir.getPath().getName()); // If we only checked the month here, then something like 07/2013 // would incorrectly not pass when the cutoff is 06/2014 if (year < cYear || month <= cMonth) { RemoteIterator<FileStatus> dateDirIt = fc.listStatus(monthDir.getPath()); while (dateDirIt.hasNext()) { FileStatus dateDir = dateDirIt.next(); try { int date = Integer.parseInt(dateDir.getPath().getName()); // If we only checked the date here, then something like // 07/21/2013 would incorrectly not pass when the cutoff is // 08/20/2013 or 07/20/2012 if (year < cYear || month < cMonth || date <= cDate) { fsList.addAll(remoteIterToList(fc.listStatus(dateDir.getPath()))); } } catch (NumberFormatException nfe) { // the directory didn't fit the format we're looking for so // skip the dir } } } } catch (NumberFormatException nfe) { // the directory didn't fit the format we're looking for so skip // the dir } } } } catch (NumberFormatException nfe) { // the directory didn't fit the format we're looking for so skip the dir } } return fsList; } }