private FSDataOutputStream create(Path f, Reporter reporter, FileStatus srcstat) throws IOException { if (destFileSys.exists(f)) { destFileSys.delete(f, false); } if (!preserve_status) { return destFileSys.create(f, true, sizeBuf, reporter); } FsPermission permission = preseved.contains(FileAttribute.PERMISSION) ? srcstat.getPermission() : null; short replication = preseved.contains(FileAttribute.REPLICATION) ? srcstat.getReplication() : destFileSys.getDefaultReplication(); long blockSize = preseved.contains(FileAttribute.BLOCK_SIZE) ? srcstat.getBlockSize() : destFileSys.getDefaultBlockSize(); return destFileSys.create(f, permission, true, sizeBuf, replication, blockSize, reporter); }
public static RawSequenceFileWriter createWriter( FileSystem fs, Configuration conf, Path name, Class keyClass, Class valClass, CompressionType compressionType) throws IOException { return new RawSequenceFileWriter( fs, conf, name, keyClass, valClass, fs.getConf().getInt("io.file.buffer.size", 65536), fs.getDefaultReplication(), fs.getDefaultBlockSize(), null, new Metadata()); }
public void testCreate() throws Exception { Configuration conf = new HdfsConfiguration(); conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true); conf.set(FsPermission.UMASK_LABEL, "000"); MiniDFSCluster cluster = null; FileSystem fs = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); cluster.waitActive(); fs = FileSystem.get(conf); FsPermission rootPerm = checkPermission(fs, "/", null); FsPermission inheritPerm = FsPermission.createImmutable((short) (rootPerm.toShort() | 0300)); FsPermission dirPerm = new FsPermission((short) 0777); fs.mkdirs(new Path("/a1/a2/a3"), dirPerm); checkPermission(fs, "/a1", dirPerm); checkPermission(fs, "/a1/a2", dirPerm); checkPermission(fs, "/a1/a2/a3", dirPerm); dirPerm = new FsPermission((short) 0123); FsPermission permission = FsPermission.createImmutable((short) (dirPerm.toShort() | 0300)); fs.mkdirs(new Path("/aa/1/aa/2/aa/3"), dirPerm); checkPermission(fs, "/aa/1", permission); checkPermission(fs, "/aa/1/aa/2", permission); checkPermission(fs, "/aa/1/aa/2/aa/3", dirPerm); FsPermission filePerm = new FsPermission((short) 0444); FSDataOutputStream out = fs.create( new Path("/b1/b2/b3.txt"), filePerm, true, conf.getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), fs.getDefaultReplication(), fs.getDefaultBlockSize(), null); out.write(123); out.close(); checkPermission(fs, "/b1", inheritPerm); checkPermission(fs, "/b1/b2", inheritPerm); checkPermission(fs, "/b1/b2/b3.txt", filePerm); conf.set(FsPermission.UMASK_LABEL, "022"); permission = FsPermission.createImmutable((short) 0666); FileSystem.mkdirs(fs, new Path("/c1"), new FsPermission(permission)); FileSystem.create(fs, new Path("/c1/c2.txt"), new FsPermission(permission)); checkPermission(fs, "/c1", permission); checkPermission(fs, "/c1/c2.txt", permission); } finally { try { if (fs != null) fs.close(); } catch (Exception e) { LOG.error(StringUtils.stringifyException(e)); } try { if (cluster != null) cluster.shutdown(); } catch (Exception e) { LOG.error(StringUtils.stringifyException(e)); } } }
@Override public short getDefaultReplication(FileSystem fs, Path path) { return fs.getDefaultReplication(path); }
/** * Log job submitted event to history. Creates a new file in history for the job. if history file * creation fails, it disables history for all other events. * * @param jobConfPath path to job conf xml file in HDFS. * @param submitTime time when job tracker received the job * @throws IOException */ public void logSubmitted(String jobConfPath, long submitTime, String jobTrackerId) throws IOException { if (disableHistory) { return; } // create output stream for logging in hadoop.job.history.location int defaultBufferSize = logDirFs.getConf().getInt("io.file.buffer.size", 4096); try { FSDataOutputStream out = null; PrintWriter writer = null; // In case the old JT is still running, but we can't connect to it, we // should ensure that it won't write to our (new JT's) job history file. if (logDirFs.exists(logFile)) { LOG.info("Remove the old history file " + logFile); logDirFs.delete(logFile, true); } out = logDirFs.create( logFile, new FsPermission(HISTORY_FILE_PERMISSION), true, defaultBufferSize, logDirFs.getDefaultReplication(), jobHistoryBlockSize, null); writer = new PrintWriter(out); fileManager.addWriter(jobId, writer); // cache it ... fileManager.setHistoryFile(jobId, logFile); writers = fileManager.getWriters(jobId); if (null != writers) { log( writers, RecordTypes.Meta, new Keys[] {Keys.VERSION}, new String[] {String.valueOf(JobHistory.VERSION)}); } String jobName = getJobName(); String user = getUserName(); // add to writer as well log( writers, RecordTypes.Job, new Keys[] { Keys.JOBID, Keys.JOBNAME, Keys.USER, Keys.SUBMIT_TIME, Keys.JOBCONF, Keys.JOBTRACKERID }, new String[] { jobId.toString(), jobName, user, String.valueOf(submitTime), jobConfPath, jobTrackerId }); } catch (IOException e) { // Disable history if we have errors other than in the user log. disableHistory = true; } /* Storing the job conf on the log dir */ Path jobFilePath = new Path(logDir, CoronaJobHistoryFilesManager.getConfFilename(jobId)); fileManager.setConfFile(jobId, jobFilePath); FSDataOutputStream jobFileOut = null; try { if (!logDirFs.exists(jobFilePath)) { jobFileOut = logDirFs.create( jobFilePath, new FsPermission(HISTORY_FILE_PERMISSION), true, defaultBufferSize, logDirFs.getDefaultReplication(), logDirFs.getDefaultBlockSize(), null); conf.writeXml(jobFileOut); jobFileOut.close(); } } catch (IOException ioe) { LOG.error("Failed to store job conf in the log dir", ioe); } finally { if (jobFileOut != null) { try { jobFileOut.close(); } catch (IOException ie) { LOG.info( "Failed to close the job configuration file " + StringUtils.stringifyException(ie)); } } } }