Esempio n. 1
0
    private FSDataOutputStream create(Path f, Reporter reporter, FileStatus srcstat)
        throws IOException {
      if (destFileSys.exists(f)) {
        destFileSys.delete(f, false);
      }
      if (!preserve_status) {
        return destFileSys.create(f, true, sizeBuf, reporter);
      }

      FsPermission permission =
          preseved.contains(FileAttribute.PERMISSION) ? srcstat.getPermission() : null;
      short replication =
          preseved.contains(FileAttribute.REPLICATION)
              ? srcstat.getReplication()
              : destFileSys.getDefaultReplication();
      long blockSize =
          preseved.contains(FileAttribute.BLOCK_SIZE)
              ? srcstat.getBlockSize()
              : destFileSys.getDefaultBlockSize();
      return destFileSys.create(f, permission, true, sizeBuf, replication, blockSize, reporter);
    }
  public static RawSequenceFileWriter createWriter(
      FileSystem fs,
      Configuration conf,
      Path name,
      Class keyClass,
      Class valClass,
      CompressionType compressionType)
      throws IOException {

    return new RawSequenceFileWriter(
        fs,
        conf,
        name,
        keyClass,
        valClass,
        fs.getConf().getInt("io.file.buffer.size", 65536),
        fs.getDefaultReplication(),
        fs.getDefaultBlockSize(),
        null,
        new Metadata());
  }
  public void testCreate() throws Exception {
    Configuration conf = new HdfsConfiguration();
    conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, true);
    conf.set(FsPermission.UMASK_LABEL, "000");
    MiniDFSCluster cluster = null;
    FileSystem fs = null;

    try {
      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
      cluster.waitActive();
      fs = FileSystem.get(conf);
      FsPermission rootPerm = checkPermission(fs, "/", null);
      FsPermission inheritPerm = FsPermission.createImmutable((short) (rootPerm.toShort() | 0300));

      FsPermission dirPerm = new FsPermission((short) 0777);
      fs.mkdirs(new Path("/a1/a2/a3"), dirPerm);
      checkPermission(fs, "/a1", dirPerm);
      checkPermission(fs, "/a1/a2", dirPerm);
      checkPermission(fs, "/a1/a2/a3", dirPerm);

      dirPerm = new FsPermission((short) 0123);
      FsPermission permission = FsPermission.createImmutable((short) (dirPerm.toShort() | 0300));
      fs.mkdirs(new Path("/aa/1/aa/2/aa/3"), dirPerm);
      checkPermission(fs, "/aa/1", permission);
      checkPermission(fs, "/aa/1/aa/2", permission);
      checkPermission(fs, "/aa/1/aa/2/aa/3", dirPerm);

      FsPermission filePerm = new FsPermission((short) 0444);
      FSDataOutputStream out =
          fs.create(
              new Path("/b1/b2/b3.txt"),
              filePerm,
              true,
              conf.getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
              fs.getDefaultReplication(),
              fs.getDefaultBlockSize(),
              null);
      out.write(123);
      out.close();
      checkPermission(fs, "/b1", inheritPerm);
      checkPermission(fs, "/b1/b2", inheritPerm);
      checkPermission(fs, "/b1/b2/b3.txt", filePerm);

      conf.set(FsPermission.UMASK_LABEL, "022");
      permission = FsPermission.createImmutable((short) 0666);
      FileSystem.mkdirs(fs, new Path("/c1"), new FsPermission(permission));
      FileSystem.create(fs, new Path("/c1/c2.txt"), new FsPermission(permission));
      checkPermission(fs, "/c1", permission);
      checkPermission(fs, "/c1/c2.txt", permission);
    } finally {
      try {
        if (fs != null) fs.close();
      } catch (Exception e) {
        LOG.error(StringUtils.stringifyException(e));
      }
      try {
        if (cluster != null) cluster.shutdown();
      } catch (Exception e) {
        LOG.error(StringUtils.stringifyException(e));
      }
    }
  }
Esempio n. 4
0
 @Override
 public short getDefaultReplication(FileSystem fs, Path path) {
   return fs.getDefaultReplication(path);
 }
  /**
   * Log job submitted event to history. Creates a new file in history for the job. if history file
   * creation fails, it disables history for all other events.
   *
   * @param jobConfPath path to job conf xml file in HDFS.
   * @param submitTime time when job tracker received the job
   * @throws IOException
   */
  public void logSubmitted(String jobConfPath, long submitTime, String jobTrackerId)
      throws IOException {

    if (disableHistory) {
      return;
    }

    // create output stream for logging in hadoop.job.history.location
    int defaultBufferSize = logDirFs.getConf().getInt("io.file.buffer.size", 4096);

    try {
      FSDataOutputStream out = null;
      PrintWriter writer = null;

      // In case the old JT is still running, but we can't connect to it, we
      // should ensure that it won't write to our (new JT's) job history file.
      if (logDirFs.exists(logFile)) {
        LOG.info("Remove the old history file " + logFile);
        logDirFs.delete(logFile, true);
      }

      out =
          logDirFs.create(
              logFile,
              new FsPermission(HISTORY_FILE_PERMISSION),
              true,
              defaultBufferSize,
              logDirFs.getDefaultReplication(),
              jobHistoryBlockSize,
              null);

      writer = new PrintWriter(out);

      fileManager.addWriter(jobId, writer);

      // cache it ...
      fileManager.setHistoryFile(jobId, logFile);

      writers = fileManager.getWriters(jobId);
      if (null != writers) {
        log(
            writers,
            RecordTypes.Meta,
            new Keys[] {Keys.VERSION},
            new String[] {String.valueOf(JobHistory.VERSION)});
      }

      String jobName = getJobName();
      String user = getUserName();

      // add to writer as well
      log(
          writers,
          RecordTypes.Job,
          new Keys[] {
            Keys.JOBID, Keys.JOBNAME, Keys.USER, Keys.SUBMIT_TIME, Keys.JOBCONF, Keys.JOBTRACKERID
          },
          new String[] {
            jobId.toString(), jobName, user,
            String.valueOf(submitTime), jobConfPath, jobTrackerId
          });

    } catch (IOException e) {
      // Disable history if we have errors other than in the user log.
      disableHistory = true;
    }

    /* Storing the job conf on the log dir */
    Path jobFilePath = new Path(logDir, CoronaJobHistoryFilesManager.getConfFilename(jobId));
    fileManager.setConfFile(jobId, jobFilePath);
    FSDataOutputStream jobFileOut = null;
    try {
      if (!logDirFs.exists(jobFilePath)) {
        jobFileOut =
            logDirFs.create(
                jobFilePath,
                new FsPermission(HISTORY_FILE_PERMISSION),
                true,
                defaultBufferSize,
                logDirFs.getDefaultReplication(),
                logDirFs.getDefaultBlockSize(),
                null);
        conf.writeXml(jobFileOut);
        jobFileOut.close();
      }
    } catch (IOException ioe) {
      LOG.error("Failed to store job conf in the log dir", ioe);
    } finally {
      if (jobFileOut != null) {
        try {
          jobFileOut.close();
        } catch (IOException ie) {
          LOG.info(
              "Failed to close the job configuration file " + StringUtils.stringifyException(ie));
        }
      }
    }
  }