コード例 #1
0
ファイル: ConfigManager.java プロジェクト: sunzz679/HadoopUSC
 public ConfigManager(Configuration conf)
     throws IOException, SAXException, RaidConfigurationException, ClassNotFoundException,
         ParserConfigurationException {
   this.conf = conf;
   this.configFileName = conf.get("raid.config.file");
   this.doReload = conf.getBoolean("raid.config.reload", true);
   this.reloadInterval = conf.getLong("raid.config.reload.interval", RELOAD_INTERVAL);
   this.periodicity = conf.getLong("raid.policy.rescan.interval", RESCAN_INTERVAL);
   this.harPartfileSize = conf.getLong("raid.har.partfile.size", HAR_PARTFILE_SIZE);
   this.maxJobsPerPolicy = conf.getInt("raid.distraid.max.jobs", DISTRAID_MAX_JOBS);
   this.maxFilesPerJob = conf.getInt("raid.distraid.max.files", DISTRAID_MAX_FILES);
   if (configFileName == null) {
     String msg =
         "No raid.config.file given in conf - "
             + "the Hadoop Raid utility cannot run. Aborting....";
     LOG.warn(msg);
     throw new IOException(msg);
   }
   reloadConfigs();
   lastSuccessfulReload = RaidNode.now();
   lastReloadAttempt = RaidNode.now();
   running = true;
 }
コード例 #2
0
ファイル: DistRaid.java プロジェクト: fire9/hadoop-20
  /**
   * create new job conf based on configuration passed.
   *
   * @param conf
   * @return
   */
  private static JobConf createJobConf(Configuration conf) {
    JobConf jobconf = new JobConf(conf, DistRaid.class);
    jobName = NAME + " " + dateForm.format(new Date(RaidNode.now()));
    jobconf.setUser(RaidNode.JOBUSER);
    jobconf.setJobName(jobName);
    jobconf.setMapSpeculativeExecution(false);
    RaidUtils.parseAndSetOptions(jobconf, SCHEDULER_OPTION_LABEL);

    jobconf.setJarByClass(DistRaid.class);
    jobconf.setInputFormat(DistRaidInputFormat.class);
    jobconf.setOutputKeyClass(Text.class);
    jobconf.setOutputValueClass(Text.class);

    jobconf.setMapperClass(DistRaidMapper.class);
    jobconf.setNumReduceTasks(0);
    return jobconf;
  }
コード例 #3
0
ファイル: ConfigManager.java プロジェクト: sunzz679/HadoopUSC
 /**
  * Reload config file if it hasn't been loaded in a while Returns true if the file was reloaded.
  */
 public synchronized boolean reloadConfigsIfNecessary() {
   long time = RaidNode.now();
   if (time > lastReloadAttempt + reloadInterval) {
     lastReloadAttempt = time;
     try {
       File file = new File(configFileName);
       long lastModified = file.lastModified();
       if (lastModified > lastSuccessfulReload && time > lastModified + RELOAD_WAIT) {
         reloadConfigs();
         lastSuccessfulReload = time;
         lastReloadAttemptFailed = false;
         return true;
       }
     } catch (Exception e) {
       if (!lastReloadAttemptFailed) {
         LOG.error("Failed to reload config file - " + "will use existing configuration.", e);
       }
       lastReloadAttemptFailed = true;
     }
   }
   return false;
 }