Beispiel #1
0
 public static String determineFileType(FileMetadata fm) throws IOException {
   // TODO: networkDataFile
   StudyFile sf = fm.getStudyFile();
   if (sf instanceof TabularDataFile) {
     return determineTabularDataFileType((TabularDataFile) sf);
   } else {
     if (sf.isRemote()) {
       return FileUtil.determineFileType(fm.getLabel());
     } else {
       return FileUtil.determineFileType(new File(sf.getFileSystemLocation()), fm.getLabel());
     }
   }
 }
Beispiel #2
0
  public static File getStudyFileDir(String authority, String studyId) {

    File file =
        new File(FileUtil.getStudyFileDir(), authority + File.separator + studyId.toUpperCase());
    if (!file.exists()) {
      file.mkdirs();
    }
    return file;
  }
Beispiel #3
0
  public static File getStudyFileDir(Study study) {

    File file =
        new File(
            FileUtil.getStudyFileDir(), study.getAuthority() + File.separator + study.getStudyId());
    if (!file.exists()) {
      file.mkdirs();
    }
    return file;
  }
Beispiel #4
0
  /** Start the JobTracker process, listen on the indicated port */
  JobTracker(Configuration conf) throws IOException {
    //
    // Grab some static constants
    //
    maxCurrentTasks = conf.getInt("mapred.tasktracker.tasks.maximum", 2);
    RETIRE_JOB_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.interval", 24 * 60 * 60 * 1000);
    RETIRE_JOB_CHECK_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.check", 60 * 1000);
    TASK_ALLOC_EPSILON = conf.getFloat("mapred.jobtracker.taskalloc.loadbalance.epsilon", 0.2f);
    PAD_FRACTION = conf.getFloat("mapred.jobtracker.taskalloc.capacitypad", 0.1f);
    MIN_SLOTS_FOR_PADDING = 3 * maxCurrentTasks;

    // This is a directory of temporary submission files.  We delete it
    // on startup, and can delete any files that we're done with
    this.conf = conf;
    JobConf jobConf = new JobConf(conf);
    this.systemDir = jobConf.getSystemDir();
    this.fs = FileSystem.get(conf);
    FileUtil.fullyDelete(fs, systemDir);
    fs.mkdirs(systemDir);

    // Same with 'localDir' except it's always on the local disk.
    jobConf.deleteLocalFiles(SUBDIR);

    // Set ports, start RPC servers, etc.
    InetSocketAddress addr = getAddress(conf);
    this.localMachine = addr.getHostName();
    this.port = addr.getPort();
    this.interTrackerServer = RPC.getServer(this, addr.getPort(), 10, false, conf);
    this.interTrackerServer.start();
    Properties p = System.getProperties();
    for (Iterator it = p.keySet().iterator(); it.hasNext(); ) {
      String key = (String) it.next();
      String val = (String) p.getProperty(key);
      LOG.info("Property '" + key + "' is " + val);
    }

    this.infoPort = conf.getInt("mapred.job.tracker.info.port", 50030);
    this.infoServer = new JobTrackerInfoServer(this, infoPort);
    this.infoServer.start();

    this.startTime = System.currentTimeMillis();

    new Thread(this.expireTrackers).start();
    new Thread(this.retireJobs).start();
    new Thread(this.initJobs).start();
  }