Ejemplo n.º 1
0
  public MiniMRCluster(
      int jobTrackerPort,
      int taskTrackerPort,
      int numTaskTrackers,
      String namenode,
      int numDir,
      String[] racks,
      String[] hosts,
      UserGroupInformation ugi,
      JobConf conf,
      int numTrackerToExclude)
      throws IOException {
    if (racks != null && racks.length < numTaskTrackers) {
      LOG.error(
          "Invalid number of racks specified. It should be at least "
              + "equal to the number of tasktrackers");
      shutdown();
    }
    if (hosts != null && numTaskTrackers > hosts.length) {
      throw new IllegalArgumentException(
          "The length of hosts ["
              + hosts.length
              + "] is less than the number of tasktrackers ["
              + numTaskTrackers
              + "].");
    }

    // Generate rack names if required
    if (racks == null) {
      System.out.println("Generating rack names for tasktrackers");
      racks = new String[numTaskTrackers];
      for (int i = 0; i < racks.length; ++i) {
        racks[i] = NetworkTopology.DEFAULT_RACK;
      }
    }

    // Generate some hostnames if required
    if (hosts == null) {
      System.out.println("Generating host names for tasktrackers");
      hosts = new String[numTaskTrackers];
      for (int i = 0; i < numTaskTrackers; i++) {
        hosts[i] = "host" + i + ".foo.com";
      }
    }
    this.jobTrackerPort = jobTrackerPort;
    this.taskTrackerPort = taskTrackerPort;
    this.jobTrackerInfoPort = 0;
    this.numTaskTrackers = 0;
    this.namenode = namenode;
    this.ugi = ugi;
    this.conf = conf; // this is the conf the mr starts with
    this.numTrackerToExclude = numTrackerToExclude;

    // start the jobtracker
    startJobTracker();

    // Create the TaskTrackers
    for (int idx = 0; idx < numTaskTrackers; idx++) {
      String rack = null;
      String host = null;
      if (racks != null) {
        rack = racks[idx];
      }
      if (hosts != null) {
        host = hosts[idx];
      }

      startTaskTracker(host, rack, idx, numDir);
    }

    this.job = createJobConf(conf);
    waitUntilIdle();
  }