Example #1
0
  /** Reads jobs from data_set file and sends them to the Scheduler entity dynamically over time. */
  public void body() {
    super.gridSimHold(10.0); // hold by 10 second

    while (current_gl < total_jobs) {

      Sim_event ev = new Sim_event();
      sim_get_next(ev);

      if (ev.get_tag() == AleaSimTags.EVENT_WAKE) {

        ComplexGridlet gl = readGridlet(current_gl);
        current_gl++;
        if (gl == null && current_gl < total_jobs) {
          super.sim_schedule(this.getEntityId(this.getEntityName()), 0.0, AleaSimTags.EVENT_WAKE);
          continue;
        } else if (gl == null && current_gl >= total_jobs) {
          continue;
        }
        // to synchronize job arrival wrt. the data set.
        double delay = Math.max(0.0, (gl.getArrival_time() - super.clock()));
        // some time is needed to transfer this job to the scheduler, i.e., delay should be delay =
        // delay - transfer_time. Fix this in the future.
        // System.out.println("Sending: "+gl.getGridletID());
        last_delay = delay;
        super.sim_schedule(
            this.getEntityId("Alea_3.0_scheduler"), delay, AleaSimTags.GRIDLET_INFO, gl);

        delay = Math.max(0.0, (gl.getArrival_time() - super.clock()));
        if (current_gl < total_jobs) {
          // use delay - next job will be loaded after the simulation time is equal to the previous
          // job arrival.
          super.sim_schedule(this.getEntityId(this.getEntityName()), delay, AleaSimTags.EVENT_WAKE);
        }

        continue;
      }
    }
    System.out.println("Shuting down - last gridlet = " + current_gl + " of " + total_jobs);
    super.sim_schedule(
        this.getEntityId("Alea_3.0_scheduler"),
        Math.round(last_delay + 2),
        AleaSimTags.SUBMISSION_DONE,
        new Integer(current_gl));
    Sim_event ev = new Sim_event();
    sim_get_next(ev);

    if (ev.get_tag() == GridSimTags.END_OF_SIMULATION) {
      System.out.println(
          "Shuting down the "
              + data_set
              + "_PWALoader... with: "
              + fail
              + " failed or skipped jobs");
    }
    shutdownUserEntity();
    super.terminateIOEntities();
  }
Example #2
0
  /** Reads one job from file. */
  private ComplexGridlet readGridlet(int j) {
    String[] values = null;
    String line = "";

    // System.out.println("Read job "+j);
    if (j == 0) {
      while (true) {
        try {
          for (int s = 0; s < ExperimentSetup.skipJob; s++) {
            line = br.readLine();
          }
          values = line.split("\t");
        } catch (IOException ex) {
          ex.printStackTrace();
        }
        if (!values[0].contains(";")) {
          if (line.charAt(0) == ' ') {
            line = line.substring(1);
          }
          if (line.charAt(0) == ' ') {
            line = line.substring(1);
          }
          if (line.charAt(0) == ' ') {
            line = line.substring(1);
          }
          if (line.charAt(0) == ' ') {
            line = line.substring(1);
          }
          values = line.split("\\s+");
          break;
        } else {
          // System.out.println("error --- "+values[0]);
        }
      }
    } else {
      try {
        line = br.readLine();
        // System.out.println(">"+line+"<");
        if (line.charAt(0) == ' ') {
          line = line.substring(1);
        }
        if (line.charAt(0) == ' ') {
          line = line.substring(1);
        }
        if (line.charAt(0) == ' ') {
          line = line.substring(1);
        }
        if (line.charAt(0) == ' ') {
          line = line.substring(1);
        }
        // System.out.println("error1 = "+line+" at gi = "+j);
        values = line.split("\\s+");

      } catch (IOException ex) {
        System.out.println("error = " + values[0] + " at gi = " + j);
        ex.printStackTrace();
      }
    }
    // such line is not a job description - it is a typo in the SWF file
    if (values.length < 5 || values[1].equals("-1")) {
      fail++;
      System.out.println(j + " returning: null " + values[0]);
      return null;
    }

    // such job failed or was cancelled and no info about runtime or numCPU is available therefore
    // we skip it
    if (values[3].equals("-1") || values[4].equals("-1")) {
      fail++;
      // System.out.println("returning: null2 ");
      return null;
    }
    // System.out.println(values[0]+"+"+values[1]+"+"+values[2] + ": Number parsing error: " +
    // values[4]);
    int id = Integer.parseInt(values[0]);
    int numCPU;
    try {
      numCPU = Integer.parseInt(values[4]);
    } catch (NumberFormatException ex) {
      System.out.println(values[0] + ": Number parsing error: " + values[4]);
      // ex.printStackTrace();
      numCPU = 1;
    }

    // we do not allow more PEs for one job than there is on the "biggest" machine.
    // Co-allocation is only supported over one cluster (GridResource) by now.
    if (numCPU > maxPE) {
      numCPU = maxPE;
    }

    long arrival = 0;
    // synchronize GridSim's arrivals with the UNIX epoch format as given in GWF
    if (start_time < 0) {
      // System.out.println("prvni: "+j+" start at:"+values[1]+" line="+line);
      start_time = Integer.parseInt(values[1]);
      arrival = 0;

    } else {

      arrival = ((Integer.parseInt(values[1]) - start_time));
      // System.out.println("pokracujeme..."+arrival);

    }
    arrival = Math.round(new Double(arrival) / ExperimentSetup.arrival_rate_multiplier);

    // minPErating is the default speed of the slowest machine in the data set
    double length = Math.round((Integer.parseInt(values[3])) * maxPErating);

    // queue name
    String queue = values[14];

    // requested RAM = KB per node (not CPU)
    long ram = Long.parseLong(values[9]);
    if (ram == -1) {
      if (Long.parseLong(values[9]) == -1) {
        // System.out.println(id + " not specified RAM, setting 1 KB...");
        ram = 1;
      } else {
        ram = Long.parseLong(values[9]);
      }
    } else {
      if (data_set.contains("zewura")
          || data_set.contains("wagap")
          || data_set.contains("meta")
          || data_set.contains("ncbr")) {

        ram = Math.round(ram / 1024.0);
        queue = values[19];
      }
      double gbram = Math.round(ram * 10 / 1048576.0) / 10.0;
      // System.out.println(id+ " requests "+ram+" KB RAM, "+gbram+" GB RAM per "+numCPU+" CPUs");
    }
    if (!ExperimentSetup.use_RAM) {
      ram = 0;
    }

    // skip such job
    /*if (data_set.contains("zewura") || data_set.contains("wagapp") || data_set.contains("meta")) {
    if (!queue.equals("short") && !queue.equals("long") && !queue.equals("normal") && !queue.equals("backfill") && !queue.equals("preemptible")) {
    fail++;
    return null;
    }
    }
    */
    // SKIP
    /*
     * if (id < 172262) { fail++; return null; }
     */

    // finally create gridlet
    // numCPU = 1;
    long job_limit = Integer.parseInt(values[8]);
    if (job_limit < 0) {
      // atlas = 432000
      // thunder = 432000
      if (data_set.equals("thunder.swf")) {
        job_limit = 48000; // 13 hours 20 min
        ExperimentSetup.max_estim++;
      } else if (data_set.equals("atlas.swf")) {
        job_limit = 73200; // 20 hours 20 minutes
        ExperimentSetup.max_estim++;
      } else if (data_set.equals("star.swf")) {
        job_limit = 64800; // 18 hours
        ExperimentSetup.max_estim++;
      } else if (data_set.equals("ctc-sp2.swf")) {
        job_limit = 64800; // 18 hours
        ExperimentSetup.max_estim++;
      } else if (data_set.equals("blue.swf")) {
        job_limit = 7200; // 2 hours
        ExperimentSetup.max_estim++;
      } else if (data_set.equals("kth-sp2.swf")) {
        job_limit = 14400; // 4 hours
        ExperimentSetup.max_estim++;
      } else if (data_set.equals("sandia.swf")) {
        job_limit = 18000; // 5 hours
        ExperimentSetup.max_estim++;
      } else {
        job_limit = Integer.parseInt(values[3]);
      }
    }

    double estimatedLength = 0.0;
    if (ExperimentSetup.estimates) {
      // roughest estimate that can be done = queue limit
      estimatedLength = Math.round(Math.max((job_limit * maxPErating), length));
      // System.out.println(id+" Estimates "+estimatedLength+" real = "+length);
    } else {
      // exact estimates
      estimatedLength = length;
      // System.out.println(id+" Exact "+estimatedLength);
    }

    String user = values[11];

    // System.out.println(id + " requests " + ram + " KB RAM per " + numCPU + " CPUs, user: "******", length: " + length + " estimatedLength: " + estimatedLength);
    int numNodes = -1;
    int ppn = -1;
    String properties = "";
    if (values.length > 19) {
      properties = values[20];

      if (data_set.contains("wagap")
          || data_set.contains("meta")
          || data_set.contains("ncbr")
          || data_set.contains("fairshare")) {
        String[] req_nodes = values[20].split(":");
        properties = values[20];
        for (int r = 0; r < req_nodes.length; r++) {
          if (req_nodes[r].contains("ppn=")) {
            String ppns = req_nodes[r].replace("ppn=", "");
            if (ppns.contains("#")) {
              int ind = ppns.indexOf("#");
              ppns = ppns.substring(0, ind);
            }
            // remove floating point values
            if (ppns.contains(".")) {
              int ind = ppns.indexOf('.');
              ppns = ppns.substring(0, ind);
            }

            // to do: 1:ppn=1+3:ppn=2
            if (ppns.contains("+")) {
              break;
            }
            if (ppns.equals("1cl_zewura")) {
              ppn = 1;
            } else {
              ppn = Integer.parseInt(ppns);
            }
          }
        }

        if (ppn != -1) {
          // korekce chyby ve workloadu
          if (numCPU < ppn) {
            System.out.println(
                id + ": CPUs mismatch CPUs = " + numCPU + " nodespec = " + properties);
            numCPU = ppn;
          }
          numNodes = numCPU / ppn;
        } else {
          numNodes = 1;
          ppn = numCPU;
        }
        // System.out.println(id+" | "+values[20]+" nodes="+numNodes+" ppn="+ppn);
      } else if (data_set.contains("zewura")) {
        numNodes = 1;
        ppn = numCPU;
      } else if (data_set.contains("hpc2n")) {
        ppn = 2;
        if (numCPU < ppn) {
          ppn = numCPU;
          numNodes = 1;
        } else if (numCPU % 2 == 1) {
          ppn = 1;
          numNodes = numCPU;
        } else {
          Long nn = Math.round(Math.ceil(numCPU / ppn));
          numNodes = nn.intValue();
        }
        if (ppn * numNodes != numCPU) {
          System.out.println(id + ": numNodes value is wrong, CPUs = " + numCPU + " ppn = " + ppn);
        }
      }

      if (ppn * numNodes != numCPU) {
        System.out.println(
            id + ": CPUs mismatch CPUs = " + numCPU + " ppn = " + ppn + " nodes = " + numNodes);
        numCPU = ppn * numNodes;
      }
    }

    // obsolete and useless
    double perc = norm.sample();

    job_limit = Math.max(1, Math.round(job_limit / ExperimentSetup.runtime_minimizer));
    length = Math.max(1, Math.round(length / ExperimentSetup.runtime_minimizer));
    estimatedLength = Math.max(1, Math.round(estimatedLength / ExperimentSetup.runtime_minimizer));

    if (data_set.contains("wagap")) {
      if (ppn > 8) {
        properties += ":^cl_zigur";
      }
      if (ppn > 12) {
        properties += ":^cl_zegox";
      }
      if (ppn > 16) {
        properties += ":^cl_zapat";
      }
    }

    if (queue.equals("backfill") && data_set.contains("meta")) {
      properties += ":^cl_manwe:^cl_mandos:^cl_skirit:^cl_ramdal:^cl_haldir:^cl_gram";
    }

    if (queue.equals("mikroskop") || queue.equals("quark")) {
      properties += ":cl_quark";
    }

    if (queue.contains("ncbr")) {
      properties += ":cl_perian";
    }

    if (!Scheduler.all_queues_names.contains(queue) && ExperimentSetup.use_queues) {
      fail++;
      System.out.println("Unknown queue " + queue + " - skipping job " + id);
      return null;
    }

    // manually established - fix it according to your needs
    double deadline = job_limit * 2;
    ComplexGridlet gl =
        new ComplexGridlet(
            id,
            user,
            job_limit,
            new Double(length),
            estimatedLength,
            10,
            10,
            "Linux",
            "Risc arch.",
            arrival,
            deadline,
            1,
            numCPU,
            0.0,
            queue,
            properties,
            perc,
            ram,
            numNodes,
            ppn);

    // and set user id to the Scheduler entity - otherwise it would be returned to the JobLoader
    // when completed.
    // System.out.println(id+" job has limit = "+(job_limit/3600.0)+" queue = "+queue);
    gl.setUserID(super.getEntityId("Alea_3.0_scheduler"));
    return gl;
  }