Пример #1
0
  public static void main(String[] args) throws IOException {

    if (args.length != 3) {
      System.out.println("Parameters: inputDir outputDir parallel");
      System.exit(1);
    }
    String inputDir = args[0];
    String outputDir = args[1];
    String parallel = args[2];
    JobConf lp = new JobConf(L10.class);
    lp.setJobName("L10 Load Page Views");
    lp.setInputFormat(TextInputFormat.class);
    lp.setOutputKeyClass(MyType.class);
    lp.setOutputValueClass(Text.class);
    lp.setMapperClass(ReadPageViews.class);
    lp.setReducerClass(Group.class);
    lp.setPartitionerClass(MyPartitioner.class);
    Properties props = System.getProperties();
    for (Map.Entry<Object, Object> entry : props.entrySet()) {
      lp.set((String) entry.getKey(), (String) entry.getValue());
    }
    FileInputFormat.addInputPath(lp, new Path(inputDir + "/page_views"));
    FileOutputFormat.setOutputPath(lp, new Path(outputDir + "/L10out"));
    // Hardcode the parallel to 40 since MyPartitioner assumes it
    lp.setNumReduceTasks(40);
    Job group = new Job(lp);

    JobControl jc = new JobControl("L10 join");
    jc.addJob(group);

    new Thread(jc).start();

    int i = 0;
    while (!jc.allFinished()) {
      ArrayList<Job> failures = jc.getFailedJobs();
      if (failures != null && failures.size() > 0) {
        for (Job failure : failures) {
          System.err.println(failure.getMessage());
        }
        break;
      }

      try {
        Thread.sleep(5000);
      } catch (InterruptedException e) {
      }

      if (i % 10000 == 0) {
        System.out.println("Running jobs");
        ArrayList<Job> running = jc.getRunningJobs();
        if (running != null && running.size() > 0) {
          for (Job r : running) {
            System.out.println(r.getJobName());
          }
        }
        System.out.println("Ready jobs");
        ArrayList<Job> ready = jc.getReadyJobs();
        if (ready != null && ready.size() > 0) {
          for (Job r : ready) {
            System.out.println(r.getJobName());
          }
        }
        System.out.println("Waiting jobs");
        ArrayList<Job> waiting = jc.getWaitingJobs();
        if (waiting != null && waiting.size() > 0) {
          for (Job r : ready) {
            System.out.println(r.getJobName());
          }
        }
        System.out.println("Successful jobs");
        ArrayList<Job> success = jc.getSuccessfulJobs();
        if (success != null && success.size() > 0) {
          for (Job r : ready) {
            System.out.println(r.getJobName());
          }
        }
      }
      i++;
    }
    ArrayList<Job> failures = jc.getFailedJobs();
    if (failures != null && failures.size() > 0) {
      for (Job failure : failures) {
        System.err.println(failure.getMessage());
      }
    }
    jc.stop();
  }
Пример #2
0
  public static void main(String[] args) throws IOException {

    JobConf lp = new JobConf(L4.class);
    lp.setJobName("Load Page Views");
    lp.setInputFormat(TextInputFormat.class);
    lp.setOutputKeyClass(Text.class);
    lp.setOutputValueClass(Text.class);
    lp.setMapperClass(ReadPageViews.class);
    lp.setCombinerClass(Combiner.class);
    lp.setReducerClass(Group.class);
    Properties props = System.getProperties();
    String dataDir = props.getProperty("PIGMIX_DIR", "/user/pig/tests/data/pigmix");
    for (Map.Entry<Object, Object> entry : props.entrySet()) {
      lp.set((String) entry.getKey(), (String) entry.getValue());
    }
    FileInputFormat.addInputPath(lp, new Path(dataDir, "page_views"));
    FileOutputFormat.setOutputPath(
        lp, new Path("/user/" + System.getProperty("user.name") + "/L4out"));
    lp.setNumReduceTasks(40);
    Job group = new Job(lp);

    JobControl jc = new JobControl("L4 join");
    jc.addJob(group);

    new Thread(jc).start();

    int i = 0;
    while (!jc.allFinished()) {
      ArrayList<Job> failures = jc.getFailedJobs();
      if (failures != null && failures.size() > 0) {
        for (Job failure : failures) {
          System.err.println(failure.getMessage());
        }
        break;
      }

      try {
        Thread.sleep(5000);
      } catch (InterruptedException e) {
      }

      if (i % 10000 == 0) {
        System.out.println("Running jobs");
        ArrayList<Job> running = jc.getRunningJobs();
        if (running != null && running.size() > 0) {
          for (Job r : running) {
            System.out.println(r.getJobName());
          }
        }
        System.out.println("Ready jobs");
        ArrayList<Job> ready = jc.getReadyJobs();
        if (ready != null && ready.size() > 0) {
          for (Job r : ready) {
            System.out.println(r.getJobName());
          }
        }
        System.out.println("Waiting jobs");
        ArrayList<Job> waiting = jc.getWaitingJobs();
        if (waiting != null && waiting.size() > 0) {
          for (Job r : ready) {
            System.out.println(r.getJobName());
          }
        }
        System.out.println("Successful jobs");
        ArrayList<Job> success = jc.getSuccessfulJobs();
        if (success != null && success.size() > 0) {
          for (Job r : ready) {
            System.out.println(r.getJobName());
          }
        }
      }
      i++;
    }
    ArrayList<Job> failures = jc.getFailedJobs();
    if (failures != null && failures.size() > 0) {
      for (Job failure : failures) {
        System.err.println(failure.getMessage());
      }
    }
    jc.stop();
  }