protected static String toInfo(JobInfo info) {
   StringBuilder sb = new StringBuilder();
   sb.append("JobInfo[ ").append(info).append("]");
   if (info != null) {
     sb.append(" --> ");
     sb.append("state - ").append(info.getJobState()).append(", ");
     sb.append("output - ").append(info.getOutput()).append(", ");
     sb.append("error - ").append(info.getError());
   }
   return sb.toString();
 }
Esempio n. 2
0
  @SuppressWarnings("unchecked")
  @Test
  public void testCountJob() throws Exception {
    List<String> payloads = Arrays.asList("capedwarf", "jboss", "redhat");
    int shardCount = 1;

    MapReduceSpecification.Builder builder = new MapReduceSpecification.Builder();
    builder.setJobName("Create MapReduce entities");
    builder.setInput(new ConsecutiveLongInput(0, payloads.size() * (long) shardCount, shardCount));
    builder.setMapper(new EntityCreator("MapReduceTest", payloads));
    builder.setKeyMarshaller(Marshallers.getVoidMarshaller());
    builder.setValueMarshaller(Marshallers.getVoidMarshaller());
    builder.setReducer(NoReducer.create());
    builder.setOutput(new NoOutput());

    final String createHandle = MapReduceJob.start(builder.build(), getSettings());

    JobInfo createJI = waitToFinish("CREATE", createHandle);
    Object create = createJI.getOutput();
    log.warning("----- Create: " + create);

    int mapShardCount = 1;

    builder = new MapReduceSpecification.Builder();
    builder.setJobName("MapReduceTest stats");
    builder.setInput(new DatastoreInput("MapReduceTest", mapShardCount));
    builder.setMapper(new CountMapper());
    builder.setKeyMarshaller(Marshallers.getStringMarshaller());
    builder.setValueMarshaller(Marshallers.getLongMarshaller());
    builder.setReducer(new CountReducer());
    builder.setOutput(new InMemoryOutput<KeyValue<String, Long>>());

    final String countHandle = MapReduceJob.start(builder.build(), getSettings());

    JobInfo countJI = waitToFinish("COUNT", countHandle);
    Object count = countJI.getOutput();
    log.warning("----- Count: " + count);

    Assert.assertTrue(count instanceof MapReduceResult);
    MapReduceResult result = MapReduceResult.class.cast(count);
    int[] chars = toChars(payloads);
    Counters counters = result.getCounters();
    for (int i = 0; i < chars.length; i++) {
      char ch = (char) ('a' + i);
      Counter c = counters.getCounter(CountMapper.toKey(ch));
      Assert.assertEquals(String.format("Invalid count for '%s'.", ch), chars[i], c.getValue());
    }
  }
 protected JobInfo waitToFinish(final String phase, final String handle) throws Exception {
   PipelineService pipelineService = PipelineServiceFactory.newPipelineService();
   JobInfo jobInfo = getJobInfo(pipelineService, phase, handle);
   JobInfo.State state = jobInfo.getJobState();
   int N = 24; // 2min
   while (isRunning(state) && N > 0) {
     N--;
     sync(5 * 1000L); // 5sec
     // new info lookup
     jobInfo = getJobInfo(pipelineService, phase, handle);
     state = jobInfo.getJobState();
   }
   if (N == 0 && isRunning(state)) {
     throw new IllegalStateException(
         "Failed to finish the job [ " + phase + " ]: " + handle + ", info: " + toInfo(jobInfo));
   }
   if (state != JobInfo.State.COMPLETED_SUCCESSFULLY) {
     throw new IllegalStateException(
         "Job " + handle + " failed [ " + phase + " ]: " + toInfo(jobInfo));
   }
   return jobInfo;
 }