protected static String toInfo(JobInfo info) {
StringBuilder sb = new StringBuilder();
sb.append("JobInfo[ ").append(info).append("]");
if (info != null) {
sb.append(" --> ");
sb.append("state - ").append(info.getJobState()).append(", ");
sb.append("output - ").append(info.getOutput()).append(", ");
sb.append("error - ").append(info.getError());
}
return sb.toString();
}
@SuppressWarnings("unchecked")
@Test
public void testCountJob() throws Exception {
List<String> payloads = Arrays.asList("capedwarf", "jboss", "redhat");
int shardCount = 1;
MapReduceSpecification.Builder builder = new MapReduceSpecification.Builder();
builder.setJobName("Create MapReduce entities");
builder.setInput(new ConsecutiveLongInput(0, payloads.size() * (long) shardCount, shardCount));
builder.setMapper(new EntityCreator("MapReduceTest", payloads));
builder.setKeyMarshaller(Marshallers.getVoidMarshaller());
builder.setValueMarshaller(Marshallers.getVoidMarshaller());
builder.setReducer(NoReducer.create());
builder.setOutput(new NoOutput());
final String createHandle = MapReduceJob.start(builder.build(), getSettings());
JobInfo createJI = waitToFinish("CREATE", createHandle);
Object create = createJI.getOutput();
log.warning("----- Create: " + create);
int mapShardCount = 1;
builder = new MapReduceSpecification.Builder();
builder.setJobName("MapReduceTest stats");
builder.setInput(new DatastoreInput("MapReduceTest", mapShardCount));
builder.setMapper(new CountMapper());
builder.setKeyMarshaller(Marshallers.getStringMarshaller());
builder.setValueMarshaller(Marshallers.getLongMarshaller());
builder.setReducer(new CountReducer());
builder.setOutput(new InMemoryOutput<KeyValue<String, Long>>());
final String countHandle = MapReduceJob.start(builder.build(), getSettings());
JobInfo countJI = waitToFinish("COUNT", countHandle);
Object count = countJI.getOutput();
log.warning("----- Count: " + count);
Assert.assertTrue(count instanceof MapReduceResult);
MapReduceResult result = MapReduceResult.class.cast(count);
int[] chars = toChars(payloads);
Counters counters = result.getCounters();
for (int i = 0; i < chars.length; i++) {
char ch = (char) ('a' + i);
Counter c = counters.getCounter(CountMapper.toKey(ch));
Assert.assertEquals(String.format("Invalid count for '%s'.", ch), chars[i], c.getValue());
}
}
protected JobInfo waitToFinish(final String phase, final String handle) throws Exception {
PipelineService pipelineService = PipelineServiceFactory.newPipelineService();
JobInfo jobInfo = getJobInfo(pipelineService, phase, handle);
JobInfo.State state = jobInfo.getJobState();
int N = 24; // 2min
while (isRunning(state) && N > 0) {
N--;
sync(5 * 1000L); // 5sec
// new info lookup
jobInfo = getJobInfo(pipelineService, phase, handle);
state = jobInfo.getJobState();
}
if (N == 0 && isRunning(state)) {
throw new IllegalStateException(
"Failed to finish the job [ " + phase + " ]: " + handle + ", info: " + toInfo(jobInfo));
}
if (state != JobInfo.State.COMPLETED_SUCCESSFULLY) {
throw new IllegalStateException(
"Job " + handle + " failed [ " + phase + " ]: " + toInfo(jobInfo));
}
return jobInfo;
}