public List<Pair<K2, V2>> run() throws IOException {
// inputs starts with the user-provided inputs.
List inputs = this.inputList;
if (mapReducePipeline.size() == 0) {
LOG.warn("No Mapper or Reducer instances in pipeline; this is a trivial test.");
}
if (inputs.size() == 0) {
LOG.warn("No inputs configured to send to MapReduce pipeline; this is a trivial test.");
}
for (Pair<Mapper, Reducer> job : mapReducePipeline) {
// Create a MapReduceDriver to run this phase of the pipeline.
MapReduceDriver mrDriver = new MapReduceDriver(job.getFirst(), job.getSecond());
mrDriver.setCounters(getCounters());
// Add the inputs from the user, or from the previous stage of the pipeline.
for (Object input : inputs) {
mrDriver.addInput((Pair) input);
}
// Run the MapReduce "job". The output of this job becomes
// the input to the next job.
inputs = mrDriver.run();
}
// The last list of values stored in "inputs" is actually the outputs.
// Unfortunately, due to the variable-length list of MR passes the user
// can test, this is not type-safe.
return (List<Pair<K2, V2>>) inputs;
}
/**
* Test characteristic set reduction
*
* @throws IOException
*/
@Test
public void characteristic_set_reducer_06() throws IOException {
MapReduceDriver<
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
CharacteristicSetWritable,
NullWritable>
driver = this.getMapReduceDriver();
this.createSet(driver, 2, 1, "http://predicate", "http://other");
this.createSet(driver, 1, 1, "http://other");
driver.runTest(false);
driver = getMapReduceDriver();
createSet(driver, 2, 1, "http://predicate", "http://other");
createSet(driver, 1, 1, "http://other");
List<Pair<CharacteristicSetWritable, NullWritable>> results = driver.run();
for (Pair<CharacteristicSetWritable, NullWritable> pair : results) {
CharacteristicSetWritable cw = pair.getFirst();
boolean expectTwo = cw.hasCharacteristic("http://predicate");
Assert.assertEquals(expectTwo ? 2 : 1, cw.getCount().get());
}
}
private List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>>
getReducerDataFromMapperInput(
final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults) {
final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> reducerInputSet =
new ArrayList<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>>();
for (final Pair<PartitionDataWritable, AdapterWithObjectWritable> pair : mapperResults) {
getListFor(pair.getFirst(), reducerInputSet).add(pair.getSecond());
}
return reducerInputSet;
}
@Test
public void testVisitPageMR() throws Exception {
// 添加模拟数据
generateVisitData();
mrPageVisitor.getConfiguration().set("custom.period", "month");
// 开始计算
List<Pair<PageVisitKey, PageVisitOutputValue>> results = mrPageVisitor.run(true);
for (int i = 0; i < results.size(); i++) {
Pair<PageVisitKey, PageVisitOutputValue> pair = results.get(i);
System.out.println(
"Key:" + pair.getFirst().toString() + ", Value:" + pair.getSecond().toString());
}
}
public List<Pair<Text, List<Text>>> getReducerPairsFromFile(String file) throws IOException {
List<Pair<Text, Text>> unsplitPairs = this.getPairsFromFile(file);
List<Pair<Text, List<Text>>> splitPairs = new LinkedList<Pair<Text, List<Text>>>();
for (Pair<Text, Text> unsplitPair : unsplitPairs) {
List<Text> splitValues = new LinkedList<Text>();
String[] splits = unsplitPair.getSecond().toString().split("(\r\n|\n)");
for (String split : splits) {
splitValues.add(new Text(split));
}
splitPairs.add(new Pair<Text, List<Text>>(unsplitPair.getFirst(), splitValues));
}
return splitPairs;
}
private PartitionData getPartitionDataFor(
final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults,
final String id,
final boolean primary) {
for (final Pair<PartitionDataWritable, AdapterWithObjectWritable> pair : mapperResults) {
if (((FeatureWritable) pair.getSecond().getObjectWritable().get())
.getFeature()
.getID()
.equals(id)
&& (pair.getFirst().getPartitionData().isPrimary() == primary)) {
return pair.getFirst().getPartitionData();
}
}
return null;
}
@Test
public void testMapReduce() throws Exception {
System.out.println(
"MapReduce,Begin------------------------------------------------------------------");
// 多个输入源
BufferedReader[] arr = {
new BufferedReader(new FileReader("D:\\hdfs\\warehouse\\appdetail.log"))
};
LongWritable longWritable = new LongWritable();
String line = null;
for (BufferedReader br : arr) {
while (null != (line = br.readLine())) {
mapReduceDriver.withInput(longWritable, new Text(line));
}
br.close();
}
List<Pair<OutFieldsBaseModel, OutFieldsBaseModel>> onlineList = mapReduceDriver.run();
Collections.sort(
onlineList,
new Comparator<Pair<OutFieldsBaseModel, OutFieldsBaseModel>>() {
@Override
public int compare(
Pair<OutFieldsBaseModel, OutFieldsBaseModel> o1,
Pair<OutFieldsBaseModel, OutFieldsBaseModel> o2) {
if (o1.getFirst().getSuffix().compareTo(o2.getFirst().getSuffix()) > 0) {
return -1;
}
return 0;
}
});
for (Pair<OutFieldsBaseModel, OutFieldsBaseModel> pair : onlineList) {
String lin =
pair.getFirst().toString()
+ "\t"
+ pair.getSecond().toString()
+ "\t"
+ pair.getFirst().getSuffix();
System.out.println(lin);
}
System.out.println(
"MapReduce,End------------------------------------------------------------------");
}
@Test
public void testPalyerOnlineHour() throws Exception {
Set<String> set = new HashSet<String>();
int count = 0;
String line = null;
LongWritable longWritable = new LongWritable();
BufferedReader reader =
new BufferedReader(new FileReader("C:\\Users\\Administrator\\Desktop\\error_pay"));
while (null != (line = reader.readLine())) {
/*String[] array = line.split(MRConstants.SEPERATOR_IN);
PaymentDayLog paymentDayLog = new PaymentDayLog(array);
set.add(paymentDayLog.getExtend().getResolution());
if("1".equals(paymentDayLog.getExtend().getResolution())){
count++;
}*/
mapReduceDriver.withInput(longWritable, new Text(line));
}
reader.close();
List<Pair<OutFieldsBaseModel, NullWritable>> list = mapReduceDriver.run();
for (Pair<OutFieldsBaseModel, NullWritable> pair : list) {
System.out.println(pair.getFirst().getSuffix() + "---" + pair.getFirst().toString());
}
}
public int compare(Pair o1, Pair o2) {
return ((Comparable) o1.getFirst()).compareTo(o2.getFirst());
}
