Esempio n. 1
0
 public EnronMail() {
   JobConf conf = new JobConf(new Configuration());
   if (MongoTool.isMapRedV1()) {
     MapredMongoConfigUtil.setInputFormat(
         conf, com.mongodb.hadoop.mapred.BSONFileInputFormat.class);
     MapredMongoConfigUtil.setOutputFormat(
         conf, com.mongodb.hadoop.mapred.MongoOutputFormat.class);
   } else {
     MongoConfigUtil.setInputFormat(conf, BSONFileInputFormat.class);
     MongoConfigUtil.setOutputFormat(conf, MongoOutputFormat.class);
   }
   FileInputFormat.addInputPath(conf, new Path("/messages"));
   MongoConfig config = new MongoConfig(conf);
   config.setInputKey("headers.From");
   config.setMapper(EnronMailMapper.class);
   config.setReducer(EnronMailReducer.class);
   config.setMapperOutputKey(MailPair.class);
   config.setMapperOutputValue(IntWritable.class);
   config.setOutputKey(MailPair.class);
   config.setOutputValue(IntWritable.class);
   config.setOutputURI("mongodb://localhost:27017/mongo_hadoop.message_pairs");
   setConf(conf);
 }
Esempio n. 2
0
  public HadoopTest() throws UnknownHostException {

    setConf(new Configuration());

    if (MongoTool.isMapRedV1()) {
      MapredMongoConfigUtil.setInputFormat(
          getConf(), com.mongodb.hadoop.mapred.MongoInputFormat.class);
      MapredMongoConfigUtil.setOutputFormat(
          getConf(), com.mongodb.hadoop.mapred.MongoOutputFormat.class);
    } else {
      MongoConfigUtil.setInputFormat(getConf(), MongoInputFormat.class);
      MongoConfigUtil.setOutputFormat(getConf(), MongoOutputFormat.class);
    }

    MongoConfigUtil.setInputURI(getConf(), "mongodb://localhost:27017/lolmaster.playerMatch");
    MongoConfigUtil.setOutputURI(getConf(), "mongodb://localhost:27017/lolmaster.out");

    MongoConfigUtil.setMapper(getConf(), HadoopMapper.class);
    MongoConfigUtil.setReducer(getConf(), HadoopReducer.class);
    MongoConfigUtil.setMapperOutputKey(getConf(), Text.class);
    MongoConfigUtil.setMapperOutputValue(getConf(), Text.class);
    MongoConfigUtil.setOutputKey(getConf(), IntWritable.class);
    MongoConfigUtil.setOutputValue(getConf(), BSONWritable.class);
  }