예제 #1
0
파일: toMysql.java 프로젝트: CHATTG1/sofa
  /**
   * ���� 1 ����·�� 2 ��������ݵ����ID 3 ������������� 4 com.mysql.jdbc.Driver 5
   * jdbc:mysql://127.0.0.1:3306/etl 6 user for mysql 7 password for mysql
   */
  public static void main(String args[])
      throws IOException, InterruptedException, ClassNotFoundException {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    DistributedCache.addFileToClassPath(
        new Path("/user/hdfs/examples/libs/mysql-connector-java-5.1.22-bin.jar"), conf);
    if (otherArgs.length != 7) {
      System.out.println("args:" + otherArgs.length + ",is should be 3");
      for (int i = 0; i < otherArgs.length; i++) {
        System.out.println(otherArgs[i]);
      }
      System.exit(7);
    }
    conf.set("datasetid", otherArgs[1]);
    conf.set("sampleNum", otherArgs[2]);
    DBConfiguration.configureDB(conf, otherArgs[3], otherArgs[4], otherArgs[5], otherArgs[6]);
    Job job = new Job(conf, "sampleToMySql");
    job.setJarByClass(toMysql.class);

    job.setMapperClass(ConnMysqlMapper.class);
    job.setReducerClass(ConnMysqlReducer.class);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(DBOutputFormat.class);
    FileInputFormat.addInputPath(job, new Path(otherArgs[0]));

    DBOutputFormat.setOutput(job, "sofa_wf_sample", "datasetid", "columns", "deltag");
    System.exit(job.waitForCompletion(true) ? 0 : 1);
  }
  public static void runJob(String mysqlJar, String output) throws Exception {
    Configuration conf = new Configuration();

    JobHelper.addJarForJob(conf, mysqlJar);

    DBConfiguration.configureDB(
        conf,
        "com.mysql.jdbc.Driver",
        "jdbc:mysql://localhost/sqoop_test" + "?user=hip_sqoop_user&password=password");

    Job job = new Job(conf);
    job.setJarByClass(DBImportExportMapReduce.class);
    job.setMapperClass(Map.class);
    job.setReducerClass(Reduce.class);

    job.setInputFormatClass(DBInputFormat.class);
    job.setOutputFormatClass(DBOutputFormat.class);

    job.setMapOutputKeyClass(StockRecord.class);
    job.setMapOutputValueClass(NullWritable.class);

    job.setOutputKeyClass(StockRecord.class);
    job.setOutputValueClass(NullWritable.class);

    job.getConfiguration().setInt("mapred.map.tasks", 4);
    job.setNumReduceTasks(4);

    DBInputFormat.setInput(
        job, StockRecord.class, "select * from stocks", "SELECT COUNT(id) FROM stocks");

    DBOutputFormat.setOutput(job, "stocks_export", StockRecord.fields);

    Path outputPath = new Path(output);

    FileOutputFormat.setOutputPath(job, outputPath);

    outputPath.getFileSystem(conf).delete(outputPath, true);

    job.waitForCompletion(true);
  }