/**
* ���� 1 ����·�� 2 ��������ݵ����ID 3 ������������� 4 com.mysql.jdbc.Driver 5
* jdbc:mysql://127.0.0.1:3306/etl 6 user for mysql 7 password for mysql
*/
public static void main(String args[])
throws IOException, InterruptedException, ClassNotFoundException {
Configuration conf = new Configuration();
String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
DistributedCache.addFileToClassPath(
new Path("/user/hdfs/examples/libs/mysql-connector-java-5.1.22-bin.jar"), conf);
if (otherArgs.length != 7) {
System.out.println("args:" + otherArgs.length + ",is should be 3");
for (int i = 0; i < otherArgs.length; i++) {
System.out.println(otherArgs[i]);
}
System.exit(7);
}
conf.set("datasetid", otherArgs[1]);
conf.set("sampleNum", otherArgs[2]);
DBConfiguration.configureDB(conf, otherArgs[3], otherArgs[4], otherArgs[5], otherArgs[6]);
Job job = new Job(conf, "sampleToMySql");
job.setJarByClass(toMysql.class);
job.setMapperClass(ConnMysqlMapper.class);
job.setReducerClass(ConnMysqlReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setInputFormatClass(TextInputFormat.class);
job.setOutputFormatClass(DBOutputFormat.class);
FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
DBOutputFormat.setOutput(job, "sofa_wf_sample", "datasetid", "columns", "deltag");
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
public static void runJob(String mysqlJar, String output) throws Exception {
Configuration conf = new Configuration();
JobHelper.addJarForJob(conf, mysqlJar);
DBConfiguration.configureDB(
conf,
"com.mysql.jdbc.Driver",
"jdbc:mysql://localhost/sqoop_test" + "?user=hip_sqoop_user&password=password");
Job job = new Job(conf);
job.setJarByClass(DBImportExportMapReduce.class);
job.setMapperClass(Map.class);
job.setReducerClass(Reduce.class);
job.setInputFormatClass(DBInputFormat.class);
job.setOutputFormatClass(DBOutputFormat.class);
job.setMapOutputKeyClass(StockRecord.class);
job.setMapOutputValueClass(NullWritable.class);
job.setOutputKeyClass(StockRecord.class);
job.setOutputValueClass(NullWritable.class);
job.getConfiguration().setInt("mapred.map.tasks", 4);
job.setNumReduceTasks(4);
DBInputFormat.setInput(
job, StockRecord.class, "select * from stocks", "SELECT COUNT(id) FROM stocks");
DBOutputFormat.setOutput(job, "stocks_export", StockRecord.fields);
Path outputPath = new Path(output);
FileOutputFormat.setOutputPath(job, outputPath);
outputPath.getFileSystem(conf).delete(outputPath, true);
job.waitForCompletion(true);
}
