public static void getData(CloudataConf conf, Path keyPath) throws IOException {
JobConf jobConf = new JobConf(TeraReadJob.class);
jobConf.set("user.name", conf.getUserId());
String libDir = CloudataMapReduceUtil.initMapReduce(jobConf);
Path tempOutputPath = new Path("ManyTableJob_Get_" + System.currentTimeMillis());
jobConf.setJobName("ManyTableJob_Get_" + "(" + new Date() + ")");
TextOutputFormat.setOutputPath(jobConf, tempOutputPath);
// <MAP>
jobConf.setMapperClass(ManyTableGetMap.class);
jobConf.setInputFormat(TextInputFormat.class);
TextInputFormat.addInputPath(jobConf, keyPath);
jobConf.setMapSpeculativeExecution(false);
jobConf.setMaxMapAttempts(0);
// </MAP>
// <REDUCE>
jobConf.setNumReduceTasks(0);
// </REDUCE>
try {
// Run Job
JobClient.runJob(jobConf);
} finally {
// delete temp output path
FileSystem fs = FileSystem.get(jobConf);
FileUtil.delete(fs, tempOutputPath, true);
CloudataMapReduceUtil.clearMapReduce(libDir);
}
}
public static Path putData() throws IOException {
CloudataConf nconf = new CloudataConf();
JobConf jobConf = new JobConf(ManyTableJob.class);
jobConf.set("user.name", nconf.getUserId());
String libDir = CloudataMapReduceUtil.initMapReduce(jobConf);
jobConf.setJobName("ManyTableJob_Put" + "(" + new Date() + ")");
jobConf.setLong("mapred.task.timeout", 30 * 60 * 1000);
Path outputPath = new Path("ManyTableJob_KEY_" + System.currentTimeMillis());
FileOutputFormat.setOutputPath(jobConf, outputPath);
// <MAP>
jobConf.setMapperClass(ManyTablePutMap.class);
jobConf.setInputFormat(SimpleInputFormat.class);
jobConf.setNumMapTasks(numOfTables);
jobConf.setMapSpeculativeExecution(false);
jobConf.setMaxMapAttempts(0);
// </MAP>
// <REDUCE>
jobConf.setNumReduceTasks(0);
// </REDUCE>
try {
// Run Job
JobClient.runJob(jobConf);
return outputPath;
} finally {
// delete temp output path
FileSystem fs = FileSystem.get(jobConf);
CloudataMapReduceUtil.clearMapReduce(libDir);
}
}
