public static void getData(CloudataConf conf, Path keyPath) throws IOException {
JobConf jobConf = new JobConf(TeraReadJob.class);
jobConf.set("user.name", conf.getUserId());
String libDir = CloudataMapReduceUtil.initMapReduce(jobConf);
Path tempOutputPath = new Path("ManyTableJob_Get_" + System.currentTimeMillis());
jobConf.setJobName("ManyTableJob_Get_" + "(" + new Date() + ")");
TextOutputFormat.setOutputPath(jobConf, tempOutputPath);
// <MAP>
jobConf.setMapperClass(ManyTableGetMap.class);
jobConf.setInputFormat(TextInputFormat.class);
TextInputFormat.addInputPath(jobConf, keyPath);
jobConf.setMapSpeculativeExecution(false);
jobConf.setMaxMapAttempts(0);
// </MAP>
// <REDUCE>
jobConf.setNumReduceTasks(0);
// </REDUCE>
try {
// Run Job
JobClient.runJob(jobConf);
} finally {
// delete temp output path
FileSystem fs = FileSystem.get(jobConf);
FileUtil.delete(fs, tempOutputPath, true);
CloudataMapReduceUtil.clearMapReduce(libDir);
}
}
public static Path putData() throws IOException {
CloudataConf nconf = new CloudataConf();
JobConf jobConf = new JobConf(ManyTableJob.class);
jobConf.set("user.name", nconf.getUserId());
String libDir = CloudataMapReduceUtil.initMapReduce(jobConf);
jobConf.setJobName("ManyTableJob_Put" + "(" + new Date() + ")");
jobConf.setLong("mapred.task.timeout", 30 * 60 * 1000);
Path outputPath = new Path("ManyTableJob_KEY_" + System.currentTimeMillis());
FileOutputFormat.setOutputPath(jobConf, outputPath);
// <MAP>
jobConf.setMapperClass(ManyTablePutMap.class);
jobConf.setInputFormat(SimpleInputFormat.class);
jobConf.setNumMapTasks(numOfTables);
jobConf.setMapSpeculativeExecution(false);
jobConf.setMaxMapAttempts(0);
// </MAP>
// <REDUCE>
jobConf.setNumReduceTasks(0);
// </REDUCE>
try {
// Run Job
JobClient.runJob(jobConf);
return outputPath;
} finally {
// delete temp output path
FileSystem fs = FileSystem.get(jobConf);
CloudataMapReduceUtil.clearMapReduce(libDir);
}
}
public void configure(JobConf jobConf) {
tableName = "T_MANY_TABLE_" + CloudataMapReduceUtil.getMapNum(jobConf);
Random rand = new Random();
this.data = new byte[10000];
rand.nextBytes(data);
try {
this.conf = new CloudataConf(jobConf);
LOG.info("TableName:" + tableName);
this.ctable = CTable.openTable(conf, tableName);
if (ctable == null) {
throw new IOException("No table:" + tableName);
}
} catch (IOException e) {
this.err = e;
}
}