protected HCatSchema getTableSchema() throws Exception {
Configuration conf = new Configuration();
Job job = new Job(conf, "hcat mapreduce read schema test");
job.setJarByClass(this.getClass());
// input/output settings
job.setInputFormatClass(HCatInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
HCatInputFormat.setInput(job, dbName, tableName);
return HCatInputFormat.getTableSchema(job.getConfiguration());
}
/**
* Run a local map reduce job to read records from HCatalog table and verify if the count is as
* expected
*
* @param readCount
* @param filter
* @return
* @throws Exception
*/
List<HCatRecord> runMRRead(int readCount, String filter) throws Exception {
MapRead.readCount = 0;
readRecords.clear();
Configuration conf = new Configuration();
conf.set(HiveConf.ConfVars.METASTORE_INTEGER_JDO_PUSHDOWN.varname, "true");
Job job = new Job(conf, "hcat mapreduce read test");
job.setJarByClass(this.getClass());
job.setMapperClass(HCatMapReduceTest.MapRead.class);
// input/output settings
job.setInputFormatClass(HCatInputFormat.class);
job.setOutputFormatClass(TextOutputFormat.class);
HCatInputFormat.setInput(job, dbName, tableName, filter);
job.setMapOutputKeyClass(BytesWritable.class);
job.setMapOutputValueClass(Text.class);
job.setNumReduceTasks(0);
Path path = new Path(fs.getWorkingDirectory(), "mapred/testHCatMapReduceOutput");
if (fs.exists(path)) {
fs.delete(path, true);
}
TextOutputFormat.setOutputPath(job, path);
job.waitForCompletion(true);
Assert.assertEquals(readCount, MapRead.readCount);
return readRecords;
}
