public void runMR(String myMultiLocs, String sortKey)
throws ParseException, IOException, Exception, org.apache.hadoop.zebra.parser.ParseException {
JobConf jobConf = new JobConf(conf);
jobConf.setJobName("TestMultipleOutputs4");
jobConf.setJarByClass(TestMultipleOutputs4.class);
jobConf.set("table.output.tfile.compression", "gz");
jobConf.set("sortKey", sortKey);
// input settings
jobConf.setInputFormat(TextInputFormat.class);
jobConf.setMapperClass(TestMultipleOutputs4.MapClass.class);
jobConf.setMapOutputKeyClass(BytesWritable.class);
jobConf.setMapOutputValueClass(ZebraTuple.class);
FileInputFormat.setInputPaths(jobConf, inputPath);
jobConf.setNumMapTasks(1);
// output settings
jobConf.setOutputFormat(BasicTableOutputFormat.class);
BasicTableOutputFormat.setMultipleOutputs(
jobConf, myMultiLocs, TestMultipleOutputs4.OutputPartitionerClass.class);
// set the logical schema with 2 columns
BasicTableOutputFormat.setSchema(jobConf, "word:string, count:int");
// for demo purposes, create 2 physical column groups
BasicTableOutputFormat.setStorageHint(jobConf, "[word];[count]");
BasicTableOutputFormat.setSortInfo(jobConf, sortKey);
System.out.println("in runMR, sortkey: " + sortKey);
// set map-only job.
jobConf.setNumReduceTasks(1);
JobClient.runJob(jobConf);
BasicTableOutputFormat.close(jobConf);
}
@Override
public void configure(JobConf job) {
bytesKey = new BytesWritable();
sortKey = job.get("sortKey");
try {
Schema outSchema = BasicTableOutputFormat.getSchema(job);
tupleRow = TypesUtils.createTuple(outSchema);
javaObj = BasicTableOutputFormat.getSortKeyGenerator(job);
} catch (IOException e) {
throw new RuntimeException(e);
} catch (org.apache.hadoop.zebra.parser.ParseException e) {
throw new RuntimeException(e);
}
}
@Override
public void map(
LongWritable key,
Text value,
OutputCollector<BytesWritable, Tuple> output,
Reporter reporter)
throws IOException {
// value should contain "word count"
String[] wdct = value.toString().split(" ");
if (wdct.length != 2) {
// LOG the error
return;
}
byte[] word = wdct[0].getBytes();
bytesKey.set(word, 0, word.length);
System.out.println("word: " + new String(word));
tupleRow.set(0, new String(word));
tupleRow.set(1, Integer.parseInt(wdct[1]));
System.out.println("count: " + Integer.parseInt(wdct[1]));
// This key has to be created by user
/*
* Tuple userKey = new DefaultTuple(); userKey.append(new String(word));
* userKey.append(Integer.parseInt(wdct[1]));
*/
System.out.println("in map, sortkey: " + sortKey);
Tuple userKey = new ZebraTuple();
if (sortKey.equalsIgnoreCase("word,count")) {
userKey.append(new String(word));
userKey.append(Integer.parseInt(wdct[1]));
}
if (sortKey.equalsIgnoreCase("count")) {
userKey.append(Integer.parseInt(wdct[1]));
}
if (sortKey.equalsIgnoreCase("word")) {
userKey.append(new String(word));
}
try {
/* New M/R Interface */
/* Converts user key to zebra BytesWritable key */
/* using sort key expr tree */
/* Returns a java base object */
/* Done for each user key */
bytesKey = BasicTableOutputFormat.getSortKey(javaObj, userKey);
} catch (Exception e) {
}
output.collect(bytesKey, tupleRow);
}
