@Override
public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
if (split.getClass().equals(TableSplit.class)) {
return tableInputFormat.createRecordReader(split, context);
} else {
return fileInputFormat.createRecordReader(split, context);
}
}
/**
* Casts an InputSplit into a HCatSplit, providing a useful error message if the cast fails.
*
* @param split the InputSplit
* @return the HCatSplit
* @throws IOException
*/
public static HCatSplit castToHCatSplit(InputSplit split) throws IOException {
if (split instanceof HCatSplit) {
return (HCatSplit) split;
} else {
throw new IOException(
"Split must be "
+ HCatSplit.class.getName()
+ " but found "
+ split.getClass().getName());
}
}
@Override
public void initialize(InputSplit split, final TaskAttemptContext context)
throws IOException, InterruptedException {
org.apache.hadoop.mapred.InputSplit oldSplit;
if (split.getClass() == FileSplit.class) {
oldSplit =
new org.apache.hadoop.mapred.FileSplit(
((FileSplit) split).getPath(),
((FileSplit) split).getStart(),
((FileSplit) split).getLength(),
split.getLocations());
} else {
oldSplit = ((InputSplitWrapper) split).realSplit;
}
@SuppressWarnings("unchecked")
Reporter reporter = new Reporter() { // Reporter interface over ctx
final TaskInputOutputContext ioCtx =
context instanceof TaskInputOutputContext ? (TaskInputOutputContext) context : null;
public void progress() {
HadoopCompat.progress(context);
}
// @Override
public float getProgress() {
return (ioCtx != null) ? ioCtx.getProgress() : 0;
}
public void setStatus(String status) {
if (ioCtx != null) HadoopCompat.setStatus(ioCtx, status);
}
public void incrCounter(String group, String counter, long amount) {
if (ioCtx != null)
HadoopCompat.incrementCounter(ioCtx.getCounter(group, counter), amount);
}
@SuppressWarnings("unchecked")
public void incrCounter(Enum<?> key, long amount) {
if (ioCtx != null) HadoopCompat.incrementCounter(ioCtx.getCounter(key), amount);
}
public org.apache.hadoop.mapred.InputSplit getInputSplit()
throws UnsupportedOperationException {
throw new UnsupportedOperationException();
}
public Counter getCounter(String group, String name) {
return ioCtx != null ? (Counter) HadoopCompat.getCounter(ioCtx, group, name) : null;
}
@SuppressWarnings("unchecked")
public Counter getCounter(Enum<?> name) {
return ioCtx != null ? (Counter) ioCtx.getCounter(name) : null;
}
};
realReader =
realInputFormat.getRecordReader(
oldSplit, (JobConf) HadoopCompat.getConfiguration(context), reporter);
keyObj = realReader.createKey();
valueObj = realReader.createValue();
}
@Override
protected void map(String key, String value, final Context context)
throws IOException, InterruptedException {
final InputSplit split = context.getInputSplit();
if (!(split instanceof DatasourceInputSplit)) {
throw new IAE(
"Unexpected split type. Expected [%s] was [%s]",
DatasourceInputSplit.class.getCanonicalName(), split.getClass().getCanonicalName());
}
final String tmpDirLoc = context.getConfiguration().get(TMP_FILE_LOC_KEY);
final File tmpDir = Paths.get(tmpDirLoc).toFile();
final DataSegment segment =
Iterables.getOnlyElement(((DatasourceInputSplit) split).getSegments()).getSegment();
final HadoopDruidConverterConfig config =
converterConfigFromConfiguration(context.getConfiguration());
context.setStatus("DOWNLOADING");
context.progress();
final Path inPath = new Path(JobHelper.getURIFromSegment(segment));
final File inDir = new File(tmpDir, "in");
if (inDir.exists() && !inDir.delete()) {
log.warn("Could not delete [%s]", inDir);
}
if (!inDir.mkdir() && (!inDir.exists() || inDir.isDirectory())) {
log.warn("Unable to make directory");
}
final long inSize =
JobHelper.unzipNoGuava(inPath, context.getConfiguration(), inDir, context);
log.debug("Loaded %d bytes into [%s] for converting", inSize, inDir.getAbsolutePath());
context.getCounter(COUNTER_GROUP, COUNTER_LOADED).increment(inSize);
context.setStatus("CONVERTING");
context.progress();
final File outDir = new File(tmpDir, "out");
if (!outDir.mkdir() && (!outDir.exists() || !outDir.isDirectory())) {
throw new IOException(String.format("Could not create output directory [%s]", outDir));
}
HadoopDruidConverterConfig.INDEX_MERGER.convert(
inDir, outDir, config.getIndexSpec(), JobHelper.progressIndicatorForContext(context));
if (config.isValidate()) {
context.setStatus("Validating");
HadoopDruidConverterConfig.INDEX_IO.validateTwoSegments(inDir, outDir);
}
context.progress();
context.setStatus("Starting PUSH");
final Path baseOutputPath = new Path(config.getSegmentOutputPath());
final FileSystem outputFS = baseOutputPath.getFileSystem(context.getConfiguration());
final DataSegment finalSegmentTemplate =
segment.withVersion(segment.getVersion() + "_converted");
final DataSegment finalSegment =
JobHelper.serializeOutIndex(
finalSegmentTemplate,
context.getConfiguration(),
context,
context.getTaskAttemptID(),
outDir,
JobHelper.makeSegmentOutputPath(baseOutputPath, outputFS, finalSegmentTemplate));
context.progress();
context.setStatus("Finished PUSH");
final String finalSegmentString =
HadoopDruidConverterConfig.jsonMapper.writeValueAsString(finalSegment);
context
.getConfiguration()
.set(ConvertingOutputFormat.PUBLISHED_SEGMENT_KEY, finalSegmentString);
context.write(new Text("dataSegment"), new Text(finalSegmentString));
context.getCounter(COUNTER_GROUP, COUNTER_WRITTEN).increment(finalSegment.getSize());
context.progress();
context.setStatus("Ready To Commit");
}
