/**
* Instantiates a new DistCp.
*
* @param configuration the hadoop configuration
* @param user the user
*/
public DistCp(Configuration configuration, String user) {
Assert.notNull(configuration, "configuration required");
this.configuration = ConfigurationUtils.createFrom(configuration, null);
// disable GenericOptionsParser
this.configuration.setBoolean("mapred.used.genericoptionsparser", true);
this.configuration.setBoolean("mapreduce.client.genericoptionsparser.used", true);
this.user = user;
}
@SuppressWarnings("rawtypes")
public void afterPropertiesSet() throws Exception {
final Configuration cfg = ConfigurationUtils.createFrom(configuration, properties);
buildGenericOptions(cfg);
if (StringUtils.hasText(user)) {
UserGroupInformation ugi =
UserGroupInformation.createProxyUser(user, UserGroupInformation.getLoginUser());
ugi.doAs(
new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
job = new Job(cfg);
return null;
}
});
} else {
job = new Job(cfg);
}
ClassLoader loader =
(beanClassLoader != null
? beanClassLoader
: org.springframework.util.ClassUtils.getDefaultClassLoader());
if (jar != null) {
JobConf conf = (JobConf) job.getConfiguration();
conf.setJar(jar.getURI().toString());
loader = ExecutionUtils.createParentLastClassLoader(jar, beanClassLoader, cfg);
conf.setClassLoader(loader);
}
// set first to enable auto-detection of K/V to skip the key/value types to be specified
if (mapper != null) {
Class<? extends Mapper> mapperClass = resolveClass(mapper, loader, Mapper.class);
job.setMapperClass(mapperClass);
configureMapperTypesIfPossible(job, mapperClass);
}
if (reducer != null) {
Class<? extends Reducer> reducerClass = resolveClass(reducer, loader, Reducer.class);
job.setReducerClass(reducerClass);
configureReducerTypesIfPossible(job, reducerClass);
}
if (StringUtils.hasText(name)) {
job.setJobName(name);
}
if (combiner != null) {
job.setCombinerClass(resolveClass(combiner, loader, Reducer.class));
}
if (groupingComparator != null) {
job.setGroupingComparatorClass(resolveClass(groupingComparator, loader, RawComparator.class));
}
if (inputFormat != null) {
job.setInputFormatClass(resolveClass(inputFormat, loader, InputFormat.class));
}
if (mapKey != null) {
job.setMapOutputKeyClass(resolveClass(mapKey, loader, Object.class));
}
if (mapValue != null) {
job.setMapOutputValueClass(resolveClass(mapValue, loader, Object.class));
}
if (numReduceTasks != null) {
job.setNumReduceTasks(numReduceTasks);
}
if (key != null) {
job.setOutputKeyClass(resolveClass(key, loader, Object.class));
}
if (value != null) {
job.setOutputValueClass(resolveClass(value, loader, Object.class));
}
if (outputFormat != null) {
job.setOutputFormatClass(resolveClass(outputFormat, loader, OutputFormat.class));
}
if (partitioner != null) {
job.setPartitionerClass(resolveClass(partitioner, loader, Partitioner.class));
}
if (sortComparator != null) {
job.setSortComparatorClass(resolveClass(sortComparator, loader, RawComparator.class));
}
if (StringUtils.hasText(workingDir)) {
job.setWorkingDirectory(new Path(workingDir));
}
if (jarClass != null) {
job.setJarByClass(jarClass);
}
if (!CollectionUtils.isEmpty(inputPaths)) {
for (String path : inputPaths) {
FileInputFormat.addInputPath(job, new Path(path));
}
}
if (StringUtils.hasText(outputPath)) {
FileOutputFormat.setOutputPath(job, new Path(outputPath));
}
if (compressOutput != null) {
FileOutputFormat.setCompressOutput(job, compressOutput);
}
if (codecClass != null) {
FileOutputFormat.setOutputCompressorClass(
job, resolveClass(codecClass, loader, CompressionCodec.class));
}
processJob(job);
}
@Override
HadoopFlow createFlow() throws IOException {
// copy flowDef
FlowDef def = FlowDef.flowDef();
if (flowDef != null) {
def.addSinks(flowDef.getSinksCopy())
.addSources(flowDef.getSourcesCopy())
.addTraps(flowDef.getTrapsCopy())
.addTails(flowDef.getTailsArray())
.setAssertionLevel(flowDef.getAssertionLevel())
.setDebugLevel(flowDef.getDebugLevel())
.addCheckpoints(flowDef.getCheckpointsCopy())
.addTags(flowDef.getTags())
.setName(flowDef.getName());
}
Set<Pipe> heads = new LinkedHashSet<Pipe>();
if (tails != null) {
for (Pipe pipe : tails) {
Collections.addAll(heads, pipe.getHeads());
}
}
Pipe pipe = null;
if (heads.size() == 1) {
pipe = heads.iterator().next();
}
if (sources != null && sources.size() == 1) {
Tap tap = sources.remove(MARKER);
if (tap != null) {
sources.put(pipe.getName(), tap);
}
}
if (sinks != null && sinks.size() == 1) {
Tap tap = sinks.remove(MARKER);
if (tap != null) {
sinks.put(pipe.getName(), tap);
}
}
def.addSources(sources).addSinks(sinks).addTraps(traps);
if (tails != null) {
def.addTails(tails);
}
if (StringUtils.hasText(beanName)) {
def.addTag(beanName);
if (!StringUtils.hasText(def.getName())) {
def.setName(beanName);
}
}
Configuration cfg = ConfigurationUtils.createFrom(configuration, properties);
Properties props = ConfigurationUtils.asProperties(cfg);
if (jarSetup) {
if (jar != null) {
AppProps.setApplicationJarPath(props, ResourceUtils.decode(jar.getURI().toString()));
} else if (jarClass != null) {
AppProps.setApplicationJarClass(props, jarClass);
} else {
// auto-detection based on the classpath
ClassLoader cascadingCL = Cascade.class.getClassLoader();
Resource cascadingCore = ResourceUtils.findContainingJar(Cascade.class);
Resource cascadingHadoop =
ResourceUtils.findContainingJar(cascadingCL, "cascading/flow/hadoop/HadoopFlow.class");
// find jgrapht
Resource jgrapht = ResourceUtils.findContainingJar(cascadingCL, "org/jgrapht/Graph.class");
Assert.notNull(cascadingCore, "Cannot find cascading-core.jar");
Assert.notNull(cascadingHadoop, "Cannot find cascading-hadoop.jar");
Assert.notNull(jgrapht, "Cannot find jgraphts-jdk.jar");
if (log.isDebugEnabled()) {
log.debug(
"Auto-detecting Cascading Libs ["
+ Arrays.toString(new Resource[] {cascadingCore, cascadingHadoop, jgrapht})
+ "]");
}
ConfigurationUtils.addLibs(cfg, cascadingCore, cascadingHadoop, jgrapht);
// config changed, reinit properties
props = ConfigurationUtils.asProperties(cfg);
}
}
if (jobPoolingInterval != null) {
FlowProps.setJobPollingInterval(props, jobPoolingInterval);
}
if (maxConcurrentSteps != null) {
FlowProps.setMaxConcurrentSteps(props, maxConcurrentSteps);
}
HadoopFlow flow = (HadoopFlow) new HadoopFlowConnector(props).connect(def);
return flow;
}