Configuration getHadoopConfiguration(List<ConfigIssue> issues) {
Configuration conf = new Configuration();
if (hdfsKerberos) {
conf.set(
CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION,
UserGroupInformation.AuthenticationMethod.KERBEROS.name());
try {
conf.set(
DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY,
"hdfs/_HOST@" + KerberosUtil.getDefaultRealm());
} catch (Exception ex) {
if (!hdfsConfigs.containsKey(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY)) {
issues.add(
getContext()
.createConfigIssue(
Groups.HADOOP_FS.name(), null, Errors.HADOOPFS_28, ex.getMessage()));
}
}
}
if (hadoopConfDir != null && !hadoopConfDir.isEmpty()) {
File hadoopConfigDir = new File(hadoopConfDir);
if (hadoopConfigDir.isAbsolute()) {
// Do not allow absolute hadoop config directory in cluster mode
issues.add(
getContext()
.createConfigIssue(
Groups.HADOOP_FS.name(), "hadoopConfDir", Errors.HADOOPFS_29, hadoopConfDir));
} else {
hadoopConfigDir =
new File(getContext().getResourcesDirectory(), hadoopConfDir).getAbsoluteFile();
}
if (!hadoopConfigDir.exists()) {
issues.add(
getContext()
.createConfigIssue(
Groups.HADOOP_FS.name(),
"hdfsConfDir",
Errors.HADOOPFS_25,
hadoopConfigDir.getPath()));
} else if (!hadoopConfigDir.isDirectory()) {
issues.add(
getContext()
.createConfigIssue(
Groups.HADOOP_FS.name(),
"hdfsConfDir",
Errors.HADOOPFS_26,
hadoopConfigDir.getPath()));
} else {
File coreSite = new File(hadoopConfigDir, "core-site.xml");
if (coreSite.exists()) {
if (!coreSite.isFile()) {
issues.add(
getContext()
.createConfigIssue(
Groups.HADOOP_FS.name(),
"hdfsConfDir",
Errors.HADOOPFS_27,
coreSite.getPath()));
}
conf.addResource(new Path(coreSite.getAbsolutePath()));
}
File hdfsSite = new File(hadoopConfigDir, "hdfs-site.xml");
if (hdfsSite.exists()) {
if (!hdfsSite.isFile()) {
issues.add(
getContext()
.createConfigIssue(
Groups.HADOOP_FS.name(),
"hdfsConfDir",
Errors.HADOOPFS_27,
hdfsSite.getPath()));
}
conf.addResource(new Path(hdfsSite.getAbsolutePath()));
}
File yarnSite = new File(hadoopConfigDir, "yarn-site.xml");
if (yarnSite.exists()) {
if (!yarnSite.isFile()) {
issues.add(
getContext()
.createConfigIssue(
Groups.HADOOP_FS.name(),
"hdfsConfDir",
Errors.HADOOPFS_27,
yarnSite.getPath()));
}
conf.addResource(new Path(yarnSite.getAbsolutePath()));
}
File mapredSite = new File(hadoopConfigDir, "mapred-site.xml");
if (mapredSite.exists()) {
if (!mapredSite.isFile()) {
issues.add(
getContext()
.createConfigIssue(
Groups.HADOOP_FS.name(),
"hdfsConfDir",
Errors.HADOOPFS_27,
mapredSite.getPath()));
}
conf.addResource(new Path(mapredSite.getAbsolutePath()));
}
}
}
for (Map.Entry<String, String> config : hdfsConfigs.entrySet()) {
conf.set(config.getKey(), config.getValue());
}
return conf;
}
