private boolean createMapredSiteXml(NodeConfig nodeConfig) {
try {
String mapredsite =
HadoopUtils.getHadoopConfDir(this.compConfig)
+ HadoopConstants.FileName.ConfigurationFile.XML_MAPRED_SITE;
String mapredsiteTemplate =
HadoopUtils.getHadoopConfDir(this.compConfig) + "mapred-site.xml.template";
AnkushTask moveFile = new Move(mapredsiteTemplate, mapredsite);
Result res = nodeConfig.getConnection().exec(moveFile);
if (res.rc != 0) {
HadoopUtils.addAndLogError(
LOG,
clusterConfig,
"Could not create Hadoop mapred-site.xml file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost());
return false;
}
return true;
} catch (Exception e) {
HadoopUtils.addAndLogError(
LOG,
clusterConfig,
"Could not create Hadoop mapred-site.xml file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost(),
e);
return false;
}
}
/*
* (non-Javadoc)
*
* @see com.impetus.ankush2.hadoop.deployer.configurator.HadoopConfigurator#
* configureCoreSiteXml(com.impetus.ankush2.framework.config.NodeConfig)
*/
@Override
protected boolean configureCoreSiteXml(NodeConfig nodeConfig) {
boolean status = true;
try {
// configuring core-site.xml file in $HADOOP_CONF_DIR
String coreSiteXmlPath =
HadoopUtils.getHadoopConfDir(this.compConfig)
+ HadoopConstants.FileName.ConfigurationFile.XML_CORE_SITE;
LOG.info(
"Configuring " + HadoopConstants.FileName.ConfigurationFile.XML_CORE_SITE + " file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost());
String valFsDefaultFS = HadoopUtils.getHdfsUri(compConfig);
Map<String, String> paramList = new HashMap<String, String>();
paramList.put("fs.defaultFS", valFsDefaultFS);
paramList.put(
"hadoop.tmp.dir",
(String)
this.compConfig.getAdvanceConfProperty(
HadoopConstants.AdvanceConfKeys.TMP_DIR_HADOOP));
// Constant.URI_FILE_PREFIX + conf.getHadoopTmpDir());
for (String propertyName : paramList.keySet()) {
status =
HadoopUtils.addPropertyToFile(
clusterConfig,
nodeConfig,
coreSiteXmlPath,
propertyName,
paramList.get(propertyName));
if (!status) {
HadoopUtils.addAndLogError(
LOG,
clusterConfig,
"Could not add " + propertyName + " property to " + coreSiteXmlPath + " file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost());
return false;
}
}
return true;
} catch (Exception e) {
HadoopUtils.addAndLogError(
LOG,
clusterConfig,
"Could not update Hadoop core-site.xml file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost(),
e);
return false;
}
}
private boolean setJavaInHadoop(NodeConfig nodeConfig) {
try {
LOG.info(
"Configuring Java in Hadoop environment scripts",
Constant.Component.Name.HADOOP,
nodeConfig.getHost());
LinkedHashMap<String, String> scriptTargetTextMap = new LinkedHashMap<String, String>();
scriptTargetTextMap.put(
HadoopUtils.getHadoopConfDir(this.compConfig)
+ HadoopConstants.FileName.ConfigurationFile.ENV_HADOOP,
AppStoreWrapper.getAnkushConfReader()
.getStringValue("hadoop2.javahome.hadoopenv.targettext"));
scriptTargetTextMap.put(
HadoopUtils.getHadoopConfDir(this.compConfig)
+ HadoopConstants.FileName.ConfigurationFile.ENV_YARN,
AppStoreWrapper.getAnkushConfReader()
.getStringValue("hadoop2.javahome.yarnenv.targettext"));
scriptTargetTextMap.put(
HadoopUtils.getHadoopConfDir(this.compConfig)
+ HadoopConstants.FileName.ConfigurationFile.ENV_MAPRED,
AppStoreWrapper.getAnkushConfReader()
.getStringValue("hadoop2.javahome.yarnenv.targettext"));
for (String hadoopEnvScript : scriptTargetTextMap.keySet()) {
if (!this.setJavaInHadoopEnvScript(
nodeConfig, hadoopEnvScript, scriptTargetTextMap.get(hadoopEnvScript))) {
return false;
}
}
return true;
} catch (Exception e) {
String errMsg = "Could not update Java in Hadoop environment scripts";
HadoopUtils.addAndLogError(
LOG, clusterConfig, errMsg, Constant.Component.Name.HADOOP, nodeConfig.getHost(), e);
return false;
}
}
private boolean configureHadoopEnvScripts(NodeConfig nodeConfig) {
Result res = null;
try {
// Set Environment Variables in Hadoop Environment Script Files
String envVariables = this.getBashrcContents();
List<String> envFileList = new ArrayList<String>();
envFileList.add(
HadoopUtils.getHadoopConfDir(this.compConfig)
+ HadoopConstants.FileName.ConfigurationFile.ENV_HADOOP);
envFileList.add(
HadoopUtils.getHadoopConfDir(this.compConfig)
+ HadoopConstants.FileName.ConfigurationFile.ENV_YARN);
envFileList.add(
HadoopUtils.getHadoopConfDir(this.compConfig)
+ HadoopConstants.FileName.ConfigurationFile.ENV_MAPRED);
for (String fileName : envFileList) {
LOG.info(
"Configuring variables in " + fileName + " file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost());
AnkushTask updateEnvFile = new AppendFileUsingEcho(envVariables, fileName);
res = nodeConfig.getConnection().exec(updateEnvFile);
if (res.rc != 0) {
String errMsg = "Could not add environment variables for Hadoop in " + fileName + " file";
HadoopUtils.addAndLogError(
LOG, clusterConfig, errMsg, Constant.Component.Name.HADOOP, nodeConfig.getHost());
return false;
}
}
return true;
} catch (Exception e) {
String errMsg = "Could not update Hadoop environment scripts";
HadoopUtils.addAndLogError(
LOG, clusterConfig, errMsg, Constant.Component.Name.HADOOP, nodeConfig.getHost(), e);
return false;
}
}
/*
* (non-Javadoc)
*
* @see com.impetus.ankush2.hadoop.deployer.configurator.HadoopConfigurator#
* configureMapredSiteXml(com.impetus.ankush2.framework.config.NodeConfig)
*/
@Override
protected boolean configureMapredSiteXml(NodeConfig nodeConfig) {
try {
if (!createMapredSiteXml(nodeConfig)) {
return false;
}
String mapredSiteXmlPath =
HadoopUtils.getHadoopConfDir(this.compConfig)
+ HadoopConstants.FileName.ConfigurationFile.XML_MAPRED_SITE;
LOG.info(
"Configuring " + HadoopConstants.FileName.ConfigurationFile.XML_MAPRED_SITE + " file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost());
Map<String, String> paramList = new HashMap<String, String>();
String mapreduceFramework =
(String)
this.compConfig.getAdvanceConfProperty(
HadoopConstants.AdvanceConfKeys.MAPREDUCE_FRAMEWORK);
if (mapreduceFramework == null) {
mapreduceFramework = Hadoop2Configurator.DEFAULT_MAPREDUCE_FRAMEWORK_YARN;
}
paramList.put("mapreduce.framework.name", mapreduceFramework);
paramList.put(
"mapreduce.cluster.temp.dir",
HadoopConstants.URI_FILE_PREFIX
+ (String)
this.compConfig.getAdvanceConfProperty(
HadoopConstants.AdvanceConfKeys.TMP_DIR_MAPRED));
boolean jobHistoryServerEnabled =
(Boolean)
this.compConfig.getAdvanceConfProperty(
HadoopConstants.AdvanceConfKeys.JOBHISTORYSERVER_ENABLED);
if (jobHistoryServerEnabled) {
paramList.put(
"mapreduce.jobhistory.address",
HadoopUtils.getJobHistoryServerHost(this.compConfig)
+ ":"
+ Hadoop2Configurator.DEFAULT_PORT_REST_JOBHISTORYSERVER);
}
for (String propertyName : paramList.keySet()) {
if (!HadoopUtils.addPropertyToFile(
clusterConfig,
nodeConfig,
mapredSiteXmlPath,
propertyName,
paramList.get(propertyName))) {
HadoopUtils.addAndLogError(
LOG,
clusterConfig,
"Could not add " + propertyName + " property to " + mapredSiteXmlPath + " file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost());
return false;
}
}
return true;
} catch (Exception e) {
HadoopUtils.addAndLogError(
LOG,
clusterConfig,
"Could not update Hadoop mapred-site.xml file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost(),
e);
return false;
}
}
/*
* (non-Javadoc)
*
* @see com.impetus.ankush2.hadoop.deployer.configurator.HadoopConfigurator#
* configureHdfsSiteXml(com.impetus.ankush2.framework.config.NodeConfig)
*/
@Override
protected boolean configureHdfsSiteXml(NodeConfig nodeConfig) {
boolean status = true;
try {
String hdfsSiteXmlPath =
HadoopUtils.getHadoopConfDir(this.compConfig)
+ HadoopConstants.FileName.ConfigurationFile.XML_HDFS_SITE;
LOG.info(
"Configuring " + HadoopConstants.FileName.ConfigurationFile.XML_HDFS_SITE + " file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost());
Map<String, String> paramList = new HashMap<String, String>();
paramList.put(
"dfs.replication",
(String)
this.compConfig.getAdvanceConfProperty(
HadoopConstants.AdvanceConfKeys.DFS_REPLICATION_FACTOR));
paramList.put("dfs.permissions.enabled", "true");
paramList.put(
"dfs.namenode.name.dir",
HadoopConstants.URI_FILE_PREFIX
+ (String)
this.compConfig.getAdvanceConfProperty(
HadoopConstants.AdvanceConfKeys.DFS_NAME_DIR));
paramList.put(
"dfs.datanode.data.dir",
HadoopConstants.URI_FILE_PREFIX
+ (String)
this.compConfig.getAdvanceConfProperty(
HadoopConstants.AdvanceConfKeys.DFS_DATA_DIR));
boolean isHaEnabled =
(Boolean)
this.compConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_ENABLED);
if (isHaEnabled) {
String nameserviceId =
(String)
this.compConfig.getAdvanceConfProperty(
HadoopConstants.AdvanceConfKeys.HA_NAMESERVICEID);
String namenodeId1 =
(String)
this.compConfig.getAdvanceConfProperty(
HadoopConstants.AdvanceConfKeys.HA_NAMENODEID1);
String namenodeId2 =
(String)
this.compConfig.getAdvanceConfProperty(
HadoopConstants.AdvanceConfKeys.HA_NAMENODEID2);
String activeNameNodeHost = HadoopUtils.getActiveNameNodeHost(this.compConfig);
String standByNameNodeHost = HadoopUtils.getStandByNameNodeHost(this.compConfig);
paramList.put("dfs.nameservices", nameserviceId);
paramList.put("dfs.ha.namenodes." + nameserviceId, namenodeId1 + "," + namenodeId2);
paramList.put(
"dfs.namenode.rpc-address." + nameserviceId + "." + namenodeId1,
activeNameNodeHost + ":" + Hadoop2Configurator.DEFAULT_PORT_RPC_NAMENODE);
paramList.put(
"dfs.namenode.rpc-address." + nameserviceId + "." + namenodeId2,
standByNameNodeHost + ":" + Hadoop2Configurator.DEFAULT_PORT_RPC_NAMENODE);
paramList.put(
"dfs.namenode.http-address." + nameserviceId + "." + namenodeId1,
activeNameNodeHost + ":" + Hadoop2Configurator.DEFAULT_PORT_HTTP_NAMENODE);
paramList.put(
"dfs.namenode.http-address." + nameserviceId + "." + namenodeId2,
standByNameNodeHost + ":" + Hadoop2Configurator.DEFAULT_PORT_HTTP_NAMENODE);
StringBuilder journalNodeList = new StringBuilder("qjournal://");
for (String journalNodeHost : HadoopUtils.getJournalNodeHosts(this.compConfig)) {
journalNodeList.append(
journalNodeHost + ":" + Hadoop2Configurator.DEFAULT_PORT_RPC_JOURNAL_NODE);
journalNodeList.append(";");
}
String valJournalNodeProp =
journalNodeList.toString().substring(0, journalNodeList.length() - 1)
+ "/"
+ nameserviceId;
paramList.put("dfs.namenode.shared.edits.dir", valJournalNodeProp);
paramList.put(
"dfs.journalnode.edits.dir",
(String)
this.compConfig.getAdvanceConfProperty(
HadoopConstants.AdvanceConfKeys.HA_JOURNALNODE_EDITS_DIR));
paramList.put(
"dfs.client.failover.proxy.provider." + nameserviceId,
"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
paramList.put("dfs.ha.fencing.methods", "shell(/bin/true)");
boolean isAutomaticFailoverEnabled =
(Boolean)
this.compConfig.getAdvanceConfProperty(
HadoopConstants.AdvanceConfKeys.HA_AUTOMATIC_FAILOVER_ENABLED);
if (isAutomaticFailoverEnabled) {
paramList.put("dfs.ha.automatic-failover.enabled." + nameserviceId, "true");
String zkEnsembleId =
(String)
this.compConfig.getAdvanceConfProperty(
HadoopConstants.AdvanceConfKeys.HA_ZK_ENSEMBLEID);
String zkQuorumValue =
ZookeeperUtils.getZookeeperConnectionString(this.clusterConfig, zkEnsembleId);
paramList.put("ha.zookeeper.quorum", zkQuorumValue);
} else {
paramList.put("dfs.ha.automatic-failover.enabled." + nameserviceId, "false");
}
} else if (HadoopUtils.getSecondaryNameNodeHost(this.compConfig) != null) {
String valAddrSNNIp = HadoopUtils.getSecondaryNameNodeHost(this.compConfig);
paramList.put(
"dfs.namenode.secondary.http-address",
valAddrSNNIp + ":" + Hadoop2Configurator.DEFAULT_PORT_HTTP_SECONDARYNAMENODE);
}
for (String propertyName : paramList.keySet()) {
status =
HadoopUtils.addPropertyToFile(
clusterConfig,
nodeConfig,
hdfsSiteXmlPath,
propertyName,
paramList.get(propertyName));
if (!status) {
HadoopUtils.addAndLogError(
LOG,
clusterConfig,
"Could not add " + propertyName + " property to " + hdfsSiteXmlPath + " file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost());
return false;
}
}
return true;
} catch (Exception e) {
HadoopUtils.addAndLogError(
LOG,
clusterConfig,
"Could not update Hadoop yarn-site.xml file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost(),
e);
return false;
}
}
/**
* Configure yarn site xml.
*
* @param nodeConfig the node config
* @return true, if successful
*/
protected boolean configureYarnSiteXml(NodeConfig nodeConfig) {
boolean status = true;
try {
String yarnSiteXmlPath =
HadoopUtils.getHadoopConfDir(this.compConfig)
+ HadoopConstants.FileName.ConfigurationFile.XML_YARN_SITE;
LOG.info(
"Configuring " + HadoopConstants.FileName.ConfigurationFile.XML_YARN_SITE + " file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost());
String valAuxServices = "mapreduce_shuffle";
String keyMRShuffleClass = "yarn.nodemanager.aux-services.mapreduce_shuffle.class";
if (this.isCdh4_4_0()) {
valAuxServices =
AppStoreWrapper.getAnkushConfReader()
.getStringValue("hadoop2.propval.auxservices." + this.compConfig.getVersion());
keyMRShuffleClass =
AppStoreWrapper.getAnkushConfReader()
.getStringValue("hadoop2.propname.mrshuffleclass." + this.compConfig.getVersion());
}
Map<String, String> paramList = new HashMap<String, String>();
String resourceManagerNode = HadoopUtils.getResourceManagerHost(this.compConfig);
paramList.put("yarn.nodemanager.aux-services", valAuxServices);
paramList.put(keyMRShuffleClass, "org.apache.hadoop.mapred.ShuffleHandler");
paramList.put("yarn.resourcemanager.hostname", resourceManagerNode);
paramList.put(
"yarn.resourcemanager.address",
resourceManagerNode + ":" + Hadoop2Configurator.DEFAULT_PORT_RPC_RESOURCEMANAGER);
paramList.put(
"yarn.resourcemanager.resource-tracker.address",
resourceManagerNode + ":" + Hadoop2Configurator.DEFAULT_PORT_RPC_RM_RESOURCETRACKER);
paramList.put(
"yarn.resourcemanager.scheduler.address",
resourceManagerNode + ":" + Hadoop2Configurator.DEFAULT_PORT_RPC_RM_SCHEDULER);
paramList.put(
"yarn.resourcemanager.scheduler.class",
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler");
boolean webAppProxyServerEnabled =
(Boolean)
this.compConfig.getAdvanceConfProperty(
HadoopConstants.AdvanceConfKeys.WEBAPPPROXYSERVER_ENABLED);
if (webAppProxyServerEnabled) {
paramList.put(
"yarn.web-proxy.address",
HadoopUtils.getWebAppProxyServerHost(this.compConfig)
+ ":"
+ Hadoop2Configurator.DEFAULT_PORT_RPC_RM_WEBAPPPROXYSERVER);
}
for (String propertyName : paramList.keySet()) {
status =
HadoopUtils.addPropertyToFile(
clusterConfig,
nodeConfig,
yarnSiteXmlPath,
propertyName,
paramList.get(propertyName));
if (!status) {
HadoopUtils.addAndLogError(
LOG,
clusterConfig,
"Could not add " + propertyName + " property to " + yarnSiteXmlPath + " file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost());
return false;
}
}
return true;
} catch (Exception e) {
HadoopUtils.addAndLogError(
LOG,
clusterConfig,
"Could not update Hadoop yarn-site.xml file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost(),
e);
return false;
}
}
private boolean configureLinuxEnvironmentFile(NodeConfig nodeConfig) {
Result res = null;
LOG.info(
"Configuring variables in " + Constant.LinuxEnvFiles.BASHRC + " file",
Constant.Component.Name.HADOOP,
nodeConfig.getHost());
try {
String componentHome = this.compConfig.getHomeDir();
// set environment variables in /etc/environment file
LinkedHashMap<String, String> envVariableMap = new LinkedHashMap<String, String>();
envVariableMap.put(HadoopUtils.KEY_HADOOP_HOME, componentHome);
envVariableMap.put(HadoopUtils.KEY_HADOOP_PREFIX, componentHome);
envVariableMap.put(
HadoopUtils.KEY_HADOOP_CONF_DIR, HadoopUtils.getHadoopConfDir(this.compConfig));
// String pathVariableVal = HadoopUtils
// .getPathVariableValue(nodeConfig)
// + ":"
// + HadoopUtils.getHadoopBinDir(this.compConfig)
// + ":"
// + HadoopUtils.getHadoopScriptDir(this.compConfig);
//
// envVariableMap.put(HadoopUtils.KEY_PATH_VARIABLE,
// pathVariableVal);
// envVariableMap.put(HadoopUtils.KEY_HADOOP_OPTS,
// "-Djava.net.preferIPv4Stack=true");
AnkushTask addEnvVariables =
new AddEnvironmentVariables(
envVariableMap, Constant.LinuxEnvFiles.BASHRC, Component.Name.HADOOP);
res = nodeConfig.getConnection().exec(addEnvVariables);
if (!res.isSuccess) {
String errMsg =
"Could not add environment variables for Hadoop in "
+ Constant.LinuxEnvFiles.BASHRC
+ " file";
HadoopUtils.addAndLogError(
LOG, clusterConfig, errMsg, Constant.Component.Name.HADOOP, nodeConfig.getHost());
return false;
}
AnkushTask sourceFile = new SourceFile(Constant.LinuxEnvFiles.BASHRC);
res = nodeConfig.getConnection().exec(sourceFile);
if (res.rc != 0) {
String errMsg = "Could not source " + Constant.LinuxEnvFiles.BASHRC + " file";
HadoopUtils.addAndLogError(
LOG, clusterConfig, errMsg, Constant.Component.Name.HADOOP, nodeConfig.getHost());
return false;
}
return true;
} catch (Exception e) {
String errMsg =
"Could not add environment variables for Hadoop in "
+ Constant.LinuxEnvFiles.BASHRC
+ " file";
HadoopUtils.addAndLogError(
LOG, clusterConfig, errMsg, Constant.Component.Name.HADOOP, nodeConfig.getHost(), e);
return false;
}
}
