@Override public boolean setupPasswordlessSSH(Set<String> nodes) { try { LOG.info( "Configuring Passwordless SSH for Hadoop 2.x cluster", Constant.Component.Name.HADOOP); if (!this.generateRsaKeysForHadoopNodes(nodes)) { return false; } Set<String> sourceHosts = new HashSet<String>(); sourceHosts.add(HadoopUtils.getNameNodeHost(this.compConfig)); sourceHosts.add(HadoopUtils.getResourceManagerHost(this.compConfig)); boolean isHaEnabled = (Boolean) this.compConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_ENABLED); if (isHaEnabled) { sourceHosts.add(HadoopUtils.getActiveNameNodeHost(this.compConfig)); sourceHosts.add(HadoopUtils.getStandByNameNodeHost(this.compConfig)); } for (String sourceHost : sourceHosts) { if (this.clusterConfig.getNodes().containsKey(sourceHost)) { if (!HadoopUtils.setupPasswordlessSSH(LOG, this.clusterConfig, sourceHost, nodes)) { return false; } } else { SSHExec connection = SSHUtils.connectToNode(sourceHost, this.clusterConfig.getAuthConf()); if (!HadoopUtils.setupPasswordlessSSH( LOG, this.clusterConfig, sourceHost, nodes, connection)) { return false; } if (connection != null) { connection.disconnect(); } } } return true; } catch (Exception e) { HadoopUtils.addAndLogError( LOG, clusterConfig, "Could not configure passwordless SSH for Hadoop", Constant.Component.Name.HADOOP, e); return false; } }
/* * (non-Javadoc) * * @see com.impetus.ankush2.hadoop.deployer.configurator.HadoopConfigurator# * configureHdfsSiteXml(com.impetus.ankush2.framework.config.NodeConfig) */ @Override protected boolean configureHdfsSiteXml(NodeConfig nodeConfig) { boolean status = true; try { String hdfsSiteXmlPath = HadoopUtils.getHadoopConfDir(this.compConfig) + HadoopConstants.FileName.ConfigurationFile.XML_HDFS_SITE; LOG.info( "Configuring " + HadoopConstants.FileName.ConfigurationFile.XML_HDFS_SITE + " file", Constant.Component.Name.HADOOP, nodeConfig.getHost()); Map<String, String> paramList = new HashMap<String, String>(); paramList.put( "dfs.replication", (String) this.compConfig.getAdvanceConfProperty( HadoopConstants.AdvanceConfKeys.DFS_REPLICATION_FACTOR)); paramList.put("dfs.permissions.enabled", "true"); paramList.put( "dfs.namenode.name.dir", HadoopConstants.URI_FILE_PREFIX + (String) this.compConfig.getAdvanceConfProperty( HadoopConstants.AdvanceConfKeys.DFS_NAME_DIR)); paramList.put( "dfs.datanode.data.dir", HadoopConstants.URI_FILE_PREFIX + (String) this.compConfig.getAdvanceConfProperty( HadoopConstants.AdvanceConfKeys.DFS_DATA_DIR)); boolean isHaEnabled = (Boolean) this.compConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_ENABLED); if (isHaEnabled) { String nameserviceId = (String) this.compConfig.getAdvanceConfProperty( HadoopConstants.AdvanceConfKeys.HA_NAMESERVICEID); String namenodeId1 = (String) this.compConfig.getAdvanceConfProperty( HadoopConstants.AdvanceConfKeys.HA_NAMENODEID1); String namenodeId2 = (String) this.compConfig.getAdvanceConfProperty( HadoopConstants.AdvanceConfKeys.HA_NAMENODEID2); String activeNameNodeHost = HadoopUtils.getActiveNameNodeHost(this.compConfig); String standByNameNodeHost = HadoopUtils.getStandByNameNodeHost(this.compConfig); paramList.put("dfs.nameservices", nameserviceId); paramList.put("dfs.ha.namenodes." + nameserviceId, namenodeId1 + "," + namenodeId2); paramList.put( "dfs.namenode.rpc-address." + nameserviceId + "." + namenodeId1, activeNameNodeHost + ":" + Hadoop2Configurator.DEFAULT_PORT_RPC_NAMENODE); paramList.put( "dfs.namenode.rpc-address." + nameserviceId + "." + namenodeId2, standByNameNodeHost + ":" + Hadoop2Configurator.DEFAULT_PORT_RPC_NAMENODE); paramList.put( "dfs.namenode.http-address." + nameserviceId + "." + namenodeId1, activeNameNodeHost + ":" + Hadoop2Configurator.DEFAULT_PORT_HTTP_NAMENODE); paramList.put( "dfs.namenode.http-address." + nameserviceId + "." + namenodeId2, standByNameNodeHost + ":" + Hadoop2Configurator.DEFAULT_PORT_HTTP_NAMENODE); StringBuilder journalNodeList = new StringBuilder("qjournal://"); for (String journalNodeHost : HadoopUtils.getJournalNodeHosts(this.compConfig)) { journalNodeList.append( journalNodeHost + ":" + Hadoop2Configurator.DEFAULT_PORT_RPC_JOURNAL_NODE); journalNodeList.append(";"); } String valJournalNodeProp = journalNodeList.toString().substring(0, journalNodeList.length() - 1) + "/" + nameserviceId; paramList.put("dfs.namenode.shared.edits.dir", valJournalNodeProp); paramList.put( "dfs.journalnode.edits.dir", (String) this.compConfig.getAdvanceConfProperty( HadoopConstants.AdvanceConfKeys.HA_JOURNALNODE_EDITS_DIR)); paramList.put( "dfs.client.failover.proxy.provider." + nameserviceId, "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"); paramList.put("dfs.ha.fencing.methods", "shell(/bin/true)"); boolean isAutomaticFailoverEnabled = (Boolean) this.compConfig.getAdvanceConfProperty( HadoopConstants.AdvanceConfKeys.HA_AUTOMATIC_FAILOVER_ENABLED); if (isAutomaticFailoverEnabled) { paramList.put("dfs.ha.automatic-failover.enabled." + nameserviceId, "true"); String zkEnsembleId = (String) this.compConfig.getAdvanceConfProperty( HadoopConstants.AdvanceConfKeys.HA_ZK_ENSEMBLEID); String zkQuorumValue = ZookeeperUtils.getZookeeperConnectionString(this.clusterConfig, zkEnsembleId); paramList.put("ha.zookeeper.quorum", zkQuorumValue); } else { paramList.put("dfs.ha.automatic-failover.enabled." + nameserviceId, "false"); } } else if (HadoopUtils.getSecondaryNameNodeHost(this.compConfig) != null) { String valAddrSNNIp = HadoopUtils.getSecondaryNameNodeHost(this.compConfig); paramList.put( "dfs.namenode.secondary.http-address", valAddrSNNIp + ":" + Hadoop2Configurator.DEFAULT_PORT_HTTP_SECONDARYNAMENODE); } for (String propertyName : paramList.keySet()) { status = HadoopUtils.addPropertyToFile( clusterConfig, nodeConfig, hdfsSiteXmlPath, propertyName, paramList.get(propertyName)); if (!status) { HadoopUtils.addAndLogError( LOG, clusterConfig, "Could not add " + propertyName + " property to " + hdfsSiteXmlPath + " file", Constant.Component.Name.HADOOP, nodeConfig.getHost()); return false; } } return true; } catch (Exception e) { HadoopUtils.addAndLogError( LOG, clusterConfig, "Could not update Hadoop yarn-site.xml file", Constant.Component.Name.HADOOP, nodeConfig.getHost(), e); return false; } }
private boolean manageHdfsServices(String action) { boolean status = true; try { HadoopServiceManager hadoopServiceManager = HadoopUtils.getServiceManagerInstance(this.clusterConfig, this.compConfig); boolean isHaEnabled = (Boolean) this.compConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_ENABLED); if (!hadoopServiceManager.manageServiceOnNodes( HadoopUtils.getJournalNodeHosts(this.compConfig), HadoopConstants.Roles.JOURNALNODE, action)) { status = false; } if (isHaEnabled) { boolean isAutomaticFailoverEnabled = (Boolean) this.compConfig.getAdvanceConfProperty( HadoopConstants.AdvanceConfKeys.HA_AUTOMATIC_FAILOVER_ENABLED); if (!hadoopServiceManager.manageServiceOnNode( HadoopUtils.getActiveNameNodeHost(this.compConfig), HadoopConstants.Roles.NAMENODE, action)) { status = false; } if (!hadoopServiceManager.manageServiceOnNode( HadoopUtils.getStandByNameNodeHost(this.compConfig), HadoopConstants.Roles.NAMENODE, action)) { status = false; } if (isAutomaticFailoverEnabled) { if (!hadoopServiceManager.manageServiceOnNodes( HadoopUtils.getHaNameNodeHosts(this.compConfig), HadoopConstants.Roles.DFSZKFAILOVERCONTROLLER, action)) { status = false; } } } else { if (hadoopServiceManager.manageServiceOnNode( HadoopUtils.getNameNodeHost(this.compConfig), HadoopConstants.Roles.NAMENODE, action)) { status = false; } } if (HadoopUtils.getSecondaryNameNodeHost(this.compConfig) != null) { if (!hadoopServiceManager.manageServiceOnNode( HadoopUtils.getSecondaryNameNodeHost(this.compConfig), HadoopConstants.Roles.SECONDARYNAMENODE, action)) { status = false; } } // Start DataNode Process on each Slave Node if (!hadoopServiceManager.manageServiceOnNodes( HadoopUtils.getSlaveHosts(this.compConfig), HadoopConstants.Roles.DATANODE, action)) { status = false; } } catch (Exception e) { HadoopUtils.addAndLogError( LOG, clusterConfig, "Could not format NameNode and start Hadoop services", Constant.Component.Name.HADOOP, e); status = false; } return status; }
private boolean initializeAndStartHdfs() { try { HadoopCommandsManager hadoopCmdManager = HadoopUtils.getCommandsManagerInstance(this.clusterConfig, this.compConfig); HadoopServiceManager hadoopServiceManager = HadoopUtils.getServiceManagerInstance(this.clusterConfig, this.compConfig); boolean isHaEnabled = (Boolean) this.compConfig.getAdvanceConfProperty(HadoopConstants.AdvanceConfKeys.HA_ENABLED); if (isHaEnabled) { if (!hadoopServiceManager.manageServiceOnNodes( HadoopUtils.getJournalNodeHosts(this.compConfig), HadoopConstants.Roles.JOURNALNODE, HadoopConstants.Command.Action.START)) { return false; } } // Format NameNode and return false if format command fails if (!hadoopCmdManager.formatNameNode()) { return false; } hadoopServiceManager.manageServiceOnNode( HadoopUtils.getNameNodeHost(this.compConfig), HadoopConstants.Roles.NAMENODE, HadoopConstants.Command.Action.START); boolean isAutomaticFailoverEnabled = false; if (isHaEnabled) { Hadoop2CommandsManager hadoop2CmdManager = (Hadoop2CommandsManager) hadoopCmdManager; isAutomaticFailoverEnabled = (Boolean) this.compConfig.getAdvanceConfProperty( HadoopConstants.AdvanceConfKeys.HA_AUTOMATIC_FAILOVER_ENABLED); hadoop2CmdManager.initializeStandByNameNode(); hadoopServiceManager.manageServiceOnNode( HadoopUtils.getStandByNameNodeHost(this.compConfig), HadoopConstants.Roles.NAMENODE, HadoopConstants.Command.Action.START); if (isAutomaticFailoverEnabled) { hadoop2CmdManager.initializeHAInZooKeeper(); hadoopServiceManager.manageServiceOnNode( HadoopUtils.getActiveNameNodeHost(this.compConfig), HadoopConstants.Roles.DFSZKFAILOVERCONTROLLER, HadoopConstants.Command.Action.START); hadoopServiceManager.manageServiceOnNode( HadoopUtils.getStandByNameNodeHost(this.compConfig), HadoopConstants.Roles.DFSZKFAILOVERCONTROLLER, HadoopConstants.Command.Action.START); } } else if (HadoopUtils.getSecondaryNameNodeHost(this.compConfig) != null) { hadoopServiceManager.manageServiceOnNode( HadoopUtils.getSecondaryNameNodeHost(this.compConfig), HadoopConstants.Roles.SECONDARYNAMENODE, HadoopConstants.Command.Action.START); } // Start DataNode Process on each Slave Node hadoopServiceManager.manageServiceOnNodes( HadoopUtils.getSlaveHosts(this.compConfig), HadoopConstants.Roles.DATANODE, HadoopConstants.Command.Action.START); return true; } catch (Exception e) { HadoopUtils.addAndLogError( LOG, clusterConfig, "Could not start Hadoop HDFS services", Constant.Component.Name.HADOOP, e); return false; } }