private boolean setJavaInHadoopEnvScript( NodeConfig nodeConfig, String filePath, String targetText) { Result res = null; try { String javaHome = this.clusterConfig.getJavaConf().getHomeDir(); String replacementText = AppStoreWrapper.getAnkushConfReader() .getStringValue("hadoop2.javahome.hadoopenv.replacementtext"); replacementText = replacementText.replaceAll( AppStoreWrapper.getAnkushConfReader() .getStringValue("hadoop2.template.javahome.value"), javaHome); AnkushTask updateJavaHome = new ReplaceText( targetText, replacementText, filePath, false, this.clusterConfig.getAuthConf().getPassword()); res = nodeConfig.getConnection().exec(updateJavaHome); if (!res.isSuccess) { if (!res.isSuccess) { String errMsg = "Could not update Java in Hadoop environment script " + filePath; HadoopUtils.addAndLogError( LOG, clusterConfig, errMsg, Constant.Component.Name.HADOOP, nodeConfig.getHost()); return false; } } return true; } catch (Exception e) { String errMsg = "Could not add environment variables for Hadoop in " + Constant.LinuxEnvFiles.ETC_ENVIRONMENT + " file"; HadoopUtils.addAndLogError( LOG, clusterConfig, errMsg, Constant.Component.Name.HADOOP, nodeConfig.getHost(), e); return false; } }
private boolean setJavaInHadoop(NodeConfig nodeConfig) { try { LOG.info( "Configuring Java in Hadoop environment scripts", Constant.Component.Name.HADOOP, nodeConfig.getHost()); LinkedHashMap<String, String> scriptTargetTextMap = new LinkedHashMap<String, String>(); scriptTargetTextMap.put( HadoopUtils.getHadoopConfDir(this.compConfig) + HadoopConstants.FileName.ConfigurationFile.ENV_HADOOP, AppStoreWrapper.getAnkushConfReader() .getStringValue("hadoop2.javahome.hadoopenv.targettext")); scriptTargetTextMap.put( HadoopUtils.getHadoopConfDir(this.compConfig) + HadoopConstants.FileName.ConfigurationFile.ENV_YARN, AppStoreWrapper.getAnkushConfReader() .getStringValue("hadoop2.javahome.yarnenv.targettext")); scriptTargetTextMap.put( HadoopUtils.getHadoopConfDir(this.compConfig) + HadoopConstants.FileName.ConfigurationFile.ENV_MAPRED, AppStoreWrapper.getAnkushConfReader() .getStringValue("hadoop2.javahome.yarnenv.targettext")); for (String hadoopEnvScript : scriptTargetTextMap.keySet()) { if (!this.setJavaInHadoopEnvScript( nodeConfig, hadoopEnvScript, scriptTargetTextMap.get(hadoopEnvScript))) { return false; } } return true; } catch (Exception e) { String errMsg = "Could not update Java in Hadoop environment scripts"; HadoopUtils.addAndLogError( LOG, clusterConfig, errMsg, Constant.Component.Name.HADOOP, nodeConfig.getHost(), e); return false; } }
/** @author hokam */ public class ServerCrashManager { // Ankush logger. private AnkushLogger log = new AnkushLogger(ServerCrashManager.class); // cluster manager. private static GenericManager<Cluster, Long> clusterManager = AppStoreWrapper.getManager(Constant.Manager.CLUSTER, Cluster.class); // node manager. private static GenericManager<Node, Long> nodeManager = AppStoreWrapper.getManager(Constant.Manager.NODE, Node.class); // node manager. private static GenericManager<Operation, Long> operationManager = AppStoreWrapper.getManager( com.impetus.ankush2.constant.Constant.Manager.OPERATION, Operation.class); /** Method to handle deploying and removing state clusters. */ public void handleDeployingRemovingClusters() { // Creating property map for deploying state. Map<String, Object> deployingStateMap = new HashMap<String, Object>(); deployingStateMap.put( com.impetus.ankush2.constant.Constant.Keys.STATE, com.impetus.ankush2.constant.Constant.Cluster.State.DEPLOYING.toString()); // Creating property map for removing state. Map<String, Object> removingStateMap = new HashMap<String, Object>(); removingStateMap.put( com.impetus.ankush2.constant.Constant.Keys.STATE, com.impetus.ankush2.constant.Constant.Cluster.State.REMOVING.toString()); // Map<String, Object> registeringStateMap = new HashMap<String, // Object>(); // registeringStateMap.put(com.impetus.ankush2.constant.Constant.Keys.STATE, // com.impetus.ankush2.constant.Constant.Cluster.State.REGISTERING); // making list of maps List<Map<String, Object>> maps = new ArrayList<Map<String, Object>>(); maps.add(deployingStateMap); maps.add(removingStateMap); // maps.add(registeringStateMap); // list of deploying + removing state clusters. List<Cluster> clusters = clusterManager.getAllByDisjunctionveNormalQuery(maps); // iterating over the all deploying/removing state clusters. for (Cluster cluster : clusters) { // getting clusterable object. try { // setting state as crashed. cluster.setState(Constant.Cluster.State.SERVER_CRASHED.toString()); // getting cluster conf. ClusterConfig conf = cluster.getClusterConfig(); // setting id of cluster inside conf. conf.setClusterId(cluster.getId()); // setting state as error. conf.setState(com.impetus.ankush2.constant.Constant.Cluster.State.ERROR); // adding error message. conf.addError("Deploy", "Server crashed unexpectedly."); // saving cluster conf. cluster.setClusterConf(conf); // saving cluster. clusterManager.save(cluster); } catch (Exception e) { log.error(e.getMessage()); try { // setting server crashed as state. cluster.setState(Constant.Cluster.State.SERVER_CRASHED.toString()); // saving in db. clusterManager.save(cluster); } catch (Exception subExe) { log.error(subExe.getMessage()); } } } } /** Method to handle deploying, removing and adding state nodes. */ public void handleDeployingRemovingNodes() { // Creating property map for deploying state. Map<String, Object> deployingStateMap = new HashMap<String, Object>(); deployingStateMap.put( com.impetus.ankush2.constant.Constant.Keys.STATE, com.impetus.ankush2.constant.Constant.Node.State.DEPLOYING.toString()); // Creating property map for removing state. Map<String, Object> removingStateMap = new HashMap<String, Object>(); removingStateMap.put( com.impetus.ankush2.constant.Constant.Keys.STATE, com.impetus.ankush2.constant.Constant.Node.State.REMOVING.toString()); Map<String, Object> addingStateMap = new HashMap<String, Object>(); addingStateMap.put( com.impetus.ankush2.constant.Constant.Keys.STATE, com.impetus.ankush2.constant.Constant.Node.State.ADDING.toString()); // making list of maps List<Map<String, Object>> maps = new ArrayList<Map<String, Object>>(); maps.add(deployingStateMap); maps.add(removingStateMap); maps.add(addingStateMap); try { // list of deploying + removing state nodes. List<Node> nodes = nodeManager.getAllByDisjunctionveNormalQuery(maps); // setting node state as Server_Crashed for (Node node : nodes) { node.setState(Constant.Node.State.SERVER_CRASHED.toString()); nodeManager.save(node); } } catch (Exception e) { log.error(e.getMessage()); } } /** Function that changed the state of all inprogress operations to ServerCrashed */ public void handleInProgressOperations() { try { // Creating property map for deploying state. Map<String, Object> inProgressStatusMap = new HashMap<String, Object>(); inProgressStatusMap.put( com.impetus.ankush2.constant.Constant.Keys.STATUS, com.impetus.ankush2.constant.Constant.Operation.Status.INPROGRESS.toString()); // making list of maps List<Map<String, Object>> maps = new ArrayList<Map<String, Object>>(); maps.add(inProgressStatusMap); List<Operation> operations = operationManager.getAllByPropertyValue(inProgressStatusMap); for (Operation operation : operations) { try { operation.setStatus( com.impetus.ankush2.constant.Constant.Operation.Status.ERROR.toString()); operation.setCompletedAt(new Date()); operation.getData().put("Error", "Server crashed unexpectedly."); operationManager.save(operation); } catch (Exception e) { log.error(e.getMessage()); try { operation.setStatus( com.impetus.ankush2.constant.Constant.Operation.Status.ERROR.toString()); operationManager.save(operation); } catch (Exception subExe) { log.error(subExe.getMessage()); } } } } catch (Exception e) { log.error(e.getMessage()); } } }
/** * Configure yarn site xml. * * @param nodeConfig the node config * @return true, if successful */ protected boolean configureYarnSiteXml(NodeConfig nodeConfig) { boolean status = true; try { String yarnSiteXmlPath = HadoopUtils.getHadoopConfDir(this.compConfig) + HadoopConstants.FileName.ConfigurationFile.XML_YARN_SITE; LOG.info( "Configuring " + HadoopConstants.FileName.ConfigurationFile.XML_YARN_SITE + " file", Constant.Component.Name.HADOOP, nodeConfig.getHost()); String valAuxServices = "mapreduce_shuffle"; String keyMRShuffleClass = "yarn.nodemanager.aux-services.mapreduce_shuffle.class"; if (this.isCdh4_4_0()) { valAuxServices = AppStoreWrapper.getAnkushConfReader() .getStringValue("hadoop2.propval.auxservices." + this.compConfig.getVersion()); keyMRShuffleClass = AppStoreWrapper.getAnkushConfReader() .getStringValue("hadoop2.propname.mrshuffleclass." + this.compConfig.getVersion()); } Map<String, String> paramList = new HashMap<String, String>(); String resourceManagerNode = HadoopUtils.getResourceManagerHost(this.compConfig); paramList.put("yarn.nodemanager.aux-services", valAuxServices); paramList.put(keyMRShuffleClass, "org.apache.hadoop.mapred.ShuffleHandler"); paramList.put("yarn.resourcemanager.hostname", resourceManagerNode); paramList.put( "yarn.resourcemanager.address", resourceManagerNode + ":" + Hadoop2Configurator.DEFAULT_PORT_RPC_RESOURCEMANAGER); paramList.put( "yarn.resourcemanager.resource-tracker.address", resourceManagerNode + ":" + Hadoop2Configurator.DEFAULT_PORT_RPC_RM_RESOURCETRACKER); paramList.put( "yarn.resourcemanager.scheduler.address", resourceManagerNode + ":" + Hadoop2Configurator.DEFAULT_PORT_RPC_RM_SCHEDULER); paramList.put( "yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"); boolean webAppProxyServerEnabled = (Boolean) this.compConfig.getAdvanceConfProperty( HadoopConstants.AdvanceConfKeys.WEBAPPPROXYSERVER_ENABLED); if (webAppProxyServerEnabled) { paramList.put( "yarn.web-proxy.address", HadoopUtils.getWebAppProxyServerHost(this.compConfig) + ":" + Hadoop2Configurator.DEFAULT_PORT_RPC_RM_WEBAPPPROXYSERVER); } for (String propertyName : paramList.keySet()) { status = HadoopUtils.addPropertyToFile( clusterConfig, nodeConfig, yarnSiteXmlPath, propertyName, paramList.get(propertyName)); if (!status) { HadoopUtils.addAndLogError( LOG, clusterConfig, "Could not add " + propertyName + " property to " + yarnSiteXmlPath + " file", Constant.Component.Name.HADOOP, nodeConfig.getHost()); return false; } } return true; } catch (Exception e) { HadoopUtils.addAndLogError( LOG, clusterConfig, "Could not update Hadoop yarn-site.xml file", Constant.Component.Name.HADOOP, nodeConfig.getHost(), e); return false; } }