/**
* Send mr job jar on agent.
*
* @param config the config
* @param jarFilepath the jar filepath
*/
public static void sendMRJobJarOnAgent(Config config, String jarFilepath) {
JobConfig jobConfig = (JobConfig) config;
String jumbuneHome = JobConfig.getJumbuneHome() + File.separator;
Remoter remoter = RemotingUtil.getRemoter(config, jumbuneHome);
File resourceDir = new File(jarFilepath);
File[] files = resourceDir.listFiles();
for (File file : files) {
if (file.getName().endsWith(".tmp")) {
file.delete();
} else {
String filename = file.getAbsolutePath();
String relativeAgentPath =
Constants.JOB_JARS_LOC
+ File.separator
+ jobConfig.getFormattedJumbuneJobName()
+ Constants.MR_RESOURCES;
String resourceFolder =
System.getenv("AGENT_HOME")
+ File.separator
+ Constants.JOB_JARS_LOC
+ File.separator
+ jobConfig.getFormattedJumbuneJobName()
+ Constants.MR_RESOURCES;
File resourceDirAgent = new File(resourceFolder);
if (!resourceDirAgent.exists()) {
resourceDirAgent.mkdirs();
}
remoter.sendJar(relativeAgentPath, filename);
}
}
remoter.close();
}
/**
* Checks if is jumbune supplied jar present.
*
* @param config the config
* @return true, if is jumbune supplied jar present
*/
public static boolean isJumbuneSuppliedJarPresent(Config config) {
JobConfig jobConfig = (JobConfig) config;
Master master = jobConfig.getMaster();
Remoter remoter = new Remoter(master.getHost(), Integer.valueOf(master.getAgentPort()));
CommandWritableBuilder builder = new CommandWritableBuilder();
builder.addCommand("ls lib/", false, null, CommandType.FS);
String result = (String) remoter.fireCommandAndGetObjectResponse(builder.getCommandWritable());
remoter.close();
return (result.length() > 0) ? true : false;
}
/**
* Checks if is mR job jar present.
*
* @param config the config
* @param jarFilepath the jar filepath
* @return true, if is mR job jar present
*/
public static boolean isMRJobJarPresent(Config config, String jarFilepath) {
JobConfig jobConfig = (JobConfig) config;
Master master = jobConfig.getMaster();
File resourceDir = new File(jarFilepath);
if (resourceDir.exists()) {
Remoter remoter = new Remoter(master.getHost(), Integer.valueOf(master.getAgentPort()));
CommandWritableBuilder builder = new CommandWritableBuilder();
builder.addCommand("ls " + jarFilepath, false, null, CommandType.FS);
String result =
(String) remoter.fireCommandAndGetObjectResponse(builder.getCommandWritable());
remoter.close();
return (result.length() > 0) ? true : false;
} else {
return false;
}
}
/**
* Make remote slave log directory.
*
* @param logCollection the log collection
* @throws JSchException the j sch exception
* @throws IOException Signals that an I/O exception has occurred.
* @throws InterruptedException the interrupted exception
*/
private static void makeRemoteSlaveLogDirectory(LogConsolidationInfo logCollection)
throws JSchException, IOException, InterruptedException {
List<Slave> listSlave = logCollection.getSlaves();
Master master = logCollection.getMaster();
String masterHost = master.getHost();
Integer agentPort = Integer.valueOf(master.getAgentPort());
Remoter remoter = new Remoter(masterHost, agentPort);
for (Slave slaveDefinition : listSlave) {
String[] hostsNode = slaveDefinition.getHosts();
String locationNode = slaveDefinition.getLocation();
for (String hostNode : hostsNode) {
String command = Constants.MKDIR_P_CMD + getFolderName(locationNode);
LOGGER.debug("Log directory generation command on WorkerNode [" + command + "]");
CommandWritableBuilder builder = new CommandWritableBuilder();
builder
.addCommand(command, false, null, CommandType.FS)
.populateFromLogConsolidationInfo(logCollection, hostNode);
String parentOfLocationNode = null;
if (locationNode.lastIndexOf(File.separator) > -1) {
parentOfLocationNode =
locationNode.substring(0, locationNode.lastIndexOf(File.separator));
LOGGER.info("Location Node " + locationNode);
LOGGER.info("Parent of Location Node " + parentOfLocationNode);
builder
.addCommand(Constants.CHMOD_CMD + parentOfLocationNode, false, null, CommandType.FS)
.populateFromLogConsolidationInfo(logCollection, hostNode);
}
remoter.fireAndForgetCommand(builder.getCommandWritable());
}
LOGGER.info("Log directory created on WorkerNodes ");
CONSOLELOGGER.info("Log directory generation on WorkerNodes ");
}
remoter.close();
}
/**
* Send jumbune supplied jar on agent.
*
* @param config the config
* @param cse the cse
* @param agentHome the agent home
*/
public static void sendJumbuneSuppliedJarOnAgent(
Config config, ClasspathElement cse, String agentHome) {
String jumbuneHome = JobConfig.getJumbuneHome();
Remoter remoter = RemotingUtil.getRemoter(config, jumbuneHome);
String hadoopHome = RemotingUtil.getHadoopHome(remoter, config);
String[] files = cse.getFiles();
for (String string : files) {
remoter.sendJar("lib/", string.replace(agentHome, jumbuneHome));
if (string.contains("log4j")) {
StringBuilder copyJarToHadoopLib =
new StringBuilder()
.append(Constants.COPY_COMMAND)
.append(string)
.append(" ")
.append(hadoopHome)
.append(Constants.LIB_DIRECTORY);
sendLibJarCommand(remoter, config, copyJarToHadoopLib.toString());
}
}
remoter.close();
}
/**
* Send lib jar command.
*
* @param remoter the remoter
* @param config the config
* @param command the command
*/
public static void sendLibJarCommand(Remoter remoter, Config config, String command) {
CommandWritableBuilder builder = new CommandWritableBuilder();
builder.addCommand(command, false, null, CommandType.FS).populate(config, null);
remoter.fireAndForgetCommand(builder.getCommandWritable());
}