Esempio n. 1
0
  /**
   * Send mr job jar on agent.
   *
   * @param config the config
   * @param jarFilepath the jar filepath
   */
  public static void sendMRJobJarOnAgent(Config config, String jarFilepath) {
    JobConfig jobConfig = (JobConfig) config;
    String jumbuneHome = JobConfig.getJumbuneHome() + File.separator;
    Remoter remoter = RemotingUtil.getRemoter(config, jumbuneHome);
    File resourceDir = new File(jarFilepath);
    File[] files = resourceDir.listFiles();
    for (File file : files) {
      if (file.getName().endsWith(".tmp")) {
        file.delete();
      } else {
        String filename = file.getAbsolutePath();

        String relativeAgentPath =
            Constants.JOB_JARS_LOC
                + File.separator
                + jobConfig.getFormattedJumbuneJobName()
                + Constants.MR_RESOURCES;
        String resourceFolder =
            System.getenv("AGENT_HOME")
                + File.separator
                + Constants.JOB_JARS_LOC
                + File.separator
                + jobConfig.getFormattedJumbuneJobName()
                + Constants.MR_RESOURCES;
        File resourceDirAgent = new File(resourceFolder);
        if (!resourceDirAgent.exists()) {
          resourceDirAgent.mkdirs();
        }
        remoter.sendJar(relativeAgentPath, filename);
      }
    }
    remoter.close();
  }
Esempio n. 2
0
 /**
  * Checks if is jumbune supplied jar present.
  *
  * @param config the config
  * @return true, if is jumbune supplied jar present
  */
 public static boolean isJumbuneSuppliedJarPresent(Config config) {
   JobConfig jobConfig = (JobConfig) config;
   Master master = jobConfig.getMaster();
   Remoter remoter = new Remoter(master.getHost(), Integer.valueOf(master.getAgentPort()));
   CommandWritableBuilder builder = new CommandWritableBuilder();
   builder.addCommand("ls lib/", false, null, CommandType.FS);
   String result = (String) remoter.fireCommandAndGetObjectResponse(builder.getCommandWritable());
   remoter.close();
   return (result.length() > 0) ? true : false;
 }
Esempio n. 3
0
 /**
  * Checks if is mR job jar present.
  *
  * @param config the config
  * @param jarFilepath the jar filepath
  * @return true, if is mR job jar present
  */
 public static boolean isMRJobJarPresent(Config config, String jarFilepath) {
   JobConfig jobConfig = (JobConfig) config;
   Master master = jobConfig.getMaster();
   File resourceDir = new File(jarFilepath);
   if (resourceDir.exists()) {
     Remoter remoter = new Remoter(master.getHost(), Integer.valueOf(master.getAgentPort()));
     CommandWritableBuilder builder = new CommandWritableBuilder();
     builder.addCommand("ls " + jarFilepath, false, null, CommandType.FS);
     String result =
         (String) remoter.fireCommandAndGetObjectResponse(builder.getCommandWritable());
     remoter.close();
     return (result.length() > 0) ? true : false;
   } else {
     return false;
   }
 }
Esempio n. 4
0
  /**
   * Make remote slave log directory.
   *
   * @param logCollection the log collection
   * @throws JSchException the j sch exception
   * @throws IOException Signals that an I/O exception has occurred.
   * @throws InterruptedException the interrupted exception
   */
  private static void makeRemoteSlaveLogDirectory(LogConsolidationInfo logCollection)
      throws JSchException, IOException, InterruptedException {

    List<Slave> listSlave = logCollection.getSlaves();
    Master master = logCollection.getMaster();
    String masterHost = master.getHost();
    Integer agentPort = Integer.valueOf(master.getAgentPort());

    Remoter remoter = new Remoter(masterHost, agentPort);
    for (Slave slaveDefinition : listSlave) {

      String[] hostsNode = slaveDefinition.getHosts();
      String locationNode = slaveDefinition.getLocation();

      for (String hostNode : hostsNode) {
        String command = Constants.MKDIR_P_CMD + getFolderName(locationNode);
        LOGGER.debug("Log directory generation command on WorkerNode [" + command + "]");

        CommandWritableBuilder builder = new CommandWritableBuilder();
        builder
            .addCommand(command, false, null, CommandType.FS)
            .populateFromLogConsolidationInfo(logCollection, hostNode);
        String parentOfLocationNode = null;
        if (locationNode.lastIndexOf(File.separator) > -1) {
          parentOfLocationNode =
              locationNode.substring(0, locationNode.lastIndexOf(File.separator));
          LOGGER.info("Location Node " + locationNode);
          LOGGER.info("Parent of Location Node " + parentOfLocationNode);
          builder
              .addCommand(Constants.CHMOD_CMD + parentOfLocationNode, false, null, CommandType.FS)
              .populateFromLogConsolidationInfo(logCollection, hostNode);
        }
        remoter.fireAndForgetCommand(builder.getCommandWritable());
      }
      LOGGER.info("Log directory created on WorkerNodes ");
      CONSOLELOGGER.info("Log directory generation on WorkerNodes ");
    }
    remoter.close();
  }
Esempio n. 5
0
  /**
   * Send jumbune supplied jar on agent.
   *
   * @param config the config
   * @param cse the cse
   * @param agentHome the agent home
   */
  public static void sendJumbuneSuppliedJarOnAgent(
      Config config, ClasspathElement cse, String agentHome) {
    String jumbuneHome = JobConfig.getJumbuneHome();
    Remoter remoter = RemotingUtil.getRemoter(config, jumbuneHome);
    String hadoopHome = RemotingUtil.getHadoopHome(remoter, config);
    String[] files = cse.getFiles();
    for (String string : files) {
      remoter.sendJar("lib/", string.replace(agentHome, jumbuneHome));

      if (string.contains("log4j")) {
        StringBuilder copyJarToHadoopLib =
            new StringBuilder()
                .append(Constants.COPY_COMMAND)
                .append(string)
                .append(" ")
                .append(hadoopHome)
                .append(Constants.LIB_DIRECTORY);
        sendLibJarCommand(remoter, config, copyJarToHadoopLib.toString());
      }
    }
    remoter.close();
  }
Esempio n. 6
0
  /**
   * Send lib jar command.
   *
   * @param remoter the remoter
   * @param config the config
   * @param command the command
   */
  public static void sendLibJarCommand(Remoter remoter, Config config, String command) {

    CommandWritableBuilder builder = new CommandWritableBuilder();
    builder.addCommand(command, false, null, CommandType.FS).populate(config, null);
    remoter.fireAndForgetCommand(builder.getCommandWritable());
  }