Example #1
0
  /**
   * Returns a Flume Node with settings from specified command line parameters. (See usage for
   * instructions)
   *
   * @param argv
   * @return
   * @throws IOException
   */
  public static FlumeNode setup(String[] argv) throws IOException {
    logVersion(LOG);
    logEnvironment(LOG);
    // Make sure the Java version is not older than 1.6
    if (!CheckJavaVersion.isVersionOk()) {
      LOG.error("Exiting because of an old Java version or Java version in bad format");
      System.exit(-1);
    }
    LOG.info("Starting flume agent on: " + NetUtils.localhost());
    LOG.info(" Working directory is: " + new File(".").getAbsolutePath());

    FlumeConfiguration.hardExitLoadConfig(); // will exit if conf file is bad.

    CommandLine cmd = null;
    Options options = new Options();
    options.addOption("c", true, "Load initial config from cmdline arg");
    options.addOption("n", true, "Set node name");
    options.addOption("s", false, "Do not start local flume status server on node");
    options.addOption("1", false, "Make flume node one shot (if closes or errors, exits)");
    options.addOption("m", false, "Have flume hard exit if in likely GC thrash situation");
    options.addOption("h", false, "Print help information");
    options.addOption("v", false, "Print version information");
    try {
      CommandLineParser parser = new PosixParser();
      cmd = parser.parse(options, argv);
    } catch (ParseException e) {
      HelpFormatter fmt = new HelpFormatter();
      fmt.printHelp("FlumeNode", options, true);
      return null;
    }

    // dump version info only
    if (cmd != null && cmd.hasOption("v")) {
      return null;
    }

    // dump help info.
    if (cmd != null && cmd.hasOption("h")) {
      HelpFormatter fmt = new HelpFormatter();
      fmt.printHelp("FlumeNode", options, true);
      return null;
    }
    // Check FlumeConfiguration file for settings that may cause node to fail.
    nodeConfigChecksOk();

    String nodename = NetUtils.localhost(); // default to local host name.
    if (cmd != null && cmd.hasOption("n")) {
      // select a different name, allow for multiple processes configured
      // differently on same node.
      nodename = cmd.getOptionValue("n");
    }

    boolean startHttp = false;
    if (cmd != null && !cmd.hasOption("s")) {
      // no -s option, start the local status server
      startHttp = true;
    }

    boolean oneshot = false;
    if (cmd != null && cmd.hasOption("1")) {
      oneshot = true;
    }

    FormatFactory.loadOutputFormatPlugins();

    // Instantiate the flume node.
    FlumeConfiguration conf = FlumeConfiguration.get();

    FlumeNode flume = new FlumeNode(nodename, conf, startHttp, oneshot);

    flume.start();

    // load an initial configuration from command line
    if (cmd != null && cmd.hasOption("c")) {
      String spec = cmd.getOptionValue("c");
      LOG.info("Loading spec from command line: '" + spec + "'");

      try {
        // node name is the default logical and physical name.
        Context ctx = new LogicalNodeContext(nodename, nodename);
        Map<String, Pair<String, String>> cfgs = FlumeBuilder.parseConf(ctx, spec);
        Pair<String, String> node = cfgs.get(nodename);
        FlumeConfigData fcd = new FlumeConfigData(0, node.getLeft(), node.getRight(), 0, 0, null);
        flume.nodesMan.spawn(ctx, nodename, fcd);
      } catch (Exception e) {
        LOG.warn("Caught exception loading node:" + e.getMessage());
        LOG.debug("Exception: ", e);
        if (oneshot) {
          System.exit(0); // exit cleanly
        }
      }

    } else {
      try {
        // default to null configurations.
        Context ctx = new LogicalNodeContext(nodename, nodename);
        FlumeConfigData fcd = new FlumeConfigData(0, "null", "null", 0, 0, null);
        flume.nodesMan.spawn(ctx, nodename, fcd);
      } catch (Exception e) {
        LOG.error("Caught exception loading node", e);
      }
    }

    if (cmd != null && cmd.hasOption("m")) {
      // setup memory use monitor
      LOG.info("Setup hard exit on memory exhaustion");
      MemoryMonitor.setupHardExitMemMonitor(FlumeConfiguration.get().getAgentMemoryThreshold());
    }

    try {
      tryKerberosLogin();
    } catch (IOException ioe) {
      LOG.error("Failed to kerberos login.", ioe);
    }

    // hangout, waiting for other agent thread to exit.
    return flume;
  }
Example #2
0
  /** This is the method that gets run when bin/flume master is executed. */
  public static void main(String[] argv) {
    FlumeNode.logVersion(LOG);
    FlumeNode.logEnvironment(LOG);
    // Make sure the Java version is not older than 1.6
    if (!CheckJavaVersion.isVersionOk()) {
      LOG.error("Exiting because of an old Java version or Java version in bad format");
      System.exit(-1);
    }
    FlumeConfiguration.hardExitLoadConfig(); // if config file is bad hardexit.

    CommandLine cmd = null;
    Options options = new Options();
    options.addOption("c", true, "Load config from file");
    options.addOption("f", false, "Use fresh (empty) flume configs");
    options.addOption("i", true, "Server id (an integer from 0 up)");

    try {
      CommandLineParser parser = new PosixParser();
      cmd = parser.parse(options, argv);
    } catch (ParseException e) {
      HelpFormatter fmt = new HelpFormatter();
      fmt.printHelp("FlumeNode", options, true);
      System.exit(1);
    }

    FlumeNode.loadOutputFormatPlugins();

    String nodeconfig = FlumeConfiguration.get().getMasterSavefile();

    if (cmd != null && cmd.hasOption("c")) {
      nodeconfig = cmd.getOptionValue("c");
    }

    if (cmd != null && cmd.hasOption("i")) {
      // if manually overridden by command line, accept it, live with
      // consequences.
      String sid = cmd.getOptionValue("i");
      LOG.info("Setting serverid from command line to be " + sid);
      try {
        int serverid = Integer.parseInt(cmd.getOptionValue("i"));
        FlumeConfiguration.get().setInt(FlumeConfiguration.MASTER_SERVER_ID, serverid);
      } catch (NumberFormatException e) {
        LOG.error("Couldn't parse server id as integer: " + sid);
        System.exit(1);
      }
    } else {
      // attempt to auto detect master id.
      try {
        if (!inferMasterHostID()) {
          System.exit(1);
        }
      } catch (Exception e) {
        // master needs to be valid to continue;
        LOG.error("Unable to resolve host '{}' ", e.getMessage());
        System.exit(1);
      }
    }

    // This will instantiate and read FlumeConfiguration - so make sure that
    // this is *after* we set the MASTER_SERVER_ID above.
    FlumeMaster config = new FlumeMaster();
    LOG.info("Starting flume master on: " + NetUtils.localhost());
    LOG.info(" Working Directory is: " + new File(".").getAbsolutePath());

    try {
      boolean autoload = FlumeConfiguration.get().getMasterSavefileAutoload();
      try {
        if (autoload && (cmd == null || (cmd != null && !cmd.hasOption("f")))) {
          // autoload a config?
          config.getSpecMan().loadConfigFile(nodeconfig);
        }
      } catch (IOException e) {
        LOG.warn("Could not autoload config from " + nodeconfig + " because " + e.getMessage());
      }
      config.serve();

    } catch (IOException e) {
      LOG.error("IO problem: " + e.getMessage());
      LOG.debug("IOException", e);
    }
  }