/**
* Returns a Flume Node with settings from specified command line parameters. (See usage for
* instructions)
*
* @param argv
* @return
* @throws IOException
*/
public static FlumeNode setup(String[] argv) throws IOException {
logVersion(LOG);
logEnvironment(LOG);
// Make sure the Java version is not older than 1.6
if (!CheckJavaVersion.isVersionOk()) {
LOG.error("Exiting because of an old Java version or Java version in bad format");
System.exit(-1);
}
LOG.info("Starting flume agent on: " + NetUtils.localhost());
LOG.info(" Working directory is: " + new File(".").getAbsolutePath());
FlumeConfiguration.hardExitLoadConfig(); // will exit if conf file is bad.
CommandLine cmd = null;
Options options = new Options();
options.addOption("c", true, "Load initial config from cmdline arg");
options.addOption("n", true, "Set node name");
options.addOption("s", false, "Do not start local flume status server on node");
options.addOption("1", false, "Make flume node one shot (if closes or errors, exits)");
options.addOption("m", false, "Have flume hard exit if in likely GC thrash situation");
options.addOption("h", false, "Print help information");
options.addOption("v", false, "Print version information");
try {
CommandLineParser parser = new PosixParser();
cmd = parser.parse(options, argv);
} catch (ParseException e) {
HelpFormatter fmt = new HelpFormatter();
fmt.printHelp("FlumeNode", options, true);
return null;
}
// dump version info only
if (cmd != null && cmd.hasOption("v")) {
return null;
}
// dump help info.
if (cmd != null && cmd.hasOption("h")) {
HelpFormatter fmt = new HelpFormatter();
fmt.printHelp("FlumeNode", options, true);
return null;
}
// Check FlumeConfiguration file for settings that may cause node to fail.
nodeConfigChecksOk();
String nodename = NetUtils.localhost(); // default to local host name.
if (cmd != null && cmd.hasOption("n")) {
// select a different name, allow for multiple processes configured
// differently on same node.
nodename = cmd.getOptionValue("n");
}
boolean startHttp = false;
if (cmd != null && !cmd.hasOption("s")) {
// no -s option, start the local status server
startHttp = true;
}
boolean oneshot = false;
if (cmd != null && cmd.hasOption("1")) {
oneshot = true;
}
FormatFactory.loadOutputFormatPlugins();
// Instantiate the flume node.
FlumeConfiguration conf = FlumeConfiguration.get();
FlumeNode flume = new FlumeNode(nodename, conf, startHttp, oneshot);
flume.start();
// load an initial configuration from command line
if (cmd != null && cmd.hasOption("c")) {
String spec = cmd.getOptionValue("c");
LOG.info("Loading spec from command line: '" + spec + "'");
try {
// node name is the default logical and physical name.
Context ctx = new LogicalNodeContext(nodename, nodename);
Map<String, Pair<String, String>> cfgs = FlumeBuilder.parseConf(ctx, spec);
Pair<String, String> node = cfgs.get(nodename);
FlumeConfigData fcd = new FlumeConfigData(0, node.getLeft(), node.getRight(), 0, 0, null);
flume.nodesMan.spawn(ctx, nodename, fcd);
} catch (Exception e) {
LOG.warn("Caught exception loading node:" + e.getMessage());
LOG.debug("Exception: ", e);
if (oneshot) {
System.exit(0); // exit cleanly
}
}
} else {
try {
// default to null configurations.
Context ctx = new LogicalNodeContext(nodename, nodename);
FlumeConfigData fcd = new FlumeConfigData(0, "null", "null", 0, 0, null);
flume.nodesMan.spawn(ctx, nodename, fcd);
} catch (Exception e) {
LOG.error("Caught exception loading node", e);
}
}
if (cmd != null && cmd.hasOption("m")) {
// setup memory use monitor
LOG.info("Setup hard exit on memory exhaustion");
MemoryMonitor.setupHardExitMemMonitor(FlumeConfiguration.get().getAgentMemoryThreshold());
}
try {
tryKerberosLogin();
} catch (IOException ioe) {
LOG.error("Failed to kerberos login.", ioe);
}
// hangout, waiting for other agent thread to exit.
return flume;
}
/** This is the method that gets run when bin/flume master is executed. */
public static void main(String[] argv) {
FlumeNode.logVersion(LOG);
FlumeNode.logEnvironment(LOG);
// Make sure the Java version is not older than 1.6
if (!CheckJavaVersion.isVersionOk()) {
LOG.error("Exiting because of an old Java version or Java version in bad format");
System.exit(-1);
}
FlumeConfiguration.hardExitLoadConfig(); // if config file is bad hardexit.
CommandLine cmd = null;
Options options = new Options();
options.addOption("c", true, "Load config from file");
options.addOption("f", false, "Use fresh (empty) flume configs");
options.addOption("i", true, "Server id (an integer from 0 up)");
try {
CommandLineParser parser = new PosixParser();
cmd = parser.parse(options, argv);
} catch (ParseException e) {
HelpFormatter fmt = new HelpFormatter();
fmt.printHelp("FlumeNode", options, true);
System.exit(1);
}
FlumeNode.loadOutputFormatPlugins();
String nodeconfig = FlumeConfiguration.get().getMasterSavefile();
if (cmd != null && cmd.hasOption("c")) {
nodeconfig = cmd.getOptionValue("c");
}
if (cmd != null && cmd.hasOption("i")) {
// if manually overridden by command line, accept it, live with
// consequences.
String sid = cmd.getOptionValue("i");
LOG.info("Setting serverid from command line to be " + sid);
try {
int serverid = Integer.parseInt(cmd.getOptionValue("i"));
FlumeConfiguration.get().setInt(FlumeConfiguration.MASTER_SERVER_ID, serverid);
} catch (NumberFormatException e) {
LOG.error("Couldn't parse server id as integer: " + sid);
System.exit(1);
}
} else {
// attempt to auto detect master id.
try {
if (!inferMasterHostID()) {
System.exit(1);
}
} catch (Exception e) {
// master needs to be valid to continue;
LOG.error("Unable to resolve host '{}' ", e.getMessage());
System.exit(1);
}
}
// This will instantiate and read FlumeConfiguration - so make sure that
// this is *after* we set the MASTER_SERVER_ID above.
FlumeMaster config = new FlumeMaster();
LOG.info("Starting flume master on: " + NetUtils.localhost());
LOG.info(" Working Directory is: " + new File(".").getAbsolutePath());
try {
boolean autoload = FlumeConfiguration.get().getMasterSavefileAutoload();
try {
if (autoload && (cmd == null || (cmd != null && !cmd.hasOption("f")))) {
// autoload a config?
config.getSpecMan().loadConfigFile(nodeconfig);
}
} catch (IOException e) {
LOG.warn("Could not autoload config from " + nodeconfig + " because " + e.getMessage());
}
config.serve();
} catch (IOException e) {
LOG.error("IO problem: " + e.getMessage());
LOG.debug("IOException", e);
}
}