/**
* Checks to make sure that nodes specified at the master get spawned at the node.
*
* @throws InterruptedException
*/
@Test
public void testZKMasterDecomission() throws IOException, InterruptedException {
// use the simple command manger, non-gossip ackmanager
cfg.set(FlumeConfiguration.MASTER_STORE, "zookeeper");
master =
new FlumeMaster(
new CommandManager(),
new ConfigManager(),
new StatusManager(),
new MasterAckManager(),
cfg);
master.serve();
MasterRPC rpc = new DirectMasterRPC(master);
FlumeNode node = new FlumeNode(rpc, false, false);
// should have nothing.
assertEquals(0, node.getLogicalNodeManager().getNodes().size());
master.getSpecMan().addLogicalNode(NetUtils.localhost(), node.getPhysicalNodeName());
master.getSpecMan().addLogicalNode(NetUtils.localhost(), "bar");
master.getSpecMan().addLogicalNode(NetUtils.localhost(), "baz");
LivenessManager liveMan = node.getLivenessManager();
liveMan.checkLogicalNodes();
// the two added nodes, plus the always present physnode/logical
assertEquals(3, node.getLogicalNodeManager().getNodes().size());
}
@Test
public void testUnmapLogicalNode() throws IOException, InterruptedException {
// use the simple command manger, non-gossip ackmanager
FlumeMaster master =
new FlumeMaster(
new CommandManager(),
new ConfigManager(),
new StatusManager(),
new MasterAckManager(),
cfg);
MasterRPC rpc = new DirectMasterRPC(master);
FlumeNode node = new FlumeNode(rpc, false, false);
// should have nothing.
assertEquals(0, node.getLogicalNodeManager().getNodes().size());
String local = NetUtils.localhost();
master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
master.getSpecMan().addLogicalNode(local, "bar");
master.getSpecMan().addLogicalNode(local, "baz");
LivenessManager liveMan = node.getLivenessManager();
liveMan.checkLogicalNodes();
assertEquals(local, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
assertNull(master.getSpecMan().getConfig("bar"));
assertNull(master.getSpecMan().getConfig("baz"));
assertNull(master.getSpecMan().getConfig(local));
master.getSpecMan().unmapLogicalNode(local, "bar");
liveMan.checkLogicalNodes();
assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
assertNull(master.getSpecMan().getConfig("bar"));
assertNull(master.getSpecMan().getConfig("baz"));
assertNull(master.getSpecMan().getConfig(local));
master.getSpecMan().unmapLogicalNode(local, "baz");
liveMan.checkLogicalNodes();
assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(null, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
assertNull(master.getSpecMan().getConfig("bar"));
assertNull(master.getSpecMan().getConfig("baz"));
assertNull(master.getSpecMan().getConfig(local));
master.getSpecMan().unmapLogicalNode(local, local);
liveMan.checkLogicalNodes();
assertNull(master.getSpecMan().getConfig("bar"));
assertNull(master.getSpecMan().getConfig("baz"));
assertNull(master.getSpecMan().getConfig(local));
assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(null, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
}
/**
* Checks to make sure that nodes specified at the master get spawned at the node.
*
* <p>Even though only two nodes are set, there should be three because we currently have an
* invariant where each physical node must have at least on logical node, which is named the same
* name as the physical node name.
*
* <p>This may go away when we "bulkify" the node-master comms into one rpc call.
*
* @throws InterruptedException
*/
@Test
public void testMasterLogicalNodeCheckAutoLogicalNode() throws IOException, InterruptedException {
FlumeMaster master = new FlumeMaster(cfg);
master.getSpecMan().addLogicalNode(NetUtils.localhost(), "bar");
master.getSpecMan().addLogicalNode(NetUtils.localhost(), "baz");
MasterRPC rpc = new DirectMasterRPC(master);
FlumeNode node = new FlumeNode(rpc, false, false);
// should have nothing.
assertEquals(0, node.getLogicalNodeManager().getNodes().size());
LivenessManager liveMan = node.getLivenessManager();
liveMan.checkLogicalNodes();
// the two added nodes, plus the always present physnode/logical
assertEquals(3, node.getLogicalNodeManager().getNodes().size());
}
/** Checks if the DFOs associated with logical node list are currently empty. */
boolean areDFOsEmpty(Collection<LogicalNode> lns) {
for (LogicalNode n : lns) {
DiskFailoverManager dfo = FlumeNode.getInstance().getDFOManager(n.getName());
if (!dfo.isEmpty()) return false;
}
return true;
}
public void doTestLogicalNodesConcurrentDFOMans(final int threads, final int events, int timeout)
throws IOException, InterruptedException, FlumeSpecException {
FlumeTestHarness.setupLocalWriteDir();
FlumeMaster master = new FlumeMaster();
FlumeNode node = new FlumeNode(new DirectMasterRPC(master), false, false);
final Reportable[] dfos = new Reportable[threads];
for (int i = 0; i < threads; i++) {
String name = "test." + i;
String report = "report." + i;
int count = events + i;
String src = "asciisynth(" + count + ",100)";
String snk = "{ diskFailover => counter(\"" + report + "\") } ";
node.getLogicalNodeManager().testingSpawn(name, src, snk);
dfos[i] = node.getLogicalNodeManager().get(name);
}
// TODO (jon) using sleep is cheating to give all threads a chance to start.
// Test seems flakey without this due to a race condition.
Thread.sleep(500);
// wait for all to be done.
waitForEmptyDFOs(node, timeout);
// check to make sure everyone got the right number of events
boolean success = true;
for (int i = 0; i < threads; i++) {
LOG.info(dfos[i].getMetrics());
}
for (int i = 0; i < threads; i++) {
CounterSink cnt = (CounterSink) ReportManager.get().getReportable("report." + i);
LOG.info(i + " expected " + (events + i) + " and got " + cnt.getCount());
success &= ((events + i) == cnt.getCount());
assertEquals(events + i, cnt.getCount());
}
assertTrue("Counts did not line up", success);
FlumeTestHarness.cleanupLocalWriteDir();
}
@Deprecated
public synchronized void getReports(Map<String, ReportEvent> reports) {
String phyName = FlumeNode.getInstance().getPhysicalNodeName();
String rprefix = phyName + "." + getName() + ".";
if (snk != null) {
snk.getReports(rprefix, reports);
}
if (src != null) {
src.getReports(rprefix, reports);
}
}
@Test
public void testDuplicateSpawn() throws IOException, InterruptedException {
// use the simple command manger, non-gossip ackmanager
FlumeMaster master =
new FlumeMaster(
new CommandManager(),
new ConfigManager(),
new StatusManager(),
new MasterAckManager(),
cfg);
MasterRPC rpc = new DirectMasterRPC(master);
FlumeNode node = new FlumeNode(rpc, false, false);
// should have nothing.
assertEquals(0, node.getLogicalNodeManager().getNodes().size());
String local = NetUtils.localhost();
// these are spawn commands
master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
master.getSpecMan().addLogicalNode(local, "bar");
master.getSpecMan().addLogicalNode(local, "baz");
// there should not be duplicates in the mapping table.
master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
master.getSpecMan().addLogicalNode(local, "bar");
master.getSpecMan().addLogicalNode(local, "baz");
Multimap<String, String> mapping = master.getSpecMan().getLogicalNodeMap();
assertEquals(3, mapping.size());
LivenessManager liveMan = node.getLivenessManager();
liveMan.checkLogicalNodes();
assertEquals(3, node.getLogicalNodeManager().getNodes().size());
}
/** Checks to make sure that an unmapped node correctly transitions to DECOMMISSIONED */
@Test
public void testUnmappedLogicalNodeGetsDecommissioned() throws IOException, InterruptedException {
FlumeMaster master = new FlumeMaster(cfg);
MasterRPC rpc = new DirectMasterRPC(master);
FlumeNode node = new FlumeNode(rpc, false, false);
// should have nothing.
assertEquals(0, node.getLogicalNodeManager().getNodes().size());
master.getSpecMan().addLogicalNode(node.getPhysicalNodeName(), "foo");
master
.getStatMan()
.updateHeartbeatStatus(
NetUtils.localhost(), node.getPhysicalNodeName(), "foo", NodeState.ACTIVE, 10);
master.getSpecMan().unmapLogicalNode(NetUtils.localhost(), "foo");
master.getStatMan().checkup();
assertEquals(NodeState.DECOMMISSIONED, master.getStatMan().getNodeStatuses().get("foo").state);
master.getSpecMan().addLogicalNode(node.getPhysicalNodeName(), "foo");
master
.getStatMan()
.updateHeartbeatStatus(
NetUtils.localhost(), node.getPhysicalNodeName(), "foo", NodeState.ACTIVE, 10);
master.getStatMan().checkup();
assertEquals(NodeState.ACTIVE, master.getStatMan().getNodeStatuses().get("foo").state);
}
/** Wait until the flume node's dfos are empty. */
private void waitForEmptyDFOs(FlumeNode node, int timeout) throws InterruptedException {
boolean done = false;
long start = System.currentTimeMillis();
while (!done) {
if (System.currentTimeMillis() - start > timeout) {
fail("Test took too long");
}
Collection<LogicalNode> lns = node.getLogicalNodeManager().getNodes();
done = areDFOsReconfigured(lns) && areDFOsEmpty(lns);
if (!done) {
Thread.sleep(250);
}
}
}
/**
* 여기에 위치하는게 맞나?
*
* @throws IOException
* @throws InterruptedException
* @throws RuntimeException
* @throws FlumeSpecException
*/
public void recover()
throws IOException, InterruptedException, RuntimeException, FlumeSpecException {
LogicalNode logicalNode = FlumeNode.getInstance().getLogicalNodeManager().get(logicalNodeName);
if (logicalNode == null) {
log.error("Failed recover [" + logicalNodeName + "] is not registed in CheckpointManager");
return;
}
log.info("Closing logicalNode[" + logicalNodeName + "]");
logicalNode.close();
log.info("Closed logicalNode[" + logicalNodeName + "]");
log.info("Restarting LogicalNode [" + logicalNodeName + "]");
logicalNode.restartNode();
log.info("Finished LogicalNode [" + logicalNodeName + "]");
}
@Test
public void testOpenClose() throws IOException, Exception {
// Set directory of webapps to build-specific dir
FlumeConfiguration.get().set(FlumeConfiguration.WEBAPPS_PATH, "build/webapps");
FlumeConfiguration conf = FlumeConfiguration.get();
String webPath = FlumeNode.getWebPath(conf);
int port = FlumeConfiguration.get().getNodeStatusPort();
StatusHttpServer http = new StatusHttpServer("flumeagent", webPath, "0.0.0.0", port, false);
for (int i = 0; i < 50; i++) {
http.start();
http.stop();
}
}
/** Creates a logical node that will accept any configuration */
public LogicalNode(Context ctx, String name) {
this.nodeName = name;
this.ctx = ctx;
// Note: version and lastSeen aren't kept up-to-date on the logical node.
// The master fills them in when it receives a NodeStatus heartbeat.
state =
new NodeStatus(
NodeState.HELLO,
0,
0,
NetUtils.localhost(),
FlumeNode.getInstance().getPhysicalNodeName());
// Set version to -1 so that all non-negative versions will be 'later'
lastGoodCfg =
new FlumeConfigData(
0,
"null",
"null",
VERSION_INFIMUM,
VERSION_INFIMUM,
FlumeConfiguration.get().getDefaultFlowName());
}
public void _jspService(HttpServletRequest request, HttpServletResponse response)
throws java.io.IOException, ServletException {
JspFactory _jspxFactory = null;
PageContext pageContext = null;
HttpSession session = null;
ServletContext application = null;
ServletConfig config = null;
JspWriter out = null;
Object page = this;
JspWriter _jspx_out = null;
PageContext _jspx_page_context = null;
try {
_jspxFactory = JspFactory.getDefaultFactory();
response.setContentType("text/html; charset=UTF-8");
pageContext = _jspxFactory.getPageContext(this, request, response, null, true, 8192, true);
_jspx_page_context = pageContext;
application = pageContext.getServletContext();
config = pageContext.getServletConfig();
session = pageContext.getSession();
out = pageContext.getOut();
_jspx_out = out;
out.write(
"<!--\n Licensed to Cloudera, Inc. under one\n or more contributor license agreements. See the NOTICE file\n distributed with this work for additional information\n regarding copyright ownership. Cloudera, Inc. licenses this file\n to you under the Apache License, Version 2.0 (the\n \"License\"); you may not use this file except in compliance\n with the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n-->\n<html><head>\n<!-- Copyright (c) 2010 Cloudera, Inc. All rights reserved. -->\n<!-- Retro web 1.0 flume Agent configuration display -->\n\n<title>Flume Node: ");
out.print(FlumeNode.getInstance().getPhysicalNodeName());
out.write(
" :: Sources and Sinks</title>\n\n<link rel=\"stylesheet\" type=\"text/css\" href=\"/flume.css\" />\n\n</head>\n<body>\n");
org.apache.jasper.runtime.JspRuntimeLibrary.include(
request, response, "menu_agent.jsp", out, false);
out.write("\n\n<h1>Flume Node: ");
out.print(FlumeNode.getInstance().getPhysicalNodeName());
out.write(" :: Extensions - Sink/Source/Decorator</h1>\n\n");
org.apache.jasper.runtime.JspRuntimeLibrary.include(
request, response, "version.jsp", out, false);
out.write("\n<hr>\n\n\n<div id=\"sinks\">\n<h2>Sinks</h2>\n\n<table>\n\n");
// treeset so that the list is sorted
for (String n : new TreeSet<String>(FlumeBuilder.getSinkNames())) {
out.write("\n <tr><td>");
out.print(n);
out.write("</td></tr>\n");
}
out.write("\n\n</table>\n</div>\n\n<div id=\"sources\">\n<h2>Sources</h2>\n<table>\n\n");
// treeset so that the list is sorted
for (String n : new TreeSet<String>(FlumeBuilder.getSourceNames())) {
out.write("\n <tr><td>");
out.print(n);
out.write("</td></tr>\n");
}
out.write("\n\n</table>\n</div>\n\n<div id=\"decos\">\n<h2>Decorators</h2>\n<table>\n\n");
// treeset so that the list is sorted
for (String n : new TreeSet<String>(FlumeBuilder.getDecoratorNames())) {
out.write("\n <tr><td>");
out.print(n);
out.write("</td></tr>\n");
}
out.write("\n\n</table>\n</div>\n</body></html>\n");
} catch (Throwable t) {
if (!(t instanceof SkipPageException)) {
out = _jspx_out;
if (out != null && out.getBufferSize() != 0) out.clearBuffer();
if (_jspx_page_context != null) _jspx_page_context.handlePageException(t);
}
} finally {
if (_jspxFactory != null) _jspxFactory.releasePageContext(_jspx_page_context);
}
}
/**
* Returns a Flume Node with settings from specified command line parameters. (See usage for
* instructions)
*
* @param argv
* @return
* @throws IOException
*/
public static FlumeNode setup(String[] argv) throws IOException {
logVersion(LOG);
logEnvironment(LOG);
// Make sure the Java version is not older than 1.6
if (!CheckJavaVersion.isVersionOk()) {
LOG.error("Exiting because of an old Java version or Java version in bad format");
System.exit(-1);
}
LOG.info("Starting flume agent on: " + NetUtils.localhost());
LOG.info(" Working directory is: " + new File(".").getAbsolutePath());
FlumeConfiguration.hardExitLoadConfig(); // will exit if conf file is bad.
CommandLine cmd = null;
Options options = new Options();
options.addOption("c", true, "Load initial config from cmdline arg");
options.addOption("n", true, "Set node name");
options.addOption("s", false, "Do not start local flume status server on node");
options.addOption("1", false, "Make flume node one shot (if closes or errors, exits)");
options.addOption("m", false, "Have flume hard exit if in likely GC thrash situation");
options.addOption("h", false, "Print help information");
options.addOption("v", false, "Print version information");
try {
CommandLineParser parser = new PosixParser();
cmd = parser.parse(options, argv);
} catch (ParseException e) {
HelpFormatter fmt = new HelpFormatter();
fmt.printHelp("FlumeNode", options, true);
return null;
}
// dump version info only
if (cmd != null && cmd.hasOption("v")) {
return null;
}
// dump help info.
if (cmd != null && cmd.hasOption("h")) {
HelpFormatter fmt = new HelpFormatter();
fmt.printHelp("FlumeNode", options, true);
return null;
}
// Check FlumeConfiguration file for settings that may cause node to fail.
nodeConfigChecksOk();
String nodename = NetUtils.localhost(); // default to local host name.
if (cmd != null && cmd.hasOption("n")) {
// select a different name, allow for multiple processes configured
// differently on same node.
nodename = cmd.getOptionValue("n");
}
boolean startHttp = false;
if (cmd != null && !cmd.hasOption("s")) {
// no -s option, start the local status server
startHttp = true;
}
boolean oneshot = false;
if (cmd != null && cmd.hasOption("1")) {
oneshot = true;
}
FormatFactory.loadOutputFormatPlugins();
// Instantiate the flume node.
FlumeConfiguration conf = FlumeConfiguration.get();
FlumeNode flume = new FlumeNode(nodename, conf, startHttp, oneshot);
flume.start();
// load an initial configuration from command line
if (cmd != null && cmd.hasOption("c")) {
String spec = cmd.getOptionValue("c");
LOG.info("Loading spec from command line: '" + spec + "'");
try {
// node name is the default logical and physical name.
Context ctx = new LogicalNodeContext(nodename, nodename);
Map<String, Pair<String, String>> cfgs = FlumeBuilder.parseConf(ctx, spec);
Pair<String, String> node = cfgs.get(nodename);
FlumeConfigData fcd = new FlumeConfigData(0, node.getLeft(), node.getRight(), 0, 0, null);
flume.nodesMan.spawn(ctx, nodename, fcd);
} catch (Exception e) {
LOG.warn("Caught exception loading node:" + e.getMessage());
LOG.debug("Exception: ", e);
if (oneshot) {
System.exit(0); // exit cleanly
}
}
} else {
try {
// default to null configurations.
Context ctx = new LogicalNodeContext(nodename, nodename);
FlumeConfigData fcd = new FlumeConfigData(0, "null", "null", 0, 0, null);
flume.nodesMan.spawn(ctx, nodename, fcd);
} catch (Exception e) {
LOG.error("Caught exception loading node", e);
}
}
if (cmd != null && cmd.hasOption("m")) {
// setup memory use monitor
LOG.info("Setup hard exit on memory exhaustion");
MemoryMonitor.setupHardExitMemMonitor(FlumeConfiguration.get().getAgentMemoryThreshold());
}
try {
tryKerberosLogin();
} catch (IOException ioe) {
LOG.error("Failed to kerberos login.", ioe);
}
// hangout, waiting for other agent thread to exit.
return flume;
}
@Test
public void testZKRemoveLogicalNode()
throws IOException, FlumeSpecException, InterruptedException {
// use the simple command manger, non-gossip ackmanager
cfg.set(FlumeConfiguration.MASTER_STORE, "zookeeper");
master =
new FlumeMaster(
new CommandManager(),
new ConfigManager(),
new StatusManager(),
new MasterAckManager(),
cfg);
master.serve();
MasterRPC rpc = new DirectMasterRPC(master);
FlumeNode node = new FlumeNode(rpc, false, false);
// should have nothing.
assertEquals(0, node.getLogicalNodeManager().getNodes().size());
String local = NetUtils.localhost();
master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
master.getSpecMan().addLogicalNode(local, "bar");
master.getSpecMan().addLogicalNode(local, "baz");
master.getSpecMan().setConfig(local, "my-test-flow", "null", "null");
master.getSpecMan().setConfig("bar", "my-test-flow", "null", "null");
master.getSpecMan().setConfig("baz", "my-test-flow", "null", "null");
LivenessManager liveMan = node.getLivenessManager();
liveMan.heartbeatChecks();
// liveMan.checkLogicalNodes();
assertEquals(local, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
assertNotNull(master.getSpecMan().getConfig("bar"));
assertNotNull(master.getSpecMan().getConfig("baz"));
assertNotNull(master.getSpecMan().getConfig(local));
master.getSpecMan().removeLogicalNode("bar");
liveMan.heartbeatChecks();
// liveMan.checkLogicalNodes();
assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
assertNull(master.getSpecMan().getConfig("bar"));
assertNotNull(master.getSpecMan().getConfig("baz"));
assertNotNull(master.getSpecMan().getConfig(local));
master.getSpecMan().removeLogicalNode("baz");
liveMan.heartbeatChecks();
// liveMan.checkLogicalNodes();
assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(null, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
assertNull(master.getSpecMan().getConfig("bar"));
assertNull(master.getSpecMan().getConfig("baz"));
assertNotNull(master.getSpecMan().getConfig(local));
master.getSpecMan().removeLogicalNode(local);
liveMan.heartbeatChecks();
// liveMan.checkLogicalNodes();
assertNull(master.getSpecMan().getConfig("bar"));
assertNull(master.getSpecMan().getConfig("baz"));
assertNull(master.getSpecMan().getConfig(local));
assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(null, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
}
@Override
public void close() throws IOException, InterruptedException {
FlumeNode.getInstance().getCheckPointManager().stopServer();
super.close();
}
@Override
public void open() throws IOException, InterruptedException {
FlumeNode.getInstance().getCheckPointManager().startServer(port);
super.open();
}
public void serve() throws IOException {
if (cfg.getMasterStore().equals(ZK_CFG_STORE)) {
try {
ZooKeeperService.getAndInit(cfg);
} catch (InterruptedException e) {
throw new IOException("Unexpected interrupt when starting ZooKeeper", e);
}
}
ReportManager.get().add(vmInfo);
ReportManager.get().add(sysInfo);
if (doHttp) {
String webPath = FlumeNode.getWebPath(cfg);
this.http =
new StatusHttpServer("flumeconfig", webPath, "0.0.0.0", cfg.getMasterHttpPort(), false);
http.addServlet(jerseyMasterServlet(), "/master/*");
http.start();
}
controlServer = new MasterClientServer(this, FlumeConfiguration.get());
configServer = new MasterAdminServer(this, FlumeConfiguration.get());
/*
* We instantiate both kinds of report servers below, but no resources are
* allocated till we call serve() on them.
*/
avroReportServer = new AvroReportServer(FlumeConfiguration.get().getReportServerPort());
thriftReportServer = new ThriftReportServer(FlumeConfiguration.get().getReportServerPort());
ReportManager.get().add(this);
try {
controlServer.serve();
configServer.serve();
/*
* Start the Avro/Thrift ReportServer based on the flag set in the
* configuration file.
*/
if (cfg.getReportServerRPC() == cfg.RPC_TYPE_AVRO) {
avroReportServer.serve();
} else {
thriftReportServer.serve();
}
} catch (TTransportException e1) {
throw new IOException("Error starting control or config server", e1);
}
cmdman.start();
ackman.start();
specman.start();
// TODO (jon) clean shutdown
reaper =
new Thread("Lost node reaper") {
@Override
public void run() {
try {
while (true) {
Thread.sleep(FlumeConfiguration.get().getConfigHeartbeatPeriod());
statman.checkup();
}
} catch (InterruptedException e) {
LOG.error("Reaper thread unexpectedly interrupted:" + e.getMessage());
LOG.debug("Lost node reaper unexpectedly interrupted", e);
}
}
};
reaper.start();
}
/** This is the method that gets run when bin/flume master is executed. */
public static void main(String[] argv) {
FlumeNode.logVersion(LOG);
FlumeNode.logEnvironment(LOG);
// Make sure the Java version is not older than 1.6
if (!CheckJavaVersion.isVersionOk()) {
LOG.error("Exiting because of an old Java version or Java version in bad format");
System.exit(-1);
}
FlumeConfiguration.hardExitLoadConfig(); // if config file is bad hardexit.
CommandLine cmd = null;
Options options = new Options();
options.addOption("c", true, "Load config from file");
options.addOption("f", false, "Use fresh (empty) flume configs");
options.addOption("i", true, "Server id (an integer from 0 up)");
try {
CommandLineParser parser = new PosixParser();
cmd = parser.parse(options, argv);
} catch (ParseException e) {
HelpFormatter fmt = new HelpFormatter();
fmt.printHelp("FlumeNode", options, true);
System.exit(1);
}
FlumeNode.loadOutputFormatPlugins();
String nodeconfig = FlumeConfiguration.get().getMasterSavefile();
if (cmd != null && cmd.hasOption("c")) {
nodeconfig = cmd.getOptionValue("c");
}
if (cmd != null && cmd.hasOption("i")) {
// if manually overridden by command line, accept it, live with
// consequences.
String sid = cmd.getOptionValue("i");
LOG.info("Setting serverid from command line to be " + sid);
try {
int serverid = Integer.parseInt(cmd.getOptionValue("i"));
FlumeConfiguration.get().setInt(FlumeConfiguration.MASTER_SERVER_ID, serverid);
} catch (NumberFormatException e) {
LOG.error("Couldn't parse server id as integer: " + sid);
System.exit(1);
}
} else {
// attempt to auto detect master id.
try {
if (!inferMasterHostID()) {
System.exit(1);
}
} catch (Exception e) {
// master needs to be valid to continue;
LOG.error("Unable to resolve host '{}' ", e.getMessage());
System.exit(1);
}
}
// This will instantiate and read FlumeConfiguration - so make sure that
// this is *after* we set the MASTER_SERVER_ID above.
FlumeMaster config = new FlumeMaster();
LOG.info("Starting flume master on: " + NetUtils.localhost());
LOG.info(" Working Directory is: " + new File(".").getAbsolutePath());
try {
boolean autoload = FlumeConfiguration.get().getMasterSavefileAutoload();
try {
if (autoload && (cmd == null || (cmd != null && !cmd.hasOption("f")))) {
// autoload a config?
config.getSpecMan().loadConfigFile(nodeconfig);
}
} catch (IOException e) {
LOG.warn("Could not autoload config from " + nodeconfig + " because " + e.getMessage());
}
config.serve();
} catch (IOException e) {
LOG.error("IO problem: " + e.getMessage());
LOG.debug("IOException", e);
}
}