/**
* Checks to make sure that nodes specified at the master get spawned at the node.
*
* @throws InterruptedException
*/
@Test
public void testZKMasterDecomission() throws IOException, InterruptedException {
// use the simple command manger, non-gossip ackmanager
cfg.set(FlumeConfiguration.MASTER_STORE, "zookeeper");
master =
new FlumeMaster(
new CommandManager(),
new ConfigManager(),
new StatusManager(),
new MasterAckManager(),
cfg);
master.serve();
MasterRPC rpc = new DirectMasterRPC(master);
FlumeNode node = new FlumeNode(rpc, false, false);
// should have nothing.
assertEquals(0, node.getLogicalNodeManager().getNodes().size());
master.getSpecMan().addLogicalNode(NetUtils.localhost(), node.getPhysicalNodeName());
master.getSpecMan().addLogicalNode(NetUtils.localhost(), "bar");
master.getSpecMan().addLogicalNode(NetUtils.localhost(), "baz");
LivenessManager liveMan = node.getLivenessManager();
liveMan.checkLogicalNodes();
// the two added nodes, plus the always present physnode/logical
assertEquals(3, node.getLogicalNodeManager().getNodes().size());
}
/** Checks to make sure that an unmapped node correctly transitions to DECOMMISSIONED */
@Test
public void testUnmappedLogicalNodeGetsDecommissioned() throws IOException, InterruptedException {
FlumeMaster master = new FlumeMaster(cfg);
MasterRPC rpc = new DirectMasterRPC(master);
FlumeNode node = new FlumeNode(rpc, false, false);
// should have nothing.
assertEquals(0, node.getLogicalNodeManager().getNodes().size());
master.getSpecMan().addLogicalNode(node.getPhysicalNodeName(), "foo");
master
.getStatMan()
.updateHeartbeatStatus(
NetUtils.localhost(), node.getPhysicalNodeName(), "foo", NodeState.ACTIVE, 10);
master.getSpecMan().unmapLogicalNode(NetUtils.localhost(), "foo");
master.getStatMan().checkup();
assertEquals(NodeState.DECOMMISSIONED, master.getStatMan().getNodeStatuses().get("foo").state);
master.getSpecMan().addLogicalNode(node.getPhysicalNodeName(), "foo");
master
.getStatMan()
.updateHeartbeatStatus(
NetUtils.localhost(), node.getPhysicalNodeName(), "foo", NodeState.ACTIVE, 10);
master.getStatMan().checkup();
assertEquals(NodeState.ACTIVE, master.getStatMan().getNodeStatuses().get("foo").state);
}
/**
* This heartbeats to provide physical node info and allows the translators to build fully
* physical configurations.
*/
@Test
public void testMasterNodeUnmapAutoUpdate() throws IOException {
// First, heart beats
String host = NetUtils.localhost();
long ver = Clock.unixTime();
flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node1", NodeState.IDLE, ver);
flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node2", NodeState.IDLE, ver);
flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node3", NodeState.IDLE, ver);
flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node4", NodeState.IDLE, ver);
flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "agent", NodeState.IDLE, ver);
// First, spawn so that all are mapped onto a node and now gets a physical
// node info
flumeMaster.getSpecMan().addLogicalNode("host", "node1");
flumeMaster.getSpecMan().addLogicalNode("host", "node2");
flumeMaster.getSpecMan().addLogicalNode("host", "node3");
flumeMaster.getSpecMan().addLogicalNode("host", "node4");
flumeMaster.getSpecMan().addLogicalNode("host", "agent");
// Now do a user initiated unmap should make the config go back to a failing
// version with logicalSinks
flumeMaster.getSpecMan().unmapAllLogicalNodes();
// Look, no explicit update call!
// check new config
Map<String, FlumeConfigData> xcfgs2 = flumeMaster.getSpecMan().getTranslatedConfigs();
FlumeConfigData agentFcd2 = xcfgs2.get("agent");
String ans2 =
"< { lazyOpen => fail( \"logicalSink( \\\"node4\\\" )\" ) } ?"
+ " < { lazyOpen => fail( \"logicalSink( \\\"node2\\\" )\" ) } ?"
+ " < { lazyOpen => fail( \"logicalSink( \\\"node1\\\" )\" ) } ? null > > >";
assertEquals(ans2, agentFcd2.sinkConfig);
}
@Test
public void testCheckup() throws IOException {
MockClock clk = new MockClock(0);
Clock.setClock(clk);
StatusManager stats = new StatusManager();
// foo is in state HELLO and has configuration numbered 0
long prev = Clock.unixTime();
boolean needsRefresh =
stats.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo", NodeState.HELLO, 0);
LOG.info(stats.getNodeStatuses());
assertTrue(needsRefresh);
// move forward in time, but not far enough to trigger being lost
clk.forward(FlumeConfiguration.get().getConfigHeartbeatPeriod() * 5);
stats.checkup();
StatusManager.NodeStatus ns = stats.getNodeStatuses().get("foo");
assertEquals(0, ns.version);
assertTrue(prev <= ns.lastseen);
assertEquals(NodeState.HELLO, ns.state);
prev = ns.lastseen;
clk.forward(FlumeConfiguration.get().getConfigHeartbeatPeriod() * 20);
stats.checkup();
ns = stats.getNodeStatuses().get("foo");
assertEquals(0, ns.version);
assertTrue(prev <= ns.lastseen);
assertEquals(NodeState.LOST, ns.state);
prev = ns.lastseen;
LOG.info(ns.toString());
LOG.info(stats.getStatus("foo").toString());
}
@Test
public void testDuplicateSpawn() throws IOException, InterruptedException {
// use the simple command manger, non-gossip ackmanager
FlumeMaster master =
new FlumeMaster(
new CommandManager(),
new ConfigManager(),
new StatusManager(),
new MasterAckManager(),
cfg);
MasterRPC rpc = new DirectMasterRPC(master);
FlumeNode node = new FlumeNode(rpc, false, false);
// should have nothing.
assertEquals(0, node.getLogicalNodeManager().getNodes().size());
String local = NetUtils.localhost();
// these are spawn commands
master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
master.getSpecMan().addLogicalNode(local, "bar");
master.getSpecMan().addLogicalNode(local, "baz");
// there should not be duplicates in the mapping table.
master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
master.getSpecMan().addLogicalNode(local, "bar");
master.getSpecMan().addLogicalNode(local, "baz");
Multimap<String, String> mapping = master.getSpecMan().getLogicalNodeMap();
assertEquals(3, mapping.size());
LivenessManager liveMan = node.getLivenessManager();
liveMan.checkLogicalNodes();
assertEquals(3, node.getLogicalNodeManager().getNodes().size());
}
/**
* Checks to make sure that nodes specified at the master get spawned at the node.
*
* <p>Even though only two nodes are set, there should be three because we currently have an
* invariant where each physical node must have at least on logical node, which is named the same
* name as the physical node name.
*
* <p>This may go away when we "bulkify" the node-master comms into one rpc call.
*
* @throws InterruptedException
*/
@Test
public void testMasterLogicalNodeCheckAutoLogicalNode() throws IOException, InterruptedException {
FlumeMaster master = new FlumeMaster(cfg);
master.getSpecMan().addLogicalNode(NetUtils.localhost(), "bar");
master.getSpecMan().addLogicalNode(NetUtils.localhost(), "baz");
MasterRPC rpc = new DirectMasterRPC(master);
FlumeNode node = new FlumeNode(rpc, false, false);
// should have nothing.
assertEquals(0, node.getLogicalNodeManager().getNodes().size());
LivenessManager liveMan = node.getLivenessManager();
liveMan.checkLogicalNodes();
// the two added nodes, plus the always present physnode/logical
assertEquals(3, node.getLogicalNodeManager().getNodes().size());
}
@Test
public void testUnmapLogicalNode() throws IOException, InterruptedException {
// use the simple command manger, non-gossip ackmanager
FlumeMaster master =
new FlumeMaster(
new CommandManager(),
new ConfigManager(),
new StatusManager(),
new MasterAckManager(),
cfg);
MasterRPC rpc = new DirectMasterRPC(master);
FlumeNode node = new FlumeNode(rpc, false, false);
// should have nothing.
assertEquals(0, node.getLogicalNodeManager().getNodes().size());
String local = NetUtils.localhost();
master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
master.getSpecMan().addLogicalNode(local, "bar");
master.getSpecMan().addLogicalNode(local, "baz");
LivenessManager liveMan = node.getLivenessManager();
liveMan.checkLogicalNodes();
assertEquals(local, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
assertNull(master.getSpecMan().getConfig("bar"));
assertNull(master.getSpecMan().getConfig("baz"));
assertNull(master.getSpecMan().getConfig(local));
master.getSpecMan().unmapLogicalNode(local, "bar");
liveMan.checkLogicalNodes();
assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
assertNull(master.getSpecMan().getConfig("bar"));
assertNull(master.getSpecMan().getConfig("baz"));
assertNull(master.getSpecMan().getConfig(local));
master.getSpecMan().unmapLogicalNode(local, "baz");
liveMan.checkLogicalNodes();
assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(null, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
assertNull(master.getSpecMan().getConfig("bar"));
assertNull(master.getSpecMan().getConfig("baz"));
assertNull(master.getSpecMan().getConfig(local));
master.getSpecMan().unmapLogicalNode(local, local);
liveMan.checkLogicalNodes();
assertNull(master.getSpecMan().getConfig("bar"));
assertNull(master.getSpecMan().getConfig("baz"));
assertNull(master.getSpecMan().getConfig(local));
assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(null, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
}
@Override
public ReportEvent getMetrics() {
ReportEvent rpt = new ReportEvent(getName());
rpt.setStringMetric(REPORTKEY_HOSTNAME, NetUtils.localhost());
rpt.setLongMetric(REPORTKEY_NODES_REPORTING_COUNT, this.getKnownNodes().size());
return rpt;
}
/**
* This returns true if the host running this process is in the list of master servers. The index
* is set in the FlumeConfiguration. If the host doesn't match, false is returned. If the
* hostnames in the master server list fail to resolve, an exception is thrown.
*/
public static boolean inferMasterHostID() throws UnknownHostException, SocketException {
String masters = FlumeConfiguration.get().getMasterServers();
String[] mtrs = masters.split(",");
int idx = NetUtils.findHostIndex(mtrs);
if (idx < 0) {
String localhost = NetUtils.localhost();
LOG.error(
"Attempted to start a master '{}' that is not " + "in the master servers list: '{}'",
localhost,
mtrs);
// localhost ips weren't in the list.
return false;
}
FlumeConfiguration.get().setInt(FlumeConfiguration.MASTER_SERVER_ID, idx);
LOG.info("Inferred master server index {}", idx);
return true;
}
/** Creates a logical node that will accept any configuration */
public LogicalNode(Context ctx, String name) {
this.nodeName = name;
this.ctx = ctx;
// Note: version and lastSeen aren't kept up-to-date on the logical node.
// The master fills them in when it receives a NodeStatus heartbeat.
state =
new NodeStatus(
NodeState.HELLO,
0,
0,
NetUtils.localhost(),
FlumeNode.getInstance().getPhysicalNodeName());
// Set version to -1 so that all non-negative versions will be 'later'
lastGoodCfg =
new FlumeConfigData(
0,
"null",
"null",
VERSION_INFIMUM,
VERSION_INFIMUM,
FlumeConfiguration.get().getDefaultFlowName());
}
/**
* Test that an autoUpdate happens when a physical node information (heartbeat) shows up and
* allows for a logicalSink/Source translation
*
* <p>This condition is assumed in the following test -- testMasterNodeUnmapAutoUpdate()
*/
@Test
public void testMasterNodeAutoUpdate() throws IOException, FlumeSpecException {
// First, heart beats
String host = NetUtils.localhost();
long ver = Clock.unixTime();
flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node1", NodeState.IDLE, ver);
flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node2", NodeState.IDLE, ver);
flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node3", NodeState.IDLE, ver);
flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node4", NodeState.IDLE, ver);
flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "agent", NodeState.IDLE, ver);
// Next spawn so that all are mapped onto a node and now gets a physical
flumeMaster.getSpecMan().addLogicalNode(host, "node1");
flumeMaster.getSpecMan().addLogicalNode(host, "node2");
flumeMaster.getSpecMan().addLogicalNode(host, "node3");
flumeMaster.getSpecMan().addLogicalNode(host, "node4");
flumeMaster.getSpecMan().addLogicalNode(host, "agent");
// Look, no explicit update call!
// check new config
Map<String, FlumeConfigData> xcfgs2 = flumeMaster.getSpecMan().getTranslatedConfigs();
FlumeConfigData agentFcd2 = xcfgs2.get("agent");
// This is wrong -- there should be a different logicalSink replacing node2
String ans2 =
"< { lazyOpen => rpcSink( \""
+ host
+ "\", 35856 ) } ?"
+ " < { lazyOpen => rpcSink( \""
+ host
+ "\", 35854 ) } ?"
+ " < { lazyOpen => rpcSink( \""
+ host
+ "\", 35853 ) } ? null > > >";
assertEquals(ans2, agentFcd2.sinkConfig);
}
public FlumeNode(FlumeConfiguration conf) {
this(NetUtils.localhost(), conf, false /* http server */, false /* oneshot */);
}
/**
* Returns a Flume Node with settings from specified command line parameters. (See usage for
* instructions)
*
* @param argv
* @return
* @throws IOException
*/
public static FlumeNode setup(String[] argv) throws IOException {
logVersion(LOG);
logEnvironment(LOG);
// Make sure the Java version is not older than 1.6
if (!CheckJavaVersion.isVersionOk()) {
LOG.error("Exiting because of an old Java version or Java version in bad format");
System.exit(-1);
}
LOG.info("Starting flume agent on: " + NetUtils.localhost());
LOG.info(" Working directory is: " + new File(".").getAbsolutePath());
FlumeConfiguration.hardExitLoadConfig(); // will exit if conf file is bad.
CommandLine cmd = null;
Options options = new Options();
options.addOption("c", true, "Load initial config from cmdline arg");
options.addOption("n", true, "Set node name");
options.addOption("s", false, "Do not start local flume status server on node");
options.addOption("1", false, "Make flume node one shot (if closes or errors, exits)");
options.addOption("m", false, "Have flume hard exit if in likely GC thrash situation");
options.addOption("h", false, "Print help information");
options.addOption("v", false, "Print version information");
try {
CommandLineParser parser = new PosixParser();
cmd = parser.parse(options, argv);
} catch (ParseException e) {
HelpFormatter fmt = new HelpFormatter();
fmt.printHelp("FlumeNode", options, true);
return null;
}
// dump version info only
if (cmd != null && cmd.hasOption("v")) {
return null;
}
// dump help info.
if (cmd != null && cmd.hasOption("h")) {
HelpFormatter fmt = new HelpFormatter();
fmt.printHelp("FlumeNode", options, true);
return null;
}
// Check FlumeConfiguration file for settings that may cause node to fail.
nodeConfigChecksOk();
String nodename = NetUtils.localhost(); // default to local host name.
if (cmd != null && cmd.hasOption("n")) {
// select a different name, allow for multiple processes configured
// differently on same node.
nodename = cmd.getOptionValue("n");
}
boolean startHttp = false;
if (cmd != null && !cmd.hasOption("s")) {
// no -s option, start the local status server
startHttp = true;
}
boolean oneshot = false;
if (cmd != null && cmd.hasOption("1")) {
oneshot = true;
}
FormatFactory.loadOutputFormatPlugins();
// Instantiate the flume node.
FlumeConfiguration conf = FlumeConfiguration.get();
FlumeNode flume = new FlumeNode(nodename, conf, startHttp, oneshot);
flume.start();
// load an initial configuration from command line
if (cmd != null && cmd.hasOption("c")) {
String spec = cmd.getOptionValue("c");
LOG.info("Loading spec from command line: '" + spec + "'");
try {
// node name is the default logical and physical name.
Context ctx = new LogicalNodeContext(nodename, nodename);
Map<String, Pair<String, String>> cfgs = FlumeBuilder.parseConf(ctx, spec);
Pair<String, String> node = cfgs.get(nodename);
FlumeConfigData fcd = new FlumeConfigData(0, node.getLeft(), node.getRight(), 0, 0, null);
flume.nodesMan.spawn(ctx, nodename, fcd);
} catch (Exception e) {
LOG.warn("Caught exception loading node:" + e.getMessage());
LOG.debug("Exception: ", e);
if (oneshot) {
System.exit(0); // exit cleanly
}
}
} else {
try {
// default to null configurations.
Context ctx = new LogicalNodeContext(nodename, nodename);
FlumeConfigData fcd = new FlumeConfigData(0, "null", "null", 0, 0, null);
flume.nodesMan.spawn(ctx, nodename, fcd);
} catch (Exception e) {
LOG.error("Caught exception loading node", e);
}
}
if (cmd != null && cmd.hasOption("m")) {
// setup memory use monitor
LOG.info("Setup hard exit on memory exhaustion");
MemoryMonitor.setupHardExitMemMonitor(FlumeConfiguration.get().getAgentMemoryThreshold());
}
try {
tryKerberosLogin();
} catch (IOException ioe) {
LOG.error("Failed to kerberos login.", ioe);
}
// hangout, waiting for other agent thread to exit.
return flume;
}
public FlumeNode(MasterRPC rpc, boolean startHttp, boolean oneshot) {
this(FlumeConfiguration.get(), NetUtils.localhost(), rpc, startHttp, oneshot);
}
@Test
public void testStatusManagerHeartbeats() throws IOException {
StatusManager stats = new StatusManager();
FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
Clock.resetDefault();
cfg.set(FlumeConfiguration.WEBAPPS_PATH, "build/webapps");
cfg.set(FlumeConfiguration.MASTER_STORE, "memory");
// avoiding gossip ack manager until it shuts down cleanly.
ConfigStore cfgStore = FlumeMaster.createConfigStore(cfg);
FlumeMaster fm =
new FlumeMaster(
new CommandManager(), new ConfigManager(cfgStore), stats, new MasterAckManager(), cfg);
fm.getSpecMan().addLogicalNode("physnode", "foo");
// foo is in state HELLO and has configuration numbered 0
long prev = Clock.unixTime();
boolean needsRefresh =
stats.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo", NodeState.HELLO, 0);
LOG.info(stats.getNodeStatuses());
assertTrue(needsRefresh);
StatusManager.NodeStatus ns = stats.getNodeStatuses().get("foo");
assertEquals(0, ns.version);
assertTrue(prev <= ns.lastseen);
assertEquals(NodeState.HELLO, ns.state);
prev = ns.lastseen;
needsRefresh =
stats.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo", NodeState.IDLE, 0);
LOG.info(stats.getNodeStatuses());
assertFalse(needsRefresh); // no new data flow version
assertEquals(0, ns.version);
assertTrue(prev <= ns.lastseen);
assertEquals(NodeState.IDLE, ns.state);
prev = ns.lastseen;
needsRefresh =
stats.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo", NodeState.ACTIVE, 10);
LOG.info(stats.getNodeStatuses());
assertFalse(needsRefresh); // node has updated version number, but master
// has not
assertEquals(10, ns.version);
assertTrue(prev <= ns.lastseen);
assertEquals(NodeState.ACTIVE, ns.state);
prev = ns.lastseen;
// same message, no refresh
needsRefresh =
stats.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo", NodeState.ACTIVE, 10);
LOG.info(stats.getNodeStatuses());
assertFalse(needsRefresh); // this is wierd, is this right?
assertEquals(10, ns.version);
assertTrue(prev <= ns.lastseen);
assertEquals(NodeState.ACTIVE, ns.state);
prev = ns.lastseen;
// regress to an older version,
needsRefresh =
stats.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo", NodeState.ACTIVE, 5);
LOG.info(stats.getNodeStatuses());
assertFalse(needsRefresh);
assertEquals(5, ns.version);
assertTrue(prev <= ns.lastseen);
assertEquals(NodeState.ACTIVE, ns.state);
prev = ns.lastseen;
LOG.info(stats.getReport());
LOG.info(stats.getName());
}
@Test
public void testZKRemoveLogicalNode()
throws IOException, FlumeSpecException, InterruptedException {
// use the simple command manger, non-gossip ackmanager
cfg.set(FlumeConfiguration.MASTER_STORE, "zookeeper");
master =
new FlumeMaster(
new CommandManager(),
new ConfigManager(),
new StatusManager(),
new MasterAckManager(),
cfg);
master.serve();
MasterRPC rpc = new DirectMasterRPC(master);
FlumeNode node = new FlumeNode(rpc, false, false);
// should have nothing.
assertEquals(0, node.getLogicalNodeManager().getNodes().size());
String local = NetUtils.localhost();
master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
master.getSpecMan().addLogicalNode(local, "bar");
master.getSpecMan().addLogicalNode(local, "baz");
master.getSpecMan().setConfig(local, "my-test-flow", "null", "null");
master.getSpecMan().setConfig("bar", "my-test-flow", "null", "null");
master.getSpecMan().setConfig("baz", "my-test-flow", "null", "null");
LivenessManager liveMan = node.getLivenessManager();
liveMan.heartbeatChecks();
// liveMan.checkLogicalNodes();
assertEquals(local, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
assertNotNull(master.getSpecMan().getConfig("bar"));
assertNotNull(master.getSpecMan().getConfig("baz"));
assertNotNull(master.getSpecMan().getConfig(local));
master.getSpecMan().removeLogicalNode("bar");
liveMan.heartbeatChecks();
// liveMan.checkLogicalNodes();
assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
assertNull(master.getSpecMan().getConfig("bar"));
assertNotNull(master.getSpecMan().getConfig("baz"));
assertNotNull(master.getSpecMan().getConfig(local));
master.getSpecMan().removeLogicalNode("baz");
liveMan.heartbeatChecks();
// liveMan.checkLogicalNodes();
assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(null, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
assertNull(master.getSpecMan().getConfig("bar"));
assertNull(master.getSpecMan().getConfig("baz"));
assertNotNull(master.getSpecMan().getConfig(local));
master.getSpecMan().removeLogicalNode(local);
liveMan.heartbeatChecks();
// liveMan.checkLogicalNodes();
assertNull(master.getSpecMan().getConfig("bar"));
assertNull(master.getSpecMan().getConfig("baz"));
assertNull(master.getSpecMan().getConfig(local));
assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
assertEquals(null, master.getSpecMan().getPhysicalNode("baz"));
assertEquals(local, master.getSpecMan().getPhysicalNode(local));
}
/** This is the method that gets run when bin/flume master is executed. */
public static void main(String[] argv) {
FlumeNode.logVersion(LOG);
FlumeNode.logEnvironment(LOG);
// Make sure the Java version is not older than 1.6
if (!CheckJavaVersion.isVersionOk()) {
LOG.error("Exiting because of an old Java version or Java version in bad format");
System.exit(-1);
}
FlumeConfiguration.hardExitLoadConfig(); // if config file is bad hardexit.
CommandLine cmd = null;
Options options = new Options();
options.addOption("c", true, "Load config from file");
options.addOption("f", false, "Use fresh (empty) flume configs");
options.addOption("i", true, "Server id (an integer from 0 up)");
try {
CommandLineParser parser = new PosixParser();
cmd = parser.parse(options, argv);
} catch (ParseException e) {
HelpFormatter fmt = new HelpFormatter();
fmt.printHelp("FlumeNode", options, true);
System.exit(1);
}
FlumeNode.loadOutputFormatPlugins();
String nodeconfig = FlumeConfiguration.get().getMasterSavefile();
if (cmd != null && cmd.hasOption("c")) {
nodeconfig = cmd.getOptionValue("c");
}
if (cmd != null && cmd.hasOption("i")) {
// if manually overridden by command line, accept it, live with
// consequences.
String sid = cmd.getOptionValue("i");
LOG.info("Setting serverid from command line to be " + sid);
try {
int serverid = Integer.parseInt(cmd.getOptionValue("i"));
FlumeConfiguration.get().setInt(FlumeConfiguration.MASTER_SERVER_ID, serverid);
} catch (NumberFormatException e) {
LOG.error("Couldn't parse server id as integer: " + sid);
System.exit(1);
}
} else {
// attempt to auto detect master id.
try {
if (!inferMasterHostID()) {
System.exit(1);
}
} catch (Exception e) {
// master needs to be valid to continue;
LOG.error("Unable to resolve host '{}' ", e.getMessage());
System.exit(1);
}
}
// This will instantiate and read FlumeConfiguration - so make sure that
// this is *after* we set the MASTER_SERVER_ID above.
FlumeMaster config = new FlumeMaster();
LOG.info("Starting flume master on: " + NetUtils.localhost());
LOG.info(" Working Directory is: " + new File(".").getAbsolutePath());
try {
boolean autoload = FlumeConfiguration.get().getMasterSavefileAutoload();
try {
if (autoload && (cmd == null || (cmd != null && !cmd.hasOption("f")))) {
// autoload a config?
config.getSpecMan().loadConfigFile(nodeconfig);
}
} catch (IOException e) {
LOG.warn("Could not autoload config from " + nodeconfig + " because " + e.getMessage());
}
config.serve();
} catch (IOException e) {
LOG.error("IO problem: " + e.getMessage());
LOG.debug("IOException", e);
}
}
/** This is only for testing. */
public AckChecksumInjector(S s) {
this(s, (NetUtils.localhost() + Clock.nanos()).getBytes(), new AckListener.Empty());
}