Ejemplo n.º 1
0
  /**
   * Checks to make sure that nodes specified at the master get spawned at the node.
   *
   * @throws InterruptedException
   */
  @Test
  public void testZKMasterDecomission() throws IOException, InterruptedException {
    // use the simple command manger, non-gossip ackmanager
    cfg.set(FlumeConfiguration.MASTER_STORE, "zookeeper");
    master =
        new FlumeMaster(
            new CommandManager(),
            new ConfigManager(),
            new StatusManager(),
            new MasterAckManager(),
            cfg);
    master.serve();
    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());

    master.getSpecMan().addLogicalNode(NetUtils.localhost(), node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "bar");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "baz");

    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();
    // the two added nodes, plus the always present physnode/logical
    assertEquals(3, node.getLogicalNodeManager().getNodes().size());
  }
Ejemplo n.º 2
0
  /** Checks to make sure that an unmapped node correctly transitions to DECOMMISSIONED */
  @Test
  public void testUnmappedLogicalNodeGetsDecommissioned() throws IOException, InterruptedException {
    FlumeMaster master = new FlumeMaster(cfg);
    MasterRPC rpc = new DirectMasterRPC(master);

    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());

    master.getSpecMan().addLogicalNode(node.getPhysicalNodeName(), "foo");

    master
        .getStatMan()
        .updateHeartbeatStatus(
            NetUtils.localhost(), node.getPhysicalNodeName(), "foo", NodeState.ACTIVE, 10);

    master.getSpecMan().unmapLogicalNode(NetUtils.localhost(), "foo");

    master.getStatMan().checkup();

    assertEquals(NodeState.DECOMMISSIONED, master.getStatMan().getNodeStatuses().get("foo").state);

    master.getSpecMan().addLogicalNode(node.getPhysicalNodeName(), "foo");
    master
        .getStatMan()
        .updateHeartbeatStatus(
            NetUtils.localhost(), node.getPhysicalNodeName(), "foo", NodeState.ACTIVE, 10);

    master.getStatMan().checkup();

    assertEquals(NodeState.ACTIVE, master.getStatMan().getNodeStatuses().get("foo").state);
  }
  /**
   * This heartbeats to provide physical node info and allows the translators to build fully
   * physical configurations.
   */
  @Test
  public void testMasterNodeUnmapAutoUpdate() throws IOException {

    // First, heart beats
    String host = NetUtils.localhost();
    long ver = Clock.unixTime();
    flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node1", NodeState.IDLE, ver);
    flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node2", NodeState.IDLE, ver);
    flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node3", NodeState.IDLE, ver);
    flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node4", NodeState.IDLE, ver);
    flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "agent", NodeState.IDLE, ver);

    // First, spawn so that all are mapped onto a node and now gets a physical
    // node info
    flumeMaster.getSpecMan().addLogicalNode("host", "node1");
    flumeMaster.getSpecMan().addLogicalNode("host", "node2");
    flumeMaster.getSpecMan().addLogicalNode("host", "node3");
    flumeMaster.getSpecMan().addLogicalNode("host", "node4");
    flumeMaster.getSpecMan().addLogicalNode("host", "agent");

    // Now do a user initiated unmap should make the config go back to a failing
    // version with logicalSinks
    flumeMaster.getSpecMan().unmapAllLogicalNodes();

    // Look, no explicit update call!

    // check new config
    Map<String, FlumeConfigData> xcfgs2 = flumeMaster.getSpecMan().getTranslatedConfigs();
    FlumeConfigData agentFcd2 = xcfgs2.get("agent");
    String ans2 =
        "< { lazyOpen => fail( \"logicalSink( \\\"node4\\\" )\" ) } ?"
            + " < { lazyOpen => fail( \"logicalSink( \\\"node2\\\" )\" ) } ?"
            + " < { lazyOpen => fail( \"logicalSink( \\\"node1\\\" )\" ) } ? null > > >";
    assertEquals(ans2, agentFcd2.sinkConfig);
  }
Ejemplo n.º 4
0
  @Test
  public void testCheckup() throws IOException {
    MockClock clk = new MockClock(0);
    Clock.setClock(clk);
    StatusManager stats = new StatusManager();

    // foo is in state HELLO and has configuration numbered 0
    long prev = Clock.unixTime();
    boolean needsRefresh =
        stats.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo", NodeState.HELLO, 0);
    LOG.info(stats.getNodeStatuses());
    assertTrue(needsRefresh);

    // move forward in time, but not far enough to trigger being lost
    clk.forward(FlumeConfiguration.get().getConfigHeartbeatPeriod() * 5);
    stats.checkup();
    StatusManager.NodeStatus ns = stats.getNodeStatuses().get("foo");
    assertEquals(0, ns.version);
    assertTrue(prev <= ns.lastseen);
    assertEquals(NodeState.HELLO, ns.state);
    prev = ns.lastseen;

    clk.forward(FlumeConfiguration.get().getConfigHeartbeatPeriod() * 20);
    stats.checkup();
    ns = stats.getNodeStatuses().get("foo");
    assertEquals(0, ns.version);
    assertTrue(prev <= ns.lastseen);
    assertEquals(NodeState.LOST, ns.state);
    prev = ns.lastseen;

    LOG.info(ns.toString());
    LOG.info(stats.getStatus("foo").toString());
  }
Ejemplo n.º 5
0
  @Test
  public void testDuplicateSpawn() throws IOException, InterruptedException {
    // use the simple command manger, non-gossip ackmanager
    FlumeMaster master =
        new FlumeMaster(
            new CommandManager(),
            new ConfigManager(),
            new StatusManager(),
            new MasterAckManager(),
            cfg);
    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());

    String local = NetUtils.localhost();

    // these are spawn commands
    master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(local, "bar");
    master.getSpecMan().addLogicalNode(local, "baz");

    // there should not be duplicates in the mapping table.
    master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(local, "bar");
    master.getSpecMan().addLogicalNode(local, "baz");

    Multimap<String, String> mapping = master.getSpecMan().getLogicalNodeMap();
    assertEquals(3, mapping.size());

    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();
    assertEquals(3, node.getLogicalNodeManager().getNodes().size());
  }
Ejemplo n.º 6
0
  /**
   * Checks to make sure that nodes specified at the master get spawned at the node.
   *
   * <p>Even though only two nodes are set, there should be three because we currently have an
   * invariant where each physical node must have at least on logical node, which is named the same
   * name as the physical node name.
   *
   * <p>This may go away when we "bulkify" the node-master comms into one rpc call.
   *
   * @throws InterruptedException
   */
  @Test
  public void testMasterLogicalNodeCheckAutoLogicalNode() throws IOException, InterruptedException {
    FlumeMaster master = new FlumeMaster(cfg);
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "bar");
    master.getSpecMan().addLogicalNode(NetUtils.localhost(), "baz");

    MasterRPC rpc = new DirectMasterRPC(master);

    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());

    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();
    // the two added nodes, plus the always present physnode/logical
    assertEquals(3, node.getLogicalNodeManager().getNodes().size());
  }
Ejemplo n.º 7
0
  @Test
  public void testUnmapLogicalNode() throws IOException, InterruptedException {
    // use the simple command manger, non-gossip ackmanager
    FlumeMaster master =
        new FlumeMaster(
            new CommandManager(),
            new ConfigManager(),
            new StatusManager(),
            new MasterAckManager(),
            cfg);
    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());

    String local = NetUtils.localhost();

    master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(local, "bar");
    master.getSpecMan().addLogicalNode(local, "baz");

    LivenessManager liveMan = node.getLivenessManager();
    liveMan.checkLogicalNodes();

    assertEquals(local, master.getSpecMan().getPhysicalNode("bar"));
    assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
    assertEquals(local, master.getSpecMan().getPhysicalNode(local));
    assertNull(master.getSpecMan().getConfig("bar"));
    assertNull(master.getSpecMan().getConfig("baz"));
    assertNull(master.getSpecMan().getConfig(local));

    master.getSpecMan().unmapLogicalNode(local, "bar");
    liveMan.checkLogicalNodes();
    assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
    assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
    assertEquals(local, master.getSpecMan().getPhysicalNode(local));
    assertNull(master.getSpecMan().getConfig("bar"));
    assertNull(master.getSpecMan().getConfig("baz"));
    assertNull(master.getSpecMan().getConfig(local));

    master.getSpecMan().unmapLogicalNode(local, "baz");
    liveMan.checkLogicalNodes();
    assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
    assertEquals(null, master.getSpecMan().getPhysicalNode("baz"));
    assertEquals(local, master.getSpecMan().getPhysicalNode(local));
    assertNull(master.getSpecMan().getConfig("bar"));
    assertNull(master.getSpecMan().getConfig("baz"));
    assertNull(master.getSpecMan().getConfig(local));

    master.getSpecMan().unmapLogicalNode(local, local);
    liveMan.checkLogicalNodes();
    assertNull(master.getSpecMan().getConfig("bar"));
    assertNull(master.getSpecMan().getConfig("baz"));
    assertNull(master.getSpecMan().getConfig(local));
    assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
    assertEquals(null, master.getSpecMan().getPhysicalNode("baz"));
    assertEquals(local, master.getSpecMan().getPhysicalNode(local));
  }
Ejemplo n.º 8
0
  @Override
  public ReportEvent getMetrics() {
    ReportEvent rpt = new ReportEvent(getName());

    rpt.setStringMetric(REPORTKEY_HOSTNAME, NetUtils.localhost());

    rpt.setLongMetric(REPORTKEY_NODES_REPORTING_COUNT, this.getKnownNodes().size());

    return rpt;
  }
Ejemplo n.º 9
0
  /**
   * This returns true if the host running this process is in the list of master servers. The index
   * is set in the FlumeConfiguration. If the host doesn't match, false is returned. If the
   * hostnames in the master server list fail to resolve, an exception is thrown.
   */
  public static boolean inferMasterHostID() throws UnknownHostException, SocketException {
    String masters = FlumeConfiguration.get().getMasterServers();
    String[] mtrs = masters.split(",");

    int idx = NetUtils.findHostIndex(mtrs);
    if (idx < 0) {

      String localhost = NetUtils.localhost();
      LOG.error(
          "Attempted to start a master '{}' that is not " + "in the master servers list: '{}'",
          localhost,
          mtrs);
      // localhost ips weren't in the list.
      return false;
    }

    FlumeConfiguration.get().setInt(FlumeConfiguration.MASTER_SERVER_ID, idx);
    LOG.info("Inferred master server index {}", idx);
    return true;
  }
Ejemplo n.º 10
0
  /** Creates a logical node that will accept any configuration */
  public LogicalNode(Context ctx, String name) {
    this.nodeName = name;
    this.ctx = ctx;

    // Note: version and lastSeen aren't kept up-to-date on the logical node.
    // The master fills them in when it receives a NodeStatus heartbeat.
    state =
        new NodeStatus(
            NodeState.HELLO,
            0,
            0,
            NetUtils.localhost(),
            FlumeNode.getInstance().getPhysicalNodeName());
    // Set version to -1 so that all non-negative versions will be 'later'
    lastGoodCfg =
        new FlumeConfigData(
            0,
            "null",
            "null",
            VERSION_INFIMUM,
            VERSION_INFIMUM,
            FlumeConfiguration.get().getDefaultFlowName());
  }
  /**
   * Test that an autoUpdate happens when a physical node information (heartbeat) shows up and
   * allows for a logicalSink/Source translation
   *
   * <p>This condition is assumed in the following test -- testMasterNodeUnmapAutoUpdate()
   */
  @Test
  public void testMasterNodeAutoUpdate() throws IOException, FlumeSpecException {

    // First, heart beats
    String host = NetUtils.localhost();
    long ver = Clock.unixTime();
    flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node1", NodeState.IDLE, ver);
    flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node2", NodeState.IDLE, ver);
    flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node3", NodeState.IDLE, ver);
    flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "node4", NodeState.IDLE, ver);
    flumeMaster.getStatMan().updateHeartbeatStatus(host, "physnode", "agent", NodeState.IDLE, ver);

    // Next spawn so that all are mapped onto a node and now gets a physical
    flumeMaster.getSpecMan().addLogicalNode(host, "node1");
    flumeMaster.getSpecMan().addLogicalNode(host, "node2");
    flumeMaster.getSpecMan().addLogicalNode(host, "node3");
    flumeMaster.getSpecMan().addLogicalNode(host, "node4");
    flumeMaster.getSpecMan().addLogicalNode(host, "agent");

    // Look, no explicit update call!

    // check new config
    Map<String, FlumeConfigData> xcfgs2 = flumeMaster.getSpecMan().getTranslatedConfigs();
    FlumeConfigData agentFcd2 = xcfgs2.get("agent");
    // This is wrong -- there should be a different logicalSink replacing node2
    String ans2 =
        "< { lazyOpen => rpcSink( \""
            + host
            + "\", 35856 ) } ?"
            + " < { lazyOpen => rpcSink( \""
            + host
            + "\", 35854 ) } ?"
            + " < { lazyOpen => rpcSink( \""
            + host
            + "\", 35853 ) } ? null > > >";
    assertEquals(ans2, agentFcd2.sinkConfig);
  }
Ejemplo n.º 12
0
 public FlumeNode(FlumeConfiguration conf) {
   this(NetUtils.localhost(), conf, false /* http server */, false /* oneshot */);
 }
Ejemplo n.º 13
0
  /**
   * Returns a Flume Node with settings from specified command line parameters. (See usage for
   * instructions)
   *
   * @param argv
   * @return
   * @throws IOException
   */
  public static FlumeNode setup(String[] argv) throws IOException {
    logVersion(LOG);
    logEnvironment(LOG);
    // Make sure the Java version is not older than 1.6
    if (!CheckJavaVersion.isVersionOk()) {
      LOG.error("Exiting because of an old Java version or Java version in bad format");
      System.exit(-1);
    }
    LOG.info("Starting flume agent on: " + NetUtils.localhost());
    LOG.info(" Working directory is: " + new File(".").getAbsolutePath());

    FlumeConfiguration.hardExitLoadConfig(); // will exit if conf file is bad.

    CommandLine cmd = null;
    Options options = new Options();
    options.addOption("c", true, "Load initial config from cmdline arg");
    options.addOption("n", true, "Set node name");
    options.addOption("s", false, "Do not start local flume status server on node");
    options.addOption("1", false, "Make flume node one shot (if closes or errors, exits)");
    options.addOption("m", false, "Have flume hard exit if in likely GC thrash situation");
    options.addOption("h", false, "Print help information");
    options.addOption("v", false, "Print version information");
    try {
      CommandLineParser parser = new PosixParser();
      cmd = parser.parse(options, argv);
    } catch (ParseException e) {
      HelpFormatter fmt = new HelpFormatter();
      fmt.printHelp("FlumeNode", options, true);
      return null;
    }

    // dump version info only
    if (cmd != null && cmd.hasOption("v")) {
      return null;
    }

    // dump help info.
    if (cmd != null && cmd.hasOption("h")) {
      HelpFormatter fmt = new HelpFormatter();
      fmt.printHelp("FlumeNode", options, true);
      return null;
    }
    // Check FlumeConfiguration file for settings that may cause node to fail.
    nodeConfigChecksOk();

    String nodename = NetUtils.localhost(); // default to local host name.
    if (cmd != null && cmd.hasOption("n")) {
      // select a different name, allow for multiple processes configured
      // differently on same node.
      nodename = cmd.getOptionValue("n");
    }

    boolean startHttp = false;
    if (cmd != null && !cmd.hasOption("s")) {
      // no -s option, start the local status server
      startHttp = true;
    }

    boolean oneshot = false;
    if (cmd != null && cmd.hasOption("1")) {
      oneshot = true;
    }

    FormatFactory.loadOutputFormatPlugins();

    // Instantiate the flume node.
    FlumeConfiguration conf = FlumeConfiguration.get();

    FlumeNode flume = new FlumeNode(nodename, conf, startHttp, oneshot);

    flume.start();

    // load an initial configuration from command line
    if (cmd != null && cmd.hasOption("c")) {
      String spec = cmd.getOptionValue("c");
      LOG.info("Loading spec from command line: '" + spec + "'");

      try {
        // node name is the default logical and physical name.
        Context ctx = new LogicalNodeContext(nodename, nodename);
        Map<String, Pair<String, String>> cfgs = FlumeBuilder.parseConf(ctx, spec);
        Pair<String, String> node = cfgs.get(nodename);
        FlumeConfigData fcd = new FlumeConfigData(0, node.getLeft(), node.getRight(), 0, 0, null);
        flume.nodesMan.spawn(ctx, nodename, fcd);
      } catch (Exception e) {
        LOG.warn("Caught exception loading node:" + e.getMessage());
        LOG.debug("Exception: ", e);
        if (oneshot) {
          System.exit(0); // exit cleanly
        }
      }

    } else {
      try {
        // default to null configurations.
        Context ctx = new LogicalNodeContext(nodename, nodename);
        FlumeConfigData fcd = new FlumeConfigData(0, "null", "null", 0, 0, null);
        flume.nodesMan.spawn(ctx, nodename, fcd);
      } catch (Exception e) {
        LOG.error("Caught exception loading node", e);
      }
    }

    if (cmd != null && cmd.hasOption("m")) {
      // setup memory use monitor
      LOG.info("Setup hard exit on memory exhaustion");
      MemoryMonitor.setupHardExitMemMonitor(FlumeConfiguration.get().getAgentMemoryThreshold());
    }

    try {
      tryKerberosLogin();
    } catch (IOException ioe) {
      LOG.error("Failed to kerberos login.", ioe);
    }

    // hangout, waiting for other agent thread to exit.
    return flume;
  }
Ejemplo n.º 14
0
 public FlumeNode(MasterRPC rpc, boolean startHttp, boolean oneshot) {
   this(FlumeConfiguration.get(), NetUtils.localhost(), rpc, startHttp, oneshot);
 }
Ejemplo n.º 15
0
  @Test
  public void testStatusManagerHeartbeats() throws IOException {
    StatusManager stats = new StatusManager();

    FlumeConfiguration cfg = FlumeConfiguration.createTestableConfiguration();
    Clock.resetDefault();
    cfg.set(FlumeConfiguration.WEBAPPS_PATH, "build/webapps");
    cfg.set(FlumeConfiguration.MASTER_STORE, "memory");

    // avoiding gossip ack manager until it shuts down cleanly.
    ConfigStore cfgStore = FlumeMaster.createConfigStore(cfg);
    FlumeMaster fm =
        new FlumeMaster(
            new CommandManager(), new ConfigManager(cfgStore), stats, new MasterAckManager(), cfg);

    fm.getSpecMan().addLogicalNode("physnode", "foo");

    // foo is in state HELLO and has configuration numbered 0
    long prev = Clock.unixTime();
    boolean needsRefresh =
        stats.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo", NodeState.HELLO, 0);
    LOG.info(stats.getNodeStatuses());
    assertTrue(needsRefresh);

    StatusManager.NodeStatus ns = stats.getNodeStatuses().get("foo");
    assertEquals(0, ns.version);
    assertTrue(prev <= ns.lastseen);
    assertEquals(NodeState.HELLO, ns.state);
    prev = ns.lastseen;

    needsRefresh =
        stats.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo", NodeState.IDLE, 0);
    LOG.info(stats.getNodeStatuses());
    assertFalse(needsRefresh); // no new data flow version
    assertEquals(0, ns.version);
    assertTrue(prev <= ns.lastseen);
    assertEquals(NodeState.IDLE, ns.state);
    prev = ns.lastseen;

    needsRefresh =
        stats.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo", NodeState.ACTIVE, 10);
    LOG.info(stats.getNodeStatuses());
    assertFalse(needsRefresh); // node has updated version number, but master
    // has not
    assertEquals(10, ns.version);
    assertTrue(prev <= ns.lastseen);
    assertEquals(NodeState.ACTIVE, ns.state);
    prev = ns.lastseen;

    // same message, no refresh
    needsRefresh =
        stats.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo", NodeState.ACTIVE, 10);
    LOG.info(stats.getNodeStatuses());
    assertFalse(needsRefresh); // this is wierd, is this right?
    assertEquals(10, ns.version);
    assertTrue(prev <= ns.lastseen);
    assertEquals(NodeState.ACTIVE, ns.state);
    prev = ns.lastseen;

    // regress to an older version,
    needsRefresh =
        stats.updateHeartbeatStatus(NetUtils.localhost(), "physnode", "foo", NodeState.ACTIVE, 5);
    LOG.info(stats.getNodeStatuses());
    assertFalse(needsRefresh);
    assertEquals(5, ns.version);
    assertTrue(prev <= ns.lastseen);
    assertEquals(NodeState.ACTIVE, ns.state);
    prev = ns.lastseen;

    LOG.info(stats.getReport());
    LOG.info(stats.getName());
  }
Ejemplo n.º 16
0
  @Test
  public void testZKRemoveLogicalNode()
      throws IOException, FlumeSpecException, InterruptedException {
    // use the simple command manger, non-gossip ackmanager
    cfg.set(FlumeConfiguration.MASTER_STORE, "zookeeper");

    master =
        new FlumeMaster(
            new CommandManager(),
            new ConfigManager(),
            new StatusManager(),
            new MasterAckManager(),
            cfg);
    master.serve();
    MasterRPC rpc = new DirectMasterRPC(master);
    FlumeNode node = new FlumeNode(rpc, false, false);
    // should have nothing.
    assertEquals(0, node.getLogicalNodeManager().getNodes().size());

    String local = NetUtils.localhost();

    master.getSpecMan().addLogicalNode(local, node.getPhysicalNodeName());
    master.getSpecMan().addLogicalNode(local, "bar");
    master.getSpecMan().addLogicalNode(local, "baz");
    master.getSpecMan().setConfig(local, "my-test-flow", "null", "null");
    master.getSpecMan().setConfig("bar", "my-test-flow", "null", "null");
    master.getSpecMan().setConfig("baz", "my-test-flow", "null", "null");

    LivenessManager liveMan = node.getLivenessManager();
    liveMan.heartbeatChecks();
    // liveMan.checkLogicalNodes();

    assertEquals(local, master.getSpecMan().getPhysicalNode("bar"));
    assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
    assertEquals(local, master.getSpecMan().getPhysicalNode(local));
    assertNotNull(master.getSpecMan().getConfig("bar"));
    assertNotNull(master.getSpecMan().getConfig("baz"));
    assertNotNull(master.getSpecMan().getConfig(local));

    master.getSpecMan().removeLogicalNode("bar");
    liveMan.heartbeatChecks();
    // liveMan.checkLogicalNodes();
    assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
    assertEquals(local, master.getSpecMan().getPhysicalNode("baz"));
    assertEquals(local, master.getSpecMan().getPhysicalNode(local));
    assertNull(master.getSpecMan().getConfig("bar"));
    assertNotNull(master.getSpecMan().getConfig("baz"));
    assertNotNull(master.getSpecMan().getConfig(local));

    master.getSpecMan().removeLogicalNode("baz");
    liveMan.heartbeatChecks();
    // liveMan.checkLogicalNodes();
    assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
    assertEquals(null, master.getSpecMan().getPhysicalNode("baz"));
    assertEquals(local, master.getSpecMan().getPhysicalNode(local));
    assertNull(master.getSpecMan().getConfig("bar"));
    assertNull(master.getSpecMan().getConfig("baz"));
    assertNotNull(master.getSpecMan().getConfig(local));

    master.getSpecMan().removeLogicalNode(local);
    liveMan.heartbeatChecks();
    // liveMan.checkLogicalNodes();
    assertNull(master.getSpecMan().getConfig("bar"));
    assertNull(master.getSpecMan().getConfig("baz"));
    assertNull(master.getSpecMan().getConfig(local));
    assertEquals(null, master.getSpecMan().getPhysicalNode("bar"));
    assertEquals(null, master.getSpecMan().getPhysicalNode("baz"));
    assertEquals(local, master.getSpecMan().getPhysicalNode(local));
  }
Ejemplo n.º 17
0
  /** This is the method that gets run when bin/flume master is executed. */
  public static void main(String[] argv) {
    FlumeNode.logVersion(LOG);
    FlumeNode.logEnvironment(LOG);
    // Make sure the Java version is not older than 1.6
    if (!CheckJavaVersion.isVersionOk()) {
      LOG.error("Exiting because of an old Java version or Java version in bad format");
      System.exit(-1);
    }
    FlumeConfiguration.hardExitLoadConfig(); // if config file is bad hardexit.

    CommandLine cmd = null;
    Options options = new Options();
    options.addOption("c", true, "Load config from file");
    options.addOption("f", false, "Use fresh (empty) flume configs");
    options.addOption("i", true, "Server id (an integer from 0 up)");

    try {
      CommandLineParser parser = new PosixParser();
      cmd = parser.parse(options, argv);
    } catch (ParseException e) {
      HelpFormatter fmt = new HelpFormatter();
      fmt.printHelp("FlumeNode", options, true);
      System.exit(1);
    }

    FlumeNode.loadOutputFormatPlugins();

    String nodeconfig = FlumeConfiguration.get().getMasterSavefile();

    if (cmd != null && cmd.hasOption("c")) {
      nodeconfig = cmd.getOptionValue("c");
    }

    if (cmd != null && cmd.hasOption("i")) {
      // if manually overridden by command line, accept it, live with
      // consequences.
      String sid = cmd.getOptionValue("i");
      LOG.info("Setting serverid from command line to be " + sid);
      try {
        int serverid = Integer.parseInt(cmd.getOptionValue("i"));
        FlumeConfiguration.get().setInt(FlumeConfiguration.MASTER_SERVER_ID, serverid);
      } catch (NumberFormatException e) {
        LOG.error("Couldn't parse server id as integer: " + sid);
        System.exit(1);
      }
    } else {
      // attempt to auto detect master id.
      try {
        if (!inferMasterHostID()) {
          System.exit(1);
        }
      } catch (Exception e) {
        // master needs to be valid to continue;
        LOG.error("Unable to resolve host '{}' ", e.getMessage());
        System.exit(1);
      }
    }

    // This will instantiate and read FlumeConfiguration - so make sure that
    // this is *after* we set the MASTER_SERVER_ID above.
    FlumeMaster config = new FlumeMaster();
    LOG.info("Starting flume master on: " + NetUtils.localhost());
    LOG.info(" Working Directory is: " + new File(".").getAbsolutePath());

    try {
      boolean autoload = FlumeConfiguration.get().getMasterSavefileAutoload();
      try {
        if (autoload && (cmd == null || (cmd != null && !cmd.hasOption("f")))) {
          // autoload a config?
          config.getSpecMan().loadConfigFile(nodeconfig);
        }
      } catch (IOException e) {
        LOG.warn("Could not autoload config from " + nodeconfig + " because " + e.getMessage());
      }
      config.serve();

    } catch (IOException e) {
      LOG.error("IO problem: " + e.getMessage());
      LOG.debug("IOException", e);
    }
  }
Ejemplo n.º 18
0
 /** This is only for testing. */
 public AckChecksumInjector(S s) {
   this(s, (NetUtils.localhost() + Clock.nanos()).getBytes(), new AckListener.Empty());
 }