Exemple #1
0
  /**
   * Start the specified number of master servers with ports starting from a specified number. Finds
   * free web and RPC ports up front for all of the masters first, then starts them on those ports,
   * populating 'masters' map.
   *
   * @param masterStartPort The starting of the port range for the masters.
   * @param numMasters Number of masters to start.
   * @param baseDirPath Kudu base directory.
   * @return Next free port.
   * @throws Exception If we are unable to start the masters.
   */
  static int startMasters(int masterStartPort, int numMasters, String baseDirPath)
      throws Exception {
    LOG.info("Starting {} masters...", numMasters);
    // Get the list of web and RPC ports to use for the master consensus configuration:
    // request NUM_MASTERS * 2 free ports as we want to also reserve the web
    // ports for the consensus configuration.
    List<Integer> ports = TestUtils.findFreePorts(masterStartPort, numMasters * 2);
    int lastFreePort = ports.get(ports.size() - 1);
    List<Integer> masterRpcPorts = Lists.newArrayListWithCapacity(numMasters);
    List<Integer> masterWebPorts = Lists.newArrayListWithCapacity(numMasters);
    masterHostPorts = Lists.newArrayListWithCapacity(numMasters);
    for (int i = 0; i < numMasters * 2; i++) {
      if (i % 2 == 0) {
        masterRpcPorts.add(ports.get(i));
        masterHostPorts.add(HostAndPort.fromParts("127.0.0.1", ports.get(i)));
      } else {
        masterWebPorts.add(ports.get(i));
      }
    }
    masterAddresses = NetUtil.hostsAndPortsToString(masterHostPorts);
    for (int i = 0; i < numMasters; i++) {
      long now = System.currentTimeMillis();
      String dataDirPath = baseDirPath + "/master-" + i + "-" + now;
      String flagsPath = TestUtils.getFlagsPath();
      // The web port must be reserved in the call to findFreePorts above and specified
      // to avoid the scenario where:
      // 1) findFreePorts finds RPC ports a, b, c for the 3 masters.
      // 2) start master 1 with RPC port and let it bind to any (specified as 0) web port.
      // 3) master 1 happens to bind to port b for the web port, as master 2 hasn't been
      // started yet and findFreePort(s) is "check-time-of-use" (it does not reserve the
      // ports, only checks that when it was last called, these ports could be used).
      List<String> masterCmdLine =
          Lists.newArrayList(
              TestUtils.findBinary("kudu-master"),
              "--flagfile=" + flagsPath,
              "--fs_wal_dir=" + dataDirPath,
              "--fs_data_dirs=" + dataDirPath,
              "--rpc_bind_addresses=127.0.0.1:" + masterRpcPorts.get(i),
              "--webserver_port=" + masterWebPorts.get(i));
      if (numMasters > 1) {
        masterCmdLine.add("--master_addresses=" + masterAddresses);
      }
      MASTERS.put(
          masterRpcPorts.get(i),
          configureAndStartProcess(masterCmdLine.toArray(new String[masterCmdLine.size()])));

      if (flagsPath.startsWith(baseDirPath)) {
        // We made a temporary copy of the flags; delete them later.
        pathsToDelete.add(flagsPath);
      }
      pathsToDelete.add(dataDirPath);
    }
    return lastFreePort + 1;
  }
Exemple #2
0
  @BeforeClass
  public static void setUpBeforeClass() throws Exception {
    LOG.info("Setting up before class...");
    // The following props are set via kudu-client's pom.
    String baseDirPath = TestUtils.getBaseDir();
    startCluster = Boolean.parseBoolean(System.getProperty(START_CLUSTER, "true"));

    if (startCluster) {
      long now = System.currentTimeMillis();
      LOG.info("Starting {} masters...", NUM_MASTERS);
      int port = startMasters(PORT_START, NUM_MASTERS, baseDirPath);
      LOG.info("Starting {} tablet servers...", NUM_TABLET_SERVERS);
      for (int i = 0; i < NUM_TABLET_SERVERS; i++) {
        port = TestUtils.findFreePort(port);
        String dataDirPath = baseDirPath + "/ts-" + i + "-" + now;
        String flagsPath = TestUtils.getFlagsPath();
        String[] tsCmdLine = {
          TestUtils.findBinary("kudu-tserver"),
          "--flagfile=" + flagsPath,
          "--fs_wal_dir=" + dataDirPath,
          "--fs_data_dirs=" + dataDirPath,
          "--tserver_master_addrs=" + masterAddresses,
          "--rpc_bind_addresses=127.0.0.1:" + port
        };
        TABLET_SERVERS.put(port, configureAndStartProcess(tsCmdLine));
        port++;

        if (flagsPath.startsWith(baseDirPath)) {
          // We made a temporary copy of the flags; delete them later.
          pathsToDelete.add(flagsPath);
        }
        pathsToDelete.add(dataDirPath);
      }
    } else {
      masterAddresses = TestUtils.getMasterAddresses();
      masterHostPorts = NetUtil.parseStrings(masterAddresses, DEFAULT_MASTER_RPC_PORT);
    }
    LOG.info("Creating new Kudu client...");
    client = new AsyncKuduClient.AsyncKuduClientBuilder(masterAddresses).build();
    syncClient = new KuduClient(client);
    LOG.info("Waiting for tablet servers...");
    if (!waitForTabletServers(NUM_TABLET_SERVERS)) {
      fail("Couldn't get " + NUM_MASTERS + " tablet servers running, aborting");
    }
  }