@BeforeClass
public static void createTribes() throws NodeValidationException {
Settings baseSettings =
Settings.builder()
.put(NetworkModule.HTTP_ENABLED.getKey(), false)
.put("transport.type", MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME)
.put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "local")
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir())
.put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), 2)
.build();
tribe1 =
new TribeClientNode(
Settings.builder()
.put(baseSettings)
.put("cluster.name", "tribe1")
.put("node.name", "tribe1_node")
.put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
.build(),
Collections.singleton(MockTcpTransportPlugin.class))
.start();
tribe2 =
new TribeClientNode(
Settings.builder()
.put(baseSettings)
.put("cluster.name", "tribe2")
.put("node.name", "tribe2_node")
.put(NodeEnvironment.NODE_ID_SEED_SETTING.getKey(), random().nextLong())
.build(),
Collections.singleton(MockTcpTransportPlugin.class))
.start();
}
public static Settings processSettings(Settings settings) {
if (settings.get(TRIBE_NAME) != null) {
// if its a node client started by this service as tribe, remove any tribe group setting
// to avoid recursive configuration
Settings.Builder sb = Settings.builder().put(settings);
for (String s : settings.getAsMap().keySet()) {
if (s.startsWith("tribe.") && !s.equals(TRIBE_NAME)) {
sb.remove(s);
}
}
return sb.build();
}
Map<String, Settings> nodesSettings = settings.getGroups("tribe", true);
if (nodesSettings.isEmpty()) {
return settings;
}
// its a tribe configured node..., force settings
Settings.Builder sb = Settings.builder().put(settings);
sb.put(Node.NODE_CLIENT_SETTING.getKey(), true); // this node should just act as a node client
sb.put(
DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(),
"local"); // a tribe node should not use zen discovery
sb.put(
DiscoveryService.INITIAL_STATE_TIMEOUT_SETTING.getKey(),
0); // nothing is going to be discovered, since no master will be elected
if (sb.get("cluster.name") == null) {
sb.put(
"cluster.name",
"tribe_"
+ Strings
.randomBase64UUID()); // make sure it won't join other tribe nodes in the same JVM
}
sb.put(TransportMasterNodeReadAction.FORCE_LOCAL_SETTING, true);
return sb.build();
}
/** Simple helper class to start external nodes to be used within a test cluster */
final class ExternalNode implements Closeable {
public static final Settings REQUIRED_SETTINGS =
Settings.builder()
.put(InternalSettingsPreparer.IGNORE_SYSTEM_PROPERTIES_SETTING.getKey(), true)
.put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "zen")
.put(Node.NODE_MODE_SETTING.getKey(), "network")
.build(); // we need network mode for this
private final Path path;
private final Random random;
private final NodeConfigurationSource nodeConfigurationSource;
private Process process;
private NodeInfo nodeInfo;
private final String clusterName;
private TransportClient client;
private final ESLogger logger = Loggers.getLogger(getClass());
private Settings externalNodeSettings;
ExternalNode(Path path, long seed, NodeConfigurationSource nodeConfigurationSource) {
this(path, null, seed, nodeConfigurationSource);
}
ExternalNode(
Path path, String clusterName, long seed, NodeConfigurationSource nodeConfigurationSource) {
if (!Files.isDirectory(path)) {
throw new IllegalArgumentException("path must be a directory");
}
this.path = path;
this.clusterName = clusterName;
this.random = new Random(seed);
this.nodeConfigurationSource = nodeConfigurationSource;
}
synchronized ExternalNode start(
Client localNode,
Settings defaultSettings,
String nodeName,
String clusterName,
int nodeOrdinal)
throws IOException, InterruptedException {
ExternalNode externalNode =
new ExternalNode(path, clusterName, random.nextLong(), nodeConfigurationSource);
Settings settings =
Settings.builder()
.put(defaultSettings)
.put(nodeConfigurationSource.nodeSettings(nodeOrdinal))
.build();
externalNode.startInternal(localNode, settings, nodeName, clusterName);
return externalNode;
}
@SuppressForbidden(reason = "needs java.io.File api to start a process")
synchronized void startInternal(
Client client, Settings settings, String nodeName, String clusterName)
throws IOException, InterruptedException {
if (process != null) {
throw new IllegalStateException("Already started");
}
List<String> params = new ArrayList<>();
if (!Constants.WINDOWS) {
params.add("bin/elasticsearch");
} else {
params.add("bin/elasticsearch.bat");
}
params.add("-Des.cluster.name=" + clusterName);
params.add("-Des.node.name=" + nodeName);
Settings.Builder externaNodeSettingsBuilder = Settings.builder();
for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
switch (entry.getKey()) {
case "cluster.name":
case "node.name":
case "path.home":
case "node.mode":
case "node.local":
case NetworkModule.TRANSPORT_TYPE_KEY:
case "discovery.type":
case NetworkModule.TRANSPORT_SERVICE_TYPE_KEY:
case "config.ignore_system_properties":
continue;
default:
externaNodeSettingsBuilder.put(entry.getKey(), entry.getValue());
}
}
this.externalNodeSettings = externaNodeSettingsBuilder.put(REQUIRED_SETTINGS).build();
for (Map.Entry<String, String> entry : externalNodeSettings.getAsMap().entrySet()) {
params.add("-Des." + entry.getKey() + "=" + entry.getValue());
}
params.add("-Des.path.home=" + PathUtils.get(".").toAbsolutePath());
params.add("-Des.path.conf=" + path + "/config");
ProcessBuilder builder = new ProcessBuilder(params);
builder.directory(path.toFile());
builder.inheritIO();
boolean success = false;
try {
logger.info("starting external node [{}] with: {}", nodeName, builder.command());
process = builder.start();
this.nodeInfo = null;
if (waitForNode(client, nodeName)) {
nodeInfo = nodeInfo(client, nodeName);
assert nodeInfo != null;
logger.info(
"external node {} found, version [{}], build {}",
nodeInfo.getNode(),
nodeInfo.getVersion(),
nodeInfo.getBuild());
} else {
throw new IllegalStateException("Node [" + nodeName + "] didn't join the cluster");
}
success = true;
} finally {
if (!success) {
stop();
}
}
}
static boolean waitForNode(final Client client, final String name) throws InterruptedException {
return ESTestCase.awaitBusy(
() -> {
final NodesInfoResponse nodeInfos = client.admin().cluster().prepareNodesInfo().get();
final NodeInfo[] nodes = nodeInfos.getNodes();
for (NodeInfo info : nodes) {
if (name.equals(info.getNode().getName())) {
return true;
}
}
return false;
},
30,
TimeUnit.SECONDS);
}
static NodeInfo nodeInfo(final Client client, final String nodeName) {
final NodesInfoResponse nodeInfos = client.admin().cluster().prepareNodesInfo().get();
final NodeInfo[] nodes = nodeInfos.getNodes();
for (NodeInfo info : nodes) {
if (nodeName.equals(info.getNode().getName())) {
return info;
}
}
return null;
}
synchronized TransportAddress getTransportAddress() {
if (nodeInfo == null) {
throw new IllegalStateException("Node has not started yet");
}
return nodeInfo.getTransport().getAddress().publishAddress();
}
synchronized Client getClient() {
if (nodeInfo == null) {
throw new IllegalStateException("Node has not started yet");
}
if (client == null) {
TransportAddress addr = nodeInfo.getTransport().getAddress().publishAddress();
// verify that the end node setting will have network enabled.
Settings clientSettings =
settingsBuilder()
.put(externalNodeSettings)
.put("client.transport.nodes_sampler_interval", "1s")
.put("name", "transport_client_" + nodeInfo.getNode().name())
.put(ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName)
.put("client.transport.sniff", false)
.build();
TransportClient client = TransportClient.builder().settings(clientSettings).build();
client.addTransportAddress(addr);
this.client = client;
}
return client;
}
synchronized void reset(long seed) {
this.random.setSeed(seed);
}
synchronized void stop() throws InterruptedException {
if (running()) {
try {
if (this.client != null) {
client.close();
}
} finally {
process.destroy();
process.waitFor();
process = null;
nodeInfo = null;
}
}
}
synchronized boolean running() {
return process != null;
}
@Override
public void close() {
try {
stop();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
synchronized String getName() {
if (nodeInfo == null) {
throw new IllegalStateException("Node has not started yet");
}
return nodeInfo.getNode().getName();
}
}
public class ClusterDiscoveryConfiguration extends NodeConfigurationSource {
static Settings DEFAULT_NODE_SETTINGS =
Settings.settingsBuilder()
.put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), "zen")
.build();
private static final String IP_ADDR = "127.0.0.1";
final int numOfNodes;
final Settings nodeSettings;
final Settings transportClientSettings;
public ClusterDiscoveryConfiguration(int numOfNodes, Settings extraSettings) {
this.numOfNodes = numOfNodes;
this.nodeSettings = Settings.builder().put(DEFAULT_NODE_SETTINGS).put(extraSettings).build();
this.transportClientSettings = Settings.builder().put(extraSettings).build();
}
@Override
public Settings nodeSettings(int nodeOrdinal) {
return nodeSettings;
}
@Override
public Settings transportClientSettings() {
return transportClientSettings;
}
public static class UnicastZen extends ClusterDiscoveryConfiguration {
// this variable is incremented on each bind attempt and will maintain the next port that should
// be tried
private static int nextPort = calcBasePort();
private final int[] unicastHostOrdinals;
private final int[] unicastHostPorts;
public UnicastZen(int numOfNodes, Settings extraSettings) {
this(numOfNodes, numOfNodes, extraSettings);
}
public UnicastZen(int numOfNodes, int numOfUnicastHosts, Settings extraSettings) {
super(numOfNodes, extraSettings);
if (numOfUnicastHosts == numOfNodes) {
unicastHostOrdinals = new int[numOfNodes];
for (int i = 0; i < numOfNodes; i++) {
unicastHostOrdinals[i] = i;
}
} else {
Set<Integer> ordinals = new HashSet<>(numOfUnicastHosts);
while (ordinals.size() != numOfUnicastHosts) {
ordinals.add(RandomizedTest.randomInt(numOfNodes - 1));
}
unicastHostOrdinals = CollectionUtils.toArray(ordinals);
}
this.unicastHostPorts = unicastHostPorts(numOfNodes);
assert unicastHostOrdinals.length <= unicastHostPorts.length;
}
public UnicastZen(int numOfNodes, int[] unicastHostOrdinals) {
this(numOfNodes, Settings.EMPTY, unicastHostOrdinals);
}
public UnicastZen(int numOfNodes, Settings extraSettings, int[] unicastHostOrdinals) {
super(numOfNodes, extraSettings);
this.unicastHostOrdinals = unicastHostOrdinals;
this.unicastHostPorts = unicastHostPorts(numOfNodes);
assert unicastHostOrdinals.length <= unicastHostPorts.length;
}
private static int calcBasePort() {
return 30000 + InternalTestCluster.JVM_BASE_PORT_OFFSET;
}
@Override
public Settings nodeSettings(int nodeOrdinal) {
Settings.Builder builder = Settings.builder();
String[] unicastHosts = new String[unicastHostOrdinals.length];
if (nodeOrdinal >= unicastHostPorts.length) {
throw new ElasticsearchException(
"nodeOrdinal ["
+ nodeOrdinal
+ "] is greater than the number unicast ports ["
+ unicastHostPorts.length
+ "]");
} else {
// we need to pin the node port & host so we'd know where to point things
builder.put(TransportSettings.PORT.getKey(), unicastHostPorts[nodeOrdinal]);
builder.put(
TransportSettings.HOST.getKey(),
IP_ADDR); // only bind on one IF we use v4 here by default
builder.put("http.enabled", false);
for (int i = 0; i < unicastHostOrdinals.length; i++) {
unicastHosts[i] = IP_ADDR + ":" + (unicastHostPorts[unicastHostOrdinals[i]]);
}
}
builder.putArray("discovery.zen.ping.unicast.hosts", unicastHosts);
return builder.put(super.nodeSettings(nodeOrdinal)).build();
}
@SuppressForbidden(reason = "we know we pass a IP address")
protected static synchronized int[] unicastHostPorts(int numHosts) {
int[] unicastHostPorts = new int[numHosts];
final int basePort = calcBasePort();
final int maxPort = basePort + InternalTestCluster.PORTS_PER_JVM;
int tries = 0;
for (int i = 0; i < unicastHostPorts.length; i++) {
boolean foundPortInRange = false;
while (tries < InternalTestCluster.PORTS_PER_JVM && !foundPortInRange) {
try (ServerSocket serverSocket = new ServerSocket()) {
// Set SO_REUSEADDR as we may bind here and not be able to reuse the address immediately
// without it.
serverSocket.setReuseAddress(NetworkUtils.defaultReuseAddress());
serverSocket.bind(new InetSocketAddress(IP_ADDR, nextPort));
// bind was a success
foundPortInRange = true;
unicastHostPorts[i] = nextPort;
} catch (IOException e) {
// Do nothing
}
nextPort++;
if (nextPort >= maxPort) {
// Roll back to the beginning of the range and do not go into another JVM's port range
nextPort = basePort;
}
tries++;
}
if (!foundPortInRange) {
throw new ElasticsearchException(
"could not find enough open ports in range ["
+ basePort
+ "-"
+ maxPort
+ "]. required ["
+ unicastHostPorts.length
+ "] ports");
}
}
return unicastHostPorts;
}
}
}
