private void waitTillNodesActive() throws Exception {
for (int i = 0; i < 60; i++) {
Thread.sleep(3000);
ZkStateReader zkStateReader = cloudClient.getZkStateReader();
ClusterState clusterState = zkStateReader.getClusterState();
DocCollection collection1 = clusterState.getCollection("collection1");
Slice slice = collection1.getSlice("shard1");
Collection<Replica> replicas = slice.getReplicas();
boolean allActive = true;
Collection<String> nodesDownNames =
nodesDown.stream().map(n -> n.coreNodeName).collect(Collectors.toList());
Collection<Replica> replicasToCheck =
replicas
.stream()
.filter(r -> !nodesDownNames.contains(r.getName()))
.collect(Collectors.toList());
for (Replica replica : replicasToCheck) {
if (!clusterState.liveNodesContain(replica.getNodeName())
|| replica.getState() != Replica.State.ACTIVE) {
allActive = false;
break;
}
}
if (allActive) {
return;
}
}
printLayout();
fail("timeout waiting to see all nodes active");
}
/** Return the jetty that a particular replica resides on */
public JettySolrRunner getReplicaJetty(Replica replica) {
for (JettySolrRunner jetty : jettys) {
if (replica.getCoreUrl().startsWith(jetty.getBaseUrl().toString())) return jetty;
}
throw new IllegalArgumentException(
"Cannot find Jetty for a replica with core url " + replica.getCoreUrl());
}
private Map<String, Object> getReplicaProps(ZkController zkController, SolrCore core) {
final String collection = core.getCoreDescriptor().getCloudDescriptor().getCollectionName();
Replica replica =
zkController
.getClusterState()
.getReplica(collection, zkController.getCoreNodeName(core.getCoreDescriptor()));
if (replica != null) {
return replica.getProperties();
}
return Collections.EMPTY_MAP;
}
private void testNodesUsedByCreate() throws Exception {
// we can use this client because we just want base url
final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("action", CollectionAction.CREATE.toString());
params.set("numShards", 2);
params.set(REPLICATION_FACTOR, 2);
String collectionName = "nodes_used_collection";
params.set("name", collectionName);
if (secondConfigSet) {
params.set("collection.configName", "conf1");
}
QueryRequest request = new QueryRequest(params);
request.setPath("/admin/collections");
createNewSolrServer("", baseUrl).request(request);
List<Integer> numShardsNumReplicaList = new ArrayList<Integer>();
numShardsNumReplicaList.add(2);
numShardsNumReplicaList.add(2);
checkForCollection("nodes_used_collection", numShardsNumReplicaList, null);
List<String> createNodeList = new ArrayList<String>();
Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();
for (String node : liveNodes) {
createNodeList.add(node);
}
DocCollection col =
cloudClient.getZkStateReader().getClusterState().getCollection("nodes_used_collection");
Collection<Slice> slices = col.getSlices();
for (Slice slice : slices) {
Collection<Replica> replicas = slice.getReplicas();
for (Replica replica : replicas) {
createNodeList.remove(replica.getNodeName());
}
}
assertEquals(createNodeList.toString(), 1, createNodeList.size());
}
public static String assignNode(String collection, ClusterState state) {
Map<String, Slice> sliceMap = state.getSlicesMap(collection);
if (sliceMap == null) {
return "core_node1";
}
int max = 0;
for (Slice slice : sliceMap.values()) {
for (Replica replica : slice.getReplicas()) {
Matcher m = COUNT.matcher(replica.getName());
if (m.matches()) {
max = Math.max(max, Integer.parseInt(m.group(1)));
}
}
}
return "core_node" + (max + 1);
}
protected void constructStreams() throws IOException {
try {
Object pushStream = ((Expressible) tupleStream).toExpression(streamFactory);
ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
ClusterState clusterState = zkStateReader.getClusterState();
Set<String> liveNodes = clusterState.getLiveNodes();
Collection<Slice> slices = clusterState.getActiveSlices(this.collection);
List<Replica> shuffler = new ArrayList();
for (Slice slice : slices) {
Collection<Replica> replicas = slice.getReplicas();
for (Replica replica : replicas) {
if (replica.getState() == Replica.State.ACTIVE
&& liveNodes.contains(replica.getNodeName())) shuffler.add(replica);
}
}
if (workers > shuffler.size()) {
throw new IOException("Number of workers exceeds nodes in the worker collection");
}
Collections.shuffle(shuffler, new Random());
for (int w = 0; w < workers; w++) {
HashMap params = new HashMap();
params.put("distrib", "false"); // We are the aggregator.
params.put("numWorkers", workers);
params.put("workerID", w);
params.put("expr", pushStream);
params.put("qt", "/stream");
Replica rep = shuffler.get(w);
ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
String url = zkProps.getCoreUrl();
SolrStream solrStream = new SolrStream(url, params);
solrStreams.add(solrStream);
}
assert (solrStreams.size() == workers);
} catch (Exception e) {
throw new IOException(e);
}
}
private Map<String, List<String>> buildUrlMap(DocCollection col) {
Map<String, List<String>> urlMap = new HashMap<>();
Collection<Slice> slices = col.getActiveSlices();
Iterator<Slice> sliceIterator = slices.iterator();
while (sliceIterator.hasNext()) {
Slice slice = sliceIterator.next();
String name = slice.getName();
List<String> urls = new ArrayList<>();
Replica leader = slice.getLeader();
if (leader == null) {
// take unoptimized general path - we cannot find a leader yet
return null;
}
ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
String url = zkProps.getCoreUrl();
urls.add(url);
Collection<Replica> replicas = slice.getReplicas();
Iterator<Replica> replicaIterator = replicas.iterator();
while (replicaIterator.hasNext()) {
Replica replica = replicaIterator.next();
if (!replica.getNodeName().equals(leader.getNodeName())
&& !replica.getName().equals(leader.getName())) {
ZkCoreNodeProps zkProps1 = new ZkCoreNodeProps(replica);
String url1 = zkProps1.getCoreUrl();
urls.add(url1);
}
}
urlMap.put(name, urls);
}
return urlMap;
}
@BeforeClass
public static void setupCluster() throws Exception {
final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");
String configName = "solrCloudCollectionConfig";
int nodeCount = 5;
configureCluster(nodeCount).addConfig(configName, configDir).configure();
Map<String, String> collectionProperties = new HashMap<>();
collectionProperties.put("config", "solrconfig-tlog.xml");
collectionProperties.put("schema", "schema.xml");
// create a collection holding data for the "to" side of the JOIN
int shards = 2;
int replicas = 2;
CollectionAdminRequest.createCollection(toColl, configName, shards, replicas)
.setProperties(collectionProperties)
.process(cluster.getSolrClient());
// get the set of nodes where replicas for the "to" collection exist
Set<String> nodeSet = new HashSet<>();
ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
ClusterState cs = zkStateReader.getClusterState();
for (Slice slice : cs.getCollection(toColl).getActiveSlices())
for (Replica replica : slice.getReplicas()) nodeSet.add(replica.getNodeName());
assertTrue(nodeSet.size() > 0);
// deploy the "from" collection to all nodes where the "to" collection exists
CollectionAdminRequest.createCollection(fromColl, configName, 1, 4)
.setCreateNodeSet(StringUtils.join(nodeSet, ","))
.setProperties(collectionProperties)
.process(cluster.getSolrClient());
toDocId = indexDoc(toColl, 1001, "a", null, "b");
indexDoc(fromColl, 2001, "a", "c", null);
Thread.sleep(1000); // so the commits fire
}
@Test
public void test() throws Exception {
final String collectionName = "customcollreplicadeletion";
CollectionAdminRequest.createCollectionWithImplicitRouter(collectionName, "conf", "a,b", 1)
.setMaxShardsPerNode(5)
.process(cluster.getSolrClient());
DocCollection collectionState = getCollectionState(collectionName);
Replica replica = getRandomReplica(collectionState.getSlice("a"));
CollectionAdminRequest.deleteReplica(collectionName, "a", replica.getName())
.process(cluster.getSolrClient());
waitForState(
"Expected shard 'a' to have no replicas",
collectionName,
(n, c) -> {
return c.getSlice("a") == null || c.getSlice("a").getReplicas().size() == 0;
});
}
private static String findLocalReplicaForFromIndex(ZkController zkController, String fromIndex) {
String fromReplica = null;
String nodeName = zkController.getNodeName();
for (Slice slice : zkController.getClusterState().getActiveSlices(fromIndex)) {
if (fromReplica != null)
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"SolrCloud join: multiple shards not yet supported " + fromIndex);
for (Replica replica : slice.getReplicas()) {
if (replica.getNodeName().equals(nodeName)) {
fromReplica = replica.getStr(ZkStateReader.CORE_NAME_PROP);
// found local replica, but is it Active?
if (replica.getState() != Replica.State.ACTIVE)
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"SolrCloud join: "
+ fromIndex
+ " has a local replica ("
+ fromReplica
+ ") on "
+ nodeName
+ ", but it is "
+ replica.getState());
break;
}
}
}
if (fromReplica == null)
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"SolrCloud join: No active replicas for " + fromIndex + " found in node " + nodeName);
return fromReplica;
}
private String getCoreUrl(
String collectionName,
String origCorename,
ClusterState clusterState,
Collection<Slice> slices,
boolean byCoreName,
boolean activeReplicas) {
String coreUrl;
Set<String> liveNodes = clusterState.getLiveNodes();
List<Slice> randomizedSlices = new ArrayList<>(slices.size());
randomizedSlices.addAll(slices);
Collections.shuffle(randomizedSlices, random);
for (Slice slice : randomizedSlices) {
List<Replica> randomizedReplicas = new ArrayList<>();
randomizedReplicas.addAll(slice.getReplicas());
Collections.shuffle(randomizedReplicas, random);
for (Replica replica : randomizedReplicas) {
if (!activeReplicas
|| (liveNodes.contains(replica.getNodeName())
&& replica.getState() == Replica.State.ACTIVE)) {
if (byCoreName && !collectionName.equals(replica.getStr(CORE_NAME_PROP))) {
// if it's by core name, make sure they match
continue;
}
if (replica.getStr(BASE_URL_PROP).equals(cores.getZkController().getBaseUrl())) {
// don't count a local core
continue;
}
if (origCorename != null) {
coreUrl = replica.getStr(BASE_URL_PROP) + "/" + origCorename;
} else {
coreUrl = replica.getCoreUrl();
if (coreUrl.endsWith("/")) {
coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
}
}
return coreUrl;
}
}
}
return null;
}
private SolrCore getCoreByCollection(String collectionName) {
ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();
ClusterState clusterState = zkStateReader.getClusterState();
DocCollection collection = clusterState.getCollectionOrNull(collectionName);
if (collection == null) {
return null;
}
Map<String, Slice> slices = collection.getActiveSlicesMap();
if (slices == null) {
return null;
}
Set<String> liveNodes = clusterState.getLiveNodes();
// look for a core on this node
Set<Map.Entry<String, Slice>> entries = slices.entrySet();
SolrCore core = null;
// Hitting the leaders is useful when it's an update request.
// For queries it doesn't matter and hence we don't distinguish here.
for (Map.Entry<String, Slice> entry : entries) {
// first see if we have the leader
Replica leaderProps = collection.getLeader(entry.getKey());
if (leaderProps != null
&& liveNodes.contains(leaderProps.getNodeName())
&& leaderProps.getState() == Replica.State.ACTIVE) {
core = checkProps(leaderProps);
if (core != null) {
return core;
}
}
// check everyone then
Map<String, Replica> shards = entry.getValue().getReplicasMap();
Set<Map.Entry<String, Replica>> shardEntries = shards.entrySet();
for (Map.Entry<String, Replica> shardEntry : shardEntries) {
Replica zkProps = shardEntry.getValue();
if (liveNodes.contains(zkProps.getNodeName())
&& zkProps.getState() == Replica.State.ACTIVE) {
core = checkProps(zkProps);
if (core != null) {
return core;
}
}
}
}
return null;
}
public static ArrayList<Node> getNodesForNewShard(
ClusterState clusterState,
String collectionName,
int numSlices,
int maxShardsPerNode,
int repFactor,
String createNodeSetStr) {
List<String> createNodeList =
createNodeSetStr == null ? null : StrUtils.splitSmart(createNodeSetStr, ",", true);
Set<String> nodes = clusterState.getLiveNodes();
List<String> nodeList = new ArrayList<String>(nodes.size());
nodeList.addAll(nodes);
if (createNodeList != null) nodeList.retainAll(createNodeList);
HashMap<String, Node> nodeNameVsShardCount = new HashMap<String, Node>();
for (String s : nodeList) nodeNameVsShardCount.put(s, new Node(s));
for (String s : clusterState.getCollections()) {
DocCollection c = clusterState.getCollection(s);
// identify suitable nodes by checking the no:of cores in each of them
for (Slice slice : c.getSlices()) {
Collection<Replica> replicas = slice.getReplicas();
for (Replica replica : replicas) {
Node count = nodeNameVsShardCount.get(replica.getNodeName());
if (count != null) {
count.totalNodes++;
if (s.equals(collectionName)) {
count.thisCollectionNodes++;
if (count.thisCollectionNodes >= maxShardsPerNode)
nodeNameVsShardCount.remove(replica.getNodeName());
}
}
}
}
}
if (nodeNameVsShardCount.size() <= 0) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Cannot create collection "
+ collectionName
+ ". No live Solr-instances"
+ ((createNodeList != null)
? " among Solr-instances specified in " + CREATE_NODE_SET + ":" + createNodeSetStr
: ""));
}
if (repFactor > nodeNameVsShardCount.size()) {
log.warn(
"Specified "
+ REPLICATION_FACTOR
+ " of "
+ repFactor
+ " on collection "
+ collectionName
+ " is higher than or equal to the number of Solr instances currently live or part of your "
+ CREATE_NODE_SET
+ "("
+ nodeList.size()
+ "). Its unusual to run two replica of the same slice on the same Solr-instance.");
}
int maxCoresAllowedToCreate = maxShardsPerNode * nodeList.size();
int requestedCoresToCreate = numSlices * repFactor;
int minCoresToCreate = requestedCoresToCreate;
if (maxCoresAllowedToCreate < minCoresToCreate) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Cannot create shards "
+ collectionName
+ ". Value of "
+ MAX_SHARDS_PER_NODE
+ " is "
+ maxShardsPerNode
+ ", and the number of live nodes is "
+ nodeList.size()
+ ". This allows a maximum of "
+ maxCoresAllowedToCreate
+ " to be created. Value of "
+ NUM_SLICES
+ " is "
+ numSlices
+ " and value of "
+ REPLICATION_FACTOR
+ " is "
+ repFactor
+ ". This requires "
+ requestedCoresToCreate
+ " shards to be created (higher than the allowed number)");
}
ArrayList<Node> sortedNodeList = new ArrayList<>(nodeNameVsShardCount.values());
Collections.sort(
sortedNodeList,
new Comparator<Node>() {
@Override
public int compare(Node x, Node y) {
return (x.weight() < y.weight()) ? -1 : ((x.weight() == y.weight()) ? 0 : 1);
}
});
return sortedNodeList;
}
@Test
public void testStoreAndRead() throws Exception {
Map<String, DocCollection> collectionStates = new HashMap<>();
Set<String> liveNodes = new HashSet<>();
liveNodes.add("node1");
liveNodes.add("node2");
Map<String, Slice> slices = new HashMap<>();
Map<String, Replica> sliceToProps = new HashMap<>();
Map<String, Object> props = new HashMap<>();
props.put("prop1", "value");
props.put("prop2", "value2");
Replica replica = new Replica("node1", props);
sliceToProps.put("node1", replica);
Slice slice = new Slice("shard1", sliceToProps, null);
slices.put("shard1", slice);
Slice slice2 = new Slice("shard2", sliceToProps, null);
slices.put("shard2", slice2);
collectionStates.put(
"collection1", new DocCollection("collection1", slices, null, DocRouter.DEFAULT));
collectionStates.put(
"collection2", new DocCollection("collection2", slices, null, DocRouter.DEFAULT));
ZkStateReader zkStateReaderMock = getMockZkStateReader(collectionStates.keySet());
ClusterState clusterState = new ClusterState(-1, liveNodes, collectionStates);
byte[] bytes = Utils.toJSON(clusterState);
// System.out.println("#################### " + new String(bytes));
ClusterState loadedClusterState = ClusterState.load(-1, bytes, liveNodes);
assertEquals(
"Provided liveNodes not used properly", 2, loadedClusterState.getLiveNodes().size());
assertEquals("No collections found", 2, loadedClusterState.getCollectionsMap().size());
assertEquals(
"Properties not copied properly",
replica.getStr("prop1"),
loadedClusterState
.getSlice("collection1", "shard1")
.getReplicasMap()
.get("node1")
.getStr("prop1"));
assertEquals(
"Properties not copied properly",
replica.getStr("prop2"),
loadedClusterState
.getSlice("collection1", "shard1")
.getReplicasMap()
.get("node1")
.getStr("prop2"));
loadedClusterState = ClusterState.load(-1, new byte[0], liveNodes);
assertEquals(
"Provided liveNodes not used properly", 2, loadedClusterState.getLiveNodes().size());
assertEquals("Should not have collections", 0, loadedClusterState.getCollectionsMap().size());
loadedClusterState = ClusterState.load(-1, (byte[]) null, liveNodes);
assertEquals(
"Provided liveNodes not used properly", 2, loadedClusterState.getLiveNodes().size());
assertEquals("Should not have collections", 0, loadedClusterState.getCollectionsMap().size());
}