private void collectStartTimes(String collectionName, Map<String, Long> urlToTime)
throws SolrServerException, IOException {
ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
// Map<String,DocCollection> collections = clusterState.getCollectionStates();
if (clusterState.hasCollection(collectionName)) {
Map<String, Slice> slices = clusterState.getSlicesMap(collectionName);
Iterator<Entry<String, Slice>> it = slices.entrySet().iterator();
while (it.hasNext()) {
Entry<String, Slice> sliceEntry = it.next();
Map<String, Replica> sliceShards = sliceEntry.getValue().getReplicasMap();
Iterator<Entry<String, Replica>> shardIt = sliceShards.entrySet().iterator();
while (shardIt.hasNext()) {
Entry<String, Replica> shardEntry = shardIt.next();
ZkCoreNodeProps coreProps = new ZkCoreNodeProps(shardEntry.getValue());
HttpSolrServer server = new HttpSolrServer(coreProps.getBaseUrl());
CoreAdminResponse mcr;
try {
mcr = CoreAdminRequest.getStatus(coreProps.getCoreName(), server);
} finally {
server.shutdown();
}
long before = mcr.getStartTime(coreProps.getCoreName()).getTime();
urlToTime.put(coreProps.getCoreUrl(), before);
}
}
} else {
throw new IllegalArgumentException(
"Could not find collection in :" + clusterState.getCollections());
}
}
private Set<String> getCollectionList(ClusterState clusterState, String collection) {
// Extract each comma separated collection name and store in a List.
List<String> rawCollectionsList = StrUtils.splitSmart(collection, ",", true);
Set<String> collectionsList = new HashSet<>();
// validate collections
for (String collectionName : rawCollectionsList) {
if (!clusterState.getCollections().contains(collectionName)) {
Aliases aliases = zkStateReader.getAliases();
String alias = aliases.getCollectionAlias(collectionName);
if (alias != null) {
List<String> aliasList = StrUtils.splitSmart(alias, ",", true);
collectionsList.addAll(aliasList);
continue;
}
throw new SolrException(ErrorCode.BAD_REQUEST, "Collection not found: " + collectionName);
}
collectionsList.add(collectionName);
}
return collectionsList;
}
public static ArrayList<Node> getNodesForNewShard(
ClusterState clusterState,
String collectionName,
int numSlices,
int maxShardsPerNode,
int repFactor,
String createNodeSetStr) {
List<String> createNodeList =
createNodeSetStr == null ? null : StrUtils.splitSmart(createNodeSetStr, ",", true);
Set<String> nodes = clusterState.getLiveNodes();
List<String> nodeList = new ArrayList<String>(nodes.size());
nodeList.addAll(nodes);
if (createNodeList != null) nodeList.retainAll(createNodeList);
HashMap<String, Node> nodeNameVsShardCount = new HashMap<String, Node>();
for (String s : nodeList) nodeNameVsShardCount.put(s, new Node(s));
for (String s : clusterState.getCollections()) {
DocCollection c = clusterState.getCollection(s);
// identify suitable nodes by checking the no:of cores in each of them
for (Slice slice : c.getSlices()) {
Collection<Replica> replicas = slice.getReplicas();
for (Replica replica : replicas) {
Node count = nodeNameVsShardCount.get(replica.getNodeName());
if (count != null) {
count.totalNodes++;
if (s.equals(collectionName)) {
count.thisCollectionNodes++;
if (count.thisCollectionNodes >= maxShardsPerNode)
nodeNameVsShardCount.remove(replica.getNodeName());
}
}
}
}
}
if (nodeNameVsShardCount.size() <= 0) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Cannot create collection "
+ collectionName
+ ". No live Solr-instances"
+ ((createNodeList != null)
? " among Solr-instances specified in " + CREATE_NODE_SET + ":" + createNodeSetStr
: ""));
}
if (repFactor > nodeNameVsShardCount.size()) {
log.warn(
"Specified "
+ REPLICATION_FACTOR
+ " of "
+ repFactor
+ " on collection "
+ collectionName
+ " is higher than or equal to the number of Solr instances currently live or part of your "
+ CREATE_NODE_SET
+ "("
+ nodeList.size()
+ "). Its unusual to run two replica of the same slice on the same Solr-instance.");
}
int maxCoresAllowedToCreate = maxShardsPerNode * nodeList.size();
int requestedCoresToCreate = numSlices * repFactor;
int minCoresToCreate = requestedCoresToCreate;
if (maxCoresAllowedToCreate < minCoresToCreate) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Cannot create shards "
+ collectionName
+ ". Value of "
+ MAX_SHARDS_PER_NODE
+ " is "
+ maxShardsPerNode
+ ", and the number of live nodes is "
+ nodeList.size()
+ ". This allows a maximum of "
+ maxCoresAllowedToCreate
+ " to be created. Value of "
+ NUM_SLICES
+ " is "
+ numSlices
+ " and value of "
+ REPLICATION_FACTOR
+ " is "
+ repFactor
+ ". This requires "
+ requestedCoresToCreate
+ " shards to be created (higher than the allowed number)");
}
ArrayList<Node> sortedNodeList = new ArrayList<>(nodeNameVsShardCount.values());
Collections.sort(
sortedNodeList,
new Comparator<Node>() {
@Override
public int compare(Node x, Node y) {
return (x.weight() < y.weight()) ? -1 : ((x.weight() == y.weight()) ? 0 : 1);
}
});
return sortedNodeList;
}
protected Set<String> commonMocks(int liveNodesCount) throws Exception {
shardHandlerFactoryMock.getShardHandler();
expectLastCall()
.andAnswer(
new IAnswer<ShardHandler>() {
@Override
public ShardHandler answer() throws Throwable {
log.info("SHARDHANDLER");
return shardHandlerMock;
}
})
.anyTimes();
workQueueMock.peekTopN(EasyMock.anyInt(), anyObject(Set.class), EasyMock.anyLong());
expectLastCall()
.andAnswer(
new IAnswer<List>() {
@Override
public List answer() throws Throwable {
Object result;
int count = 0;
while ((result = queue.peek()) == null) {
Thread.sleep(1000);
count++;
if (count > 1) return null;
}
return Arrays.asList(result);
}
})
.anyTimes();
workQueueMock.getTailId();
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
Object result = null;
Iterator iter = queue.iterator();
while (iter.hasNext()) {
result = iter.next();
}
return result == null ? null : ((QueueEvent) result).getId();
}
})
.anyTimes();
workQueueMock.peek(true);
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
Object result;
while ((result = queue.peek()) == null) {
Thread.sleep(1000);
}
return result;
}
})
.anyTimes();
workQueueMock.remove(anyObject(QueueEvent.class));
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
queue.remove((QueueEvent) getCurrentArguments()[0]);
return null;
}
})
.anyTimes();
workQueueMock.poll();
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return queue.poll();
}
})
.anyTimes();
zkStateReaderMock.getClusterState();
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return clusterStateMock;
}
})
.anyTimes();
zkStateReaderMock.getZkClient();
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return solrZkClientMock;
}
})
.anyTimes();
zkStateReaderMock.updateClusterState(anyBoolean());
clusterStateMock.getCollections();
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return collectionsSet;
}
})
.anyTimes();
final Set<String> liveNodes = new HashSet<>();
for (int i = 0; i < liveNodesCount; i++) {
final String address = "localhost:" + (8963 + i) + "_solr";
liveNodes.add(address);
zkStateReaderMock.getBaseUrlForNodeName(address);
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
// This works as long as this test does not use a
// webapp context with an underscore in it
return address.replaceAll("_", "/");
}
})
.anyTimes();
}
zkStateReaderMock.getClusterProps();
expectLastCall()
.andAnswer(
new IAnswer<Map>() {
@Override
public Map answer() throws Throwable {
return new HashMap();
}
});
solrZkClientMock.getZkClientTimeout();
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return 30000;
}
})
.anyTimes();
clusterStateMock.hasCollection(anyObject(String.class));
expectLastCall()
.andAnswer(
new IAnswer<Boolean>() {
@Override
public Boolean answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
return collectionsSet.contains(key);
}
})
.anyTimes();
clusterStateMock.getLiveNodes();
expectLastCall()
.andAnswer(
new IAnswer<Object>() {
@Override
public Object answer() throws Throwable {
return liveNodes;
}
})
.anyTimes();
solrZkClientMock.create(
anyObject(String.class),
anyObject(byte[].class),
anyObject(CreateMode.class),
anyBoolean());
expectLastCall()
.andAnswer(
new IAnswer<String>() {
@Override
public String answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
zkMap.put(key, null);
handleCreateCollMessage((byte[]) getCurrentArguments()[1]);
return key;
}
})
.anyTimes();
solrZkClientMock.makePath(anyObject(String.class), anyObject(byte[].class), anyBoolean());
expectLastCall()
.andAnswer(
new IAnswer<String>() {
@Override
public String answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
return key;
}
})
.anyTimes();
solrZkClientMock.makePath(
anyObject(String.class),
anyObject(byte[].class),
anyObject(CreateMode.class),
anyBoolean());
expectLastCall()
.andAnswer(
new IAnswer<String>() {
@Override
public String answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
return key;
}
})
.anyTimes();
solrZkClientMock.exists(anyObject(String.class), anyBoolean());
expectLastCall()
.andAnswer(
new IAnswer<Boolean>() {
@Override
public Boolean answer() throws Throwable {
String key = (String) getCurrentArguments()[0];
return zkMap.containsKey(key);
}
})
.anyTimes();
zkMap.put("/configs/myconfig", null);
return liveNodes;
}