private Map<String, List<String>> buildUrlMap(DocCollection col) {
Map<String, List<String>> urlMap = new HashMap<>();
Collection<Slice> slices = col.getActiveSlices();
Iterator<Slice> sliceIterator = slices.iterator();
while (sliceIterator.hasNext()) {
Slice slice = sliceIterator.next();
String name = slice.getName();
List<String> urls = new ArrayList<>();
Replica leader = slice.getLeader();
if (leader == null) {
// take unoptimized general path - we cannot find a leader yet
return null;
}
ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
String url = zkProps.getCoreUrl();
urls.add(url);
Collection<Replica> replicas = slice.getReplicas();
Iterator<Replica> replicaIterator = replicas.iterator();
while (replicaIterator.hasNext()) {
Replica replica = replicaIterator.next();
if (!replica.getNodeName().equals(leader.getNodeName())
&& !replica.getName().equals(leader.getName())) {
ZkCoreNodeProps zkProps1 = new ZkCoreNodeProps(replica);
String url1 = zkProps1.getCoreUrl();
urls.add(url1);
}
}
urlMap.put(name, urls);
}
return urlMap;
}
/**
* Walks the NamedList response after performing an update request looking for the replication
* factor that was achieved in each shard involved in the request. For single doc updates, there
* will be only one shard in the return value.
*/
@SuppressWarnings("rawtypes")
public Map<String, Integer> getShardReplicationFactor(String collection, NamedList resp) {
connect();
Map<String, Integer> results = new HashMap<String, Integer>();
if (resp instanceof CloudSolrServer.RouteResponse) {
NamedList routes = ((CloudSolrServer.RouteResponse) resp).getRouteResponses();
ClusterState clusterState = zkStateReader.getClusterState();
Map<String, String> leaders = new HashMap<String, String>();
for (Slice slice : clusterState.getActiveSlices(collection)) {
Replica leader = slice.getLeader();
if (leader != null) {
ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
String leaderUrl = zkProps.getBaseUrl() + "/" + zkProps.getCoreName();
leaders.put(leaderUrl, slice.getName());
String altLeaderUrl = zkProps.getBaseUrl() + "/" + collection;
leaders.put(altLeaderUrl, slice.getName());
}
}
Iterator<Map.Entry<String, Object>> routeIter = routes.iterator();
while (routeIter.hasNext()) {
Map.Entry<String, Object> next = routeIter.next();
String host = next.getKey();
NamedList hostResp = (NamedList) next.getValue();
Integer rf =
(Integer) ((NamedList) hostResp.get("responseHeader")).get(UpdateRequest.REPFACT);
if (rf != null) {
String shard = leaders.get(host);
if (shard == null) {
if (host.endsWith("/")) shard = leaders.get(host.substring(0, host.length() - 1));
if (shard == null) {
shard = host;
}
}
results.put(shard, rf);
}
}
}
return results;
}
protected final synchronized int getShardIndex(String shardId, DocCollection dc) {
if (shardIndexCache == null) shardIndexCache = new HashMap<String, Integer>(20);
Integer idx = shardIndexCache.get(shardId);
if (idx != null) return idx.intValue(); // meh auto-boxing
int s = 0;
for (Slice slice : dc.getSlices()) {
if (shardId.equals(slice.getName())) {
shardIndexCache.put(shardId, new Integer(s));
return s;
}
++s;
}
throw new IllegalStateException(
"Cannot find index of shard '" + shardId + "' in collection: " + collection);
}
protected List<String> buildShardList(CloudSolrClient cloudSolrServer) {
ZkStateReader zkStateReader = cloudSolrServer.getZkStateReader();
ClusterState clusterState = zkStateReader.getClusterState();
String[] collections = null;
if (clusterState.hasCollection(collection)) {
collections = new String[] {collection};
} else {
// might be a collection alias?
Aliases aliases = zkStateReader.getAliases();
String aliasedCollections = aliases.getCollectionAlias(collection);
if (aliasedCollections == null)
throw new IllegalArgumentException("Collection " + collection + " not found!");
collections = aliasedCollections.split(",");
}
Set<String> liveNodes = clusterState.getLiveNodes();
Random random = new Random(5150);
List<String> shards = new ArrayList<String>();
for (String coll : collections) {
for (Slice slice : clusterState.getSlices(coll)) {
List<String> replicas = new ArrayList<String>();
for (Replica r : slice.getReplicas()) {
ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(r);
if (liveNodes.contains(replicaCoreProps.getNodeName()))
replicas.add(replicaCoreProps.getCoreUrl());
}
int numReplicas = replicas.size();
if (numReplicas == 0)
throw new IllegalStateException(
"Shard " + slice.getName() + " does not have any active replicas!");
String replicaUrl =
(numReplicas == 1) ? replicas.get(0) : replicas.get(random.nextInt(replicas.size()));
shards.add(replicaUrl);
}
}
return shards;
}
@Override
public int getPartition(Object o) {
Object docId = null;
if (o instanceof SolrInputDocument) {
SolrInputDocument doc = (SolrInputDocument) o;
docId = doc.getFieldValue(idField);
if (docId == null)
throw new IllegalArgumentException(
"SolrInputDocument must contain a non-null value for " + idField);
} else {
docId = o;
}
if (!(docId instanceof String))
throw new IllegalArgumentException(
"Only String document IDs are supported by this Partitioner!");
DocCollection dc = getDocCollection();
Slice slice = dc.getRouter().getTargetSlice((String) docId, null, null, null, dc);
return getShardIndex(slice.getName(), dc);
}
public String getShardId(String docId) {
DocCollection dc = getDocCollection();
Slice slice = dc.getRouter().getTargetSlice(docId, null, null, null, dc);
return slice.getName();
}
public int createSubRequests(ResponseBuilder rb) throws IOException {
SolrParams params = rb.req.getParams();
String id1[] = params.getParams("id");
String ids[] = params.getParams("ids");
if (id1 == null && ids == null) {
return ResponseBuilder.STAGE_DONE;
}
List<String> allIds = new ArrayList<String>();
if (id1 != null) {
for (String s : id1) {
allIds.add(s);
}
}
if (ids != null) {
for (String s : ids) {
allIds.addAll(StrUtils.splitSmart(s, ",", true));
}
}
// TODO: handle collection=...?
ZkController zkController =
rb.req.getCore().getCoreDescriptor().getCoreContainer().getZkController();
// if shards=... then use that
if (zkController != null && params.get("shards") == null) {
CloudDescriptor cloudDescriptor = rb.req.getCore().getCoreDescriptor().getCloudDescriptor();
String collection = cloudDescriptor.getCollectionName();
ClusterState clusterState = zkController.getClusterState();
DocCollection coll = clusterState.getCollection(collection);
Map<String, List<String>> sliceToId = new HashMap<String, List<String>>();
for (String id : allIds) {
Slice slice = coll.getRouter().getTargetSlice(id, null, params, coll);
List<String> idsForShard = sliceToId.get(slice.getName());
if (idsForShard == null) {
idsForShard = new ArrayList<String>(2);
sliceToId.put(slice.getName(), idsForShard);
}
idsForShard.add(id);
}
for (Map.Entry<String, List<String>> entry : sliceToId.entrySet()) {
String shard = entry.getKey();
String shardIdList = StrUtils.join(entry.getValue(), ',');
ShardRequest sreq = new ShardRequest();
sreq.purpose = 1;
// sreq.shards = new String[]{shard}; // TODO: would be nice if this would work...
sreq.shards = sliceToShards(rb, collection, shard);
sreq.actualShards = sreq.shards;
sreq.params = new ModifiableSolrParams();
sreq.params.set(
ShardParams.SHARDS_QT,
"/get"); // TODO: how to avoid hardcoding this and hit the same handler?
sreq.params.set("distrib", false);
sreq.params.set("ids", shardIdList);
rb.addRequest(this, sreq);
}
} else {
String shardIdList = StrUtils.join(allIds, ',');
ShardRequest sreq = new ShardRequest();
sreq.purpose = 1;
sreq.shards = null; // ALL
sreq.actualShards = sreq.shards;
sreq.params = new ModifiableSolrParams();
sreq.params.set(
ShardParams.SHARDS_QT,
"/get"); // TODO: how to avoid hardcoding this and hit the same handler?
sreq.params.set("distrib", false);
sreq.params.set("ids", shardIdList);
rb.addRequest(this, sreq);
}
return ResponseBuilder.STAGE_DONE;
}
