private Map<String, List<String>> buildUrlMap(DocCollection col) {
   Map<String, List<String>> urlMap = new HashMap<>();
   Collection<Slice> slices = col.getActiveSlices();
   Iterator<Slice> sliceIterator = slices.iterator();
   while (sliceIterator.hasNext()) {
     Slice slice = sliceIterator.next();
     String name = slice.getName();
     List<String> urls = new ArrayList<>();
     Replica leader = slice.getLeader();
     if (leader == null) {
       // take unoptimized general path - we cannot find a leader yet
       return null;
     }
     ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
     String url = zkProps.getCoreUrl();
     urls.add(url);
     Collection<Replica> replicas = slice.getReplicas();
     Iterator<Replica> replicaIterator = replicas.iterator();
     while (replicaIterator.hasNext()) {
       Replica replica = replicaIterator.next();
       if (!replica.getNodeName().equals(leader.getNodeName())
           && !replica.getName().equals(leader.getName())) {
         ZkCoreNodeProps zkProps1 = new ZkCoreNodeProps(replica);
         String url1 = zkProps1.getCoreUrl();
         urls.add(url1);
       }
     }
     urlMap.put(name, urls);
   }
   return urlMap;
 }
  /**
   * Walks the NamedList response after performing an update request looking for the replication
   * factor that was achieved in each shard involved in the request. For single doc updates, there
   * will be only one shard in the return value.
   */
  @SuppressWarnings("rawtypes")
  public Map<String, Integer> getShardReplicationFactor(String collection, NamedList resp) {
    connect();

    Map<String, Integer> results = new HashMap<String, Integer>();
    if (resp instanceof CloudSolrServer.RouteResponse) {
      NamedList routes = ((CloudSolrServer.RouteResponse) resp).getRouteResponses();
      ClusterState clusterState = zkStateReader.getClusterState();
      Map<String, String> leaders = new HashMap<String, String>();
      for (Slice slice : clusterState.getActiveSlices(collection)) {
        Replica leader = slice.getLeader();
        if (leader != null) {
          ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
          String leaderUrl = zkProps.getBaseUrl() + "/" + zkProps.getCoreName();
          leaders.put(leaderUrl, slice.getName());
          String altLeaderUrl = zkProps.getBaseUrl() + "/" + collection;
          leaders.put(altLeaderUrl, slice.getName());
        }
      }

      Iterator<Map.Entry<String, Object>> routeIter = routes.iterator();
      while (routeIter.hasNext()) {
        Map.Entry<String, Object> next = routeIter.next();
        String host = next.getKey();
        NamedList hostResp = (NamedList) next.getValue();
        Integer rf =
            (Integer) ((NamedList) hostResp.get("responseHeader")).get(UpdateRequest.REPFACT);
        if (rf != null) {
          String shard = leaders.get(host);
          if (shard == null) {
            if (host.endsWith("/")) shard = leaders.get(host.substring(0, host.length() - 1));
            if (shard == null) {
              shard = host;
            }
          }
          results.put(shard, rf);
        }
      }
    }
    return results;
  }
Example #3
0
  protected final synchronized int getShardIndex(String shardId, DocCollection dc) {
    if (shardIndexCache == null) shardIndexCache = new HashMap<String, Integer>(20);

    Integer idx = shardIndexCache.get(shardId);
    if (idx != null) return idx.intValue(); // meh auto-boxing

    int s = 0;
    for (Slice slice : dc.getSlices()) {
      if (shardId.equals(slice.getName())) {
        shardIndexCache.put(shardId, new Integer(s));
        return s;
      }
      ++s;
    }
    throw new IllegalStateException(
        "Cannot find index of shard '" + shardId + "' in collection: " + collection);
  }
Example #4
0
  protected List<String> buildShardList(CloudSolrClient cloudSolrServer) {
    ZkStateReader zkStateReader = cloudSolrServer.getZkStateReader();

    ClusterState clusterState = zkStateReader.getClusterState();

    String[] collections = null;
    if (clusterState.hasCollection(collection)) {
      collections = new String[] {collection};
    } else {
      // might be a collection alias?
      Aliases aliases = zkStateReader.getAliases();
      String aliasedCollections = aliases.getCollectionAlias(collection);
      if (aliasedCollections == null)
        throw new IllegalArgumentException("Collection " + collection + " not found!");
      collections = aliasedCollections.split(",");
    }

    Set<String> liveNodes = clusterState.getLiveNodes();
    Random random = new Random(5150);

    List<String> shards = new ArrayList<String>();
    for (String coll : collections) {
      for (Slice slice : clusterState.getSlices(coll)) {
        List<String> replicas = new ArrayList<String>();
        for (Replica r : slice.getReplicas()) {
          ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(r);
          if (liveNodes.contains(replicaCoreProps.getNodeName()))
            replicas.add(replicaCoreProps.getCoreUrl());
        }
        int numReplicas = replicas.size();
        if (numReplicas == 0)
          throw new IllegalStateException(
              "Shard " + slice.getName() + " does not have any active replicas!");

        String replicaUrl =
            (numReplicas == 1) ? replicas.get(0) : replicas.get(random.nextInt(replicas.size()));
        shards.add(replicaUrl);
      }
    }
    return shards;
  }
Example #5
0
  @Override
  public int getPartition(Object o) {

    Object docId = null;
    if (o instanceof SolrInputDocument) {
      SolrInputDocument doc = (SolrInputDocument) o;
      docId = doc.getFieldValue(idField);
      if (docId == null)
        throw new IllegalArgumentException(
            "SolrInputDocument must contain a non-null value for " + idField);
    } else {
      docId = o;
    }

    if (!(docId instanceof String))
      throw new IllegalArgumentException(
          "Only String document IDs are supported by this Partitioner!");

    DocCollection dc = getDocCollection();
    Slice slice = dc.getRouter().getTargetSlice((String) docId, null, null, null, dc);
    return getShardIndex(slice.getName(), dc);
  }
Example #6
0
 public String getShardId(String docId) {
   DocCollection dc = getDocCollection();
   Slice slice = dc.getRouter().getTargetSlice(docId, null, null, null, dc);
   return slice.getName();
 }
  public int createSubRequests(ResponseBuilder rb) throws IOException {
    SolrParams params = rb.req.getParams();
    String id1[] = params.getParams("id");
    String ids[] = params.getParams("ids");

    if (id1 == null && ids == null) {
      return ResponseBuilder.STAGE_DONE;
    }

    List<String> allIds = new ArrayList<String>();
    if (id1 != null) {
      for (String s : id1) {
        allIds.add(s);
      }
    }
    if (ids != null) {
      for (String s : ids) {
        allIds.addAll(StrUtils.splitSmart(s, ",", true));
      }
    }

    // TODO: handle collection=...?

    ZkController zkController =
        rb.req.getCore().getCoreDescriptor().getCoreContainer().getZkController();

    // if shards=... then use that
    if (zkController != null && params.get("shards") == null) {
      CloudDescriptor cloudDescriptor = rb.req.getCore().getCoreDescriptor().getCloudDescriptor();

      String collection = cloudDescriptor.getCollectionName();
      ClusterState clusterState = zkController.getClusterState();
      DocCollection coll = clusterState.getCollection(collection);

      Map<String, List<String>> sliceToId = new HashMap<String, List<String>>();
      for (String id : allIds) {
        Slice slice = coll.getRouter().getTargetSlice(id, null, params, coll);

        List<String> idsForShard = sliceToId.get(slice.getName());
        if (idsForShard == null) {
          idsForShard = new ArrayList<String>(2);
          sliceToId.put(slice.getName(), idsForShard);
        }
        idsForShard.add(id);
      }

      for (Map.Entry<String, List<String>> entry : sliceToId.entrySet()) {
        String shard = entry.getKey();
        String shardIdList = StrUtils.join(entry.getValue(), ',');

        ShardRequest sreq = new ShardRequest();

        sreq.purpose = 1;
        // sreq.shards = new String[]{shard};    // TODO: would be nice if this would work...
        sreq.shards = sliceToShards(rb, collection, shard);
        sreq.actualShards = sreq.shards;
        sreq.params = new ModifiableSolrParams();
        sreq.params.set(
            ShardParams.SHARDS_QT,
            "/get"); // TODO: how to avoid hardcoding this and hit the same handler?
        sreq.params.set("distrib", false);
        sreq.params.set("ids", shardIdList);

        rb.addRequest(this, sreq);
      }
    } else {
      String shardIdList = StrUtils.join(allIds, ',');
      ShardRequest sreq = new ShardRequest();

      sreq.purpose = 1;
      sreq.shards = null; // ALL
      sreq.actualShards = sreq.shards;
      sreq.params = new ModifiableSolrParams();
      sreq.params.set(
          ShardParams.SHARDS_QT,
          "/get"); // TODO: how to avoid hardcoding this and hit the same handler?
      sreq.params.set("distrib", false);
      sreq.params.set("ids", shardIdList);

      rb.addRequest(this, sreq);
    }

    return ResponseBuilder.STAGE_DONE;
  }