private Map<String, List<String>> buildUrlMap(DocCollection col) {
   Map<String, List<String>> urlMap = new HashMap<>();
   Collection<Slice> slices = col.getActiveSlices();
   Iterator<Slice> sliceIterator = slices.iterator();
   while (sliceIterator.hasNext()) {
     Slice slice = sliceIterator.next();
     String name = slice.getName();
     List<String> urls = new ArrayList<>();
     Replica leader = slice.getLeader();
     if (leader == null) {
       // take unoptimized general path - we cannot find a leader yet
       return null;
     }
     ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
     String url = zkProps.getCoreUrl();
     urls.add(url);
     Collection<Replica> replicas = slice.getReplicas();
     Iterator<Replica> replicaIterator = replicas.iterator();
     while (replicaIterator.hasNext()) {
       Replica replica = replicaIterator.next();
       if (!replica.getNodeName().equals(leader.getNodeName())
           && !replica.getName().equals(leader.getName())) {
         ZkCoreNodeProps zkProps1 = new ZkCoreNodeProps(replica);
         String url1 = zkProps1.getCoreUrl();
         urls.add(url1);
       }
     }
     urlMap.put(name, urls);
   }
   return urlMap;
 }
  private List<Node> getCollectionUrls(
      SolrQueryRequest req, String collection, String shardZkNodeName) {
    ClusterState clusterState =
        req.getCore().getCoreDescriptor().getCoreContainer().getZkController().getClusterState();
    List<Node> urls = new ArrayList<Node>();
    Map<String, Slice> slices = clusterState.getSlices(collection);
    if (slices == null) {
      throw new ZooKeeperException(
          ErrorCode.BAD_REQUEST, "Could not find collection in zk: " + clusterState);
    }
    for (Map.Entry<String, Slice> sliceEntry : slices.entrySet()) {
      Slice replicas = slices.get(sliceEntry.getKey());

      Map<String, Replica> shardMap = replicas.getReplicasMap();

      for (Entry<String, Replica> entry : shardMap.entrySet()) {
        ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(entry.getValue());
        if (clusterState.liveNodesContain(nodeProps.getNodeName())
            && !entry.getKey().equals(shardZkNodeName)) {
          urls.add(new StdNode(nodeProps));
        }
      }
    }
    if (urls.size() == 0) {
      return null;
    }
    return urls;
  }
  private void waitTillNodesActive() throws Exception {
    for (int i = 0; i < 60; i++) {
      Thread.sleep(3000);
      ZkStateReader zkStateReader = cloudClient.getZkStateReader();
      ClusterState clusterState = zkStateReader.getClusterState();
      DocCollection collection1 = clusterState.getCollection("collection1");
      Slice slice = collection1.getSlice("shard1");
      Collection<Replica> replicas = slice.getReplicas();
      boolean allActive = true;

      Collection<String> nodesDownNames =
          nodesDown.stream().map(n -> n.coreNodeName).collect(Collectors.toList());

      Collection<Replica> replicasToCheck =
          replicas
              .stream()
              .filter(r -> !nodesDownNames.contains(r.getName()))
              .collect(Collectors.toList());

      for (Replica replica : replicasToCheck) {
        if (!clusterState.liveNodesContain(replica.getNodeName())
            || replica.getState() != Replica.State.ACTIVE) {
          allActive = false;
          break;
        }
      }
      if (allActive) {
        return;
      }
    }
    printLayout();
    fail("timeout waiting to see all nodes active");
  }
示例#4
0
  @Override
  public Collection<Slice> getSearchSlicesSingle(
      String shardKey, SolrParams params, DocCollection collection) {
    if (shardKey == null) {
      // search across whole collection
      // TODO: this may need modification in the future when shard splitting could cause an overlap
      return collection.getSlices();
    }
    String id = shardKey;

    int idx = shardKey.indexOf(separator);
    if (idx < 0) {
      // shardKey is a simple id, so don't do a range
      return Collections.singletonList(
          hashToSlice(Hash.murmurhash3_x86_32(id, 0, id.length(), 0), collection));
    }

    int m1 = mask1;
    int m2 = mask2;

    String part1 = id.substring(0, idx);
    int bitsSepIdx = part1.indexOf(bitsSepartor);
    if (bitsSepIdx > 0) {
      int firstBits = getBits(part1, bitsSepIdx);
      if (firstBits >= 0) {
        m1 = firstBits == 0 ? 0 : (-1 << (32 - firstBits));
        m2 = firstBits == 32 ? 0 : (-1 >>> firstBits);
        part1 = part1.substring(0, bitsSepIdx);
      }
    }

    //  If the upper bits are 0xF0000000, the range we want to cover is
    //  0xF0000000 0xFfffffff

    int hash1 = Hash.murmurhash3_x86_32(part1, 0, part1.length(), 0);
    int upperBits = hash1 & m1;
    int lowerBound = upperBits;
    int upperBound = upperBits | m2;

    if (m1 == 0) {
      // no bits used from first part of key.. the code above will produce 0x000000000->0xffffffff
      // which only works on unsigned space, but we're using signed space.
      lowerBound = Integer.MIN_VALUE;
      upperBound = Integer.MAX_VALUE;
    }

    Range completeRange = new Range(lowerBound, upperBound);

    List<Slice> targetSlices = new ArrayList<Slice>(1);
    for (Slice slice : collection.getSlices()) {
      Range range = slice.getRange();
      if (range != null && range.overlaps(completeRange)) {
        targetSlices.add(slice);
      }
    }

    return targetSlices;
  }
示例#5
0
  private String getCoreUrl(
      String collectionName,
      String origCorename,
      ClusterState clusterState,
      Collection<Slice> slices,
      boolean byCoreName,
      boolean activeReplicas) {
    String coreUrl;
    Set<String> liveNodes = clusterState.getLiveNodes();
    List<Slice> randomizedSlices = new ArrayList<>(slices.size());
    randomizedSlices.addAll(slices);
    Collections.shuffle(randomizedSlices, random);

    for (Slice slice : randomizedSlices) {
      List<Replica> randomizedReplicas = new ArrayList<>();
      randomizedReplicas.addAll(slice.getReplicas());
      Collections.shuffle(randomizedReplicas, random);

      for (Replica replica : randomizedReplicas) {
        if (!activeReplicas
            || (liveNodes.contains(replica.getNodeName())
                && replica.getState() == Replica.State.ACTIVE)) {

          if (byCoreName && !collectionName.equals(replica.getStr(CORE_NAME_PROP))) {
            // if it's by core name, make sure they match
            continue;
          }
          if (replica.getStr(BASE_URL_PROP).equals(cores.getZkController().getBaseUrl())) {
            // don't count a local core
            continue;
          }

          if (origCorename != null) {
            coreUrl = replica.getStr(BASE_URL_PROP) + "/" + origCorename;
          } else {
            coreUrl = replica.getCoreUrl();
            if (coreUrl.endsWith("/")) {
              coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
            }
          }

          return coreUrl;
        }
      }
    }
    return null;
  }
示例#6
0
  protected final synchronized int getShardIndex(String shardId, DocCollection dc) {
    if (shardIndexCache == null) shardIndexCache = new HashMap<String, Integer>(20);

    Integer idx = shardIndexCache.get(shardId);
    if (idx != null) return idx.intValue(); // meh auto-boxing

    int s = 0;
    for (Slice slice : dc.getSlices()) {
      if (shardId.equals(slice.getName())) {
        shardIndexCache.put(shardId, new Integer(s));
        return s;
      }
      ++s;
    }
    throw new IllegalStateException(
        "Cannot find index of shard '" + shardId + "' in collection: " + collection);
  }
  private void testNodesUsedByCreate() throws Exception {
    // we can use this client because we just want base url
    final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));

    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set("action", CollectionAction.CREATE.toString());

    params.set("numShards", 2);
    params.set(REPLICATION_FACTOR, 2);
    String collectionName = "nodes_used_collection";

    params.set("name", collectionName);

    if (secondConfigSet) {
      params.set("collection.configName", "conf1");
    }

    QueryRequest request = new QueryRequest(params);
    request.setPath("/admin/collections");
    createNewSolrServer("", baseUrl).request(request);

    List<Integer> numShardsNumReplicaList = new ArrayList<Integer>();
    numShardsNumReplicaList.add(2);
    numShardsNumReplicaList.add(2);
    checkForCollection("nodes_used_collection", numShardsNumReplicaList, null);

    List<String> createNodeList = new ArrayList<String>();

    Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();

    for (String node : liveNodes) {
      createNodeList.add(node);
    }

    DocCollection col =
        cloudClient.getZkStateReader().getClusterState().getCollection("nodes_used_collection");
    Collection<Slice> slices = col.getSlices();
    for (Slice slice : slices) {
      Collection<Replica> replicas = slice.getReplicas();
      for (Replica replica : replicas) {
        createNodeList.remove(replica.getNodeName());
      }
    }
    assertEquals(createNodeList.toString(), 1, createNodeList.size());
  }
示例#8
0
  public static String assignNode(String collection, ClusterState state) {
    Map<String, Slice> sliceMap = state.getSlicesMap(collection);
    if (sliceMap == null) {
      return "core_node1";
    }

    int max = 0;
    for (Slice slice : sliceMap.values()) {
      for (Replica replica : slice.getReplicas()) {
        Matcher m = COUNT.matcher(replica.getName());
        if (m.matches()) {
          max = Math.max(max, Integer.parseInt(m.group(1)));
        }
      }
    }

    return "core_node" + (max + 1);
  }
示例#9
0
  protected void constructStreams() throws IOException {

    try {
      Object pushStream = ((Expressible) tupleStream).toExpression(streamFactory);

      ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
      ClusterState clusterState = zkStateReader.getClusterState();
      Set<String> liveNodes = clusterState.getLiveNodes();
      Collection<Slice> slices = clusterState.getActiveSlices(this.collection);
      List<Replica> shuffler = new ArrayList();
      for (Slice slice : slices) {
        Collection<Replica> replicas = slice.getReplicas();
        for (Replica replica : replicas) {
          if (replica.getState() == Replica.State.ACTIVE
              && liveNodes.contains(replica.getNodeName())) shuffler.add(replica);
        }
      }

      if (workers > shuffler.size()) {
        throw new IOException("Number of workers exceeds nodes in the worker collection");
      }

      Collections.shuffle(shuffler, new Random());

      for (int w = 0; w < workers; w++) {
        HashMap params = new HashMap();
        params.put("distrib", "false"); // We are the aggregator.
        params.put("numWorkers", workers);
        params.put("workerID", w);
        params.put("expr", pushStream);
        params.put("qt", "/stream");
        Replica rep = shuffler.get(w);
        ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
        String url = zkProps.getCoreUrl();
        SolrStream solrStream = new SolrStream(url, params);
        solrStreams.add(solrStream);
      }

      assert (solrStreams.size() == workers);

    } catch (Exception e) {
      throw new IOException(e);
    }
  }
    /**
     * Retrieve all requests recorded by this queue which were sent to given collection and shard
     *
     * @param zkStateReader the {@link org.apache.solr.common.cloud.ZkStateReader} from which
     *     cluster state is read
     * @param collectionName the given collection name for which requests have to be extracted
     * @param shardId the given shard name for which requests have to be extracted
     * @return a list of {@link
     *     org.apache.solr.handler.component.TrackingShardHandlerFactory.ShardRequestAndParams} or
     *     empty list if none are found
     */
    public List<ShardRequestAndParams> getShardRequests(
        ZkStateReader zkStateReader, String collectionName, String shardId) {
      DocCollection collection = zkStateReader.getClusterState().getCollection(collectionName);
      assert collection != null;
      Slice slice = collection.getSlice(shardId);
      assert slice != null;

      for (Map.Entry<String, List<ShardRequestAndParams>> entry : requests.entrySet()) {
        // multiple shard addresses may be present separated by '|'
        List<String> list = StrUtils.splitSmart(entry.getKey(), '|');
        for (Map.Entry<String, Replica> replica : slice.getReplicasMap().entrySet()) {
          String coreUrl = new ZkCoreNodeProps(replica.getValue()).getCoreUrl();
          if (list.contains(coreUrl)) {
            return new ArrayList<>(entry.getValue());
          }
        }
      }
      return Collections.emptyList();
    }
示例#11
0
  protected List<String> buildShardList(CloudSolrClient cloudSolrServer) {
    ZkStateReader zkStateReader = cloudSolrServer.getZkStateReader();

    ClusterState clusterState = zkStateReader.getClusterState();

    String[] collections = null;
    if (clusterState.hasCollection(collection)) {
      collections = new String[] {collection};
    } else {
      // might be a collection alias?
      Aliases aliases = zkStateReader.getAliases();
      String aliasedCollections = aliases.getCollectionAlias(collection);
      if (aliasedCollections == null)
        throw new IllegalArgumentException("Collection " + collection + " not found!");
      collections = aliasedCollections.split(",");
    }

    Set<String> liveNodes = clusterState.getLiveNodes();
    Random random = new Random(5150);

    List<String> shards = new ArrayList<String>();
    for (String coll : collections) {
      for (Slice slice : clusterState.getSlices(coll)) {
        List<String> replicas = new ArrayList<String>();
        for (Replica r : slice.getReplicas()) {
          ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(r);
          if (liveNodes.contains(replicaCoreProps.getNodeName()))
            replicas.add(replicaCoreProps.getCoreUrl());
        }
        int numReplicas = replicas.size();
        if (numReplicas == 0)
          throw new IllegalStateException(
              "Shard " + slice.getName() + " does not have any active replicas!");

        String replicaUrl =
            (numReplicas == 1) ? replicas.get(0) : replicas.get(random.nextInt(replicas.size()));
        shards.add(replicaUrl);
      }
    }
    return shards;
  }
  /**
   * Walks the NamedList response after performing an update request looking for the replication
   * factor that was achieved in each shard involved in the request. For single doc updates, there
   * will be only one shard in the return value.
   */
  @SuppressWarnings("rawtypes")
  public Map<String, Integer> getShardReplicationFactor(String collection, NamedList resp) {
    connect();

    Map<String, Integer> results = new HashMap<String, Integer>();
    if (resp instanceof CloudSolrServer.RouteResponse) {
      NamedList routes = ((CloudSolrServer.RouteResponse) resp).getRouteResponses();
      ClusterState clusterState = zkStateReader.getClusterState();
      Map<String, String> leaders = new HashMap<String, String>();
      for (Slice slice : clusterState.getActiveSlices(collection)) {
        Replica leader = slice.getLeader();
        if (leader != null) {
          ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
          String leaderUrl = zkProps.getBaseUrl() + "/" + zkProps.getCoreName();
          leaders.put(leaderUrl, slice.getName());
          String altLeaderUrl = zkProps.getBaseUrl() + "/" + collection;
          leaders.put(altLeaderUrl, slice.getName());
        }
      }

      Iterator<Map.Entry<String, Object>> routeIter = routes.iterator();
      while (routeIter.hasNext()) {
        Map.Entry<String, Object> next = routeIter.next();
        String host = next.getKey();
        NamedList hostResp = (NamedList) next.getValue();
        Integer rf =
            (Integer) ((NamedList) hostResp.get("responseHeader")).get(UpdateRequest.REPFACT);
        if (rf != null) {
          String shard = leaders.get(host);
          if (shard == null) {
            if (host.endsWith("/")) shard = leaders.get(host.substring(0, host.length() - 1));
            if (shard == null) {
              shard = host;
            }
          }
          results.put(shard, rf);
        }
      }
    }
    return results;
  }
示例#13
0
  @Override
  public int getPartition(Object o) {

    Object docId = null;
    if (o instanceof SolrInputDocument) {
      SolrInputDocument doc = (SolrInputDocument) o;
      docId = doc.getFieldValue(idField);
      if (docId == null)
        throw new IllegalArgumentException(
            "SolrInputDocument must contain a non-null value for " + idField);
    } else {
      docId = o;
    }

    if (!(docId instanceof String))
      throw new IllegalArgumentException(
          "Only String document IDs are supported by this Partitioner!");

    DocCollection dc = getDocCollection();
    Slice slice = dc.getRouter().getTargetSlice((String) docId, null, null, null, dc);
    return getShardIndex(slice.getName(), dc);
  }
  @BeforeClass
  public static void setupCluster() throws Exception {
    final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");

    String configName = "solrCloudCollectionConfig";
    int nodeCount = 5;
    configureCluster(nodeCount).addConfig(configName, configDir).configure();

    Map<String, String> collectionProperties = new HashMap<>();
    collectionProperties.put("config", "solrconfig-tlog.xml");
    collectionProperties.put("schema", "schema.xml");

    // create a collection holding data for the "to" side of the JOIN

    int shards = 2;
    int replicas = 2;
    CollectionAdminRequest.createCollection(toColl, configName, shards, replicas)
        .setProperties(collectionProperties)
        .process(cluster.getSolrClient());

    // get the set of nodes where replicas for the "to" collection exist
    Set<String> nodeSet = new HashSet<>();
    ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
    ClusterState cs = zkStateReader.getClusterState();
    for (Slice slice : cs.getCollection(toColl).getActiveSlices())
      for (Replica replica : slice.getReplicas()) nodeSet.add(replica.getNodeName());
    assertTrue(nodeSet.size() > 0);

    // deploy the "from" collection to all nodes where the "to" collection exists
    CollectionAdminRequest.createCollection(fromColl, configName, 1, 4)
        .setCreateNodeSet(StringUtils.join(nodeSet, ","))
        .setProperties(collectionProperties)
        .process(cluster.getSolrClient());

    toDocId = indexDoc(toColl, 1001, "a", null, "b");
    indexDoc(fromColl, 2001, "a", "c", null);

    Thread.sleep(1000); // so the commits fire
  }
  private static String findLocalReplicaForFromIndex(ZkController zkController, String fromIndex) {
    String fromReplica = null;

    String nodeName = zkController.getNodeName();
    for (Slice slice : zkController.getClusterState().getActiveSlices(fromIndex)) {
      if (fromReplica != null)
        throw new SolrException(
            SolrException.ErrorCode.BAD_REQUEST,
            "SolrCloud join: multiple shards not yet supported " + fromIndex);

      for (Replica replica : slice.getReplicas()) {
        if (replica.getNodeName().equals(nodeName)) {
          fromReplica = replica.getStr(ZkStateReader.CORE_NAME_PROP);
          // found local replica, but is it Active?
          if (replica.getState() != Replica.State.ACTIVE)
            throw new SolrException(
                SolrException.ErrorCode.BAD_REQUEST,
                "SolrCloud join: "
                    + fromIndex
                    + " has a local replica ("
                    + fromReplica
                    + ") on "
                    + nodeName
                    + ", but it is "
                    + replica.getState());

          break;
        }
      }
    }

    if (fromReplica == null)
      throw new SolrException(
          SolrException.ErrorCode.BAD_REQUEST,
          "SolrCloud join: No active replicas for " + fromIndex + " found in node " + nodeName);

    return fromReplica;
  }
示例#16
0
  private void getShardLeaders() {
    Set<Entry<String, Map<String, Slice>>> collections = collectionStates.entrySet();
    for (Entry<String, Map<String, Slice>> collection : collections) {
      Map<String, Slice> state = collection.getValue();
      Set<Entry<String, Slice>> slices = state.entrySet();
      for (Entry<String, Slice> sliceEntry : slices) {
        Slice slice = sliceEntry.getValue();
        Map<String, ZkNodeProps> shards = slice.getShards();
        Set<Entry<String, ZkNodeProps>> shardsEntries = shards.entrySet();
        for (Entry<String, ZkNodeProps> shardEntry : shardsEntries) {
          ZkNodeProps props = shardEntry.getValue();
          if (props.containsKey(ZkStateReader.LEADER_PROP)) {
            Map<String, ZkNodeProps> leadersForCollection = leaders.get(collection.getKey());
            if (leadersForCollection == null) {
              leadersForCollection = new HashMap<String, ZkNodeProps>();

              leaders.put(collection.getKey(), leadersForCollection);
            }
            leadersForCollection.put(sliceEntry.getKey(), props);
          }
        }
      }
    }
  }
  private String getUrlFromZk(String collection) {
    ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
    Map<String, Slice> slices = clusterState.getSlicesMap(collection);

    if (slices == null) {
      throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection:" + collection);
    }

    for (Map.Entry<String, Slice> entry : slices.entrySet()) {
      Slice slice = entry.getValue();
      Map<String, Replica> shards = slice.getReplicasMap();
      Set<Map.Entry<String, Replica>> shardEntries = shards.entrySet();
      for (Map.Entry<String, Replica> shardEntry : shardEntries) {
        final ZkNodeProps node = shardEntry.getValue();
        if (clusterState.liveNodesContain(node.getStr(ZkStateReader.NODE_NAME_PROP))) {
          return ZkCoreNodeProps.getCoreUrl(
              node.getStr(ZkStateReader.BASE_URL_PROP),
              collection); // new ZkCoreNodeProps(node).getCoreUrl();
        }
      }
    }

    throw new RuntimeException("Could not find a live node for collection:" + collection);
  }
示例#18
0
  public int createSubRequests(ResponseBuilder rb) throws IOException {
    SolrParams params = rb.req.getParams();
    String id1[] = params.getParams("id");
    String ids[] = params.getParams("ids");

    if (id1 == null && ids == null) {
      return ResponseBuilder.STAGE_DONE;
    }

    List<String> allIds = new ArrayList<String>();
    if (id1 != null) {
      for (String s : id1) {
        allIds.add(s);
      }
    }
    if (ids != null) {
      for (String s : ids) {
        allIds.addAll(StrUtils.splitSmart(s, ",", true));
      }
    }

    // TODO: handle collection=...?

    ZkController zkController =
        rb.req.getCore().getCoreDescriptor().getCoreContainer().getZkController();

    // if shards=... then use that
    if (zkController != null && params.get("shards") == null) {
      CloudDescriptor cloudDescriptor = rb.req.getCore().getCoreDescriptor().getCloudDescriptor();

      String collection = cloudDescriptor.getCollectionName();
      ClusterState clusterState = zkController.getClusterState();
      DocCollection coll = clusterState.getCollection(collection);

      Map<String, List<String>> sliceToId = new HashMap<String, List<String>>();
      for (String id : allIds) {
        Slice slice = coll.getRouter().getTargetSlice(id, null, params, coll);

        List<String> idsForShard = sliceToId.get(slice.getName());
        if (idsForShard == null) {
          idsForShard = new ArrayList<String>(2);
          sliceToId.put(slice.getName(), idsForShard);
        }
        idsForShard.add(id);
      }

      for (Map.Entry<String, List<String>> entry : sliceToId.entrySet()) {
        String shard = entry.getKey();
        String shardIdList = StrUtils.join(entry.getValue(), ',');

        ShardRequest sreq = new ShardRequest();

        sreq.purpose = 1;
        // sreq.shards = new String[]{shard};    // TODO: would be nice if this would work...
        sreq.shards = sliceToShards(rb, collection, shard);
        sreq.actualShards = sreq.shards;
        sreq.params = new ModifiableSolrParams();
        sreq.params.set(
            ShardParams.SHARDS_QT,
            "/get"); // TODO: how to avoid hardcoding this and hit the same handler?
        sreq.params.set("distrib", false);
        sreq.params.set("ids", shardIdList);

        rb.addRequest(this, sreq);
      }
    } else {
      String shardIdList = StrUtils.join(allIds, ',');
      ShardRequest sreq = new ShardRequest();

      sreq.purpose = 1;
      sreq.shards = null; // ALL
      sreq.actualShards = sreq.shards;
      sreq.params = new ModifiableSolrParams();
      sreq.params.set(
          ShardParams.SHARDS_QT,
          "/get"); // TODO: how to avoid hardcoding this and hit the same handler?
      sreq.params.set("distrib", false);
      sreq.params.set("ids", shardIdList);

      rb.addRequest(this, sreq);
    }

    return ResponseBuilder.STAGE_DONE;
  }
  protected NamedList<Object> sendRequest(SolrRequest request)
      throws SolrServerException, IOException {
    connect();

    ClusterState clusterState = zkStateReader.getClusterState();

    boolean sendToLeaders = false;
    List<String> replicas = null;

    if (request instanceof IsUpdateRequest) {
      if (request instanceof UpdateRequest) {
        NamedList<Object> response = directUpdate((AbstractUpdateRequest) request, clusterState);
        if (response != null) {
          return response;
        }
      }
      sendToLeaders = true;
      replicas = new ArrayList<>();
    }

    SolrParams reqParams = request.getParams();
    if (reqParams == null) {
      reqParams = new ModifiableSolrParams();
    }
    List<String> theUrlList = new ArrayList<>();
    if (request.getPath().equals("/admin/collections")
        || request.getPath().equals("/admin/cores")) {
      Set<String> liveNodes = clusterState.getLiveNodes();
      for (String liveNode : liveNodes) {
        theUrlList.add(zkStateReader.getBaseUrlForNodeName(liveNode));
      }
    } else {
      String collection = reqParams.get(UpdateParams.COLLECTION, defaultCollection);

      if (collection == null) {
        throw new SolrServerException(
            "No collection param specified on request and no default collection has been set.");
      }

      Set<String> collectionsList = getCollectionList(clusterState, collection);
      if (collectionsList.size() == 0) {
        throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection: " + collection);
      }

      String shardKeys = reqParams.get(ShardParams._ROUTE_);
      if (shardKeys == null) {
        shardKeys = reqParams.get(ShardParams.SHARD_KEYS); // deprecated
      }

      // TODO: not a big deal because of the caching, but we could avoid looking
      // at every shard
      // when getting leaders if we tweaked some things

      // Retrieve slices from the cloud state and, for each collection
      // specified,
      // add it to the Map of slices.
      Map<String, Slice> slices = new HashMap<>();
      for (String collectionName : collectionsList) {
        DocCollection col = getDocCollection(clusterState, collectionName);
        Collection<Slice> routeSlices = col.getRouter().getSearchSlices(shardKeys, reqParams, col);
        ClientUtils.addSlices(slices, collectionName, routeSlices, true);
      }
      Set<String> liveNodes = clusterState.getLiveNodes();

      List<String> leaderUrlList = null;
      List<String> urlList = null;
      List<String> replicasList = null;

      // build a map of unique nodes
      // TODO: allow filtering by group, role, etc
      Map<String, ZkNodeProps> nodes = new HashMap<>();
      List<String> urlList2 = new ArrayList<>();
      for (Slice slice : slices.values()) {
        for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) {
          ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
          String node = coreNodeProps.getNodeName();
          if (!liveNodes.contains(coreNodeProps.getNodeName())
              || !coreNodeProps.getState().equals(ZkStateReader.ACTIVE)) continue;
          if (nodes.put(node, nodeProps) == null) {
            if (!sendToLeaders || (sendToLeaders && coreNodeProps.isLeader())) {
              String url;
              if (reqParams.get(UpdateParams.COLLECTION) == null) {
                url =
                    ZkCoreNodeProps.getCoreUrl(
                        nodeProps.getStr(ZkStateReader.BASE_URL_PROP), defaultCollection);
              } else {
                url = coreNodeProps.getCoreUrl();
              }
              urlList2.add(url);
            } else if (sendToLeaders) {
              String url;
              if (reqParams.get(UpdateParams.COLLECTION) == null) {
                url =
                    ZkCoreNodeProps.getCoreUrl(
                        nodeProps.getStr(ZkStateReader.BASE_URL_PROP), defaultCollection);
              } else {
                url = coreNodeProps.getCoreUrl();
              }
              replicas.add(url);
            }
          }
        }
      }

      if (sendToLeaders) {
        leaderUrlList = urlList2;
        replicasList = replicas;
      } else {
        urlList = urlList2;
      }

      if (sendToLeaders) {
        theUrlList = new ArrayList<>(leaderUrlList.size());
        theUrlList.addAll(leaderUrlList);
      } else {
        theUrlList = new ArrayList<>(urlList.size());
        theUrlList.addAll(urlList);
      }
      if (theUrlList.isEmpty()) {
        throw new SolrException(
            SolrException.ErrorCode.INVALID_STATE, "Not enough nodes to handle the request");
      }

      Collections.shuffle(theUrlList, rand);
      if (sendToLeaders) {
        ArrayList<String> theReplicas = new ArrayList<>(replicasList.size());
        theReplicas.addAll(replicasList);
        Collections.shuffle(theReplicas, rand);
        theUrlList.addAll(theReplicas);
      }
    }

    LBHttpSolrServer.Req req = new LBHttpSolrServer.Req(request, theUrlList);
    LBHttpSolrServer.Rsp rsp = lbServer.request(req);
    return rsp.getResponse();
  }
示例#20
0
  public static ArrayList<Node> getNodesForNewShard(
      ClusterState clusterState,
      String collectionName,
      int numSlices,
      int maxShardsPerNode,
      int repFactor,
      String createNodeSetStr) {
    List<String> createNodeList =
        createNodeSetStr == null ? null : StrUtils.splitSmart(createNodeSetStr, ",", true);

    Set<String> nodes = clusterState.getLiveNodes();

    List<String> nodeList = new ArrayList<String>(nodes.size());
    nodeList.addAll(nodes);
    if (createNodeList != null) nodeList.retainAll(createNodeList);

    HashMap<String, Node> nodeNameVsShardCount = new HashMap<String, Node>();
    for (String s : nodeList) nodeNameVsShardCount.put(s, new Node(s));
    for (String s : clusterState.getCollections()) {
      DocCollection c = clusterState.getCollection(s);
      // identify suitable nodes  by checking the no:of cores in each of them
      for (Slice slice : c.getSlices()) {
        Collection<Replica> replicas = slice.getReplicas();
        for (Replica replica : replicas) {
          Node count = nodeNameVsShardCount.get(replica.getNodeName());
          if (count != null) {
            count.totalNodes++;
            if (s.equals(collectionName)) {
              count.thisCollectionNodes++;
              if (count.thisCollectionNodes >= maxShardsPerNode)
                nodeNameVsShardCount.remove(replica.getNodeName());
            }
          }
        }
      }
    }

    if (nodeNameVsShardCount.size() <= 0) {
      throw new SolrException(
          SolrException.ErrorCode.BAD_REQUEST,
          "Cannot create collection "
              + collectionName
              + ". No live Solr-instances"
              + ((createNodeList != null)
                  ? " among Solr-instances specified in " + CREATE_NODE_SET + ":" + createNodeSetStr
                  : ""));
    }

    if (repFactor > nodeNameVsShardCount.size()) {
      log.warn(
          "Specified "
              + REPLICATION_FACTOR
              + " of "
              + repFactor
              + " on collection "
              + collectionName
              + " is higher than or equal to the number of Solr instances currently live or part of your "
              + CREATE_NODE_SET
              + "("
              + nodeList.size()
              + "). Its unusual to run two replica of the same slice on the same Solr-instance.");
    }

    int maxCoresAllowedToCreate = maxShardsPerNode * nodeList.size();
    int requestedCoresToCreate = numSlices * repFactor;
    int minCoresToCreate = requestedCoresToCreate;
    if (maxCoresAllowedToCreate < minCoresToCreate) {
      throw new SolrException(
          SolrException.ErrorCode.BAD_REQUEST,
          "Cannot create shards "
              + collectionName
              + ". Value of "
              + MAX_SHARDS_PER_NODE
              + " is "
              + maxShardsPerNode
              + ", and the number of live nodes is "
              + nodeList.size()
              + ". This allows a maximum of "
              + maxCoresAllowedToCreate
              + " to be created. Value of "
              + NUM_SLICES
              + " is "
              + numSlices
              + " and value of "
              + REPLICATION_FACTOR
              + " is "
              + repFactor
              + ". This requires "
              + requestedCoresToCreate
              + " shards to be created (higher than the allowed number)");
    }

    ArrayList<Node> sortedNodeList = new ArrayList<>(nodeNameVsShardCount.values());
    Collections.sort(
        sortedNodeList,
        new Comparator<Node>() {
          @Override
          public int compare(Node x, Node y) {
            return (x.weight() < y.weight()) ? -1 : ((x.weight() == y.weight()) ? 0 : 1);
          }
        });
    return sortedNodeList;
  }
  @Test
  public void testReadShards() throws Exception {
    String zkDir = dataDir.getAbsolutePath() + File.separator + "zookeeper/server1/data";
    ZkTestServer server = null;
    SolrZkClient zkClient = null;
    ZkController zkController = null;
    try {
      server = new ZkTestServer(zkDir);
      server.run();

      AbstractZkTestCase.makeSolrZkNode(server.getZkHost());

      zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
      String shardsPath = "/collections/collection1/shards/shardid1";
      zkClient.makePath(shardsPath);

      addShardToZk(zkClient, shardsPath, SHARD1, URL1);
      addShardToZk(zkClient, shardsPath, SHARD2, URL2);
      addShardToZk(zkClient, shardsPath, SHARD3, URL3);

      if (DEBUG) {
        zkClient.printLayoutToStdOut();
      }

      zkController =
          new ZkController(server.getZkAddress(), TIMEOUT, 1000, "localhost", "8983", "solr");

      zkController.getZkStateReader().updateCloudState(true);
      CloudState cloudInfo = zkController.getCloudState();
      Map<String, Slice> slices = cloudInfo.getSlices("collection1");
      assertNotNull(slices);

      for (Slice slice : slices.values()) {
        Map<String, ZkNodeProps> shards = slice.getShards();
        if (DEBUG) {
          for (String shardName : shards.keySet()) {
            ZkNodeProps props = shards.get(shardName);
            System.out.println("shard:" + shardName);
            System.out.println("props:" + props.toString());
          }
        }
        assertNotNull(shards.get(SHARD1));
        assertNotNull(shards.get(SHARD2));
        assertNotNull(shards.get(SHARD3));

        ZkNodeProps props = shards.get(SHARD1);
        assertEquals(URL1, props.get(ZkStateReader.URL_PROP));
        assertEquals(TEST_NODE_NAME, props.get(ZkStateReader.NODE_NAME));

        props = shards.get(SHARD2);
        assertEquals(URL2, props.get(ZkStateReader.URL_PROP));
        assertEquals(TEST_NODE_NAME, props.get(ZkStateReader.NODE_NAME));

        props = shards.get(SHARD3);
        assertEquals(URL3, props.get(ZkStateReader.URL_PROP));
        assertEquals(TEST_NODE_NAME, props.get(ZkStateReader.NODE_NAME));
      }

    } finally {
      if (zkClient != null) {
        zkClient.close();
      }
      if (zkController != null) {
        zkController.close();
      }
      if (server != null) {
        server.shutdown();
      }
    }
  }
示例#22
0
 public String getShardId(String docId) {
   DocCollection dc = getDocCollection();
   Slice slice = dc.getRouter().getTargetSlice(docId, null, null, null, dc);
   return slice.getName();
 }