private Map<String, List<String>> buildUrlMap(DocCollection col) {
   Map<String, List<String>> urlMap = new HashMap<>();
   Collection<Slice> slices = col.getActiveSlices();
   Iterator<Slice> sliceIterator = slices.iterator();
   while (sliceIterator.hasNext()) {
     Slice slice = sliceIterator.next();
     String name = slice.getName();
     List<String> urls = new ArrayList<>();
     Replica leader = slice.getLeader();
     if (leader == null) {
       // take unoptimized general path - we cannot find a leader yet
       return null;
     }
     ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
     String url = zkProps.getCoreUrl();
     urls.add(url);
     Collection<Replica> replicas = slice.getReplicas();
     Iterator<Replica> replicaIterator = replicas.iterator();
     while (replicaIterator.hasNext()) {
       Replica replica = replicaIterator.next();
       if (!replica.getNodeName().equals(leader.getNodeName())
           && !replica.getName().equals(leader.getName())) {
         ZkCoreNodeProps zkProps1 = new ZkCoreNodeProps(replica);
         String url1 = zkProps1.getCoreUrl();
         urls.add(url1);
       }
     }
     urlMap.put(name, urls);
   }
   return urlMap;
 }
  private void waitTillNodesActive() throws Exception {
    for (int i = 0; i < 60; i++) {
      Thread.sleep(3000);
      ZkStateReader zkStateReader = cloudClient.getZkStateReader();
      ClusterState clusterState = zkStateReader.getClusterState();
      DocCollection collection1 = clusterState.getCollection("collection1");
      Slice slice = collection1.getSlice("shard1");
      Collection<Replica> replicas = slice.getReplicas();
      boolean allActive = true;

      Collection<String> nodesDownNames =
          nodesDown.stream().map(n -> n.coreNodeName).collect(Collectors.toList());

      Collection<Replica> replicasToCheck =
          replicas
              .stream()
              .filter(r -> !nodesDownNames.contains(r.getName()))
              .collect(Collectors.toList());

      for (Replica replica : replicasToCheck) {
        if (!clusterState.liveNodesContain(replica.getNodeName())
            || replica.getState() != Replica.State.ACTIVE) {
          allActive = false;
          break;
        }
      }
      if (allActive) {
        return;
      }
    }
    printLayout();
    fail("timeout waiting to see all nodes active");
  }
  private String getCoreUrl(
      String collectionName,
      String origCorename,
      ClusterState clusterState,
      Collection<Slice> slices,
      boolean byCoreName,
      boolean activeReplicas) {
    String coreUrl;
    Set<String> liveNodes = clusterState.getLiveNodes();
    List<Slice> randomizedSlices = new ArrayList<>(slices.size());
    randomizedSlices.addAll(slices);
    Collections.shuffle(randomizedSlices, random);

    for (Slice slice : randomizedSlices) {
      List<Replica> randomizedReplicas = new ArrayList<>();
      randomizedReplicas.addAll(slice.getReplicas());
      Collections.shuffle(randomizedReplicas, random);

      for (Replica replica : randomizedReplicas) {
        if (!activeReplicas
            || (liveNodes.contains(replica.getNodeName())
                && replica.getState() == Replica.State.ACTIVE)) {

          if (byCoreName && !collectionName.equals(replica.getStr(CORE_NAME_PROP))) {
            // if it's by core name, make sure they match
            continue;
          }
          if (replica.getStr(BASE_URL_PROP).equals(cores.getZkController().getBaseUrl())) {
            // don't count a local core
            continue;
          }

          if (origCorename != null) {
            coreUrl = replica.getStr(BASE_URL_PROP) + "/" + origCorename;
          } else {
            coreUrl = replica.getCoreUrl();
            if (coreUrl.endsWith("/")) {
              coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
            }
          }

          return coreUrl;
        }
      }
    }
    return null;
  }
  private void testNodesUsedByCreate() throws Exception {
    // we can use this client because we just want base url
    final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));

    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set("action", CollectionAction.CREATE.toString());

    params.set("numShards", 2);
    params.set(REPLICATION_FACTOR, 2);
    String collectionName = "nodes_used_collection";

    params.set("name", collectionName);

    if (secondConfigSet) {
      params.set("collection.configName", "conf1");
    }

    QueryRequest request = new QueryRequest(params);
    request.setPath("/admin/collections");
    createNewSolrServer("", baseUrl).request(request);

    List<Integer> numShardsNumReplicaList = new ArrayList<Integer>();
    numShardsNumReplicaList.add(2);
    numShardsNumReplicaList.add(2);
    checkForCollection("nodes_used_collection", numShardsNumReplicaList, null);

    List<String> createNodeList = new ArrayList<String>();

    Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();

    for (String node : liveNodes) {
      createNodeList.add(node);
    }

    DocCollection col =
        cloudClient.getZkStateReader().getClusterState().getCollection("nodes_used_collection");
    Collection<Slice> slices = col.getSlices();
    for (Slice slice : slices) {
      Collection<Replica> replicas = slice.getReplicas();
      for (Replica replica : replicas) {
        createNodeList.remove(replica.getNodeName());
      }
    }
    assertEquals(createNodeList.toString(), 1, createNodeList.size());
  }
Beispiel #5
0
  public static String assignNode(String collection, ClusterState state) {
    Map<String, Slice> sliceMap = state.getSlicesMap(collection);
    if (sliceMap == null) {
      return "core_node1";
    }

    int max = 0;
    for (Slice slice : sliceMap.values()) {
      for (Replica replica : slice.getReplicas()) {
        Matcher m = COUNT.matcher(replica.getName());
        if (m.matches()) {
          max = Math.max(max, Integer.parseInt(m.group(1)));
        }
      }
    }

    return "core_node" + (max + 1);
  }
Beispiel #6
0
  protected void constructStreams() throws IOException {

    try {
      Object pushStream = ((Expressible) tupleStream).toExpression(streamFactory);

      ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
      ClusterState clusterState = zkStateReader.getClusterState();
      Set<String> liveNodes = clusterState.getLiveNodes();
      Collection<Slice> slices = clusterState.getActiveSlices(this.collection);
      List<Replica> shuffler = new ArrayList();
      for (Slice slice : slices) {
        Collection<Replica> replicas = slice.getReplicas();
        for (Replica replica : replicas) {
          if (replica.getState() == Replica.State.ACTIVE
              && liveNodes.contains(replica.getNodeName())) shuffler.add(replica);
        }
      }

      if (workers > shuffler.size()) {
        throw new IOException("Number of workers exceeds nodes in the worker collection");
      }

      Collections.shuffle(shuffler, new Random());

      for (int w = 0; w < workers; w++) {
        HashMap params = new HashMap();
        params.put("distrib", "false"); // We are the aggregator.
        params.put("numWorkers", workers);
        params.put("workerID", w);
        params.put("expr", pushStream);
        params.put("qt", "/stream");
        Replica rep = shuffler.get(w);
        ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
        String url = zkProps.getCoreUrl();
        SolrStream solrStream = new SolrStream(url, params);
        solrStreams.add(solrStream);
      }

      assert (solrStreams.size() == workers);

    } catch (Exception e) {
      throw new IOException(e);
    }
  }
Beispiel #7
0
  protected List<String> buildShardList(CloudSolrClient cloudSolrServer) {
    ZkStateReader zkStateReader = cloudSolrServer.getZkStateReader();

    ClusterState clusterState = zkStateReader.getClusterState();

    String[] collections = null;
    if (clusterState.hasCollection(collection)) {
      collections = new String[] {collection};
    } else {
      // might be a collection alias?
      Aliases aliases = zkStateReader.getAliases();
      String aliasedCollections = aliases.getCollectionAlias(collection);
      if (aliasedCollections == null)
        throw new IllegalArgumentException("Collection " + collection + " not found!");
      collections = aliasedCollections.split(",");
    }

    Set<String> liveNodes = clusterState.getLiveNodes();
    Random random = new Random(5150);

    List<String> shards = new ArrayList<String>();
    for (String coll : collections) {
      for (Slice slice : clusterState.getSlices(coll)) {
        List<String> replicas = new ArrayList<String>();
        for (Replica r : slice.getReplicas()) {
          ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(r);
          if (liveNodes.contains(replicaCoreProps.getNodeName()))
            replicas.add(replicaCoreProps.getCoreUrl());
        }
        int numReplicas = replicas.size();
        if (numReplicas == 0)
          throw new IllegalStateException(
              "Shard " + slice.getName() + " does not have any active replicas!");

        String replicaUrl =
            (numReplicas == 1) ? replicas.get(0) : replicas.get(random.nextInt(replicas.size()));
        shards.add(replicaUrl);
      }
    }
    return shards;
  }
  @BeforeClass
  public static void setupCluster() throws Exception {
    final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");

    String configName = "solrCloudCollectionConfig";
    int nodeCount = 5;
    configureCluster(nodeCount).addConfig(configName, configDir).configure();

    Map<String, String> collectionProperties = new HashMap<>();
    collectionProperties.put("config", "solrconfig-tlog.xml");
    collectionProperties.put("schema", "schema.xml");

    // create a collection holding data for the "to" side of the JOIN

    int shards = 2;
    int replicas = 2;
    CollectionAdminRequest.createCollection(toColl, configName, shards, replicas)
        .setProperties(collectionProperties)
        .process(cluster.getSolrClient());

    // get the set of nodes where replicas for the "to" collection exist
    Set<String> nodeSet = new HashSet<>();
    ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
    ClusterState cs = zkStateReader.getClusterState();
    for (Slice slice : cs.getCollection(toColl).getActiveSlices())
      for (Replica replica : slice.getReplicas()) nodeSet.add(replica.getNodeName());
    assertTrue(nodeSet.size() > 0);

    // deploy the "from" collection to all nodes where the "to" collection exists
    CollectionAdminRequest.createCollection(fromColl, configName, 1, 4)
        .setCreateNodeSet(StringUtils.join(nodeSet, ","))
        .setProperties(collectionProperties)
        .process(cluster.getSolrClient());

    toDocId = indexDoc(toColl, 1001, "a", null, "b");
    indexDoc(fromColl, 2001, "a", "c", null);

    Thread.sleep(1000); // so the commits fire
  }
  private static String findLocalReplicaForFromIndex(ZkController zkController, String fromIndex) {
    String fromReplica = null;

    String nodeName = zkController.getNodeName();
    for (Slice slice : zkController.getClusterState().getActiveSlices(fromIndex)) {
      if (fromReplica != null)
        throw new SolrException(
            SolrException.ErrorCode.BAD_REQUEST,
            "SolrCloud join: multiple shards not yet supported " + fromIndex);

      for (Replica replica : slice.getReplicas()) {
        if (replica.getNodeName().equals(nodeName)) {
          fromReplica = replica.getStr(ZkStateReader.CORE_NAME_PROP);
          // found local replica, but is it Active?
          if (replica.getState() != Replica.State.ACTIVE)
            throw new SolrException(
                SolrException.ErrorCode.BAD_REQUEST,
                "SolrCloud join: "
                    + fromIndex
                    + " has a local replica ("
                    + fromReplica
                    + ") on "
                    + nodeName
                    + ", but it is "
                    + replica.getState());

          break;
        }
      }
    }

    if (fromReplica == null)
      throw new SolrException(
          SolrException.ErrorCode.BAD_REQUEST,
          "SolrCloud join: No active replicas for " + fromIndex + " found in node " + nodeName);

    return fromReplica;
  }
Beispiel #10
0
  public static ArrayList<Node> getNodesForNewShard(
      ClusterState clusterState,
      String collectionName,
      int numSlices,
      int maxShardsPerNode,
      int repFactor,
      String createNodeSetStr) {
    List<String> createNodeList =
        createNodeSetStr == null ? null : StrUtils.splitSmart(createNodeSetStr, ",", true);

    Set<String> nodes = clusterState.getLiveNodes();

    List<String> nodeList = new ArrayList<String>(nodes.size());
    nodeList.addAll(nodes);
    if (createNodeList != null) nodeList.retainAll(createNodeList);

    HashMap<String, Node> nodeNameVsShardCount = new HashMap<String, Node>();
    for (String s : nodeList) nodeNameVsShardCount.put(s, new Node(s));
    for (String s : clusterState.getCollections()) {
      DocCollection c = clusterState.getCollection(s);
      // identify suitable nodes  by checking the no:of cores in each of them
      for (Slice slice : c.getSlices()) {
        Collection<Replica> replicas = slice.getReplicas();
        for (Replica replica : replicas) {
          Node count = nodeNameVsShardCount.get(replica.getNodeName());
          if (count != null) {
            count.totalNodes++;
            if (s.equals(collectionName)) {
              count.thisCollectionNodes++;
              if (count.thisCollectionNodes >= maxShardsPerNode)
                nodeNameVsShardCount.remove(replica.getNodeName());
            }
          }
        }
      }
    }

    if (nodeNameVsShardCount.size() <= 0) {
      throw new SolrException(
          SolrException.ErrorCode.BAD_REQUEST,
          "Cannot create collection "
              + collectionName
              + ". No live Solr-instances"
              + ((createNodeList != null)
                  ? " among Solr-instances specified in " + CREATE_NODE_SET + ":" + createNodeSetStr
                  : ""));
    }

    if (repFactor > nodeNameVsShardCount.size()) {
      log.warn(
          "Specified "
              + REPLICATION_FACTOR
              + " of "
              + repFactor
              + " on collection "
              + collectionName
              + " is higher than or equal to the number of Solr instances currently live or part of your "
              + CREATE_NODE_SET
              + "("
              + nodeList.size()
              + "). Its unusual to run two replica of the same slice on the same Solr-instance.");
    }

    int maxCoresAllowedToCreate = maxShardsPerNode * nodeList.size();
    int requestedCoresToCreate = numSlices * repFactor;
    int minCoresToCreate = requestedCoresToCreate;
    if (maxCoresAllowedToCreate < minCoresToCreate) {
      throw new SolrException(
          SolrException.ErrorCode.BAD_REQUEST,
          "Cannot create shards "
              + collectionName
              + ". Value of "
              + MAX_SHARDS_PER_NODE
              + " is "
              + maxShardsPerNode
              + ", and the number of live nodes is "
              + nodeList.size()
              + ". This allows a maximum of "
              + maxCoresAllowedToCreate
              + " to be created. Value of "
              + NUM_SLICES
              + " is "
              + numSlices
              + " and value of "
              + REPLICATION_FACTOR
              + " is "
              + repFactor
              + ". This requires "
              + requestedCoresToCreate
              + " shards to be created (higher than the allowed number)");
    }

    ArrayList<Node> sortedNodeList = new ArrayList<>(nodeNameVsShardCount.values());
    Collections.sort(
        sortedNodeList,
        new Comparator<Node>() {
          @Override
          public int compare(Node x, Node y) {
            return (x.weight() < y.weight()) ? -1 : ((x.weight() == y.weight()) ? 0 : 1);
          }
        });
    return sortedNodeList;
  }