private void waitTillNodesActive() throws Exception {
    for (int i = 0; i < 60; i++) {
      Thread.sleep(3000);
      ZkStateReader zkStateReader = cloudClient.getZkStateReader();
      ClusterState clusterState = zkStateReader.getClusterState();
      DocCollection collection1 = clusterState.getCollection("collection1");
      Slice slice = collection1.getSlice("shard1");
      Collection<Replica> replicas = slice.getReplicas();
      boolean allActive = true;

      Collection<String> nodesDownNames =
          nodesDown.stream().map(n -> n.coreNodeName).collect(Collectors.toList());

      Collection<Replica> replicasToCheck =
          replicas
              .stream()
              .filter(r -> !nodesDownNames.contains(r.getName()))
              .collect(Collectors.toList());

      for (Replica replica : replicasToCheck) {
        if (!clusterState.liveNodesContain(replica.getNodeName())
            || replica.getState() != Replica.State.ACTIVE) {
          allActive = false;
          break;
        }
      }
      if (allActive) {
        return;
      }
    }
    printLayout();
    fail("timeout waiting to see all nodes active");
  }
Exemplo n.º 2
0
  private String getRemotCoreUrl(String collectionName, String origCorename) {
    ClusterState clusterState = cores.getZkController().getClusterState();
    Collection<Slice> slices = clusterState.getActiveSlices(collectionName);
    boolean byCoreName = false;

    if (slices == null) {
      slices = new ArrayList<>();
      // look by core name
      byCoreName = true;
      getSlicesForCollections(clusterState, slices, true);
      if (slices.isEmpty()) {
        getSlicesForCollections(clusterState, slices, false);
      }
    }

    if (slices.isEmpty()) {
      return null;
    }

    if (collectionsList == null) collectionsList = new ArrayList<>();

    collectionsList.add(collectionName);
    String coreUrl =
        getCoreUrl(collectionName, origCorename, clusterState, slices, byCoreName, true);

    if (coreUrl == null) {
      coreUrl = getCoreUrl(collectionName, origCorename, clusterState, slices, byCoreName, false);
    }

    return coreUrl;
  }
  private void collectStartTimes(String collectionName, Map<String, Long> urlToTime)
      throws SolrServerException, IOException {
    ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
    //    Map<String,DocCollection> collections = clusterState.getCollectionStates();
    if (clusterState.hasCollection(collectionName)) {
      Map<String, Slice> slices = clusterState.getSlicesMap(collectionName);

      Iterator<Entry<String, Slice>> it = slices.entrySet().iterator();
      while (it.hasNext()) {
        Entry<String, Slice> sliceEntry = it.next();
        Map<String, Replica> sliceShards = sliceEntry.getValue().getReplicasMap();
        Iterator<Entry<String, Replica>> shardIt = sliceShards.entrySet().iterator();
        while (shardIt.hasNext()) {
          Entry<String, Replica> shardEntry = shardIt.next();
          ZkCoreNodeProps coreProps = new ZkCoreNodeProps(shardEntry.getValue());
          HttpSolrServer server = new HttpSolrServer(coreProps.getBaseUrl());
          CoreAdminResponse mcr;
          try {
            mcr = CoreAdminRequest.getStatus(coreProps.getCoreName(), server);
          } finally {
            server.shutdown();
          }
          long before = mcr.getStartTime(coreProps.getCoreName()).getTime();
          urlToTime.put(coreProps.getCoreUrl(), before);
        }
      }
    } else {
      throw new IllegalArgumentException(
          "Could not find collection in :" + clusterState.getCollections());
    }
  }
  private List<Node> getCollectionUrls(
      SolrQueryRequest req, String collection, String shardZkNodeName) {
    ClusterState clusterState =
        req.getCore().getCoreDescriptor().getCoreContainer().getZkController().getClusterState();
    List<Node> urls = new ArrayList<Node>();
    Map<String, Slice> slices = clusterState.getSlices(collection);
    if (slices == null) {
      throw new ZooKeeperException(
          ErrorCode.BAD_REQUEST, "Could not find collection in zk: " + clusterState);
    }
    for (Map.Entry<String, Slice> sliceEntry : slices.entrySet()) {
      Slice replicas = slices.get(sliceEntry.getKey());

      Map<String, Replica> shardMap = replicas.getReplicasMap();

      for (Entry<String, Replica> entry : shardMap.entrySet()) {
        ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(entry.getValue());
        if (clusterState.liveNodesContain(nodeProps.getNodeName())
            && !entry.getKey().equals(shardZkNodeName)) {
          urls.add(new StdNode(nodeProps));
        }
      }
    }
    if (urls.size() == 0) {
      return null;
    }
    return urls;
  }
Exemplo n.º 5
0
  private SolrCore getCoreByCollection(String collectionName) {
    ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();

    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection collection = clusterState.getCollectionOrNull(collectionName);
    if (collection == null) {
      return null;
    }
    Map<String, Slice> slices = collection.getActiveSlicesMap();
    if (slices == null) {
      return null;
    }
    Set<String> liveNodes = clusterState.getLiveNodes();
    // look for a core on this node
    Set<Map.Entry<String, Slice>> entries = slices.entrySet();
    SolrCore core = null;

    // Hitting the leaders is useful when it's an update request.
    // For queries it doesn't matter and hence we don't distinguish here.
    for (Map.Entry<String, Slice> entry : entries) {
      // first see if we have the leader
      Replica leaderProps = collection.getLeader(entry.getKey());
      if (leaderProps != null
          && liveNodes.contains(leaderProps.getNodeName())
          && leaderProps.getState() == Replica.State.ACTIVE) {
        core = checkProps(leaderProps);
        if (core != null) {
          return core;
        }
      }

      // check everyone then
      Map<String, Replica> shards = entry.getValue().getReplicasMap();
      Set<Map.Entry<String, Replica>> shardEntries = shards.entrySet();
      for (Map.Entry<String, Replica> shardEntry : shardEntries) {
        Replica zkProps = shardEntry.getValue();
        if (liveNodes.contains(zkProps.getNodeName())
            && zkProps.getState() == Replica.State.ACTIVE) {
          core = checkProps(zkProps);
          if (core != null) {
            return core;
          }
        }
      }
    }
    return null;
  }
Exemplo n.º 6
0
 private void getSlicesForCollections(
     ClusterState clusterState, Collection<Slice> slices, boolean activeSlices) {
   if (activeSlices) {
     for (Map.Entry<String, DocCollection> entry : clusterState.getCollectionsMap().entrySet()) {
       final Collection<Slice> activeCollectionSlices = entry.getValue().getActiveSlices();
       if (activeCollectionSlices != null) {
         slices.addAll(activeCollectionSlices);
       }
     }
   } else {
     for (Map.Entry<String, DocCollection> entry : clusterState.getCollectionsMap().entrySet()) {
       final Collection<Slice> collectionSlices = entry.getValue().getSlices();
       if (collectionSlices != null) {
         slices.addAll(collectionSlices);
       }
     }
   }
 }
Exemplo n.º 7
0
  protected void constructStreams() throws IOException {

    try {
      Object pushStream = ((Expressible) tupleStream).toExpression(streamFactory);

      ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();
      ClusterState clusterState = zkStateReader.getClusterState();
      Set<String> liveNodes = clusterState.getLiveNodes();
      Collection<Slice> slices = clusterState.getActiveSlices(this.collection);
      List<Replica> shuffler = new ArrayList();
      for (Slice slice : slices) {
        Collection<Replica> replicas = slice.getReplicas();
        for (Replica replica : replicas) {
          if (replica.getState() == Replica.State.ACTIVE
              && liveNodes.contains(replica.getNodeName())) shuffler.add(replica);
        }
      }

      if (workers > shuffler.size()) {
        throw new IOException("Number of workers exceeds nodes in the worker collection");
      }

      Collections.shuffle(shuffler, new Random());

      for (int w = 0; w < workers; w++) {
        HashMap params = new HashMap();
        params.put("distrib", "false"); // We are the aggregator.
        params.put("numWorkers", workers);
        params.put("workerID", w);
        params.put("expr", pushStream);
        params.put("qt", "/stream");
        Replica rep = shuffler.get(w);
        ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep);
        String url = zkProps.getCoreUrl();
        SolrStream solrStream = new SolrStream(url, params);
        solrStreams.add(solrStream);
      }

      assert (solrStreams.size() == workers);

    } catch (Exception e) {
      throw new IOException(e);
    }
  }
 private void checkForMissingCollection(String collectionName) throws Exception {
   // check for a  collection - we poll the state
   long timeoutAt = System.currentTimeMillis() + 45000;
   boolean found = true;
   while (System.currentTimeMillis() < timeoutAt) {
     getCommonCloudSolrServer().getZkStateReader().updateClusterState(true);
     ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
     //      Map<String,DocCollection> collections = clusterState
     //          .getCollectionStates();
     if (!clusterState.hasCollection(collectionName)) {
       found = false;
       break;
     }
     Thread.sleep(100);
   }
   if (found) {
     fail("Found collection that should be gone " + collectionName);
   }
 }
Exemplo n.º 9
0
  protected List<String> buildShardList(CloudSolrClient cloudSolrServer) {
    ZkStateReader zkStateReader = cloudSolrServer.getZkStateReader();

    ClusterState clusterState = zkStateReader.getClusterState();

    String[] collections = null;
    if (clusterState.hasCollection(collection)) {
      collections = new String[] {collection};
    } else {
      // might be a collection alias?
      Aliases aliases = zkStateReader.getAliases();
      String aliasedCollections = aliases.getCollectionAlias(collection);
      if (aliasedCollections == null)
        throw new IllegalArgumentException("Collection " + collection + " not found!");
      collections = aliasedCollections.split(",");
    }

    Set<String> liveNodes = clusterState.getLiveNodes();
    Random random = new Random(5150);

    List<String> shards = new ArrayList<String>();
    for (String coll : collections) {
      for (Slice slice : clusterState.getSlices(coll)) {
        List<String> replicas = new ArrayList<String>();
        for (Replica r : slice.getReplicas()) {
          ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(r);
          if (liveNodes.contains(replicaCoreProps.getNodeName()))
            replicas.add(replicaCoreProps.getCoreUrl());
        }
        int numReplicas = replicas.size();
        if (numReplicas == 0)
          throw new IllegalStateException(
              "Shard " + slice.getName() + " does not have any active replicas!");

        String replicaUrl =
            (numReplicas == 1) ? replicas.get(0) : replicas.get(random.nextInt(replicas.size()));
        shards.add(replicaUrl);
      }
    }
    return shards;
  }
  /**
   * Walks the NamedList response after performing an update request looking for the replication
   * factor that was achieved in each shard involved in the request. For single doc updates, there
   * will be only one shard in the return value.
   */
  @SuppressWarnings("rawtypes")
  public Map<String, Integer> getShardReplicationFactor(String collection, NamedList resp) {
    connect();

    Map<String, Integer> results = new HashMap<String, Integer>();
    if (resp instanceof CloudSolrServer.RouteResponse) {
      NamedList routes = ((CloudSolrServer.RouteResponse) resp).getRouteResponses();
      ClusterState clusterState = zkStateReader.getClusterState();
      Map<String, String> leaders = new HashMap<String, String>();
      for (Slice slice : clusterState.getActiveSlices(collection)) {
        Replica leader = slice.getLeader();
        if (leader != null) {
          ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
          String leaderUrl = zkProps.getBaseUrl() + "/" + zkProps.getCoreName();
          leaders.put(leaderUrl, slice.getName());
          String altLeaderUrl = zkProps.getBaseUrl() + "/" + collection;
          leaders.put(altLeaderUrl, slice.getName());
        }
      }

      Iterator<Map.Entry<String, Object>> routeIter = routes.iterator();
      while (routeIter.hasNext()) {
        Map.Entry<String, Object> next = routeIter.next();
        String host = next.getKey();
        NamedList hostResp = (NamedList) next.getValue();
        Integer rf =
            (Integer) ((NamedList) hostResp.get("responseHeader")).get(UpdateRequest.REPFACT);
        if (rf != null) {
          String shard = leaders.get(host);
          if (shard == null) {
            if (host.endsWith("/")) shard = leaders.get(host.substring(0, host.length() - 1));
            if (shard == null) {
              shard = host;
            }
          }
          results.put(shard, rf);
        }
      }
    }
    return results;
  }
  @BeforeClass
  public static void setupCluster() throws Exception {
    final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");

    String configName = "solrCloudCollectionConfig";
    int nodeCount = 5;
    configureCluster(nodeCount).addConfig(configName, configDir).configure();

    Map<String, String> collectionProperties = new HashMap<>();
    collectionProperties.put("config", "solrconfig-tlog.xml");
    collectionProperties.put("schema", "schema.xml");

    // create a collection holding data for the "to" side of the JOIN

    int shards = 2;
    int replicas = 2;
    CollectionAdminRequest.createCollection(toColl, configName, shards, replicas)
        .setProperties(collectionProperties)
        .process(cluster.getSolrClient());

    // get the set of nodes where replicas for the "to" collection exist
    Set<String> nodeSet = new HashSet<>();
    ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
    ClusterState cs = zkStateReader.getClusterState();
    for (Slice slice : cs.getCollection(toColl).getActiveSlices())
      for (Replica replica : slice.getReplicas()) nodeSet.add(replica.getNodeName());
    assertTrue(nodeSet.size() > 0);

    // deploy the "from" collection to all nodes where the "to" collection exists
    CollectionAdminRequest.createCollection(fromColl, configName, 1, 4)
        .setCreateNodeSet(StringUtils.join(nodeSet, ","))
        .setProperties(collectionProperties)
        .process(cluster.getSolrClient());

    toDocId = indexDoc(toColl, 1001, "a", null, "b");
    indexDoc(fromColl, 2001, "a", "c", null);

    Thread.sleep(1000); // so the commits fire
  }
  protected DocCollection getDocCollection(ClusterState clusterState, String collection)
      throws SolrException {
    ExpiringCachedDocCollection cachedState =
        collectionStateCache != null ? collectionStateCache.get(collection) : null;
    if (cachedState != null && cachedState.cached != null) {
      return cachedState.cached;
    }

    DocCollection col = clusterState.getCollectionOrNull(collection);
    if (col == null) return null;
    if (col.getStateFormat() > 1)
      collectionStateCache.put(collection, new ExpiringCachedDocCollection(col));
    return col;
  }
  private String getUrlFromZk(String collection) {
    ClusterState clusterState = getCommonCloudSolrServer().getZkStateReader().getClusterState();
    Map<String, Slice> slices = clusterState.getSlicesMap(collection);

    if (slices == null) {
      throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection:" + collection);
    }

    for (Map.Entry<String, Slice> entry : slices.entrySet()) {
      Slice slice = entry.getValue();
      Map<String, Replica> shards = slice.getReplicasMap();
      Set<Map.Entry<String, Replica>> shardEntries = shards.entrySet();
      for (Map.Entry<String, Replica> shardEntry : shardEntries) {
        final ZkNodeProps node = shardEntry.getValue();
        if (clusterState.liveNodesContain(node.getStr(ZkStateReader.NODE_NAME_PROP))) {
          return ZkCoreNodeProps.getCoreUrl(
              node.getStr(ZkStateReader.BASE_URL_PROP),
              collection); // new ZkCoreNodeProps(node).getCoreUrl();
        }
      }
    }

    throw new RuntimeException("Could not find a live node for collection:" + collection);
  }
Exemplo n.º 14
0
  private String getCoreUrl(
      String collectionName,
      String origCorename,
      ClusterState clusterState,
      Collection<Slice> slices,
      boolean byCoreName,
      boolean activeReplicas) {
    String coreUrl;
    Set<String> liveNodes = clusterState.getLiveNodes();
    List<Slice> randomizedSlices = new ArrayList<>(slices.size());
    randomizedSlices.addAll(slices);
    Collections.shuffle(randomizedSlices, random);

    for (Slice slice : randomizedSlices) {
      List<Replica> randomizedReplicas = new ArrayList<>();
      randomizedReplicas.addAll(slice.getReplicas());
      Collections.shuffle(randomizedReplicas, random);

      for (Replica replica : randomizedReplicas) {
        if (!activeReplicas
            || (liveNodes.contains(replica.getNodeName())
                && replica.getState() == Replica.State.ACTIVE)) {

          if (byCoreName && !collectionName.equals(replica.getStr(CORE_NAME_PROP))) {
            // if it's by core name, make sure they match
            continue;
          }
          if (replica.getStr(BASE_URL_PROP).equals(cores.getZkController().getBaseUrl())) {
            // don't count a local core
            continue;
          }

          if (origCorename != null) {
            coreUrl = replica.getStr(BASE_URL_PROP) + "/" + origCorename;
          } else {
            coreUrl = replica.getCoreUrl();
            if (coreUrl.endsWith("/")) {
              coreUrl = coreUrl.substring(0, coreUrl.length() - 1);
            }
          }

          return coreUrl;
        }
      }
    }
    return null;
  }
Exemplo n.º 15
0
  public static String assignNode(String collection, ClusterState state) {
    Map<String, Slice> sliceMap = state.getSlicesMap(collection);
    if (sliceMap == null) {
      return "core_node1";
    }

    int max = 0;
    for (Slice slice : sliceMap.values()) {
      for (Replica replica : slice.getReplicas()) {
        Matcher m = COUNT.matcher(replica.getName());
        if (m.matches()) {
          max = Math.max(max, Integer.parseInt(m.group(1)));
        }
      }
    }

    return "core_node" + (max + 1);
  }
Exemplo n.º 16
0
  /**
   * Assign a new unique id up to slices count - then add replicas evenly.
   *
   * @return the assigned shard id
   */
  public static String assignShard(String collection, ClusterState state, Integer numShards) {
    if (numShards == null) {
      numShards = 1;
    }
    String returnShardId = null;
    Map<String, Slice> sliceMap = state.getActiveSlicesMap(collection);

    // TODO: now that we create shards ahead of time, is this code needed?  Esp since hash ranges
    // aren't assigned when creating via this method?

    if (sliceMap == null) {
      return "shard1";
    }

    List<String> shardIdNames = new ArrayList<String>(sliceMap.keySet());

    if (shardIdNames.size() < numShards) {
      return "shard" + (shardIdNames.size() + 1);
    }

    // TODO: don't need to sort to find shard with fewest replicas!

    // else figure out which shard needs more replicas
    final Map<String, Integer> map = new HashMap<String, Integer>();
    for (String shardId : shardIdNames) {
      int cnt = sliceMap.get(shardId).getReplicasMap().size();
      map.put(shardId, cnt);
    }

    Collections.sort(
        shardIdNames,
        new Comparator<String>() {

          @Override
          public int compare(String o1, String o2) {
            Integer one = map.get(o1);
            Integer two = map.get(o2);
            return one.compareTo(two);
          }
        });

    returnShardId = shardIdNames.get(0);
    return returnShardId;
  }
  private Set<String> getCollectionList(ClusterState clusterState, String collection) {
    // Extract each comma separated collection name and store in a List.
    List<String> rawCollectionsList = StrUtils.splitSmart(collection, ",", true);
    Set<String> collectionsList = new HashSet<>();
    // validate collections
    for (String collectionName : rawCollectionsList) {
      if (!clusterState.getCollections().contains(collectionName)) {
        Aliases aliases = zkStateReader.getAliases();
        String alias = aliases.getCollectionAlias(collectionName);
        if (alias != null) {
          List<String> aliasList = StrUtils.splitSmart(alias, ",", true);
          collectionsList.addAll(aliasList);
          continue;
        }

        throw new SolrException(ErrorCode.BAD_REQUEST, "Collection not found: " + collectionName);
      }

      collectionsList.add(collectionName);
    }
    return collectionsList;
  }
Exemplo n.º 18
0
  public int createSubRequests(ResponseBuilder rb) throws IOException {
    SolrParams params = rb.req.getParams();
    String id1[] = params.getParams("id");
    String ids[] = params.getParams("ids");

    if (id1 == null && ids == null) {
      return ResponseBuilder.STAGE_DONE;
    }

    List<String> allIds = new ArrayList<String>();
    if (id1 != null) {
      for (String s : id1) {
        allIds.add(s);
      }
    }
    if (ids != null) {
      for (String s : ids) {
        allIds.addAll(StrUtils.splitSmart(s, ",", true));
      }
    }

    // TODO: handle collection=...?

    ZkController zkController =
        rb.req.getCore().getCoreDescriptor().getCoreContainer().getZkController();

    // if shards=... then use that
    if (zkController != null && params.get("shards") == null) {
      CloudDescriptor cloudDescriptor = rb.req.getCore().getCoreDescriptor().getCloudDescriptor();

      String collection = cloudDescriptor.getCollectionName();
      ClusterState clusterState = zkController.getClusterState();
      DocCollection coll = clusterState.getCollection(collection);

      Map<String, List<String>> sliceToId = new HashMap<String, List<String>>();
      for (String id : allIds) {
        Slice slice = coll.getRouter().getTargetSlice(id, null, params, coll);

        List<String> idsForShard = sliceToId.get(slice.getName());
        if (idsForShard == null) {
          idsForShard = new ArrayList<String>(2);
          sliceToId.put(slice.getName(), idsForShard);
        }
        idsForShard.add(id);
      }

      for (Map.Entry<String, List<String>> entry : sliceToId.entrySet()) {
        String shard = entry.getKey();
        String shardIdList = StrUtils.join(entry.getValue(), ',');

        ShardRequest sreq = new ShardRequest();

        sreq.purpose = 1;
        // sreq.shards = new String[]{shard};    // TODO: would be nice if this would work...
        sreq.shards = sliceToShards(rb, collection, shard);
        sreq.actualShards = sreq.shards;
        sreq.params = new ModifiableSolrParams();
        sreq.params.set(
            ShardParams.SHARDS_QT,
            "/get"); // TODO: how to avoid hardcoding this and hit the same handler?
        sreq.params.set("distrib", false);
        sreq.params.set("ids", shardIdList);

        rb.addRequest(this, sreq);
      }
    } else {
      String shardIdList = StrUtils.join(allIds, ',');
      ShardRequest sreq = new ShardRequest();

      sreq.purpose = 1;
      sreq.shards = null; // ALL
      sreq.actualShards = sreq.shards;
      sreq.params = new ModifiableSolrParams();
      sreq.params.set(
          ShardParams.SHARDS_QT,
          "/get"); // TODO: how to avoid hardcoding this and hit the same handler?
      sreq.params.set("distrib", false);
      sreq.params.set("ids", shardIdList);

      rb.addRequest(this, sreq);
    }

    return ResponseBuilder.STAGE_DONE;
  }
Exemplo n.º 19
0
  public static ArrayList<Node> getNodesForNewShard(
      ClusterState clusterState,
      String collectionName,
      int numSlices,
      int maxShardsPerNode,
      int repFactor,
      String createNodeSetStr) {
    List<String> createNodeList =
        createNodeSetStr == null ? null : StrUtils.splitSmart(createNodeSetStr, ",", true);

    Set<String> nodes = clusterState.getLiveNodes();

    List<String> nodeList = new ArrayList<String>(nodes.size());
    nodeList.addAll(nodes);
    if (createNodeList != null) nodeList.retainAll(createNodeList);

    HashMap<String, Node> nodeNameVsShardCount = new HashMap<String, Node>();
    for (String s : nodeList) nodeNameVsShardCount.put(s, new Node(s));
    for (String s : clusterState.getCollections()) {
      DocCollection c = clusterState.getCollection(s);
      // identify suitable nodes  by checking the no:of cores in each of them
      for (Slice slice : c.getSlices()) {
        Collection<Replica> replicas = slice.getReplicas();
        for (Replica replica : replicas) {
          Node count = nodeNameVsShardCount.get(replica.getNodeName());
          if (count != null) {
            count.totalNodes++;
            if (s.equals(collectionName)) {
              count.thisCollectionNodes++;
              if (count.thisCollectionNodes >= maxShardsPerNode)
                nodeNameVsShardCount.remove(replica.getNodeName());
            }
          }
        }
      }
    }

    if (nodeNameVsShardCount.size() <= 0) {
      throw new SolrException(
          SolrException.ErrorCode.BAD_REQUEST,
          "Cannot create collection "
              + collectionName
              + ". No live Solr-instances"
              + ((createNodeList != null)
                  ? " among Solr-instances specified in " + CREATE_NODE_SET + ":" + createNodeSetStr
                  : ""));
    }

    if (repFactor > nodeNameVsShardCount.size()) {
      log.warn(
          "Specified "
              + REPLICATION_FACTOR
              + " of "
              + repFactor
              + " on collection "
              + collectionName
              + " is higher than or equal to the number of Solr instances currently live or part of your "
              + CREATE_NODE_SET
              + "("
              + nodeList.size()
              + "). Its unusual to run two replica of the same slice on the same Solr-instance.");
    }

    int maxCoresAllowedToCreate = maxShardsPerNode * nodeList.size();
    int requestedCoresToCreate = numSlices * repFactor;
    int minCoresToCreate = requestedCoresToCreate;
    if (maxCoresAllowedToCreate < minCoresToCreate) {
      throw new SolrException(
          SolrException.ErrorCode.BAD_REQUEST,
          "Cannot create shards "
              + collectionName
              + ". Value of "
              + MAX_SHARDS_PER_NODE
              + " is "
              + maxShardsPerNode
              + ", and the number of live nodes is "
              + nodeList.size()
              + ". This allows a maximum of "
              + maxCoresAllowedToCreate
              + " to be created. Value of "
              + NUM_SLICES
              + " is "
              + numSlices
              + " and value of "
              + REPLICATION_FACTOR
              + " is "
              + repFactor
              + ". This requires "
              + requestedCoresToCreate
              + " shards to be created (higher than the allowed number)");
    }

    ArrayList<Node> sortedNodeList = new ArrayList<>(nodeNameVsShardCount.values());
    Collections.sort(
        sortedNodeList,
        new Comparator<Node>() {
          @Override
          public int compare(Node x, Node y) {
            return (x.weight() < y.weight()) ? -1 : ((x.weight() == y.weight()) ? 0 : 1);
          }
        });
    return sortedNodeList;
  }
  protected NamedList<Object> sendRequest(SolrRequest request)
      throws SolrServerException, IOException {
    connect();

    ClusterState clusterState = zkStateReader.getClusterState();

    boolean sendToLeaders = false;
    List<String> replicas = null;

    if (request instanceof IsUpdateRequest) {
      if (request instanceof UpdateRequest) {
        NamedList<Object> response = directUpdate((AbstractUpdateRequest) request, clusterState);
        if (response != null) {
          return response;
        }
      }
      sendToLeaders = true;
      replicas = new ArrayList<>();
    }

    SolrParams reqParams = request.getParams();
    if (reqParams == null) {
      reqParams = new ModifiableSolrParams();
    }
    List<String> theUrlList = new ArrayList<>();
    if (request.getPath().equals("/admin/collections")
        || request.getPath().equals("/admin/cores")) {
      Set<String> liveNodes = clusterState.getLiveNodes();
      for (String liveNode : liveNodes) {
        theUrlList.add(zkStateReader.getBaseUrlForNodeName(liveNode));
      }
    } else {
      String collection = reqParams.get(UpdateParams.COLLECTION, defaultCollection);

      if (collection == null) {
        throw new SolrServerException(
            "No collection param specified on request and no default collection has been set.");
      }

      Set<String> collectionsList = getCollectionList(clusterState, collection);
      if (collectionsList.size() == 0) {
        throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection: " + collection);
      }

      String shardKeys = reqParams.get(ShardParams._ROUTE_);
      if (shardKeys == null) {
        shardKeys = reqParams.get(ShardParams.SHARD_KEYS); // deprecated
      }

      // TODO: not a big deal because of the caching, but we could avoid looking
      // at every shard
      // when getting leaders if we tweaked some things

      // Retrieve slices from the cloud state and, for each collection
      // specified,
      // add it to the Map of slices.
      Map<String, Slice> slices = new HashMap<>();
      for (String collectionName : collectionsList) {
        DocCollection col = getDocCollection(clusterState, collectionName);
        Collection<Slice> routeSlices = col.getRouter().getSearchSlices(shardKeys, reqParams, col);
        ClientUtils.addSlices(slices, collectionName, routeSlices, true);
      }
      Set<String> liveNodes = clusterState.getLiveNodes();

      List<String> leaderUrlList = null;
      List<String> urlList = null;
      List<String> replicasList = null;

      // build a map of unique nodes
      // TODO: allow filtering by group, role, etc
      Map<String, ZkNodeProps> nodes = new HashMap<>();
      List<String> urlList2 = new ArrayList<>();
      for (Slice slice : slices.values()) {
        for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) {
          ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
          String node = coreNodeProps.getNodeName();
          if (!liveNodes.contains(coreNodeProps.getNodeName())
              || !coreNodeProps.getState().equals(ZkStateReader.ACTIVE)) continue;
          if (nodes.put(node, nodeProps) == null) {
            if (!sendToLeaders || (sendToLeaders && coreNodeProps.isLeader())) {
              String url;
              if (reqParams.get(UpdateParams.COLLECTION) == null) {
                url =
                    ZkCoreNodeProps.getCoreUrl(
                        nodeProps.getStr(ZkStateReader.BASE_URL_PROP), defaultCollection);
              } else {
                url = coreNodeProps.getCoreUrl();
              }
              urlList2.add(url);
            } else if (sendToLeaders) {
              String url;
              if (reqParams.get(UpdateParams.COLLECTION) == null) {
                url =
                    ZkCoreNodeProps.getCoreUrl(
                        nodeProps.getStr(ZkStateReader.BASE_URL_PROP), defaultCollection);
              } else {
                url = coreNodeProps.getCoreUrl();
              }
              replicas.add(url);
            }
          }
        }
      }

      if (sendToLeaders) {
        leaderUrlList = urlList2;
        replicasList = replicas;
      } else {
        urlList = urlList2;
      }

      if (sendToLeaders) {
        theUrlList = new ArrayList<>(leaderUrlList.size());
        theUrlList.addAll(leaderUrlList);
      } else {
        theUrlList = new ArrayList<>(urlList.size());
        theUrlList.addAll(urlList);
      }
      if (theUrlList.isEmpty()) {
        throw new SolrException(
            SolrException.ErrorCode.INVALID_STATE, "Not enough nodes to handle the request");
      }

      Collections.shuffle(theUrlList, rand);
      if (sendToLeaders) {
        ArrayList<String> theReplicas = new ArrayList<>(replicasList.size());
        theReplicas.addAll(replicasList);
        Collections.shuffle(theReplicas, rand);
        theUrlList.addAll(theReplicas);
      }
    }

    LBHttpSolrServer.Req req = new LBHttpSolrServer.Req(request, theUrlList);
    LBHttpSolrServer.Rsp rsp = lbServer.request(req);
    return rsp.getResponse();
  }
Exemplo n.º 21
0
  @Test
  public void testStoreAndRead() throws Exception {
    Map<String, DocCollection> collectionStates = new HashMap<>();
    Set<String> liveNodes = new HashSet<>();
    liveNodes.add("node1");
    liveNodes.add("node2");

    Map<String, Slice> slices = new HashMap<>();
    Map<String, Replica> sliceToProps = new HashMap<>();
    Map<String, Object> props = new HashMap<>();

    props.put("prop1", "value");
    props.put("prop2", "value2");
    Replica replica = new Replica("node1", props);
    sliceToProps.put("node1", replica);
    Slice slice = new Slice("shard1", sliceToProps, null);
    slices.put("shard1", slice);
    Slice slice2 = new Slice("shard2", sliceToProps, null);
    slices.put("shard2", slice2);
    collectionStates.put(
        "collection1", new DocCollection("collection1", slices, null, DocRouter.DEFAULT));
    collectionStates.put(
        "collection2", new DocCollection("collection2", slices, null, DocRouter.DEFAULT));
    ZkStateReader zkStateReaderMock = getMockZkStateReader(collectionStates.keySet());

    ClusterState clusterState = new ClusterState(-1, liveNodes, collectionStates);
    byte[] bytes = Utils.toJSON(clusterState);
    // System.out.println("#################### " + new String(bytes));
    ClusterState loadedClusterState = ClusterState.load(-1, bytes, liveNodes);

    assertEquals(
        "Provided liveNodes not used properly", 2, loadedClusterState.getLiveNodes().size());
    assertEquals("No collections found", 2, loadedClusterState.getCollectionsMap().size());
    assertEquals(
        "Properties not copied properly",
        replica.getStr("prop1"),
        loadedClusterState
            .getSlice("collection1", "shard1")
            .getReplicasMap()
            .get("node1")
            .getStr("prop1"));
    assertEquals(
        "Properties not copied properly",
        replica.getStr("prop2"),
        loadedClusterState
            .getSlice("collection1", "shard1")
            .getReplicasMap()
            .get("node1")
            .getStr("prop2"));

    loadedClusterState = ClusterState.load(-1, new byte[0], liveNodes);

    assertEquals(
        "Provided liveNodes not used properly", 2, loadedClusterState.getLiveNodes().size());
    assertEquals("Should not have collections", 0, loadedClusterState.getCollectionsMap().size());

    loadedClusterState = ClusterState.load(-1, (byte[]) null, liveNodes);

    assertEquals(
        "Provided liveNodes not used properly", 2, loadedClusterState.getLiveNodes().size());
    assertEquals("Should not have collections", 0, loadedClusterState.getCollectionsMap().size());
  }
  protected Set<String> commonMocks(int liveNodesCount) throws Exception {

    shardHandlerFactoryMock.getShardHandler();
    expectLastCall()
        .andAnswer(
            new IAnswer<ShardHandler>() {
              @Override
              public ShardHandler answer() throws Throwable {
                log.info("SHARDHANDLER");
                return shardHandlerMock;
              }
            })
        .anyTimes();
    workQueueMock.peekTopN(EasyMock.anyInt(), anyObject(Set.class), EasyMock.anyLong());
    expectLastCall()
        .andAnswer(
            new IAnswer<List>() {
              @Override
              public List answer() throws Throwable {
                Object result;
                int count = 0;
                while ((result = queue.peek()) == null) {
                  Thread.sleep(1000);
                  count++;
                  if (count > 1) return null;
                }

                return Arrays.asList(result);
              }
            })
        .anyTimes();

    workQueueMock.getTailId();
    expectLastCall()
        .andAnswer(
            new IAnswer<Object>() {
              @Override
              public Object answer() throws Throwable {
                Object result = null;
                Iterator iter = queue.iterator();
                while (iter.hasNext()) {
                  result = iter.next();
                }
                return result == null ? null : ((QueueEvent) result).getId();
              }
            })
        .anyTimes();

    workQueueMock.peek(true);
    expectLastCall()
        .andAnswer(
            new IAnswer<Object>() {
              @Override
              public Object answer() throws Throwable {
                Object result;
                while ((result = queue.peek()) == null) {
                  Thread.sleep(1000);
                }
                return result;
              }
            })
        .anyTimes();

    workQueueMock.remove(anyObject(QueueEvent.class));
    expectLastCall()
        .andAnswer(
            new IAnswer<Object>() {
              @Override
              public Object answer() throws Throwable {
                queue.remove((QueueEvent) getCurrentArguments()[0]);
                return null;
              }
            })
        .anyTimes();

    workQueueMock.poll();
    expectLastCall()
        .andAnswer(
            new IAnswer<Object>() {
              @Override
              public Object answer() throws Throwable {
                return queue.poll();
              }
            })
        .anyTimes();

    zkStateReaderMock.getClusterState();
    expectLastCall()
        .andAnswer(
            new IAnswer<Object>() {
              @Override
              public Object answer() throws Throwable {
                return clusterStateMock;
              }
            })
        .anyTimes();

    zkStateReaderMock.getZkClient();
    expectLastCall()
        .andAnswer(
            new IAnswer<Object>() {
              @Override
              public Object answer() throws Throwable {
                return solrZkClientMock;
              }
            })
        .anyTimes();

    zkStateReaderMock.updateClusterState(anyBoolean());

    clusterStateMock.getCollections();
    expectLastCall()
        .andAnswer(
            new IAnswer<Object>() {
              @Override
              public Object answer() throws Throwable {
                return collectionsSet;
              }
            })
        .anyTimes();
    final Set<String> liveNodes = new HashSet<>();
    for (int i = 0; i < liveNodesCount; i++) {
      final String address = "localhost:" + (8963 + i) + "_solr";
      liveNodes.add(address);

      zkStateReaderMock.getBaseUrlForNodeName(address);
      expectLastCall()
          .andAnswer(
              new IAnswer<Object>() {
                @Override
                public Object answer() throws Throwable {
                  // This works as long as this test does not use a
                  // webapp context with an underscore in it
                  return address.replaceAll("_", "/");
                }
              })
          .anyTimes();
    }
    zkStateReaderMock.getClusterProps();
    expectLastCall()
        .andAnswer(
            new IAnswer<Map>() {
              @Override
              public Map answer() throws Throwable {
                return new HashMap();
              }
            });

    solrZkClientMock.getZkClientTimeout();
    expectLastCall()
        .andAnswer(
            new IAnswer<Object>() {
              @Override
              public Object answer() throws Throwable {
                return 30000;
              }
            })
        .anyTimes();

    clusterStateMock.hasCollection(anyObject(String.class));
    expectLastCall()
        .andAnswer(
            new IAnswer<Boolean>() {
              @Override
              public Boolean answer() throws Throwable {
                String key = (String) getCurrentArguments()[0];
                return collectionsSet.contains(key);
              }
            })
        .anyTimes();

    clusterStateMock.getLiveNodes();
    expectLastCall()
        .andAnswer(
            new IAnswer<Object>() {
              @Override
              public Object answer() throws Throwable {
                return liveNodes;
              }
            })
        .anyTimes();
    solrZkClientMock.create(
        anyObject(String.class),
        anyObject(byte[].class),
        anyObject(CreateMode.class),
        anyBoolean());
    expectLastCall()
        .andAnswer(
            new IAnswer<String>() {
              @Override
              public String answer() throws Throwable {
                String key = (String) getCurrentArguments()[0];
                zkMap.put(key, null);
                handleCreateCollMessage((byte[]) getCurrentArguments()[1]);
                return key;
              }
            })
        .anyTimes();

    solrZkClientMock.makePath(anyObject(String.class), anyObject(byte[].class), anyBoolean());
    expectLastCall()
        .andAnswer(
            new IAnswer<String>() {
              @Override
              public String answer() throws Throwable {
                String key = (String) getCurrentArguments()[0];
                return key;
              }
            })
        .anyTimes();

    solrZkClientMock.makePath(
        anyObject(String.class),
        anyObject(byte[].class),
        anyObject(CreateMode.class),
        anyBoolean());
    expectLastCall()
        .andAnswer(
            new IAnswer<String>() {
              @Override
              public String answer() throws Throwable {
                String key = (String) getCurrentArguments()[0];
                return key;
              }
            })
        .anyTimes();

    solrZkClientMock.exists(anyObject(String.class), anyBoolean());
    expectLastCall()
        .andAnswer(
            new IAnswer<Boolean>() {
              @Override
              public Boolean answer() throws Throwable {
                String key = (String) getCurrentArguments()[0];
                return zkMap.containsKey(key);
              }
            })
        .anyTimes();

    zkMap.put("/configs/myconfig", null);

    return liveNodes;
  }
  protected void testTemplate(
      Integer numberOfNodes,
      Integer numberOfNodesToCreateOn,
      CreateNodeListOptions createNodeListOption,
      Integer replicationFactor,
      Integer numberOfSlices,
      Integer maxShardsPerNode,
      boolean collectionExceptedToBeCreated)
      throws Exception {
    assertTrue(
        "Wrong usage of testTemplate. numberOfNodesToCreateOn "
            + numberOfNodesToCreateOn
            + " is not allowed to be higher than numberOfNodes "
            + numberOfNodes,
        numberOfNodes.intValue() >= numberOfNodesToCreateOn.intValue());
    assertTrue(
        "Wrong usage of testTemplage. createNodeListOption has to be "
            + CreateNodeListOptions.SEND
            + " when numberOfNodes and numberOfNodesToCreateOn are unequal",
        ((createNodeListOption == CreateNodeListOptions.SEND)
            || (numberOfNodes.intValue() == numberOfNodesToCreateOn.intValue())));

    Set<String> liveNodes = commonMocks(numberOfNodes);
    List<String> createNodeList = new ArrayList<>();
    int i = 0;
    for (String node : liveNodes) {
      if (i++ < numberOfNodesToCreateOn) {
        createNodeList.add(node);
      }
    }

    if (random().nextBoolean())
      Collections.shuffle(createNodeList, OverseerCollectionProcessor.RANDOM);

    List<SubmitCapture> submitCaptures = null;
    if (collectionExceptedToBeCreated) {
      submitCaptures = mockShardHandlerForCreateJob(numberOfSlices, replicationFactor);
    }

    replay(workQueueMock);
    replay(solrZkClientMock);
    replay(zkStateReaderMock);
    replay(clusterStateMock);
    replay(shardHandlerFactoryMock);
    replay(shardHandlerMock);

    log.info("clusterstate " + clusterStateMock.hashCode());

    startComponentUnderTest();

    final List<String> createNodeListToSend =
        ((createNodeListOption != CreateNodeListOptions.SEND_NULL) ? createNodeList : null);
    final boolean sendCreateNodeList = (createNodeListOption != CreateNodeListOptions.DONT_SEND);
    final boolean dontShuffleCreateNodeSet =
        (createNodeListToSend != null) && sendCreateNodeList && random().nextBoolean();
    issueCreateJob(
        numberOfSlices,
        replicationFactor,
        maxShardsPerNode,
        createNodeListToSend,
        sendCreateNodeList,
        !dontShuffleCreateNodeSet);
    waitForEmptyQueue(10000);

    if (collectionExceptedToBeCreated) {
      assertNotNull(lastProcessMessageResult.getResponse().toString(), lastProcessMessageResult);
    }
    verify(shardHandlerFactoryMock);
    verify(shardHandlerMock);

    if (collectionExceptedToBeCreated) {
      verifySubmitCaptures(
          submitCaptures,
          numberOfSlices,
          replicationFactor,
          createNodeList,
          dontShuffleCreateNodeSet);
    }
  }
  private String getShard(int hash, String collection, ClusterState clusterState) {
    // ranges should be part of the cloud state and eventually gotten from zk

    // get the shard names
    return clusterState.getShard(hash, collection);
  }