private void reviveNodes(Set<Integer> failedNodes) {
    for (int node : failedNodes) {
      ForceFailStore<ByteArray, byte[], byte[]> forceFailStore = getForceFailStore(node);
      forceFailStore.setFail(false);

      if (logger.isTraceEnabled()) logger.trace("Stopped failing requests to " + node);
    }

    while (!failedNodes.isEmpty()) {
      for (int node : failedNodes)
        if (failureDetector.isAvailable(cluster.getNodeById(node))) failedNodes.remove(node);
    }
  }
  private Set<Integer> getFailedNodes() {
    Set<Integer> failedNodes = new CopyOnWriteArraySet<Integer>();
    Random rand = new Random();
    int offset = rand.nextInt(NUM_NODES_TOTAL);

    for (int i = 0; i < NUM_NODES_FAILED; i++) failedNodes.add((offset + i) % NUM_NODES_TOTAL);

    for (int node : failedNodes) {
      ForceFailStore<ByteArray, byte[], byte[]> forceFailStore = getForceFailStore(node);
      forceFailStore.setFail(true);

      if (logger.isTraceEnabled()) logger.trace("Started failing requests to " + node);
    }
    return failedNodes;
  }