protected Exception postJsonToPipelineWithRetry(
      String endpoint, List docs, ArrayList<String> mutable, Exception lastExc, int requestId)
      throws Exception {
    Exception retryAfterException = null;

    try {
      postJsonToPipeline(endpoint, docs, requestId);
      if (lastExc != null)
        log.info(
            "Re-try request "
                + requestId
                + " to "
                + endpoint
                + " succeeded after seeing a "
                + lastExc.getMessage());
    } catch (Exception exc) {
      log.warn("Failed to send request " + requestId + " to '" + endpoint + "' due to: " + exc);
      if (mutable.size() > 1) {
        // try another endpoint but update the cloned list to avoid re-hitting the one having an
        // error
        if (log.isDebugEnabled())
          log.debug("Will re-try failed request " + requestId + " on next endpoint in the list");

        mutable.remove(endpoint);
        retryAfterException = exc;
      } else {
        // no other endpoints to try ... brief wait and then retry
        log.warn(
            "No more endpoints available to try ... will retry to send request "
                + requestId
                + " to "
                + endpoint
                + " after waiting 1 sec");
        try {
          Thread.sleep(1000);
        } catch (InterruptedException ignore) {
          Thread.interrupted();
        }
        // note we want the exception to propagate from here up the stack since we re-tried and it
        // didn't work
        postJsonToPipeline(endpoint, docs, requestId);
        log.info("Re-try request " + requestId + " to " + endpoint + " succeeded");
        retryAfterException = null; // return success condition
      }
    }

    return retryAfterException;
  }
  public void postBatchToPipeline(List docs) throws Exception {
    int numDocs = docs.size();

    int requestId = requestCounter.incrementAndGet();
    ArrayList<String> mutable = null;
    synchronized (this) {
      mutable = new ArrayList<String>(sessions.keySet());
    }

    if (mutable.isEmpty()) {
      // completely hosed ... try to re-establish all sessions
      synchronized (this) {
        try {
          Thread.sleep(2000);
        } catch (InterruptedException ie) {
          Thread.interrupted();
        }

        sessions = establishSessions(originalEndpoints, fusionUser, fusionPass, fusionRealm);
        mutable = new ArrayList<String>(sessions.keySet());
      }
      if (mutable.isEmpty())
        throw new IllegalStateException(
            "No available endpoints! "
                + "Check log for previous errors as to why there are no more endpoints available. This is a fatal error.");
    }

    if (mutable.size() > 1) {
      Exception lastExc = null;

      // try all the endpoints until success is reached ... or we run out of endpoints to try ...
      while (!mutable.isEmpty()) {
        String endpoint = getLbEndpoint(mutable);
        if (endpoint == null) {
          // no more endpoints available ... fail
          if (lastExc != null) {
            log.error(
                "No more endpoints available to retry failed request ("
                    + requestId
                    + ")! raising last seen error: "
                    + lastExc);
            throw lastExc;
          } else {
            throw new RuntimeException(
                "No Fusion pipeline endpoints available to process request "
                    + requestId
                    + "! Check logs for previous errors.");
          }
        }

        if (log.isDebugEnabled())
          log.debug(
              "POSTing batch of "
                  + numDocs
                  + " input docs to "
                  + endpoint
                  + " as request "
                  + requestId);

        Exception retryAfterException =
            postJsonToPipelineWithRetry(endpoint, docs, mutable, lastExc, requestId);
        if (retryAfterException == null) {
          lastExc = null;
          break; // request succeeded ...
        }

        lastExc = retryAfterException; // try next endpoint (if available) after seeing an exception
      }

      if (lastExc != null) {
        // request failed and we exhausted the list of endpoints to try ...
        log.error("Failing request " + requestId + " due to: " + lastExc);
        throw lastExc;
      }

    } else {
      String endpoint = getLbEndpoint(mutable);
      if (log.isDebugEnabled())
        log.debug(
            "POSTing batch of "
                + numDocs
                + " input docs to "
                + endpoint
                + " as request "
                + requestId);

      Exception exc = postJsonToPipelineWithRetry(endpoint, docs, mutable, null, requestId);
      if (exc != null) throw exc;
    }
  }