protected Exception postJsonToPipelineWithRetry( String endpoint, List docs, ArrayList<String> mutable, Exception lastExc, int requestId) throws Exception { Exception retryAfterException = null; try { postJsonToPipeline(endpoint, docs, requestId); if (lastExc != null) log.info( "Re-try request " + requestId + " to " + endpoint + " succeeded after seeing a " + lastExc.getMessage()); } catch (Exception exc) { log.warn("Failed to send request " + requestId + " to '" + endpoint + "' due to: " + exc); if (mutable.size() > 1) { // try another endpoint but update the cloned list to avoid re-hitting the one having an // error if (log.isDebugEnabled()) log.debug("Will re-try failed request " + requestId + " on next endpoint in the list"); mutable.remove(endpoint); retryAfterException = exc; } else { // no other endpoints to try ... brief wait and then retry log.warn( "No more endpoints available to try ... will retry to send request " + requestId + " to " + endpoint + " after waiting 1 sec"); try { Thread.sleep(1000); } catch (InterruptedException ignore) { Thread.interrupted(); } // note we want the exception to propagate from here up the stack since we re-tried and it // didn't work postJsonToPipeline(endpoint, docs, requestId); log.info("Re-try request " + requestId + " to " + endpoint + " succeeded"); retryAfterException = null; // return success condition } } return retryAfterException; }
public void postBatchToPipeline(List docs) throws Exception { int numDocs = docs.size(); int requestId = requestCounter.incrementAndGet(); ArrayList<String> mutable = null; synchronized (this) { mutable = new ArrayList<String>(sessions.keySet()); } if (mutable.isEmpty()) { // completely hosed ... try to re-establish all sessions synchronized (this) { try { Thread.sleep(2000); } catch (InterruptedException ie) { Thread.interrupted(); } sessions = establishSessions(originalEndpoints, fusionUser, fusionPass, fusionRealm); mutable = new ArrayList<String>(sessions.keySet()); } if (mutable.isEmpty()) throw new IllegalStateException( "No available endpoints! " + "Check log for previous errors as to why there are no more endpoints available. This is a fatal error."); } if (mutable.size() > 1) { Exception lastExc = null; // try all the endpoints until success is reached ... or we run out of endpoints to try ... while (!mutable.isEmpty()) { String endpoint = getLbEndpoint(mutable); if (endpoint == null) { // no more endpoints available ... fail if (lastExc != null) { log.error( "No more endpoints available to retry failed request (" + requestId + ")! raising last seen error: " + lastExc); throw lastExc; } else { throw new RuntimeException( "No Fusion pipeline endpoints available to process request " + requestId + "! Check logs for previous errors."); } } if (log.isDebugEnabled()) log.debug( "POSTing batch of " + numDocs + " input docs to " + endpoint + " as request " + requestId); Exception retryAfterException = postJsonToPipelineWithRetry(endpoint, docs, mutable, lastExc, requestId); if (retryAfterException == null) { lastExc = null; break; // request succeeded ... } lastExc = retryAfterException; // try next endpoint (if available) after seeing an exception } if (lastExc != null) { // request failed and we exhausted the list of endpoints to try ... log.error("Failing request " + requestId + " due to: " + lastExc); throw lastExc; } } else { String endpoint = getLbEndpoint(mutable); if (log.isDebugEnabled()) log.debug( "POSTing batch of " + numDocs + " input docs to " + endpoint + " as request " + requestId); Exception exc = postJsonToPipelineWithRetry(endpoint, docs, mutable, null, requestId); if (exc != null) throw exc; } }