protected Exception postJsonToPipelineWithRetry(
String endpoint, List docs, ArrayList<String> mutable, Exception lastExc, int requestId)
throws Exception {
Exception retryAfterException = null;
try {
postJsonToPipeline(endpoint, docs, requestId);
if (lastExc != null)
log.info(
"Re-try request "
+ requestId
+ " to "
+ endpoint
+ " succeeded after seeing a "
+ lastExc.getMessage());
} catch (Exception exc) {
log.warn("Failed to send request " + requestId + " to '" + endpoint + "' due to: " + exc);
if (mutable.size() > 1) {
// try another endpoint but update the cloned list to avoid re-hitting the one having an
// error
if (log.isDebugEnabled())
log.debug("Will re-try failed request " + requestId + " on next endpoint in the list");
mutable.remove(endpoint);
retryAfterException = exc;
} else {
// no other endpoints to try ... brief wait and then retry
log.warn(
"No more endpoints available to try ... will retry to send request "
+ requestId
+ " to "
+ endpoint
+ " after waiting 1 sec");
try {
Thread.sleep(1000);
} catch (InterruptedException ignore) {
Thread.interrupted();
}
// note we want the exception to propagate from here up the stack since we re-tried and it
// didn't work
postJsonToPipeline(endpoint, docs, requestId);
log.info("Re-try request " + requestId + " to " + endpoint + " succeeded");
retryAfterException = null; // return success condition
}
}
return retryAfterException;
}
public static void usleep(int msec) {
try {
Thread.sleep(msec);
} catch (InterruptedException e) {
}
}
public void postBatchToPipeline(List docs) throws Exception {
int numDocs = docs.size();
int requestId = requestCounter.incrementAndGet();
ArrayList<String> mutable = null;
synchronized (this) {
mutable = new ArrayList<String>(sessions.keySet());
}
if (mutable.isEmpty()) {
// completely hosed ... try to re-establish all sessions
synchronized (this) {
try {
Thread.sleep(2000);
} catch (InterruptedException ie) {
Thread.interrupted();
}
sessions = establishSessions(originalEndpoints, fusionUser, fusionPass, fusionRealm);
mutable = new ArrayList<String>(sessions.keySet());
}
if (mutable.isEmpty())
throw new IllegalStateException(
"No available endpoints! "
+ "Check log for previous errors as to why there are no more endpoints available. This is a fatal error.");
}
if (mutable.size() > 1) {
Exception lastExc = null;
// try all the endpoints until success is reached ... or we run out of endpoints to try ...
while (!mutable.isEmpty()) {
String endpoint = getLbEndpoint(mutable);
if (endpoint == null) {
// no more endpoints available ... fail
if (lastExc != null) {
log.error(
"No more endpoints available to retry failed request ("
+ requestId
+ ")! raising last seen error: "
+ lastExc);
throw lastExc;
} else {
throw new RuntimeException(
"No Fusion pipeline endpoints available to process request "
+ requestId
+ "! Check logs for previous errors.");
}
}
if (log.isDebugEnabled())
log.debug(
"POSTing batch of "
+ numDocs
+ " input docs to "
+ endpoint
+ " as request "
+ requestId);
Exception retryAfterException =
postJsonToPipelineWithRetry(endpoint, docs, mutable, lastExc, requestId);
if (retryAfterException == null) {
lastExc = null;
break; // request succeeded ...
}
lastExc = retryAfterException; // try next endpoint (if available) after seeing an exception
}
if (lastExc != null) {
// request failed and we exhausted the list of endpoints to try ...
log.error("Failing request " + requestId + " due to: " + lastExc);
throw lastExc;
}
} else {
String endpoint = getLbEndpoint(mutable);
if (log.isDebugEnabled())
log.debug(
"POSTing batch of "
+ numDocs
+ " input docs to "
+ endpoint
+ " as request "
+ requestId);
Exception exc = postJsonToPipelineWithRetry(endpoint, docs, mutable, null, requestId);
if (exc != null) throw exc;
}
}