/**
* Send a document to Solr for indexing with re-try support in case of communication exception.
*/
protected void addDocWithRetry(CloudSolrServer solr, SolrInputDocument doc, int retryInSecs)
throws Exception {
try {
solr.add(doc);
} catch (Exception solrExc) {
// add some basic re-try logic in the event of a communication error
Throwable rootCause = SolrException.getRootCause(solrExc);
if (rootCause instanceof IOException) {
log.error(
rootCause.getClass().getSimpleName()
+ " when trying to send a document to SolrCloud, will re-try after waiting "
+ retryInSecs
+ " seconds; "
+ rootCause);
try {
Thread.sleep(retryInSecs * 1000);
} catch (InterruptedException ignoreMe) {
}
// re-try this doc
solr.add(doc);
}
}
}
public static QueryResponse querySolr(
SolrClient solrServer,
SolrQuery solrQuery,
int startIndex,
String cursorMark,
StreamingResponseCallback callback)
throws SolrServerException {
QueryResponse resp = null;
try {
if (cursorMark != null) {
solrQuery.setStart(0);
solrQuery.set("cursorMark", cursorMark);
} else {
solrQuery.setStart(startIndex);
}
if (callback != null) {
resp = solrServer.queryAndStreamResponse(solrQuery, callback);
} else {
resp = solrServer.query(solrQuery);
}
} catch (Exception exc) {
log.error("Query [" + solrQuery + "] failed due to: " + exc);
// re-try once in the event of a communications error with the server
Throwable rootCause = SolrException.getRootCause(exc);
boolean wasCommError =
(rootCause instanceof ConnectException
|| rootCause instanceof IOException
|| rootCause instanceof SocketException);
if (wasCommError) {
try {
Thread.sleep(2000L);
} catch (InterruptedException ie) {
Thread.interrupted();
}
try {
if (callback != null) {
resp = solrServer.queryAndStreamResponse(solrQuery, callback);
} else {
resp = solrServer.query(solrQuery);
}
} catch (Exception excOnRetry) {
if (excOnRetry instanceof SolrServerException) {
throw (SolrServerException) excOnRetry;
} else {
throw new SolrServerException(excOnRetry);
}
}
} else {
if (exc instanceof SolrServerException) {
throw (SolrServerException) exc;
} else {
throw new SolrServerException(exc);
}
}
}
return resp;
}
private static boolean shouldRetry(Exception exc) {
Throwable rootCause = SolrException.getRootCause(exc);
return (rootCause instanceof ConnectException || rootCause instanceof SocketException);
}
/**
* As this class doesn't watch external collections on the client side, there's a chance that the
* request will fail due to cached stale state, which means the state must be refreshed from ZK
* and retried.
*/
protected NamedList<Object> requestWithRetryOnStaleState(
SolrRequest request, int retryCount, String collection)
throws SolrServerException, IOException {
connect(); // important to call this before you start working with the ZkStateReader
// build up a _stateVer_ param to pass to the server containing all of the
// external collection state versions involved in this request, which allows
// the server to notify us that our cached state for one or more of the external
// collections is stale and needs to be refreshed ... this code has no impact on internal
// collections
String stateVerParam = null;
List<DocCollection> requestedCollections = null;
if (collection != null
&& !request
.getPath()
.startsWith("/admin")) { // don't do _stateVer_ checking for admin requests
Set<String> requestedCollectionNames =
getCollectionList(getZkStateReader().getClusterState(), collection);
StringBuilder stateVerParamBuilder = null;
for (String requestedCollection : requestedCollectionNames) {
// track the version of state we're using on the client side using the _stateVer_ param
DocCollection coll =
getDocCollection(getZkStateReader().getClusterState(), requestedCollection);
int collVer = coll.getZNodeVersion();
if (coll.getStateFormat() > 1) {
if (requestedCollections == null)
requestedCollections = new ArrayList<>(requestedCollectionNames.size());
requestedCollections.add(coll);
if (stateVerParamBuilder == null) {
stateVerParamBuilder = new StringBuilder();
} else {
stateVerParamBuilder.append(
"|"); // hopefully pipe is not an allowed char in a collection name
}
stateVerParamBuilder.append(coll.getName()).append(":").append(collVer);
}
}
if (stateVerParamBuilder != null) {
stateVerParam = stateVerParamBuilder.toString();
}
}
if (request.getParams() instanceof ModifiableSolrParams) {
ModifiableSolrParams params = (ModifiableSolrParams) request.getParams();
if (stateVerParam != null) {
params.set(STATE_VERSION, stateVerParam);
} else {
params.remove(STATE_VERSION);
}
} // else: ??? how to set this ???
NamedList<Object> resp = null;
try {
resp = sendRequest(request);
} catch (Exception exc) {
Throwable rootCause = SolrException.getRootCause(exc);
// don't do retry support for admin requests or if the request doesn't have a collection
// specified
if (collection == null || request.getPath().startsWith("/admin")) {
if (exc instanceof SolrServerException) {
throw (SolrServerException) exc;
} else if (exc instanceof IOException) {
throw (IOException) exc;
} else if (exc instanceof RuntimeException) {
throw (RuntimeException) exc;
} else {
throw new SolrServerException(rootCause);
}
}
int errorCode =
(rootCause instanceof SolrException)
? ((SolrException) rootCause).code()
: SolrException.ErrorCode.UNKNOWN.code;
log.error(
"Request to collection {} failed due to (" + errorCode + ") {}, retry? " + retryCount,
collection,
rootCause.toString());
boolean wasCommError =
(rootCause instanceof ConnectException
|| rootCause instanceof ConnectTimeoutException
|| rootCause instanceof NoHttpResponseException
|| rootCause instanceof SocketException);
boolean stateWasStale = false;
if (retryCount < MAX_STALE_RETRIES
&& requestedCollections != null
&& !requestedCollections.isEmpty()
&& SolrException.ErrorCode.getErrorCode(errorCode)
== SolrException.ErrorCode.INVALID_STATE) {
// cached state for one or more external collections was stale
// re-issue request using updated state
stateWasStale = true;
// just re-read state for all of them, which is a little heavy handed but hopefully a rare
// occurrence
for (DocCollection ext : requestedCollections) {
collectionStateCache.remove(ext.getName());
}
}
// if we experienced a communication error, it's worth checking the state
// with ZK just to make sure the node we're trying to hit is still part of the collection
if (retryCount < MAX_STALE_RETRIES
&& !stateWasStale
&& requestedCollections != null
&& !requestedCollections.isEmpty()
&& wasCommError) {
for (DocCollection ext : requestedCollections) {
DocCollection latestStateFromZk =
getDocCollection(zkStateReader.getClusterState(), ext.getName());
if (latestStateFromZk.getZNodeVersion() != ext.getZNodeVersion()) {
// looks like we couldn't reach the server because the state was stale == retry
stateWasStale = true;
// we just pulled state from ZK, so update the cache so that the retry uses it
collectionStateCache.put(
ext.getName(), new ExpiringCachedDocCollection(latestStateFromZk));
}
}
}
if (requestedCollections != null) {
requestedCollections.clear(); // done with this
}
// if the state was stale, then we retry the request once with new state pulled from Zk
if (stateWasStale) {
log.warn(
"Re-trying request to collection(s) "
+ collection
+ " after stale state error from server.");
resp = requestWithRetryOnStaleState(request, retryCount + 1, collection);
} else {
if (exc instanceof SolrServerException) {
throw (SolrServerException) exc;
} else if (exc instanceof IOException) {
throw (IOException) exc;
} else {
throw new SolrServerException(rootCause);
}
}
}
return resp;
}
