/**
* Convenience utility: creates a retry loop calling the given proc and retrying if needed
*
* @param client Zookeeper
* @param proc procedure to call with retry
* @param <T> return type
* @return procedure result
* @throws Exception any non-retriable errors
*/
public static <T> T callWithRetry(CuratorZookeeperClient client, Callable<T> proc)
throws Exception {
T result = null;
RetryLoop retryLoop = client.newRetryLoop();
while (retryLoop.shouldContinue()) {
try {
client.internalBlockUntilConnectedOrTimedOut();
result = proc.call();
retryLoop.markComplete();
} catch (Exception e) {
retryLoop.takeException(e);
}
}
return result;
}
private <DATA_TYPE> void handleBackgroundOperationException(
OperationAndData<DATA_TYPE> operationAndData, Throwable e) {
do {
if ((operationAndData != null) && RetryLoop.isRetryException(e)) {
if (!Boolean.getBoolean(DebugUtils.PROPERTY_DONT_LOG_CONNECTION_ISSUES)) {
log.debug("Retry-able exception received", e);
}
if (client
.getRetryPolicy()
.allowRetry(
operationAndData.getThenIncrementRetryCount(),
operationAndData.getElapsedTimeMs(),
operationAndData)) {
if (!Boolean.getBoolean(DebugUtils.PROPERTY_DONT_LOG_CONNECTION_ISSUES)) {
log.debug("Retrying operation");
}
backgroundOperations.offer(operationAndData);
break;
} else {
if (!Boolean.getBoolean(DebugUtils.PROPERTY_DONT_LOG_CONNECTION_ISSUES)) {
log.debug("Retry policy did not allow retry");
}
if (operationAndData.getErrorCallback() != null) {
operationAndData.getErrorCallback().retriesExhausted(operationAndData);
}
}
}
logError("Background exception was not retry-able or retry gave up", e);
} while (false);
}
private void tryWithMutex(MutableAtomicValue<byte[]> result, MakeValue makeValue)
throws Exception {
long startMs = System.currentTimeMillis();
int retryCount = 0;
if (mutex.acquire(promotedToLock.getMaxLockTime(), promotedToLock.getMaxLockTimeUnit())) {
try {
boolean done = false;
while (!done) {
result.stats.incrementPromotedTries();
if (tryOnce(result, makeValue)) {
result.succeeded = true;
done = true;
} else {
if (!promotedToLock
.getRetryPolicy()
.allowRetry(
retryCount++,
System.currentTimeMillis() - startMs,
RetryLoop.getDefaultRetrySleeper())) {
done = true;
}
}
}
} finally {
mutex.release();
}
}
result.stats.setPromotedTimeMs(System.currentTimeMillis() - startMs);
}
private Map<String, String> queryExhibitors(Exhibitors localExhibitors) {
Map<String, String> values = newValues();
long start = System.currentTimeMillis();
int retries = 0;
boolean done = false;
while (!done) {
List<String> hostnames = Lists.newArrayList(localExhibitors.getHostnames());
if (hostnames.size() == 0) {
done = true;
} else {
String hostname = hostnames.get(random.nextInt(hostnames.size()));
try {
String encoded =
restClient.getRaw(hostname, localExhibitors.getRestPort(), restUriPath, MIME_TYPE);
values.putAll(decodeExhibitorList(encoded));
done = true;
} catch (Throwable e) {
if (retryPolicy.allowRetry(
retries++, System.currentTimeMillis() - start, RetryLoop.getDefaultRetrySleeper())) {
log.warn("Couldn't get servers from Exhibitor. Retrying.", e);
} else {
log.error("Couldn't get servers from Exhibitor. Giving up.", e);
done = true;
}
}
}
}
return values;
}
private byte[] get(final String pathSuffix) throws Exception {
return RetryLoop.callWithRetry(
zk.getZookeeperClient(),
new Callable<byte[]>() {
@Override
public byte[] call() throws Exception {
return zk.getData().forPath(buildZookeeperPath(pathSuffix));
}
});
}
private void watch(final String pathSuffix, final Watcher callback) throws Exception {
RetryLoop.callWithRetry(
zk.getZookeeperClient(),
new Callable<Void>() {
@Override
public Void call() throws Exception {
zk.checkExists().usingWatcher(callback).forPath(buildZookeeperPath(pathSuffix));
return null;
}
});
}
@SuppressWarnings({"ThrowableResultOfMethodCallIgnored"})
<DATA_TYPE> void processBackgroundOperation(
OperationAndData<DATA_TYPE> operationAndData, CuratorEvent event) {
boolean isInitialExecution = (event == null);
if (isInitialExecution) {
performBackgroundOperation(operationAndData);
return;
}
boolean doQueueOperation = false;
do {
if (RetryLoop.shouldRetry(event.getResultCode())) {
if (client
.getRetryPolicy()
.allowRetry(
operationAndData.getThenIncrementRetryCount(),
operationAndData.getElapsedTimeMs(),
operationAndData)) {
doQueueOperation = true;
} else {
if (operationAndData.getErrorCallback() != null) {
operationAndData.getErrorCallback().retriesExhausted(operationAndData);
}
KeeperException.Code code = KeeperException.Code.get(event.getResultCode());
Exception e = null;
try {
e = (code != null) ? KeeperException.create(code) : null;
} catch (Throwable ignore) {
}
if (e == null) {
e = new Exception("Unknown result code: " + event.getResultCode());
}
logError("Background operation retry gave up", e);
}
break;
}
if (operationAndData.getCallback() != null) {
sendToBackgroundCallback(operationAndData, event);
break;
}
processEvent(event);
} while (false);
if (doQueueOperation) {
queueOperation(operationAndData);
}
}
/**
* Optimistic concurrency scheme for tryAtomicUpdate. Try to update, and keep trying until
* successful.
*
* @param pathSuffix suffix to use to build path in ZooKeeper.
* @param f function used to initialize the node, or transform the data already there.
* @throws Exception
*/
private void atomicUpdate(final String pathSuffix, final NodeFunction f) throws Exception {
boolean done = false;
do {
done =
RetryLoop.callWithRetry(
zk.getZookeeperClient(),
new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
return tryAtomicUpdate(pathSuffix, f);
}
});
Thread.sleep(BASE_OPTIMISTIC_RETRY_TIME_MS + rand.nextInt(BASE_OPTIMISTIC_RETRY_TIME_MS));
} while (!done);
}
private Stat pathInForeground(final String path) throws Exception {
TimeTrace trace = client.getZookeeperClient().startTracer("ExistsBuilderImpl-Foreground");
Stat returnStat =
RetryLoop.callWithRetry(
client.getZookeeperClient(),
new Callable<Stat>() {
@Override
public Stat call() throws Exception {
Stat returnStat;
if (watching.isWatched()) {
returnStat = client.getZooKeeper().exists(path, true);
} else {
returnStat = client.getZooKeeper().exists(path, watching.getWatcher());
}
return returnStat;
}
});
trace.commit();
return returnStat;
}
private void tryOptimistic(MutableAtomicValue<byte[]> result, MakeValue makeValue)
throws Exception {
long startMs = System.currentTimeMillis();
int retryCount = 0;
boolean done = false;
while (!done) {
result.stats.incrementOptimisticTries();
if (tryOnce(result, makeValue)) {
result.succeeded = true;
done = true;
} else {
if (!retryPolicy.allowRetry(
retryCount++,
System.currentTimeMillis() - startMs,
RetryLoop.getDefaultRetrySleeper())) {
done = true;
}
}
}
result.stats.setOptimisticTimeMs(System.currentTimeMillis() - startMs);
}