Example #1
0
  /**
   * Convenience utility: creates a retry loop calling the given proc and retrying if needed
   *
   * @param client Zookeeper
   * @param proc procedure to call with retry
   * @param <T> return type
   * @return procedure result
   * @throws Exception any non-retriable errors
   */
  public static <T> T callWithRetry(CuratorZookeeperClient client, Callable<T> proc)
      throws Exception {
    T result = null;
    RetryLoop retryLoop = client.newRetryLoop();
    while (retryLoop.shouldContinue()) {
      try {
        client.internalBlockUntilConnectedOrTimedOut();

        result = proc.call();
        retryLoop.markComplete();
      } catch (Exception e) {
        retryLoop.takeException(e);
      }
    }
    return result;
  }
Example #2
0
  private <DATA_TYPE> void handleBackgroundOperationException(
      OperationAndData<DATA_TYPE> operationAndData, Throwable e) {
    do {
      if ((operationAndData != null) && RetryLoop.isRetryException(e)) {
        if (!Boolean.getBoolean(DebugUtils.PROPERTY_DONT_LOG_CONNECTION_ISSUES)) {
          log.debug("Retry-able exception received", e);
        }
        if (client
            .getRetryPolicy()
            .allowRetry(
                operationAndData.getThenIncrementRetryCount(),
                operationAndData.getElapsedTimeMs(),
                operationAndData)) {
          if (!Boolean.getBoolean(DebugUtils.PROPERTY_DONT_LOG_CONNECTION_ISSUES)) {
            log.debug("Retrying operation");
          }
          backgroundOperations.offer(operationAndData);
          break;
        } else {
          if (!Boolean.getBoolean(DebugUtils.PROPERTY_DONT_LOG_CONNECTION_ISSUES)) {
            log.debug("Retry policy did not allow retry");
          }
          if (operationAndData.getErrorCallback() != null) {
            operationAndData.getErrorCallback().retriesExhausted(operationAndData);
          }
        }
      }

      logError("Background exception was not retry-able or retry gave up", e);
    } while (false);
  }
  private void tryWithMutex(MutableAtomicValue<byte[]> result, MakeValue makeValue)
      throws Exception {
    long startMs = System.currentTimeMillis();
    int retryCount = 0;

    if (mutex.acquire(promotedToLock.getMaxLockTime(), promotedToLock.getMaxLockTimeUnit())) {
      try {
        boolean done = false;
        while (!done) {
          result.stats.incrementPromotedTries();
          if (tryOnce(result, makeValue)) {
            result.succeeded = true;
            done = true;
          } else {
            if (!promotedToLock
                .getRetryPolicy()
                .allowRetry(
                    retryCount++,
                    System.currentTimeMillis() - startMs,
                    RetryLoop.getDefaultRetrySleeper())) {
              done = true;
            }
          }
        }
      } finally {
        mutex.release();
      }
    }

    result.stats.setPromotedTimeMs(System.currentTimeMillis() - startMs);
  }
  private Map<String, String> queryExhibitors(Exhibitors localExhibitors) {
    Map<String, String> values = newValues();

    long start = System.currentTimeMillis();
    int retries = 0;
    boolean done = false;
    while (!done) {
      List<String> hostnames = Lists.newArrayList(localExhibitors.getHostnames());
      if (hostnames.size() == 0) {
        done = true;
      } else {
        String hostname = hostnames.get(random.nextInt(hostnames.size()));
        try {
          String encoded =
              restClient.getRaw(hostname, localExhibitors.getRestPort(), restUriPath, MIME_TYPE);
          values.putAll(decodeExhibitorList(encoded));
          done = true;
        } catch (Throwable e) {
          if (retryPolicy.allowRetry(
              retries++, System.currentTimeMillis() - start, RetryLoop.getDefaultRetrySleeper())) {
            log.warn("Couldn't get servers from Exhibitor. Retrying.", e);
          } else {
            log.error("Couldn't get servers from Exhibitor. Giving up.", e);
            done = true;
          }
        }
      }
    }

    return values;
  }
 private byte[] get(final String pathSuffix) throws Exception {
   return RetryLoop.callWithRetry(
       zk.getZookeeperClient(),
       new Callable<byte[]>() {
         @Override
         public byte[] call() throws Exception {
           return zk.getData().forPath(buildZookeeperPath(pathSuffix));
         }
       });
 }
 private void watch(final String pathSuffix, final Watcher callback) throws Exception {
   RetryLoop.callWithRetry(
       zk.getZookeeperClient(),
       new Callable<Void>() {
         @Override
         public Void call() throws Exception {
           zk.checkExists().usingWatcher(callback).forPath(buildZookeeperPath(pathSuffix));
           return null;
         }
       });
 }
Example #7
0
  @SuppressWarnings({"ThrowableResultOfMethodCallIgnored"})
  <DATA_TYPE> void processBackgroundOperation(
      OperationAndData<DATA_TYPE> operationAndData, CuratorEvent event) {
    boolean isInitialExecution = (event == null);
    if (isInitialExecution) {
      performBackgroundOperation(operationAndData);
      return;
    }

    boolean doQueueOperation = false;
    do {
      if (RetryLoop.shouldRetry(event.getResultCode())) {
        if (client
            .getRetryPolicy()
            .allowRetry(
                operationAndData.getThenIncrementRetryCount(),
                operationAndData.getElapsedTimeMs(),
                operationAndData)) {
          doQueueOperation = true;
        } else {
          if (operationAndData.getErrorCallback() != null) {
            operationAndData.getErrorCallback().retriesExhausted(operationAndData);
          }

          KeeperException.Code code = KeeperException.Code.get(event.getResultCode());
          Exception e = null;
          try {
            e = (code != null) ? KeeperException.create(code) : null;
          } catch (Throwable ignore) {
          }
          if (e == null) {
            e = new Exception("Unknown result code: " + event.getResultCode());
          }
          logError("Background operation retry gave up", e);
        }
        break;
      }

      if (operationAndData.getCallback() != null) {
        sendToBackgroundCallback(operationAndData, event);
        break;
      }

      processEvent(event);
    } while (false);

    if (doQueueOperation) {
      queueOperation(operationAndData);
    }
  }
  /**
   * Optimistic concurrency scheme for tryAtomicUpdate. Try to update, and keep trying until
   * successful.
   *
   * @param pathSuffix suffix to use to build path in ZooKeeper.
   * @param f function used to initialize the node, or transform the data already there.
   * @throws Exception
   */
  private void atomicUpdate(final String pathSuffix, final NodeFunction f) throws Exception {
    boolean done = false;

    do {
      done =
          RetryLoop.callWithRetry(
              zk.getZookeeperClient(),
              new Callable<Boolean>() {
                @Override
                public Boolean call() throws Exception {
                  return tryAtomicUpdate(pathSuffix, f);
                }
              });
      Thread.sleep(BASE_OPTIMISTIC_RETRY_TIME_MS + rand.nextInt(BASE_OPTIMISTIC_RETRY_TIME_MS));
    } while (!done);
  }
Example #9
0
 private Stat pathInForeground(final String path) throws Exception {
   TimeTrace trace = client.getZookeeperClient().startTracer("ExistsBuilderImpl-Foreground");
   Stat returnStat =
       RetryLoop.callWithRetry(
           client.getZookeeperClient(),
           new Callable<Stat>() {
             @Override
             public Stat call() throws Exception {
               Stat returnStat;
               if (watching.isWatched()) {
                 returnStat = client.getZooKeeper().exists(path, true);
               } else {
                 returnStat = client.getZooKeeper().exists(path, watching.getWatcher());
               }
               return returnStat;
             }
           });
   trace.commit();
   return returnStat;
 }
  private void tryOptimistic(MutableAtomicValue<byte[]> result, MakeValue makeValue)
      throws Exception {
    long startMs = System.currentTimeMillis();
    int retryCount = 0;

    boolean done = false;
    while (!done) {
      result.stats.incrementOptimisticTries();
      if (tryOnce(result, makeValue)) {
        result.succeeded = true;
        done = true;
      } else {
        if (!retryPolicy.allowRetry(
            retryCount++,
            System.currentTimeMillis() - startMs,
            RetryLoop.getDefaultRetrySleeper())) {
          done = true;
        }
      }
    }

    result.stats.setOptimisticTimeMs(System.currentTimeMillis() - startMs);
  }