コード例 #1
0
 /**
  * 这里会判断当前调用服务的状态,分析判断是否需要进入降级状态 如果服务在指定的时间区间内累积的错误,达到了配置的次数,则进入服务降级
  * 如果满足上面条件,并且满足重试机制,则也不会进入降级流程,而是触发远程服务调用
  *
  * @param invoker
  * @param invocation
  * @return
  */
 private boolean checkNeedCircuitBreak(Invoker<?> invoker, Invocation invocation) {
   String interfaceName = invoker.getUrl().getParameter(Constants.INTERFACE_KEY);
   String method = invocation.getMethodName();
   String methodKey = Config.getMethodPropertyName(invoker, invocation).toString();
   int limit = Config.getBreakLimit(invoker, invocation);
   BreakCounter breakCounter = breakCounterMap.get(methodKey);
   if (breakCounter != null && breakCounter.isEnable()) {
     long currentExceptionCount = breakCounter.getCurrentExceptionCount();
     long currentBreakCount = breakCounter.getCurrentBreakCount();
     logger.info(
         "[{}] check invoke [{}.{}] circuit break,current exception count [{}]  limit [{}]",
         localHost,
         interfaceName,
         method,
         currentExceptionCount,
         limit);
     if (limit <= currentExceptionCount) {
       if (currentBreakCount > 0 && needRetry(invoker, invocation, currentBreakCount)) {
         logger.info(
             "[{}] retry invoke [{}.{}] current break count [{}]",
             localHost,
             interfaceName,
             method,
             currentBreakCount);
         breakCounter.incrementRetryTimes();
         return false;
       }
       return true;
     }
   }
   return false;
 }
コード例 #2
0
 private void caughtException(Invoker<?> invoker, Invocation invocation, Exception e) {
   String interfaceName = invoker.getUrl().getParameter(Constants.INTERFACE_KEY);
   String method = invocation.getMethodName();
   StringBuffer interfaceConfig = new StringBuffer(Config.DUBBO_REFERENCE_PREFIX);
   interfaceConfig.append(interfaceName);
   StringBuffer methodConfig = new StringBuffer(interfaceConfig.toString());
   methodConfig.append(".").append(method);
   String methodKey = methodConfig.toString();
   int timeout =
       invoker
           .getUrl()
           .getMethodParameter(
               invocation.getMethodName(), Constants.TIMEOUT_KEY, Constants.DEFAULT_TIMEOUT);
   int limit = Config.getBreakLimit(invoker, invocation);
   // 一个异常的有效期,是通过连续出现异常数量乘以每个调用的超时时间,比如你配置连续出现10次异常之后进行服务降级,并且每次服务调用的超时时间是2000ms的话,同时
   // 每个服务重试次数是为2次,那么就是在(2+1)*2000*10
   ExceptionMarker breakMarker =
       new ExceptionMarker(System.currentTimeMillis(), limit * timeout, e);
   if (!breakCounterMap.containsKey(methodKey)) {
     BreakCounter oldValue = breakCounterMap.putIfAbsent(methodKey, new BreakCounter(methodKey));
     // 返回的oldValue为空,表示之前没有创建了赌赢的异常计数器,则需要对它分配一个loop
     if (oldValue == null) {
       nextLoop().register(breakCounterMap.get(methodKey));
     }
   }
   BreakCounter counter = breakCounterMap.get(methodKey);
   counter.addExceptionMarker(breakMarker);
   logger.info(
       "[{}] caught exception for rpc invoke [{}.{}],current exception count [{}]",
       localHost,
       interfaceName,
       method,
       counter.getCurrentExceptionCount());
 }
コード例 #3
0
 public Result invoke(Invoker<?> invoker, Invocation invocation) throws RpcException {
   if (Config.checkFunctionSwitch(invoker, invocation)) {
     logger.info("[{}] had [{}] breaker", localHost, breakCounterMap.size());
     return wrapBreakerInvoke(invoker, invocation);
   }
   Result result = invoker.invoke(invocation);
   toBeNormal(invoker, invocation);
   return result;
 }
コード例 #4
0
 private boolean needRetry(Invoker<?> invoker, Invocation invocation, long currentBreakCount) {
   String interfaceName = invoker.getUrl().getParameter(Constants.INTERFACE_KEY);
   String method = invocation.getMethodName();
   int frequency = Config.getRetryFrequency(invoker, invocation);
   logger.info(
       "[{}] check invoke [{}.{}] need retry,current break count [{}],retry frequency [{}]",
       localHost,
       interfaceName,
       method,
       currentBreakCount,
       frequency);
   if (currentBreakCount % frequency == 0) {
     logger.info("[{}] retry invoke [{}.{}]", localHost, interfaceName, method);
     return true;
   }
   return false;
 }
コード例 #5
0
/**
 * Created by bieber on 2015/4/30. Dubbo服务降级Filter 通过拦截每个方法的请求,并且读取每个方法对服务降级条件的配置 从而自动进行服务降级,以及服务恢复
 */
@Activate(group = {Constants.CONSUMER})
public class RemoteFacadeCircuitBreaker implements Filter {

  private static final Logger logger = LoggerFactory.getLogger("CIRCUITBREAKER");

  private static final InetAddress localHost = Config.getLocalAddress();
  // 用于存储某个方法出现异常的计数器key:interfaceName.methodName,value:对应的异常计数器
  private volatile ConcurrentHashMap<String, BreakCounter> breakCounterMap =
      new ConcurrentHashMap<String, BreakCounter>();
  // 对某个方法异常计数器处理器,用于分析记录的异常到当前实现是否失效了(为了满足在某个时间内出现异常次数),如果失效将从BreakCounter中移除
  private BreakCounterLoop[] breakCounterLoops =
      new BreakCounterLoop[Runtime.getRuntime().availableProcessors()];
  // 获取dubbo的代理工程扩展
  private static final ProxyFactory proxyFactory =
      ExtensionLoader.getExtensionLoader(ProxyFactory.class).getAdaptiveExtension();
  // 缓存服务降级代理类的Invoker,避免重复创建
  private static final ConcurrentHashMap<String, Invoker> CIRCUIT_BREAKER_INVOKER_CACHE =
      new ConcurrentHashMap<String, Invoker>();

  private volatile AtomicLong loopCount = new AtomicLong(0);

  public RemoteFacadeCircuitBreaker() {
    String intervalConf =
        ConfigUtils.getProperty("dubbo.reference.check.break.marker.interval", "60000");
    logger.info(
        "[{}] has already been initialized circuit breaker,check break marker interval [{}]",
        localHost,
        intervalConf);
    long interval = Long.parseLong(intervalConf);
    for (int i = 0; i < breakCounterLoops.length; i++) {
      BreakCounterLoop loop = new BreakCounterLoop(interval);
      breakCounterLoops[i] = loop;
    }
  }

  // 获取下一个遍历器
  private BreakCounterLoop nextLoop() {
    return breakCounterLoops[((int) (loopCount.incrementAndGet() % breakCounterLoops.length))];
  }

  public Result invoke(Invoker<?> invoker, Invocation invocation) throws RpcException {
    if (Config.checkFunctionSwitch(invoker, invocation)) {
      logger.info("[{}] had [{}] breaker", localHost, breakCounterMap.size());
      return wrapBreakerInvoke(invoker, invocation);
    }
    Result result = invoker.invoke(invocation);
    toBeNormal(invoker, invocation);
    return result;
  }

  private Result wrapBreakerInvoke(Invoker<?> invoker, Invocation invocation) throws RpcException {
    // 首先检查是否需要进入服务降级流程
    if (checkNeedCircuitBreak(invoker, invocation)) {
      logger.info(
          "[{}] activate the circuit break for url [{}],invoke method [{}]",
          localHost,
          invoker.getUrl(),
          invocation.getMethodName());
      // 进入服务降级
      return doCircuitBreak(invoker, invocation);
    }
    try {
      Result result = invoker.invoke(invocation);
      // 将该服务从服务降级中恢复出来
      toBeNormal(invoker, invocation);
      return result;
    } catch (RpcException e) {
      // 如果是请求超时或者网络异常,进行异常统计
      if (!e.isBiz()) {
        caughtException(invoker, invocation, e);
      }
      throw e;
    }
  }

  /**
   * 将服务恢复正常
   *
   * @param invoker
   * @param invocation
   */
  private void toBeNormal(Invoker<?> invoker, Invocation invocation) {
    String interfaceName = invoker.getUrl().getParameter(Constants.INTERFACE_KEY);
    String method = invocation.getMethodName();
    StringBuffer methodConfig = new StringBuffer(Config.DUBBO_REFERENCE_PREFIX);
    methodConfig.append(interfaceName).append(".").append(method);
    String methodKey = methodConfig.toString();
    // 从其中删除对应的异常计数器
    BreakCounter counter = breakCounterMap.remove(methodKey);
    if (counter != null) {
      logger.info("[{}] [{}.{}] to be normal", localHost, interfaceName, methodKey);
      // 将这个counter设置为失效
      counter.disable();
    }
  }

  /**
   * 这里会判断当前调用服务的状态,分析判断是否需要进入降级状态 如果服务在指定的时间区间内累积的错误,达到了配置的次数,则进入服务降级
   * 如果满足上面条件,并且满足重试机制,则也不会进入降级流程,而是触发远程服务调用
   *
   * @param invoker
   * @param invocation
   * @return
   */
  private boolean checkNeedCircuitBreak(Invoker<?> invoker, Invocation invocation) {
    String interfaceName = invoker.getUrl().getParameter(Constants.INTERFACE_KEY);
    String method = invocation.getMethodName();
    String methodKey = Config.getMethodPropertyName(invoker, invocation).toString();
    int limit = Config.getBreakLimit(invoker, invocation);
    BreakCounter breakCounter = breakCounterMap.get(methodKey);
    if (breakCounter != null && breakCounter.isEnable()) {
      long currentExceptionCount = breakCounter.getCurrentExceptionCount();
      long currentBreakCount = breakCounter.getCurrentBreakCount();
      logger.info(
          "[{}] check invoke [{}.{}] circuit break,current exception count [{}]  limit [{}]",
          localHost,
          interfaceName,
          method,
          currentExceptionCount,
          limit);
      if (limit <= currentExceptionCount) {
        if (currentBreakCount > 0 && needRetry(invoker, invocation, currentBreakCount)) {
          logger.info(
              "[{}] retry invoke [{}.{}] current break count [{}]",
              localHost,
              interfaceName,
              method,
              currentBreakCount);
          breakCounter.incrementRetryTimes();
          return false;
        }
        return true;
      }
    }
    return false;
  }

  private boolean needRetry(Invoker<?> invoker, Invocation invocation, long currentBreakCount) {
    String interfaceName = invoker.getUrl().getParameter(Constants.INTERFACE_KEY);
    String method = invocation.getMethodName();
    int frequency = Config.getRetryFrequency(invoker, invocation);
    logger.info(
        "[{}] check invoke [{}.{}] need retry,current break count [{}],retry frequency [{}]",
        localHost,
        interfaceName,
        method,
        currentBreakCount,
        frequency);
    if (currentBreakCount % frequency == 0) {
      logger.info("[{}] retry invoke [{}.{}]", localHost, interfaceName, method);
      return true;
    }
    return false;
  }

  private <T extends Object> Result doCircuitBreak(Invoker<?> invoker, Invocation invocation)
      throws RpcException {
    String interfaceName = invoker.getUrl().getParameter(Constants.INTERFACE_KEY);
    String circuitBreaker = interfaceName + "CircuitBreak";
    incrementBreakCount(invoker, invocation);
    try {
      logger.info("[{}] check has class [{}] to handle circuit break", localHost, circuitBreaker);
      Invoker<?> breakerInvoker = null;
      if (CIRCUIT_BREAKER_INVOKER_CACHE.containsKey(circuitBreaker)) {
        breakerInvoker = CIRCUIT_BREAKER_INVOKER_CACHE.get(circuitBreaker);
      } else {
        Class<T> breakerType = (Class<T>) Class.forName(circuitBreaker);
        Class<T> interfaceType = (Class<T>) Class.forName(interfaceName);
        if (interfaceType.isAssignableFrom(breakerType)) {
          logger.info("[{}] handle circuit break by class [{}]", localHost, circuitBreaker);
          T breaker = breakerType.newInstance();
          breakerInvoker = proxyFactory.getInvoker(breaker, interfaceType, invoker.getUrl());
          Invoker<?> oldInvoker =
              CIRCUIT_BREAKER_INVOKER_CACHE.putIfAbsent(circuitBreaker, breakerInvoker);
          if (oldInvoker != null) {
            breakerInvoker = oldInvoker;
          }
        }
      }
      if (breakerInvoker != null) {
        return breakerInvoker.invoke(invocation);
      }
    } catch (Exception e) {
      logger.error("failed to invoke circuit breaker", e);
    }
    logger.info("[{}] handle circuit break by exception", localHost);
    CircuitBreakerException baseBusinessException =
        new CircuitBreakerException(interfaceName, invocation.getMethodName());
    throw baseBusinessException;
  }

  private void incrementBreakCount(Invoker<?> invoker, Invocation invocation) {
    String interfaceName = invoker.getUrl().getParameter(Constants.INTERFACE_KEY);
    String method = invocation.getMethodName();
    StringBuffer interfaceConfig = new StringBuffer(Config.DUBBO_REFERENCE_PREFIX);
    interfaceConfig.append(interfaceName);
    StringBuffer methodConfig = new StringBuffer(interfaceConfig.toString());
    methodConfig.append(".").append(method);
    String methodKey = methodConfig.toString();
    BreakCounter counter = breakCounterMap.get(methodKey);
    counter.incrementBreakCount();
  }

  private void caughtException(Invoker<?> invoker, Invocation invocation, Exception e) {
    String interfaceName = invoker.getUrl().getParameter(Constants.INTERFACE_KEY);
    String method = invocation.getMethodName();
    StringBuffer interfaceConfig = new StringBuffer(Config.DUBBO_REFERENCE_PREFIX);
    interfaceConfig.append(interfaceName);
    StringBuffer methodConfig = new StringBuffer(interfaceConfig.toString());
    methodConfig.append(".").append(method);
    String methodKey = methodConfig.toString();
    int timeout =
        invoker
            .getUrl()
            .getMethodParameter(
                invocation.getMethodName(), Constants.TIMEOUT_KEY, Constants.DEFAULT_TIMEOUT);
    int limit = Config.getBreakLimit(invoker, invocation);
    // 一个异常的有效期,是通过连续出现异常数量乘以每个调用的超时时间,比如你配置连续出现10次异常之后进行服务降级,并且每次服务调用的超时时间是2000ms的话,同时
    // 每个服务重试次数是为2次,那么就是在(2+1)*2000*10
    ExceptionMarker breakMarker =
        new ExceptionMarker(System.currentTimeMillis(), limit * timeout, e);
    if (!breakCounterMap.containsKey(methodKey)) {
      BreakCounter oldValue = breakCounterMap.putIfAbsent(methodKey, new BreakCounter(methodKey));
      // 返回的oldValue为空,表示之前没有创建了赌赢的异常计数器,则需要对它分配一个loop
      if (oldValue == null) {
        nextLoop().register(breakCounterMap.get(methodKey));
      }
    }
    BreakCounter counter = breakCounterMap.get(methodKey);
    counter.addExceptionMarker(breakMarker);
    logger.info(
        "[{}] caught exception for rpc invoke [{}.{}],current exception count [{}]",
        localHost,
        interfaceName,
        method,
        counter.getCurrentExceptionCount());
  }
}