예제 #1
0
public class Publisher {

  private static final Logger logger = LoggerFactory.getLogger(Publisher.class);

  private final Jedis publisherJedis;

  private final String channel;

  public Publisher(Jedis publisherJedis, String channel) {
    this.publisherJedis = publisherJedis;
    this.channel = channel;
  }

  public void start() {
    logger.info("Type your message (quit for terminate)");
    try {
      BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));

      while (true) {
        String line = reader.readLine();

        if (!"quit".equals(line)) {
          publisherJedis.publish(channel, line);
        } else {
          break;
        }
      }

    } catch (IOException e) {
      logger.error("IO failure while reading input, e");
    }
  }
}
/** 通信层一些辅助方法 */
public class RemotingHelper {

  private static final Logger LOGGER = LoggerFactory.getLogger(RemotingHelper.class);

  /** IP:PORT */
  public static SocketAddress string2SocketAddress(final String addr) {
    String[] s = addr.split(":");
    return new InetSocketAddress(s[0], Integer.valueOf(s[1]));
  }

  public static String parseChannelRemoteAddr(final Channel channel) {
    if (null == channel) {
      return "";
    }
    final SocketAddress remote = channel.remoteAddress();
    final String addr = remote != null ? remote.toString() : "";

    if (addr.length() > 0) {
      int index = addr.lastIndexOf("/");
      if (index >= 0) {
        return addr.substring(index + 1);
      }

      return addr;
    }

    return "";
  }

  public static void closeChannel(Channel channel) {
    final String addrRemote = RemotingHelper.parseChannelRemoteAddr(channel);
    channel
        .close()
        .addListener(
            new ChannelHandlerListener() {
              @Override
              public void operationComplete(Future future) throws Exception {
                LOGGER.info(
                    "closeChannel: close the connection to remote address[{}] result: {}",
                    addrRemote,
                    future.isSuccess());
              }
            });
  }
}
예제 #3
0
/** @author Robert HG ([email protected]) on 11/6/15. */
public class MinaHandler extends IoHandlerAdapter {

  private static final Logger LOGGER = LoggerFactory.getLogger(RemotingHelper.RemotingLogName);

  private AbstractRemoting remoting;
  private String sideType; // SERVER , CLIENT

  public MinaHandler(AbstractRemoting remoting) {
    this.remoting = remoting;
    if (remoting instanceof MinaRemotingClient) {
      sideType = "CLIENT";
    } else {
      sideType = "SERVER";
    }
  }

  @Override
  public void sessionCreated(IoSession session) throws Exception {
    final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(new MinaChannel(session));
    LOGGER.info("{} : sessionCreated {}", sideType, remoteAddress);
    super.sessionCreated(session);
  }

  @Override
  public void sessionOpened(IoSession session) throws Exception {
    Channel channel = new MinaChannel(session);
    final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(channel);
    LOGGER.info("{}: sessionOpened, the channel[{}]", sideType, remoteAddress);

    if (remoting.getChannelEventListener() != null) {
      remoting.putRemotingEvent(
          new RemotingEvent(RemotingEventType.CONNECT, remoteAddress, channel));
    }
  }

  @Override
  public void sessionClosed(IoSession session) throws Exception {
    com.github.ltsopensource.remoting.Channel channel = new MinaChannel(session);

    final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(channel);
    LOGGER.info("{}: sessionClosed, the channel[{}]", sideType, remoteAddress);

    if (remoting.getChannelEventListener() != null) {
      remoting.putRemotingEvent(new RemotingEvent(RemotingEventType.CLOSE, remoteAddress, channel));
    }
  }

  @Override
  public void sessionIdle(IoSession session, IdleStatus status) throws Exception {
    com.github.ltsopensource.remoting.Channel channel = new MinaChannel(session);

    final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(channel);

    if (IdleStatus.BOTH_IDLE == status) {
      LOGGER.info("{}: IDLE [{}]", sideType, remoteAddress);
      RemotingHelper.closeChannel(channel);
    }

    if (remoting.getChannelEventListener() != null) {
      RemotingEventType remotingEventType = null;
      if (IdleStatus.BOTH_IDLE == status) {
        remotingEventType = RemotingEventType.ALL_IDLE;
      } else if (IdleStatus.READER_IDLE == status) {
        remotingEventType = RemotingEventType.READER_IDLE;
      } else if (IdleStatus.WRITER_IDLE == status) {
        remotingEventType = RemotingEventType.WRITER_IDLE;
      }
      remoting.putRemotingEvent(new RemotingEvent(remotingEventType, remoteAddress, channel));
    }
  }

  @Override
  public void exceptionCaught(IoSession session, Throwable cause) throws Exception {
    com.github.ltsopensource.remoting.Channel channel = new MinaChannel(session);

    final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(channel);
    LOGGER.warn("{}: exceptionCaught {}. ", sideType, remoteAddress, cause);

    if (remoting.getChannelEventListener() != null) {
      remoting.putRemotingEvent(
          new RemotingEvent(RemotingEventType.EXCEPTION, remoteAddress, channel));
    }

    RemotingHelper.closeChannel(channel);
  }

  @Override
  public void messageReceived(IoSession session, Object message) throws Exception {
    if (message != null && message instanceof RemotingCommand) {
      remoting.processMessageReceived(new MinaChannel(session), (RemotingCommand) message);
    }
  }
}
/** @author Robert HG ([email protected]) on 5/17/15. */
public class RedisRegistry extends FailbackRegistry {

  private static final Logger LOGGER = LoggerFactory.getLogger(RedisRegistry.class);

  private final Map<String, JedisPool> jedisPools = new ConcurrentHashMap<String, JedisPool>();

  private String clusterName;
  private final ScheduledExecutorService expireExecutor =
      Executors.newScheduledThreadPool(
          1, new NamedThreadFactory("LTSRedisRegistryExpireTimer", true));
  private final ScheduledFuture<?> expireFuture;
  private final int expirePeriod;
  private boolean replicate;
  private final int reconnectPeriod;
  private final ConcurrentMap<String, Notifier> notifiers =
      new ConcurrentHashMap<String, Notifier>();
  private RedisLock lock;

  public RedisRegistry(AppContext appContext) {
    super(appContext);
    Config config = appContext.getConfig();
    this.clusterName = config.getClusterName();
    this.lock = new RedisLock("LTS_CLEAN_LOCK_KEY", config.getIdentity(), 2 * 60); // 锁两分钟过期

    JedisPoolConfig redisConfig = new JedisPoolConfig();
    // TODO 可以设置n多参数
    String address = NodeRegistryUtils.getRealRegistryAddress(config.getRegistryAddress());

    String cluster = config.getParameter("cluster", "failover");
    if (!"failover".equals(cluster) && !"replicate".equals(cluster)) {
      throw new IllegalArgumentException(
          "Unsupported redis cluster: "
              + cluster
              + ". The redis cluster only supported failover or replicate.");
    }
    replicate = "replicate".equals(cluster);

    this.reconnectPeriod =
        config.getParameter(
            ExtConfig.REGISTRY_RECONNECT_PERIOD_KEY, Constants.DEFAULT_REGISTRY_RECONNECT_PERIOD);

    String[] addrs = address.split(",");
    for (String addr : addrs) {
      int i = addr.indexOf(':');
      String host = addr.substring(0, i);
      int port = Integer.parseInt(addr.substring(i + 1));
      this.jedisPools.put(addr, new JedisPool(redisConfig, host, port, Constants.DEFAULT_TIMEOUT));
    }

    this.expirePeriod =
        config.getParameter(ExtConfig.REDIS_SESSION_TIMEOUT, Constants.DEFAULT_SESSION_TIMEOUT);

    this.expireFuture =
        expireExecutor.scheduleWithFixedDelay(
            new Runnable() {
              public void run() {
                try {
                  deferExpired(); // 延长过期时间
                } catch (Throwable t) { // 防御性容错
                  LOGGER.error(
                      "Unexpected exception occur at defer expire time, cause: " + t.getMessage(),
                      t);
                }
              }
            },
            expirePeriod / 2,
            expirePeriod / 2,
            TimeUnit.MILLISECONDS);
  }

  private void deferExpired() {
    for (Map.Entry<String, JedisPool> entry : jedisPools.entrySet()) {
      JedisPool jedisPool = entry.getValue();
      try {
        Jedis jedis = jedisPool.getResource();
        try {
          for (Node node : new HashSet<Node>(getRegistered())) {
            String key = NodeRegistryUtils.getNodeTypePath(clusterName, node.getNodeType());
            if (jedis.hset(
                    key, node.toFullString(), String.valueOf(SystemClock.now() + expirePeriod))
                == 1) {
              jedis.publish(key, Constants.REGISTER);
            }
          }
          if (lock.acquire(jedis)) {
            clean(jedis);
          }
          if (!replicate) {
            break; //  如果服务器端已同步数据,只需写入单台机器
          }
        } finally {
          jedis.close();
        }
      } catch (Throwable t) {
        LOGGER.warn(
            "Failed to write provider heartbeat to redis registry. registry: "
                + entry.getKey()
                + ", cause: "
                + t.getMessage(),
            t);
      }
    }
  }

  private void clean(Jedis jedis) {
    // /LTS/{集群名字}/NODES/
    Set<String> nodeTypePaths =
        jedis.keys(NodeRegistryUtils.getRootPath(appContext.getConfig().getClusterName()) + "/*");
    if (CollectionUtils.isNotEmpty(nodeTypePaths)) {
      for (String nodeTypePath : nodeTypePaths) {
        // /LTS/{集群名字}/NODES/JOB_TRACKER
        Set<String> nodePaths = jedis.keys(nodeTypePath);
        if (CollectionUtils.isNotEmpty(nodePaths)) {
          for (String nodePath : nodePaths) {
            Map<String, String> nodes = jedis.hgetAll(nodePath);
            if (CollectionUtils.isNotEmpty(nodes)) {
              boolean delete = false;
              long now = SystemClock.now();
              for (Map.Entry<String, String> entry : nodes.entrySet()) {
                String key = entry.getKey();
                long expire = Long.parseLong(entry.getValue());
                if (expire < now) {
                  jedis.hdel(nodePath, key);
                  delete = true;
                  if (LOGGER.isWarnEnabled()) {
                    LOGGER.warn(
                        "Delete expired key: "
                            + nodePath
                            + " -> value: "
                            + entry.getKey()
                            + ", expire: "
                            + new Date(expire)
                            + ", now: "
                            + new Date(now));
                  }
                }
              }
              if (delete) {
                jedis.publish(nodePath, Constants.UNREGISTER);
              }
            }
          }
        }
      }
    }
  }

  @Override
  protected void doRegister(Node node) {
    String key = NodeRegistryUtils.getNodeTypePath(clusterName, node.getNodeType());
    String expire = String.valueOf(SystemClock.now() + expirePeriod);
    boolean success = false;
    NodeRegistryException exception = null;
    for (Map.Entry<String, JedisPool> entry : jedisPools.entrySet()) {
      JedisPool jedisPool = entry.getValue();
      try {
        Jedis jedis = jedisPool.getResource();
        try {
          jedis.hset(key, node.toFullString(), expire);
          jedis.publish(key, Constants.REGISTER);
          success = true;
          if (!replicate) {
            break; //  如果服务器端已同步数据,只需写入单台机器
          }
        } finally {
          jedis.close();
        }
      } catch (Throwable t) {
        exception =
            new NodeRegistryException(
                "Failed to register node to redis registry. registry: "
                    + entry.getKey()
                    + ", node: "
                    + node
                    + ", cause: "
                    + t.getMessage(),
                t);
      }
    }
    if (exception != null) {
      if (success) {
        LOGGER.warn(exception.getMessage(), exception);
      } else {
        throw exception;
      }
    }
  }

  @Override
  protected void doUnRegister(Node node) {
    String key = NodeRegistryUtils.getNodeTypePath(clusterName, node.getNodeType());
    boolean success = false;
    NodeRegistryException exception = null;
    for (Map.Entry<String, JedisPool> entry : jedisPools.entrySet()) {
      JedisPool jedisPool = entry.getValue();
      try {
        Jedis jedis = jedisPool.getResource();
        try {
          jedis.hdel(key, node.toFullString());
          jedis.publish(key, Constants.UNREGISTER);
          success = true;
          if (!replicate) {
            break; //  如果服务器端已同步数据,只需写入单台机器
          }
        } finally {
          jedis.close();
        }
      } catch (Throwable t) {
        exception =
            new NodeRegistryException(
                "Failed to unregister node to redis registry. registry: "
                    + entry.getKey()
                    + ", node: "
                    + node
                    + ", cause: "
                    + t.getMessage(),
                t);
      }
    }
    if (exception != null) {
      if (success) {
        LOGGER.warn(exception.getMessage(), exception);
      } else {
        throw exception;
      }
    }
  }

  @Override
  protected void doSubscribe(Node node, NotifyListener listener) {

    List<NodeType> listenNodeTypes = node.getListenNodeTypes();
    if (CollectionUtils.isEmpty(listenNodeTypes)) {
      return;
    }
    for (NodeType listenNodeType : listenNodeTypes) {
      String listenNodePath = NodeRegistryUtils.getNodeTypePath(clusterName, listenNodeType);

      Notifier notifier = notifiers.get(listenNodePath);
      if (notifier == null) {
        Notifier newNotifier = new Notifier(listenNodePath);
        notifiers.putIfAbsent(listenNodePath, newNotifier);
        notifier = notifiers.get(listenNodePath);
        if (notifier == newNotifier) {
          notifier.start();
        }
      }

      boolean success = false;
      NodeRegistryException exception = null;
      for (Map.Entry<String, JedisPool> entry : jedisPools.entrySet()) {
        JedisPool jedisPool = entry.getValue();
        try {
          Jedis jedis = jedisPool.getResource();
          try {
            doNotify(
                jedis,
                Collections.singletonList(listenNodePath),
                Collections.singletonList(listener));
            success = true;
            break; // 只需读一个服务器的数据

          } finally {
            jedis.close();
          }
        } catch (Throwable t) {
          exception =
              new NodeRegistryException(
                  "Failed to unregister node to redis registry. registry: "
                      + entry.getKey()
                      + ", node: "
                      + node
                      + ", cause: "
                      + t.getMessage(),
                  t);
        }
      }
      if (exception != null) {
        if (success) {
          LOGGER.warn(exception.getMessage(), exception);
        } else {
          throw exception;
        }
      }
    }
  }

  @Override
  protected void doUnsubscribe(Node node, NotifyListener listener) {}

  @Override
  public void destroy() {
    super.destroy();
    try {
      expireFuture.cancel(true);
    } catch (Throwable t) {
      LOGGER.warn(t.getMessage(), t);
    }
    try {
      for (Notifier notifier : notifiers.values()) {
        notifier.shutdown();
      }
    } catch (Throwable t) {
      LOGGER.warn(t.getMessage(), t);
    }
    for (Map.Entry<String, JedisPool> entry : jedisPools.entrySet()) {
      JedisPool jedisPool = entry.getValue();
      try {
        jedisPool.destroy();
      } catch (Throwable t) {
        LOGGER.warn(
            "Failed to destroy the redis registry client. registry: "
                + entry.getKey()
                + ", cause: "
                + t.getMessage(),
            t);
      }
    }
  }

  private ConcurrentHashMap<String /*key*/, List<String>> cachedNodeMap =
      new ConcurrentHashMap<String, List<String>>();

  private void doNotify(
      Jedis jedis, Collection<String> keys, Collection<NotifyListener> listeners) {
    if (CollectionUtils.isEmpty(keys) && CollectionUtils.isEmpty(listeners)) {
      return;
    }

    for (String key : keys) {

      Map<String, String> values = jedis.hgetAll(key);
      List<String> currentChildren =
          values == null ? new ArrayList<String>(0) : new ArrayList<String>(values.keySet());
      List<String> oldChildren = cachedNodeMap.get(key);

      // 1. 找出增加的 节点
      List<String> addChildren = CollectionUtils.getLeftDiff(currentChildren, oldChildren);
      // 2. 找出减少的 节点
      List<String> decChildren = CollectionUtils.getLeftDiff(oldChildren, currentChildren);

      if (CollectionUtils.isNotEmpty(addChildren)) {
        List<Node> nodes = new ArrayList<Node>(addChildren.size());
        for (String child : addChildren) {
          Node node = NodeRegistryUtils.parse(child);
          nodes.add(node);
        }
        for (NotifyListener listener : listeners) {
          notify(NotifyEvent.ADD, nodes, listener);
        }
      }
      if (CollectionUtils.isNotEmpty(decChildren)) {
        List<Node> nodes = new ArrayList<Node>(decChildren.size());
        for (String child : decChildren) {
          Node node = NodeRegistryUtils.parse(child);
          nodes.add(node);
        }
        for (NotifyListener listener : listeners) {
          notify(NotifyEvent.REMOVE, nodes, listener);
        }
      }
      cachedNodeMap.put(key, currentChildren);
    }
  }

  private void doNotify(Jedis jedis, String key) {
    for (Map.Entry<Node, Set<NotifyListener>> entry :
        new HashMap<Node, Set<NotifyListener>>(getSubscribed()).entrySet()) {
      doNotify(
          jedis, Collections.singletonList(key), new HashSet<NotifyListener>(entry.getValue()));
    }
  }

  private class NotifySub extends JedisPubSub {

    private final JedisPool jedisPool;

    public NotifySub(JedisPool jedisPool) {
      this.jedisPool = jedisPool;
    }

    @Override
    public void onMessage(String key, String msg) {
      if (LOGGER.isInfoEnabled()) {
        LOGGER.info("redis event: " + key + " = " + msg);
      }
      if (msg.equals(Constants.REGISTER) || msg.equals(Constants.UNREGISTER)) {
        try {
          Jedis jedis = jedisPool.getResource();
          try {
            doNotify(jedis, key);
          } finally {
            jedis.close();
          }
        } catch (Throwable t) {
          LOGGER.error(t.getMessage(), t);
        }
      }
    }
  }

  // 用这个线程来监控redis是否可用
  private volatile String monitorId;
  private volatile boolean redisAvailable = false;

  private class Notifier extends Thread {

    private final String listenNodePath;

    private volatile Jedis jedis;

    private volatile boolean running = true;

    public Notifier(String listenNodePath) {
      super.setDaemon(true);
      super.setName("LTSRedisSubscribe");
      this.listenNodePath = listenNodePath;
      if (monitorId == null) {
        monitorId = listenNodePath;
      }
    }

    @Override
    public void run() {
      try {
        while (running) {
          int retryTimes = 0;
          for (Map.Entry<String, JedisPool> entry : jedisPools.entrySet()) {
            try {
              JedisPool jedisPool = entry.getValue();
              jedis = jedisPool.getResource();
              if (listenNodePath.equals(monitorId) && !redisAvailable) {
                redisAvailable = true;
                appContext.getRegistryStatMonitor().setAvailable(redisAvailable);
              }
              try {
                retryTimes = 0;
                jedis.subscribe(new NotifySub(jedisPool), listenNodePath); // 阻塞
                break;
              } finally {
                jedis.close();
              }
            } catch (Throwable t) { // 重试另一台
              LOGGER.warn(
                  "Failed to subscribe node from redis registry. registry: " + entry.getKey(), t);
              if (++retryTimes % jedisPools.size() == 0) {
                // 如果在所有redis都不可用,需要休息一会,避免空转占用过多cpu资源
                sleep(reconnectPeriod);
                if (listenNodePath.equals(monitorId) && redisAvailable) {
                  redisAvailable = false;
                  appContext.getRegistryStatMonitor().setAvailable(redisAvailable);
                }
              }
            }
          }
        }
      } catch (Throwable t) {
        LOGGER.error(t.getMessage(), t);
      }
    }

    public void shutdown() {
      try {
        running = false;
        jedis.disconnect();
      } catch (Throwable t) {
        LOGGER.warn(t.getMessage(), t);
      }
    }
  }
}