@Transactional
  private List<PortForwardingRuleVO> findRulesForThisRouter(
      VirtualRouterVmInventory vr, Map<String, Object> data, boolean isNewCreated) {
    if (!isNewCreated) {
      String sql =
          "select rule from PortForwardingRuleVO rule, VirtualRouterPortForwardingRuleRefVO ref, VmNicVO nic, VmInstanceVO vm where vm.state = :vmState and nic.vmInstanceUuid = vm.uuid and rule.vmNicUuid = nic.uuid and rule.uuid = ref.uuid and ref.virtualRouterVmUuid = :vrUuid";
      TypedQuery<PortForwardingRuleVO> q =
          dbf.getEntityManager().createQuery(sql, PortForwardingRuleVO.class);
      q.setParameter("vrUuid", vr.getUuid());
      q.setParameter("vmState", VmInstanceState.Running);
      return q.getResultList();
    } else {
      VmNicInventory publicNic = vr.getPublicNic();
      VmNicInventory guestNic = vr.getGuestNic();
      String sql =
          "select rule from PortForwardingRuleVO rule, VipVO vip, VmNicVO nic, VmInstanceVO vm where vm.uuid = nic.vmInstanceUuid and vm.state = :vmState and rule.vipUuid = vip.uuid and rule.vmNicUuid = nic.uuid and vip.l3NetworkUuid = :vipL3Uuid and nic.l3NetworkUuid = :guestL3Uuid";
      TypedQuery<PortForwardingRuleVO> q =
          dbf.getEntityManager().createQuery(sql, PortForwardingRuleVO.class);
      q.setParameter("vipL3Uuid", publicNic.getL3NetworkUuid());
      q.setParameter("guestL3Uuid", guestNic.getL3NetworkUuid());
      q.setParameter("vmState", VmInstanceState.Running);

      List<PortForwardingRuleVO> rules = q.getResultList();

      if (!rules.isEmpty()) {
        List<VirtualRouterPortForwardingRuleRefVO> refs =
            new ArrayList<VirtualRouterPortForwardingRuleRefVO>();
        for (PortForwardingRuleVO rule : rules) {
          VirtualRouterPortForwardingRuleRefVO ref = new VirtualRouterPortForwardingRuleRefVO();
          ref.setVirtualRouterVmUuid(vr.getUuid());
          ref.setVipUuid(rule.getVipUuid());
          ref.setUuid(rule.getUuid());
          dbf.getEntityManager().persist(ref);
          refs.add(ref);
        }

        data.put(VirtualRouterSyncPortForwardingRulesOnStartFlow.class.getName(), refs);
      }

      return rules;
    }
  }
  @Override
  public void run(final FlowTrigger chain, final Map data) {
    final VirtualRouterVmInventory vr =
        (VirtualRouterVmInventory) data.get(VirtualRouterConstant.Param.VR.toString());

    List<String> nwServed = vr.getGuestL3Networks();
    List<String> l3Uuids =
        vrMgr.selectL3NetworksNeedingSpecificNetworkService(nwServed, NetworkServiceType.DNS);
    if (l3Uuids.isEmpty()) {
      chain.next();
      return;
    }

    if (VirtualRouterSystemTags.DEDICATED_ROLE_VR.hasTag(vr.getUuid())
        && !VirtualRouterSystemTags.VR_DNS_ROLE.hasTag(vr.getUuid())) {
      chain.next();
      return;
    }

    new VirtualRouterRoleManager().makeDnsRole(vr.getUuid());

    SimpleQuery<L3NetworkDnsVO> query = dbf.createQuery(L3NetworkDnsVO.class);
    query.select(L3NetworkDnsVO_.dns);
    query.add(L3NetworkDnsVO_.l3NetworkUuid, Op.IN, l3Uuids);
    List<String> lst = query.listValue();
    if (lst.isEmpty()) {
      chain.next();
      return;
    }

    Set<String> dnsAddresses = new HashSet<String>(lst.size());
    dnsAddresses.addAll(lst);

    final List<DnsInfo> dns = new ArrayList<DnsInfo>(dnsAddresses.size());
    for (String d : dnsAddresses) {
      DnsInfo dinfo = new DnsInfo();
      dinfo.setDnsAddress(d);
      dns.add(dinfo);
    }

    SetDnsCmd cmd = new SetDnsCmd();
    cmd.setDns(dns);

    VirtualRouterAsyncHttpCallMsg msg = new VirtualRouterAsyncHttpCallMsg();
    msg.setVmInstanceUuid(vr.getUuid());
    msg.setPath(VirtualRouterConstant.VR_SET_DNS_PATH);
    msg.setCommand(cmd);
    msg.setCommandTimeout(apiTimeoutManager.getTimeout(cmd.getClass(), "5m"));
    bus.makeTargetServiceIdByResourceUuid(msg, VmInstanceConstant.SERVICE_ID, vr.getUuid());
    bus.send(
        msg,
        new CloudBusCallBack(chain) {
          @Override
          public void run(MessageReply reply) {
            if (!reply.isSuccess()) {
              chain.fail(reply.getError());
              return;
            }

            VirtualRouterAsyncHttpCallReply re = reply.castReply();
            SetDnsRsp ret = re.toResponse(SetDnsRsp.class);
            if (ret.isSuccess()) {
              chain.next();
            } else {
              String err =
                  String.format(
                      "virtual router[name: %s, uuid: %s] failed to configure dns%s, %s ",
                      vr.getName(), vr.getUuid(), JSONObjectUtil.toJsonString(dns), ret.getError());
              logger.warn(err);
              chain.fail(errf.stringToOperationError(err));
            }
          }
        });
  }
  private void applyEip(
      final VirtualRouterVmInventory vr, final EipStruct struct, final Completion completion) {
    FlowChain chain = FlowChainBuilder.newSimpleFlowChain();
    chain.setName(String.format("apply-eip-%s-vr-%s", struct.getEip().getUuid(), vr.getUuid()));
    chain
        .then(
            new Flow() {
              @Override
              public void run(final FlowTrigger trigger, Map data) {
                asf.openFirewall(
                    vr.getUuid(),
                    struct.getVip().getL3NetworkUuid(),
                    getFirewallRules(struct),
                    new Completion(trigger) {
                      @Override
                      public void success() {
                        trigger.next();
                      }

                      @Override
                      public void fail(ErrorCode errorCode) {
                        trigger.fail(errorCode);
                      }
                    });
              }

              @Override
              public void rollback(final FlowRollback trigger, Map data) {
                asf.removeFirewall(
                    vr.getUuid(),
                    struct.getVip().getL3NetworkUuid(),
                    getFirewallRules(struct),
                    new Completion(trigger) {
                      @Override
                      public void success() {
                        trigger.rollback();
                      }

                      @Override
                      public void fail(ErrorCode errorCode) {
                        logger.warn(
                            String.format(
                                "failed to remove firewall rules on virtual router[uuid:%s, l3Network uuid:%s], %s",
                                vr.getUuid(), struct.getVip().getL3NetworkUuid(), errorCode));
                        trigger.rollback();
                      }
                    });
              }
            })
        .then(
            new NoRollbackFlow() {
              @Override
              public void run(final FlowTrigger trigger, Map data) {
                EipTO to = new EipTO();
                String priMac =
                    CollectionUtils.find(
                        vr.getVmNics(),
                        new Function<String, VmNicInventory>() {
                          @Override
                          public String call(VmNicInventory arg) {
                            if (arg.getL3NetworkUuid().equals(struct.getNic().getL3NetworkUuid())) {
                              return arg.getMac();
                            }
                            return null;
                          }
                        });
                to.setPrivateMac(priMac);
                to.setVipIp(struct.getVip().getIp());
                to.setGuestIp(struct.getNic().getIp());
                to.setSnatInboundTraffic(struct.isSnatInboundTraffic());

                VirtualRouterCommands.CreateEipCmd cmd = new VirtualRouterCommands.CreateEipCmd();
                cmd.setEip(to);
                VirtualRouterAsyncHttpCallMsg msg = new VirtualRouterAsyncHttpCallMsg();
                msg.setCheckStatus(true);
                msg.setPath(VirtualRouterConstant.VR_CREATE_EIP);
                msg.setCommand(cmd);
                msg.setVmInstanceUuid(vr.getUuid());
                bus.makeTargetServiceIdByResourceUuid(
                    msg, VmInstanceConstant.SERVICE_ID, vr.getUuid());
                bus.send(
                    msg,
                    new CloudBusCallBack(completion) {
                      @Override
                      public void run(MessageReply reply) {
                        if (!reply.isSuccess()) {
                          trigger.fail(reply.getError());
                          return;
                        }

                        VirtualRouterAsyncHttpCallReply re = reply.castReply();
                        CreateEipRsp ret = re.toResponse(CreateEipRsp.class);
                        if (ret.isSuccess()) {
                          trigger.next();
                        } else {
                          trigger.fail(
                              errf.stringToOperationError(
                                  String.format(
                                      "failed to create eip[uuid:%s, name:%s, ip:%s] for vm nic[uuid:%s] on virtual router[uuid:%s], %s",
                                      struct.getEip().getUuid(),
                                      struct.getEip().getName(),
                                      struct.getVip().getIp(),
                                      struct.getNic().getUuid(),
                                      vr.getUuid(),
                                      ret.getError())));
                        }
                      }
                    });
              }
            })
        .done(
            new FlowDoneHandler(completion) {
              @Override
              public void handle(Map data) {
                String info =
                    String.format(
                        "successfully created eip[uuid:%s, name:%s, ip:%s] for vm nic[uuid:%s] on virtual router[uuid:%s]",
                        struct.getEip().getUuid(),
                        struct.getEip().getName(),
                        struct.getVip().getIp(),
                        struct.getNic().getUuid(),
                        vr.getUuid());
                new VirtualRouterRoleManager().makeEipRole(vr.getUuid());
                logger.debug(info);
                completion.success();
              }
            })
        .error(
            new FlowErrorHandler(completion) {
              @Override
              public void handle(ErrorCode errCode, Map data) {
                completion.fail(errCode);
              }
            })
        .start();
  }
  @Override
  public void run(final FlowTrigger chain, Map data) {
    final VirtualRouterVmInventory vr =
        (VirtualRouterVmInventory) data.get(VirtualRouterConstant.Param.VR.toString());
    VmNicInventory guestNic = vr.getGuestNic();
    if (!vrMgr.isL3NetworkNeedingNetworkServiceByVirtualRouter(
        guestNic.getL3NetworkUuid(), PortForwardingConstant.PORTFORWARDING_NETWORK_SERVICE_TYPE)) {
      chain.next();
      return;
    }

    boolean isNewCreated = data.containsKey(Param.IS_NEW_CREATED.toString());

    List<PortForwardingRuleVO> ruleVOs = findRulesForThisRouter(vr, data, isNewCreated);
    if (ruleVOs.isEmpty()) {
      chain.next();
      return;
    }

    Map<String, PortForwardingRuleVO> ruleMap =
        new HashMap<String, PortForwardingRuleVO>(ruleVOs.size());
    for (PortForwardingRuleVO rvo : ruleVOs) {
      ruleMap.put(rvo.getUuid(), rvo);
    }

    Collection<PortForwardingRuleTO> tos = calculateAllRules(ruleMap, vr.getUuid());
    List<PortForwardingRuleTO> toList = new ArrayList<PortForwardingRuleTO>(tos.size());
    toList.addAll(tos);

    SyncPortForwardingRuleCmd cmd = new SyncPortForwardingRuleCmd();
    cmd.setRules(toList);

    VirtualRouterAsyncHttpCallMsg msg = new VirtualRouterAsyncHttpCallMsg();
    msg.setCommand(cmd);
    msg.setPath(VirtualRouterConstant.VR_SYNC_PORT_FORWARDING);
    msg.setVmInstanceUuid(vr.getUuid());
    bus.makeTargetServiceIdByResourceUuid(msg, VmInstanceConstant.SERVICE_ID, vr.getUuid());
    bus.send(
        msg,
        new CloudBusCallBack(chain) {
          @Override
          public void run(MessageReply reply) {
            if (!reply.isSuccess()) {
              chain.fail(reply.getError());
              return;
            }

            VirtualRouterAsyncHttpCallReply re = reply.castReply();
            SyncPortForwardingRuleRsp ret = re.toResponse(SyncPortForwardingRuleRsp.class);
            if (ret.isSuccess()) {
              String info =
                  String.format(
                      "successfully sync port forwarding rules served by virtual router[name: %s uuid: %s]",
                      vr.getName(), vr.getUuid());
              logger.debug(info);
              chain.next();
            } else {
              String err =
                  String.format(
                      "failed to sync port forwarding rules served by virtual router[name: %s, uuid: %s], because %s",
                      vr.getName(), vr.getUuid(), ret.getError());
              logger.warn(err);
              chain.fail(errf.stringToOperationError(err));
            }
          }
        });
  }
  @Override
  public void revokeEip(final EipStruct struct, final Completion completion) {
    SimpleQuery<VirtualRouterEipRefVO> q = dbf.createQuery(VirtualRouterEipRefVO.class);
    q.add(VirtualRouterEipRefVO_.eipUuid, SimpleQuery.Op.EQ, struct.getEip().getUuid());
    final VirtualRouterEipRefVO ref = q.find();
    if (ref == null) {
      // vr may have been deleted
      completion.success();
      return;
    }

    VirtualRouterVmVO vrvo = dbf.findByUuid(ref.getVirtualRouterVmUuid(), VirtualRouterVmVO.class);
    if (vrvo.getState() != VmInstanceState.Running) {
      // rule will be synced when vr state changes to Running
      completion.success();
      return;
    }

    final VirtualRouterVmInventory vr = VirtualRouterVmInventory.valueOf(vrvo);

    // TODO: how to cleanup on failure
    final FlowChain chain = FlowChainBuilder.newSimpleFlowChain();
    chain.setName(String.format("revoke-eip-%s-vr-%s", struct.getEip().getUuid(), vr.getUuid()));
    chain
        .then(
            new NoRollbackFlow() {
              @Override
              public void run(final FlowTrigger trigger, Map data) {
                VirtualRouterCommands.RemoveEipCmd cmd = new VirtualRouterCommands.RemoveEipCmd();
                EipTO to = new EipTO();
                String priMac =
                    CollectionUtils.find(
                        vr.getVmNics(),
                        new Function<String, VmNicInventory>() {
                          @Override
                          public String call(VmNicInventory arg) {
                            if (arg.getL3NetworkUuid().equals(struct.getNic().getL3NetworkUuid())) {
                              return arg.getMac();
                            }
                            return null;
                          }
                        });

                to.setPrivateMac(priMac);
                to.setSnatInboundTraffic(struct.isSnatInboundTraffic());
                to.setVipIp(struct.getVip().getIp());
                to.setGuestIp(struct.getNic().getIp());
                cmd.setEip(to);

                VirtualRouterAsyncHttpCallMsg msg = new VirtualRouterAsyncHttpCallMsg();
                msg.setVmInstanceUuid(vr.getUuid());
                msg.setCommand(cmd);
                msg.setCheckStatus(true);
                msg.setPath(VirtualRouterConstant.VR_REMOVE_EIP);
                bus.makeTargetServiceIdByResourceUuid(
                    msg, VmInstanceConstant.SERVICE_ID, vr.getUuid());
                bus.send(
                    msg,
                    new CloudBusCallBack(trigger) {
                      @Override
                      public void run(MessageReply reply) {
                        if (!reply.isSuccess()) {
                          trigger.setError(reply.getError());
                        } else {
                          VirtualRouterAsyncHttpCallReply re = reply.castReply();
                          RemoveEipRsp ret = re.toResponse(RemoveEipRsp.class);
                          if (!ret.isSuccess()) {
                            String err =
                                String.format(
                                    "failed to remove eip[uuid:%s, name:%s, ip:%s] for vm nic[uuid:%s] on virtual router[uuid:%s], %s",
                                    struct.getEip().getUuid(),
                                    struct.getEip().getName(),
                                    struct.getVip().getIp(),
                                    struct.getNic().getUuid(),
                                    vr.getUuid(),
                                    ret.getError());
                            trigger.setError(errf.stringToOperationError(err));
                          }
                        }

                        trigger.next();
                      }
                    });
              }
            })
        .then(
            new NoRollbackFlow() {
              @Override
              public void run(final FlowTrigger trigger, Map data) {
                asf.removeFirewall(
                    vr.getUuid(),
                    struct.getVip().getL3NetworkUuid(),
                    getFirewallRules(struct),
                    new Completion() {
                      @Override
                      public void success() {
                        trigger.next();
                      }

                      @Override
                      public void fail(ErrorCode errorCode) {
                        logger.warn(
                            String.format(
                                "failed to remove firewall rules on virtual router[uuid:%s, l3Network uuid:%s], %s",
                                vr.getUuid(), struct.getVip().getL3NetworkUuid(), errorCode));
                        trigger.next();
                      }
                    });
              }
            })
        .done(
            new FlowDoneHandler(completion) {
              @Override
              public void handle(Map data) {
                String info =
                    String.format(
                        "successfully removed eip[uuid:%s, name:%s, ip:%s] for vm nic[uuid:%s] on virtual router[uuid:%s]",
                        struct.getEip().getUuid(),
                        struct.getEip().getName(),
                        struct.getVip().getIp(),
                        struct.getNic().getUuid(),
                        vr.getUuid());
                logger.debug(info);
                dbf.remove(ref);
                completion.success();
              }
            })
        .error(
            new FlowErrorHandler(completion) {
              @Override
              public void handle(ErrorCode errCode, Map data) {
                completion.fail(errCode);
              }
            })
        .start();
  }