Example #1
0
  private void processChildTables(
      Map<String, TableConfig> tables,
      TableConfig parentTable,
      String dataNodes,
      Element tableNode) {
    // parse child tables
    NodeList childNodeList = tableNode.getChildNodes();
    for (int j = 0; j < childNodeList.getLength(); j++) {
      Node theNode = childNodeList.item(j);
      if (!theNode.getNodeName().equals("childTable")) {
        continue;
      }
      Element childTbElement = (Element) theNode;

      String cdTbName = childTbElement.getAttribute("name").toUpperCase();
      String primaryKey =
          childTbElement.hasAttribute("primaryKey")
              ? childTbElement.getAttribute("primaryKey").toUpperCase()
              : null;

      boolean autoIncrement = false;
      if (childTbElement.hasAttribute("autoIncrement")) {
        autoIncrement = Boolean.parseBoolean(childTbElement.getAttribute("autoIncrement"));
      }
      boolean needAddLimit = true;
      if (childTbElement.hasAttribute("needAddLimit")) {
        needAddLimit = Boolean.parseBoolean(childTbElement.getAttribute("needAddLimit"));
      }
      String joinKey = childTbElement.getAttribute("joinKey").toUpperCase();
      String parentKey = childTbElement.getAttribute("parentKey").toUpperCase();
      TableConfig table =
          new TableConfig(
              cdTbName,
              primaryKey,
              autoIncrement,
              needAddLimit,
              TableConfig.TYPE_GLOBAL_DEFAULT,
              dataNodes,
              getDbType(dataNodes),
              null,
              false,
              parentTable,
              true,
              joinKey,
              parentKey);
      if (tables.containsKey(table.getName())) {
        throw new ConfigException("table " + table.getName() + " duplicated!");
      }
      tables.put(table.getName(), table);
      processChildTables(tables, table, dataNodes, childTbElement);
    }
  }
Example #2
0
  /**
   * @param schema
   * @param ctx
   * @param tc
   * @return true表示校验通过,false表示检验不通过
   */
  public static boolean checkRuleRequired(
      SchemaConfig schema,
      DruidShardingParseInfo ctx,
      RouteCalculateUnit routeUnit,
      TableConfig tc) {
    if (!tc.isRuleRequired()) {
      return true;
    }
    boolean hasRequiredValue = false;
    String tableName = tc.getName();
    if (routeUnit.getTablesAndConditions().get(tableName) == null
        || routeUnit.getTablesAndConditions().get(tableName).size() == 0) {
      hasRequiredValue = false;
    } else {
      for (Map.Entry<String, Set<ColumnRoutePair>> condition :
          routeUnit.getTablesAndConditions().get(tableName).entrySet()) {

        String colName = condition.getKey();
        // 条件字段是拆分字段
        if (colName.equals(tc.getPartitionColumn())) {
          hasRequiredValue = true;
          break;
        }
      }
    }
    return hasRequiredValue;
  }
Example #3
0
  /**
   * 处理分库表路由
   *
   * @param schema
   * @param tablesAndConditions
   * @param tablesRouteMap
   * @throws SQLNonTransientException
   */
  public static void findRouteWithcConditionsForTables(
      SchemaConfig schema,
      RouteResultset rrs,
      Map<String, Map<String, Set<ColumnRoutePair>>> tablesAndConditions,
      Map<String, Set<String>> tablesRouteMap,
      String sql,
      LayerCachePool cachePool,
      boolean isSelect)
      throws SQLNonTransientException {
    // 为分库表找路由
    for (Map.Entry<String, Map<String, Set<ColumnRoutePair>>> entry :
        tablesAndConditions.entrySet()) {
      String tableName = entry.getKey().toUpperCase();
      TableConfig tableConfig = schema.getTables().get(tableName);
      if (tableConfig == null) {
        String msg =
            "can't find table define in schema " + tableName + " schema:" + schema.getName();
        LOGGER.warn(msg);
        throw new SQLNonTransientException(msg);
      }
      // 全局表或者不分库的表略过(全局表后面再计算)
      if (tableConfig.isGlobalTable()
          || schema.getTables().get(tableName).getDataNodes().size() == 1) {
        continue;
      } else { // 非全局表:分库表、childTable、其他
        Map<String, Set<ColumnRoutePair>> columnsMap = entry.getValue();
        String joinKey = tableConfig.getJoinKey();
        String partionCol = tableConfig.getPartitionColumn();
        String primaryKey = tableConfig.getPrimaryKey();
        boolean isFoundPartitionValue =
            partionCol != null && entry.getValue().get(partionCol) != null;
        boolean isLoadData = false;
        if (LOGGER.isDebugEnabled()) {
          if (sql.startsWith(LoadData.loadDataHint)
              || rrs.isLoadData()) { // 由于load data一次会计算很多路由数据,如果输出此日志会极大降低load data的性能
            isLoadData = true;
          }
        }
        if (entry.getValue().get(primaryKey) != null
            && entry.getValue().size() == 1
            && !isLoadData) { // 主键查找
          // try by primary key if found in cache
          Set<ColumnRoutePair> primaryKeyPairs = entry.getValue().get(primaryKey);
          if (primaryKeyPairs != null) {
            if (LOGGER.isDebugEnabled()) {
              LOGGER.debug("try to find cache by primary key ");
            }
            String tableKey = schema.getName() + '_' + tableName;
            boolean allFound = true;
            for (ColumnRoutePair pair : primaryKeyPairs) { // 可能id in(1,2,3)多主键
              String cacheKey = pair.colValue;
              String dataNode = (String) cachePool.get(tableKey, cacheKey);
              if (dataNode == null) {
                allFound = false;
                continue;
              } else {
                if (tablesRouteMap.get(tableName) == null) {
                  tablesRouteMap.put(tableName, new HashSet<String>());
                }
                tablesRouteMap.get(tableName).add(dataNode);
                continue;
              }
            }
            if (!allFound) {
              // need cache primary key ->datanode relation
              if (isSelect && tableConfig.getPrimaryKey() != null) {
                rrs.setPrimaryKey(tableKey + '.' + tableConfig.getPrimaryKey());
              }
            } else { // 主键缓存中找到了就执行循环的下一轮
              continue;
            }
          }
        }
        if (isFoundPartitionValue) { // 分库表
          Set<ColumnRoutePair> partitionValue = columnsMap.get(partionCol);
          if (partitionValue == null || partitionValue.size() == 0) {
            if (tablesRouteMap.get(tableName) == null) {
              tablesRouteMap.put(tableName, new HashSet<String>());
            }
            tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
          } else {
            for (ColumnRoutePair pair : partitionValue) {
              if (pair.colValue != null) {
                Integer nodeIndex =
                    tableConfig.getRule().getRuleAlgorithm().calculate(pair.colValue);
                if (nodeIndex == null) {
                  String msg =
                      "can't find any valid datanode :"
                          + tableConfig.getName()
                          + " -> "
                          + tableConfig.getPartitionColumn()
                          + " -> "
                          + pair.colValue;
                  LOGGER.warn(msg);
                  throw new SQLNonTransientException(msg);
                }
                String node = tableConfig.getDataNodes().get(nodeIndex);
                if (node != null) {
                  if (tablesRouteMap.get(tableName) == null) {
                    tablesRouteMap.put(tableName, new HashSet<String>());
                  }
                  tablesRouteMap.get(tableName).add(node);
                }
              }
              if (pair.rangeValue != null) {
                Integer[] nodeIndexs =
                    tableConfig
                        .getRule()
                        .getRuleAlgorithm()
                        .calculateRange(
                            pair.rangeValue.beginValue.toString(),
                            pair.rangeValue.endValue.toString());
                for (Integer idx : nodeIndexs) {
                  String node = tableConfig.getDataNodes().get(idx);
                  if (node != null) {
                    if (tablesRouteMap.get(tableName) == null) {
                      tablesRouteMap.put(tableName, new HashSet<String>());
                    }
                    tablesRouteMap.get(tableName).add(node);
                  }
                }
              }
            }
          }
        } else if (joinKey != null
            && columnsMap.get(joinKey) != null
            && columnsMap.get(joinKey).size()
                != 0) { // childTable  (如果是select 语句的父子表join)之前要找到root table,将childTable移除,只留下root
                        // table
          Set<ColumnRoutePair> joinKeyValue = columnsMap.get(joinKey);

          ColumnRoutePair joinCol = null;

          Set<String> dataNodeSet = ruleByJoinValueCalculate(rrs, tableConfig, joinKeyValue);

          if (dataNodeSet.isEmpty()) {
            throw new SQLNonTransientException("parent key can't find any valid datanode ");
          }
          if (LOGGER.isDebugEnabled()) {
            LOGGER.debug(
                "found partion nodes (using parent partion rule directly) for child table to update  "
                    + Arrays.toString(dataNodeSet.toArray())
                    + " sql :"
                    + sql);
          }
          if (dataNodeSet.size() > 1) {
            routeToMultiNode(rrs.isCacheAble(), rrs, dataNodeSet, sql);
            rrs.setFinishedRoute(true);
            return;
          } else {
            rrs.setCacheAble(true);
            routeToSingleNode(rrs, dataNodeSet.iterator().next(), sql);
            return;
          }

        } else {
          // 没找到拆分字段,该表的所有节点都路由
          if (tablesRouteMap.get(tableName) == null) {
            tablesRouteMap.put(tableName, new HashSet<String>());
          }
          tablesRouteMap.get(tableName).addAll(tableConfig.getDataNodes());
        }
      }
    }
  }
Example #4
0
  /**
   * 单表路由
   *
   * @param schema
   * @param ctx
   * @param tableName
   * @param rrs
   * @param isSelect
   * @return
   * @throws SQLNonTransientException
   */
  public static RouteResultset tryRouteForOneTable(
      SchemaConfig schema,
      DruidShardingParseInfo ctx,
      RouteCalculateUnit routeUnit,
      String tableName,
      RouteResultset rrs,
      boolean isSelect,
      LayerCachePool cachePool)
      throws SQLNonTransientException {
    if (isNoSharding(schema, tableName)) {
      return routeToSingleNode(rrs, schema.getDataNode(), ctx.getSql());
    }

    TableConfig tc = schema.getTables().get(tableName);
    if (tc == null) {
      String msg = "can't find table define in schema " + tableName + " schema:" + schema.getName();
      LOGGER.warn(msg);
      throw new SQLNonTransientException(msg);
    }
    if (tc.isGlobalTable()) { // 全局表
      if (isSelect) {
        // global select ,not cache route result
        rrs.setCacheAble(false);
        return routeToSingleNode(rrs, tc.getRandomDataNode(), ctx.getSql());
      } else { // insert into 全局表的记录
        return routeToMultiNode(false, rrs, tc.getDataNodes(), ctx.getSql(), true);
      }
    } else { // 单表或者分库表
      if (!checkRuleRequired(schema, ctx, routeUnit, tc)) {
        throw new IllegalArgumentException(
            "route rule for table " + tc.getName() + " is required: " + ctx.getSql());
      }
      if (tc.getPartitionColumn() == null && !tc.isSecondLevel()) { // 单表且不是childTable
        //				return RouterUtil.routeToSingleNode(rrs, tc.getDataNodes().get(0),ctx.getSql());
        return routeToMultiNode(rrs.isCacheAble(), rrs, tc.getDataNodes(), ctx.getSql());
      } else {
        // 每个表对应的路由映射
        Map<String, Set<String>> tablesRouteMap = new HashMap<String, Set<String>>();
        if (routeUnit.getTablesAndConditions() != null
            && routeUnit.getTablesAndConditions().size() > 0) {
          RouterUtil.findRouteWithcConditionsForTables(
              schema,
              rrs,
              routeUnit.getTablesAndConditions(),
              tablesRouteMap,
              ctx.getSql(),
              cachePool,
              isSelect);
          if (rrs.isFinishedRoute()) {
            return rrs;
          }
        }

        if (tablesRouteMap.get(tableName) == null) {
          return routeToMultiNode(rrs.isCacheAble(), rrs, tc.getDataNodes(), ctx.getSql());
        } else {
          //					boolean isCache = rrs.isCacheAble();
          //					if(tablesRouteMap.get(tableName).size() > 1) {
          //
          //					}
          return routeToMultiNode(
              rrs.isCacheAble(), rrs, tablesRouteMap.get(tableName), ctx.getSql());
        }
      }
    }
  }
Example #5
0
  private Map<String, TableConfig> loadTables(Element node) {
    // Map<String, TableConfig> tables = new HashMap<String, TableConfig>();

    // 支持表名中包含引号[`] BEN GONG
    Map<String, TableConfig> tables = new TableConfigMap();

    NodeList nodeList = node.getElementsByTagName("table");
    for (int i = 0; i < nodeList.getLength(); i++) {
      Element tableElement = (Element) nodeList.item(i);
      String tableNameElement = tableElement.getAttribute("name").toUpperCase();
      String[] tableNames = tableNameElement.split(",");

      String primaryKey =
          tableElement.hasAttribute("primaryKey")
              ? tableElement.getAttribute("primaryKey").toUpperCase()
              : null;

      boolean autoIncrement = false;
      if (tableElement.hasAttribute("autoIncrement")) {
        autoIncrement = Boolean.parseBoolean(tableElement.getAttribute("autoIncrement"));
      }
      boolean needAddLimit = true;
      if (tableElement.hasAttribute("needAddLimit")) {
        needAddLimit = Boolean.parseBoolean(tableElement.getAttribute("needAddLimit"));
      }
      String tableTypeStr =
          tableElement.hasAttribute("type") ? tableElement.getAttribute("type") : null;
      int tableType = TableConfig.TYPE_GLOBAL_DEFAULT;
      if ("global".equalsIgnoreCase(tableTypeStr)) {
        tableType = TableConfig.TYPE_GLOBAL_TABLE;
      }
      String dataNode = tableElement.getAttribute("dataNode");
      TableRuleConfig tableRule = null;
      if (tableElement.hasAttribute("rule")) {
        String ruleName = tableElement.getAttribute("rule");
        tableRule = tableRules.get(ruleName);
        if (tableRule == null) {
          throw new ConfigException("rule " + ruleName + " is not found!");
        }
      }
      boolean ruleRequired = false;
      if (tableElement.hasAttribute("ruleRequired")) {
        ruleRequired = Boolean.parseBoolean(tableElement.getAttribute("ruleRequired"));
      }

      if (tableNames == null) {
        throw new ConfigException("table name is not found!");
      }
      String distPrex = "distribute(";
      boolean distTableDns = dataNode.startsWith(distPrex);
      if (distTableDns) {
        dataNode = dataNode.substring(distPrex.length(), dataNode.length() - 1);
      }
      for (int j = 0; j < tableNames.length; j++) {
        String tableName = tableNames[j];
        TableConfig table =
            new TableConfig(
                tableName,
                primaryKey,
                autoIncrement,
                needAddLimit,
                tableType,
                dataNode,
                getDbType(dataNode),
                (tableRule != null) ? tableRule.getRule() : null,
                ruleRequired,
                null,
                false,
                null,
                null);
        checkDataNodeExists(table.getDataNodes());
        if (distTableDns) {
          distributeDataNodes(table.getDataNodes());
        }
        if (tables.containsKey(table.getName())) {
          throw new ConfigException("table " + tableName + " duplicated!");
        }
        tables.put(table.getName(), table);
      }

      if (tableNames.length == 1) {
        TableConfig table = tables.get(tableNames[0]);

        // process child tables
        processChildTables(tables, table, dataNode, tableElement);
      }
    }

    return tables;
  }