예제 #1
0
 private String[] getInsertUpdateSql(
     ColumnTypeHelper helper, WriteRowsEvent event, RDBSchnauzer rep)
     throws UnsupportedEncodingException {
   List<Row> rows = event.getRows();
   BitColumn bCol = event.getUsedColumns().clone();
   String InsertSql = rep.getInsertFields(bCol);
   String[] sql = new String[rows.size()];
   rep.setUsedColumn(bCol);
   byte[] unsigntags = rep.getUnsignedTags();
   String rowSql = "";
   if (Tags.Verbose) LOGGER.info(rows.size() + Infos.Row);
   int index = 0;
   String cuid = "";
   for (int j = 0; j < rows.size(); j++) {
     rowSql = "";
     Row row = rows.get(j);
     List<Column> columns = row.getColumns();
     if (!rep.needReplicate(columns)) continue;
     cuid = getCuid(rep);
     rowSql =
         InsertSql
             + "("
             + helper.getValueStr(columns, bCol, unsigntags, rep.getFullFields())
             + rep.getDefStr()
             + cuid
             + ")";
     rowSql +=
         "  ON duplicate KEY UPDATE "
             + helper.getUpdataStr(columns, rep.getColumnNames(), unsigntags, rep.getFullFields());
     sql[index++] = rowSql;
   }
   return sql;
 }
예제 #2
0
 private String[] getCommInsertSql(ColumnTypeHelper helper, WriteRowsEvent event, RDBSchnauzer rep)
     throws UnsupportedEncodingException {
   List<Row> rows = event.getRows();
   BitColumn bCol = event.getUsedColumns().clone();
   String[] sql = new String[1];
   sql[0] = rep.getInsertFields(bCol);
   rep.setUsedColumn(bCol);
   byte[] unsigntags = rep.getUnsignedTags();
   String sqlValue = "";
   if (Tags.Verbose) LOGGER.info(rows.size() + Infos.Row);
   String cuid = "";
   for (int j = 0; j < rows.size(); j++) {
     Row row = rows.get(j);
     List<Column> columns = row.getColumns();
     if (!rep.needReplicate(columns)) continue;
     if (!sqlValue.isEmpty()) sqlValue += ",";
     cuid = getCuid(rep);
     sqlValue +=
         "("
             + helper.getValueStr(columns, bCol, unsigntags, rep.getFullFields())
             + rep.getDefStr()
             + cuid
             + ")";
   }
   if (sqlValue.isEmpty()) {
     sql[0] = "";
     return sql;
   }
   sql[0] += sqlValue;
   return sql;
 }
예제 #3
0
  @Override
  public boolean doDelete(ColumnTypeHelper helper, DeleteRowsEvent event) {
    boolean need = false;
    String[] sqls = null;
    int index = 0;
    doBeforeDelete(helper, event);
    for (int i = 0; i < tables.size(); i++) {
      RDBSchnauzer rep = (RDBSchnauzer) tables.get(i);
      if (notEqualS(helper.databaseName, masterDb.dbname)) continue;
      if (notEqualS(helper.tableName, rep.getMasterTableName())) continue;
      List<Row> rows = event.getRows();
      for (int j = 0; j < rows.size(); j++) {
        Row row = rows.get(j);
        List<Column> columns = row.getColumns();
        if (sqls == null) {
          sqls = new String[rows.size() * tables.size() + 1];
          sqls[0] = getStatusUpdateSql(helper);
        }
        need = true;
        try {
          sqls[++index] = rep.getDelete() + getWhereStr(rep, columns, helper);
        } catch (Exception e) {
          mailAndLog(Infos.Create + Infos.DoDelete + Infos.Failed, e.getMessage());
          return false;
        }
      }
    }

    String[] errInfo = new String[1];

    if (!need || !logSql(sqls)) {
      autoSetPosition(helper);
      return true;
    }

    if (dbhelper.excuteSqlByTransaction(sqls, errInfo)) {
      slaveDb.binlog = this.getBinLogName(helper);
      slaveDb.pos = helper.position;
      slaveDb.tablepos = helper.tableMapPos;
      LOGGER.info(Infos.DoDelete + Infos.OK);
      return true;
    }

    String sInfo = getWarning(sqls, errInfo);
    mailAndLog(Infos.DoDelete + Infos.Failed, sInfo);
    rollBackToTableEvent(helper);
    return false;
  }
예제 #4
0
 private String[] getInsUpSqlSql(ColumnTypeHelper helper, WriteRowsEvent event, RDBSchnauzer rep)
     throws UnsupportedEncodingException {
   List<Row> rows = event.getRows();
   BitColumn bCol = event.getUsedColumns().clone();
   String InsertSql = rep.getInsertFields(bCol);
   String sIfExistPre =
       " set nocount off  if not exists(select 1 from " + rep.getSlaveTableName() + " where ";
   String updatePre = "   UPDATE " + rep.getSlaveTableName() + " SET ";
   String[] sql = new String[rows.size() * 4];
   rep.setUsedColumn(bCol);
   byte[] unsigntags = rep.getUnsignedTags();
   String rowSql = "";
   if (Tags.Verbose) LOGGER.info(rows.size() + Infos.Row);
   int index = 0;
   for (int j = 0; j < rows.size(); j++) {
     rowSql = "";
     Row row = rows.get(j);
     List<Column> columns = row.getColumns();
     if (!rep.needReplicate(columns)) continue;
     rowSql = sIfExistPre + getWhereStr(rep, columns, helper) + ")  ";
     String cuid = getCuid(rep);
     rowSql +=
         InsertSql
             + "("
             + helper.getValueStr(columns, bCol, unsigntags, rep.getFullFields())
             + rep.getDefStr()
             + cuid
             + ")";
     rowSql += " else ";
     rowSql +=
         updatePre
             + helper.getUpdataStr(columns, rep.getColumnNames(), unsigntags, rep.getFullFields())
             + " where "
             + getWhereStr(rep, columns, helper)
             + "   ";
     sql[index++] = rowSql;
   }
   return sql;
 }
예제 #5
0
  private void innerDelete(ColumnTypeHelper helper, DeleteRowsEvent event, RedisSchnauzer table)
      throws UnsupportedEncodingException {
    List<Row> rows = event.getRows();
    LOGGER.info(rows.size() + Infos.Row);
    int scoreIndex = table.getMemberIndex();
    int memberIndex = table.getScoreIndex();
    int valueIndex = table.getValueIndex();
    Double score = 0.0;
    String member = "";
    String value = "";
    for (int j = 0; j < rows.size(); j++) {
      Row row = rows.get(j);
      List<Column> columns = row.getColumns();
      if (!table.needReplicate(columns, helper)) continue;
      switch (table.getType()) {
        case String:
          dbhelper.delete(table.getKey(columns));
          dbhelper.setBinlogKey(
              getBinLogName(helper), Long.toString(helper.position), helper.tableName);
          break;

        case Set:
          score = -1.00;
          value =
              helper.getColStr(
                  valueIndex,
                  columns.get(valueIndex),
                  (byte) 1,
                  table.masterfields.get(valueIndex));
          dbhelper.zincrby(table.getKey(columns), score, value);
          dbhelper.setBinlogKey(
              getBinLogName(helper), Long.toString(helper.position), helper.tableName);
          break;
        case SortedSet:
          score =
              Double.parseDouble(
                      helper.getColStr(
                          scoreIndex,
                          columns.get(scoreIndex),
                          (byte) 1,
                          table.masterfields.get(scoreIndex)))
                  * table.scorefield.multiplier;
          member =
              helper.getColStr(
                  memberIndex,
                  columns.get(memberIndex),
                  (byte) 1,
                  table.masterfields.get(memberIndex));
          dbhelper.zincrby(table.getKey(columns), -1 * score, member);
          dbhelper.setBinlogKey(
              getBinLogName(helper), Long.toString(helper.position), helper.tableName);
          break;
        case List:
          ErrorHelper.errExit(
              "innerDelete====" + Infos.Illegal + "RedisStructure:" + table.getType().toString());
          break;
        default:
          ErrorHelper.errExit(
              "innerDelete====" + Infos.Illegal + "RedisStructure:" + table.getType().toString());
          break;
      }
    }
  }
  @Override
  public List<SourceRecord> poll() throws InterruptedException {

    try {
      MaxwellAbstractRowsEvent event = replicator.getEvent();
      this.maxwellContext.ensurePositionThread();

      if (event == null) {
        return null;
      }

      if (event.getTable().getDatabase().getName().equals("maxwell")) {
        return null;
      }

      String databaseName = event.getDatabase().getName();
      String tableName = event.getTable().getName();

      String topicName = databaseName + "." + tableName;

      ArrayList<SourceRecord> records = new ArrayList<>();

      Table table = event.getTable();

      List<Row> rows = event.filteredRows();
      // databaseName.tableName
      // create schema for primary key
      Schema pkSchema = DataConverter.convertPrimaryKeySchema(table);

      List<Struct> primaryKeys = new ArrayList<Struct>();

      for (Row row : rows) {
        // make primary key schema
        Struct pkStruct = new Struct(pkSchema);

        for (String pk : table.getPKList()) {
          int idx = table.findColumnIndex(pk);

          Column column = row.getColumns().get(idx);
          ColumnDef def = table.getColumnList().get(idx);

          switch (def.getType()) {
            case "int":
              IntColumnDef intDef = (IntColumnDef) def;
              Long l = intDef.toLong(column.getValue());
              pkStruct.put(pk, l.intValue());
              break;
            default:
              throw new RuntimeException("unsupported type");
          }
        }
        primaryKeys.add(pkStruct);
      }

      Iterator<String> jsonIter = event.toJSONStrings().iterator();
      //            Iterator<String> keysIter = event.getPKStrings().iterator();

      Iterator<Struct> pkIter = primaryKeys.iterator();

      while (jsonIter.hasNext() && pkIter.hasNext()) {
        String json = jsonIter.next();
        Struct key = pkIter.next();

        System.out.print("got a maxwell event!");
        System.out.println(json);
        SourceRecord rec =
            new SourceRecord(
                sourcePartition(),
                sourceOffset(event),
                topicName,
                null, // partition
                pkSchema,
                key,
                VALUE_SCHEMA,
                json);
        records.add(rec);
      }
      return records;

      // return records;
    } catch (Exception e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    return null;
  }
예제 #7
0
  @Override
  public boolean doUpdate(ColumnTypeHelper helper, UpdateRowsEvent event) {
    String preSql = "";
    String aftSql = "";
    boolean doOne = false;
    String[] sqls = null;
    int sqlsIndex = 0;
    doBeforeUpdate(helper, event);
    for (int i = 0; i < tables.size(); i++) {
      RDBSchnauzer rep = (RDBSchnauzer) tables.get(i);
      if (notEqualS(helper.databaseName, this.masterDb.dbname)) continue;
      if (notEqualS(helper.tableName, rep.getMasterTableName())) continue;
      List<Pair<Row>> rows = event.getRows();
      String[] tableSqls = new String[rows.size() + 1];
      int pos = 1;
      byte[] unsigntags = rep.getUnsignedTags();
      for (int j = 0; j < rows.size(); j++) {
        String sql = "Update " + rep.getSlaveTableName() + "  set ";
        Pair<Row> pRow = rows.get(j);
        Row row = pRow.getAfter();
        Row preRow = pRow.getBefore();

        List<Column> columns = row.getColumns();
        List<Column> pcolumns = preRow.getColumns();
        if (!rep.needReplicate(columns)) continue;
        try {
          preSql =
              helper.getUpdataStr(pcolumns, rep.getColumnNames(), unsigntags, rep.getFullFields());
          aftSql =
              helper.getUpdataStr(columns, rep.getColumnNames(), unsigntags, rep.getFullFields());
        } catch (Exception e) {
          mailAndLog(Infos.Create + Infos.DoUpdate + Infos.Failed, e.getMessage());
          return false;
        }

        if (isEqualS(preSql, aftSql)) continue;
        sql += aftSql;
        try {
          sql += " where " + getWhereStr(rep, columns, helper);
        } catch (Exception e) {
          mailAndLog(Infos.Create + Infos.DoUpdate + Infos.Failed, e.getMessage());
          return false;
        }

        tableSqls[pos++] = sql;
        doOne = true;
      }
      if (sqls == null) {
        sqls = new String[tables.size() * rows.size() + 1];
        for (int k = 0; k < sqls.length; k++) sqls[k] = "";
        sqls[0] = getStatusUpdateSql(helper);
      }
      int sIndex = 0;
      for (int j = 0; j < tableSqls.length; j++) {
        sIndex = (sqlsIndex + 1);
        if ((tableSqls[j] == null) || (tableSqls[j].isEmpty())) continue;
        if ((sqls[sIndex] != null) && (!sqls[sIndex].isEmpty())) sqls[i + 1] += ";";
        sqls[++sqlsIndex] += tableSqls[j];
      }
    }

    String[] errInfo = new String[1];

    if (!doOne || !logSql(sqls)) {
      autoSetPosition(helper);
      return true;
    }

    if (dbhelper.excuteSqlByTransaction(sqls, errInfo)) {
      this.slaveDb.binlog = this.getBinLogName(helper);
      this.slaveDb.pos = helper.position;
      this.slaveDb.tablepos = helper.tableMapPos;
      LOGGER.info(Infos.DoUpdate + Infos.OK);
      return true;
    }

    String sInfo = getWarning(sqls, errInfo);
    mailAndLog(Infos.DoUpdate + Infos.Failed, sInfo);
    this.slaveDb.errorMsg = "Update command faild";
    HeartBeatInfo hinfo = new HeartBeatInfo();
    ConfigGetHelper conf = new ConfigGetHelper(Integer.toString(this.slaveDb.masterID));
    conf.getHeartBeatSet(hinfo);
    if (hinfo.port > 0 && !hinfo.host.isEmpty()) {
      LOGGER.info(Infos.SendErrorInfo);
      LocalInfo info = LocalInfoGetter.getLocalInfo();
      hinfo.SerialNo = info.getSNStr();
      HeartBeatSender beatSender = new HeartBeatSender(this.masterDb, this.slaveDb, hinfo);
      beatSender.sendError();
    }
    rollBackToTableEvent(helper);
    return false;
  }