private String[] getInsertUpdateSql( ColumnTypeHelper helper, WriteRowsEvent event, RDBSchnauzer rep) throws UnsupportedEncodingException { List<Row> rows = event.getRows(); BitColumn bCol = event.getUsedColumns().clone(); String InsertSql = rep.getInsertFields(bCol); String[] sql = new String[rows.size()]; rep.setUsedColumn(bCol); byte[] unsigntags = rep.getUnsignedTags(); String rowSql = ""; if (Tags.Verbose) LOGGER.info(rows.size() + Infos.Row); int index = 0; String cuid = ""; for (int j = 0; j < rows.size(); j++) { rowSql = ""; Row row = rows.get(j); List<Column> columns = row.getColumns(); if (!rep.needReplicate(columns)) continue; cuid = getCuid(rep); rowSql = InsertSql + "(" + helper.getValueStr(columns, bCol, unsigntags, rep.getFullFields()) + rep.getDefStr() + cuid + ")"; rowSql += " ON duplicate KEY UPDATE " + helper.getUpdataStr(columns, rep.getColumnNames(), unsigntags, rep.getFullFields()); sql[index++] = rowSql; } return sql; }
private String[] getCommInsertSql(ColumnTypeHelper helper, WriteRowsEvent event, RDBSchnauzer rep) throws UnsupportedEncodingException { List<Row> rows = event.getRows(); BitColumn bCol = event.getUsedColumns().clone(); String[] sql = new String[1]; sql[0] = rep.getInsertFields(bCol); rep.setUsedColumn(bCol); byte[] unsigntags = rep.getUnsignedTags(); String sqlValue = ""; if (Tags.Verbose) LOGGER.info(rows.size() + Infos.Row); String cuid = ""; for (int j = 0; j < rows.size(); j++) { Row row = rows.get(j); List<Column> columns = row.getColumns(); if (!rep.needReplicate(columns)) continue; if (!sqlValue.isEmpty()) sqlValue += ","; cuid = getCuid(rep); sqlValue += "(" + helper.getValueStr(columns, bCol, unsigntags, rep.getFullFields()) + rep.getDefStr() + cuid + ")"; } if (sqlValue.isEmpty()) { sql[0] = ""; return sql; } sql[0] += sqlValue; return sql; }
@Override public boolean doDelete(ColumnTypeHelper helper, DeleteRowsEvent event) { boolean need = false; String[] sqls = null; int index = 0; doBeforeDelete(helper, event); for (int i = 0; i < tables.size(); i++) { RDBSchnauzer rep = (RDBSchnauzer) tables.get(i); if (notEqualS(helper.databaseName, masterDb.dbname)) continue; if (notEqualS(helper.tableName, rep.getMasterTableName())) continue; List<Row> rows = event.getRows(); for (int j = 0; j < rows.size(); j++) { Row row = rows.get(j); List<Column> columns = row.getColumns(); if (sqls == null) { sqls = new String[rows.size() * tables.size() + 1]; sqls[0] = getStatusUpdateSql(helper); } need = true; try { sqls[++index] = rep.getDelete() + getWhereStr(rep, columns, helper); } catch (Exception e) { mailAndLog(Infos.Create + Infos.DoDelete + Infos.Failed, e.getMessage()); return false; } } } String[] errInfo = new String[1]; if (!need || !logSql(sqls)) { autoSetPosition(helper); return true; } if (dbhelper.excuteSqlByTransaction(sqls, errInfo)) { slaveDb.binlog = this.getBinLogName(helper); slaveDb.pos = helper.position; slaveDb.tablepos = helper.tableMapPos; LOGGER.info(Infos.DoDelete + Infos.OK); return true; } String sInfo = getWarning(sqls, errInfo); mailAndLog(Infos.DoDelete + Infos.Failed, sInfo); rollBackToTableEvent(helper); return false; }
private String[] getInsUpSqlSql(ColumnTypeHelper helper, WriteRowsEvent event, RDBSchnauzer rep) throws UnsupportedEncodingException { List<Row> rows = event.getRows(); BitColumn bCol = event.getUsedColumns().clone(); String InsertSql = rep.getInsertFields(bCol); String sIfExistPre = " set nocount off if not exists(select 1 from " + rep.getSlaveTableName() + " where "; String updatePre = " UPDATE " + rep.getSlaveTableName() + " SET "; String[] sql = new String[rows.size() * 4]; rep.setUsedColumn(bCol); byte[] unsigntags = rep.getUnsignedTags(); String rowSql = ""; if (Tags.Verbose) LOGGER.info(rows.size() + Infos.Row); int index = 0; for (int j = 0; j < rows.size(); j++) { rowSql = ""; Row row = rows.get(j); List<Column> columns = row.getColumns(); if (!rep.needReplicate(columns)) continue; rowSql = sIfExistPre + getWhereStr(rep, columns, helper) + ") "; String cuid = getCuid(rep); rowSql += InsertSql + "(" + helper.getValueStr(columns, bCol, unsigntags, rep.getFullFields()) + rep.getDefStr() + cuid + ")"; rowSql += " else "; rowSql += updatePre + helper.getUpdataStr(columns, rep.getColumnNames(), unsigntags, rep.getFullFields()) + " where " + getWhereStr(rep, columns, helper) + " "; sql[index++] = rowSql; } return sql; }
private void innerDelete(ColumnTypeHelper helper, DeleteRowsEvent event, RedisSchnauzer table) throws UnsupportedEncodingException { List<Row> rows = event.getRows(); LOGGER.info(rows.size() + Infos.Row); int scoreIndex = table.getMemberIndex(); int memberIndex = table.getScoreIndex(); int valueIndex = table.getValueIndex(); Double score = 0.0; String member = ""; String value = ""; for (int j = 0; j < rows.size(); j++) { Row row = rows.get(j); List<Column> columns = row.getColumns(); if (!table.needReplicate(columns, helper)) continue; switch (table.getType()) { case String: dbhelper.delete(table.getKey(columns)); dbhelper.setBinlogKey( getBinLogName(helper), Long.toString(helper.position), helper.tableName); break; case Set: score = -1.00; value = helper.getColStr( valueIndex, columns.get(valueIndex), (byte) 1, table.masterfields.get(valueIndex)); dbhelper.zincrby(table.getKey(columns), score, value); dbhelper.setBinlogKey( getBinLogName(helper), Long.toString(helper.position), helper.tableName); break; case SortedSet: score = Double.parseDouble( helper.getColStr( scoreIndex, columns.get(scoreIndex), (byte) 1, table.masterfields.get(scoreIndex))) * table.scorefield.multiplier; member = helper.getColStr( memberIndex, columns.get(memberIndex), (byte) 1, table.masterfields.get(memberIndex)); dbhelper.zincrby(table.getKey(columns), -1 * score, member); dbhelper.setBinlogKey( getBinLogName(helper), Long.toString(helper.position), helper.tableName); break; case List: ErrorHelper.errExit( "innerDelete====" + Infos.Illegal + "RedisStructure:" + table.getType().toString()); break; default: ErrorHelper.errExit( "innerDelete====" + Infos.Illegal + "RedisStructure:" + table.getType().toString()); break; } } }
@Override public List<SourceRecord> poll() throws InterruptedException { try { MaxwellAbstractRowsEvent event = replicator.getEvent(); this.maxwellContext.ensurePositionThread(); if (event == null) { return null; } if (event.getTable().getDatabase().getName().equals("maxwell")) { return null; } String databaseName = event.getDatabase().getName(); String tableName = event.getTable().getName(); String topicName = databaseName + "." + tableName; ArrayList<SourceRecord> records = new ArrayList<>(); Table table = event.getTable(); List<Row> rows = event.filteredRows(); // databaseName.tableName // create schema for primary key Schema pkSchema = DataConverter.convertPrimaryKeySchema(table); List<Struct> primaryKeys = new ArrayList<Struct>(); for (Row row : rows) { // make primary key schema Struct pkStruct = new Struct(pkSchema); for (String pk : table.getPKList()) { int idx = table.findColumnIndex(pk); Column column = row.getColumns().get(idx); ColumnDef def = table.getColumnList().get(idx); switch (def.getType()) { case "int": IntColumnDef intDef = (IntColumnDef) def; Long l = intDef.toLong(column.getValue()); pkStruct.put(pk, l.intValue()); break; default: throw new RuntimeException("unsupported type"); } } primaryKeys.add(pkStruct); } Iterator<String> jsonIter = event.toJSONStrings().iterator(); // Iterator<String> keysIter = event.getPKStrings().iterator(); Iterator<Struct> pkIter = primaryKeys.iterator(); while (jsonIter.hasNext() && pkIter.hasNext()) { String json = jsonIter.next(); Struct key = pkIter.next(); System.out.print("got a maxwell event!"); System.out.println(json); SourceRecord rec = new SourceRecord( sourcePartition(), sourceOffset(event), topicName, null, // partition pkSchema, key, VALUE_SCHEMA, json); records.add(rec); } return records; // return records; } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } return null; }
@Override public boolean doUpdate(ColumnTypeHelper helper, UpdateRowsEvent event) { String preSql = ""; String aftSql = ""; boolean doOne = false; String[] sqls = null; int sqlsIndex = 0; doBeforeUpdate(helper, event); for (int i = 0; i < tables.size(); i++) { RDBSchnauzer rep = (RDBSchnauzer) tables.get(i); if (notEqualS(helper.databaseName, this.masterDb.dbname)) continue; if (notEqualS(helper.tableName, rep.getMasterTableName())) continue; List<Pair<Row>> rows = event.getRows(); String[] tableSqls = new String[rows.size() + 1]; int pos = 1; byte[] unsigntags = rep.getUnsignedTags(); for (int j = 0; j < rows.size(); j++) { String sql = "Update " + rep.getSlaveTableName() + " set "; Pair<Row> pRow = rows.get(j); Row row = pRow.getAfter(); Row preRow = pRow.getBefore(); List<Column> columns = row.getColumns(); List<Column> pcolumns = preRow.getColumns(); if (!rep.needReplicate(columns)) continue; try { preSql = helper.getUpdataStr(pcolumns, rep.getColumnNames(), unsigntags, rep.getFullFields()); aftSql = helper.getUpdataStr(columns, rep.getColumnNames(), unsigntags, rep.getFullFields()); } catch (Exception e) { mailAndLog(Infos.Create + Infos.DoUpdate + Infos.Failed, e.getMessage()); return false; } if (isEqualS(preSql, aftSql)) continue; sql += aftSql; try { sql += " where " + getWhereStr(rep, columns, helper); } catch (Exception e) { mailAndLog(Infos.Create + Infos.DoUpdate + Infos.Failed, e.getMessage()); return false; } tableSqls[pos++] = sql; doOne = true; } if (sqls == null) { sqls = new String[tables.size() * rows.size() + 1]; for (int k = 0; k < sqls.length; k++) sqls[k] = ""; sqls[0] = getStatusUpdateSql(helper); } int sIndex = 0; for (int j = 0; j < tableSqls.length; j++) { sIndex = (sqlsIndex + 1); if ((tableSqls[j] == null) || (tableSqls[j].isEmpty())) continue; if ((sqls[sIndex] != null) && (!sqls[sIndex].isEmpty())) sqls[i + 1] += ";"; sqls[++sqlsIndex] += tableSqls[j]; } } String[] errInfo = new String[1]; if (!doOne || !logSql(sqls)) { autoSetPosition(helper); return true; } if (dbhelper.excuteSqlByTransaction(sqls, errInfo)) { this.slaveDb.binlog = this.getBinLogName(helper); this.slaveDb.pos = helper.position; this.slaveDb.tablepos = helper.tableMapPos; LOGGER.info(Infos.DoUpdate + Infos.OK); return true; } String sInfo = getWarning(sqls, errInfo); mailAndLog(Infos.DoUpdate + Infos.Failed, sInfo); this.slaveDb.errorMsg = "Update command faild"; HeartBeatInfo hinfo = new HeartBeatInfo(); ConfigGetHelper conf = new ConfigGetHelper(Integer.toString(this.slaveDb.masterID)); conf.getHeartBeatSet(hinfo); if (hinfo.port > 0 && !hinfo.host.isEmpty()) { LOGGER.info(Infos.SendErrorInfo); LocalInfo info = LocalInfoGetter.getLocalInfo(); hinfo.SerialNo = info.getSNStr(); HeartBeatSender beatSender = new HeartBeatSender(this.masterDb, this.slaveDb, hinfo); beatSender.sendError(); } rollBackToTableEvent(helper); return false; }