private String[] getInsertUpdateSql(
ColumnTypeHelper helper, WriteRowsEvent event, RDBSchnauzer rep)
throws UnsupportedEncodingException {
List<Row> rows = event.getRows();
BitColumn bCol = event.getUsedColumns().clone();
String InsertSql = rep.getInsertFields(bCol);
String[] sql = new String[rows.size()];
rep.setUsedColumn(bCol);
byte[] unsigntags = rep.getUnsignedTags();
String rowSql = "";
if (Tags.Verbose) LOGGER.info(rows.size() + Infos.Row);
int index = 0;
String cuid = "";
for (int j = 0; j < rows.size(); j++) {
rowSql = "";
Row row = rows.get(j);
List<Column> columns = row.getColumns();
if (!rep.needReplicate(columns)) continue;
cuid = getCuid(rep);
rowSql =
InsertSql
+ "("
+ helper.getValueStr(columns, bCol, unsigntags, rep.getFullFields())
+ rep.getDefStr()
+ cuid
+ ")";
rowSql +=
" ON duplicate KEY UPDATE "
+ helper.getUpdataStr(columns, rep.getColumnNames(), unsigntags, rep.getFullFields());
sql[index++] = rowSql;
}
return sql;
}
private String[] getCommInsertSql(ColumnTypeHelper helper, WriteRowsEvent event, RDBSchnauzer rep)
throws UnsupportedEncodingException {
List<Row> rows = event.getRows();
BitColumn bCol = event.getUsedColumns().clone();
String[] sql = new String[1];
sql[0] = rep.getInsertFields(bCol);
rep.setUsedColumn(bCol);
byte[] unsigntags = rep.getUnsignedTags();
String sqlValue = "";
if (Tags.Verbose) LOGGER.info(rows.size() + Infos.Row);
String cuid = "";
for (int j = 0; j < rows.size(); j++) {
Row row = rows.get(j);
List<Column> columns = row.getColumns();
if (!rep.needReplicate(columns)) continue;
if (!sqlValue.isEmpty()) sqlValue += ",";
cuid = getCuid(rep);
sqlValue +=
"("
+ helper.getValueStr(columns, bCol, unsigntags, rep.getFullFields())
+ rep.getDefStr()
+ cuid
+ ")";
}
if (sqlValue.isEmpty()) {
sql[0] = "";
return sql;
}
sql[0] += sqlValue;
return sql;
}
@Override
public boolean doDelete(ColumnTypeHelper helper, DeleteRowsEvent event) {
boolean need = false;
String[] sqls = null;
int index = 0;
doBeforeDelete(helper, event);
for (int i = 0; i < tables.size(); i++) {
RDBSchnauzer rep = (RDBSchnauzer) tables.get(i);
if (notEqualS(helper.databaseName, masterDb.dbname)) continue;
if (notEqualS(helper.tableName, rep.getMasterTableName())) continue;
List<Row> rows = event.getRows();
for (int j = 0; j < rows.size(); j++) {
Row row = rows.get(j);
List<Column> columns = row.getColumns();
if (sqls == null) {
sqls = new String[rows.size() * tables.size() + 1];
sqls[0] = getStatusUpdateSql(helper);
}
need = true;
try {
sqls[++index] = rep.getDelete() + getWhereStr(rep, columns, helper);
} catch (Exception e) {
mailAndLog(Infos.Create + Infos.DoDelete + Infos.Failed, e.getMessage());
return false;
}
}
}
String[] errInfo = new String[1];
if (!need || !logSql(sqls)) {
autoSetPosition(helper);
return true;
}
if (dbhelper.excuteSqlByTransaction(sqls, errInfo)) {
slaveDb.binlog = this.getBinLogName(helper);
slaveDb.pos = helper.position;
slaveDb.tablepos = helper.tableMapPos;
LOGGER.info(Infos.DoDelete + Infos.OK);
return true;
}
String sInfo = getWarning(sqls, errInfo);
mailAndLog(Infos.DoDelete + Infos.Failed, sInfo);
rollBackToTableEvent(helper);
return false;
}
private String[] getInsUpSqlSql(ColumnTypeHelper helper, WriteRowsEvent event, RDBSchnauzer rep)
throws UnsupportedEncodingException {
List<Row> rows = event.getRows();
BitColumn bCol = event.getUsedColumns().clone();
String InsertSql = rep.getInsertFields(bCol);
String sIfExistPre =
" set nocount off if not exists(select 1 from " + rep.getSlaveTableName() + " where ";
String updatePre = " UPDATE " + rep.getSlaveTableName() + " SET ";
String[] sql = new String[rows.size() * 4];
rep.setUsedColumn(bCol);
byte[] unsigntags = rep.getUnsignedTags();
String rowSql = "";
if (Tags.Verbose) LOGGER.info(rows.size() + Infos.Row);
int index = 0;
for (int j = 0; j < rows.size(); j++) {
rowSql = "";
Row row = rows.get(j);
List<Column> columns = row.getColumns();
if (!rep.needReplicate(columns)) continue;
rowSql = sIfExistPre + getWhereStr(rep, columns, helper) + ") ";
String cuid = getCuid(rep);
rowSql +=
InsertSql
+ "("
+ helper.getValueStr(columns, bCol, unsigntags, rep.getFullFields())
+ rep.getDefStr()
+ cuid
+ ")";
rowSql += " else ";
rowSql +=
updatePre
+ helper.getUpdataStr(columns, rep.getColumnNames(), unsigntags, rep.getFullFields())
+ " where "
+ getWhereStr(rep, columns, helper)
+ " ";
sql[index++] = rowSql;
}
return sql;
}
private void innerDelete(ColumnTypeHelper helper, DeleteRowsEvent event, RedisSchnauzer table)
throws UnsupportedEncodingException {
List<Row> rows = event.getRows();
LOGGER.info(rows.size() + Infos.Row);
int scoreIndex = table.getMemberIndex();
int memberIndex = table.getScoreIndex();
int valueIndex = table.getValueIndex();
Double score = 0.0;
String member = "";
String value = "";
for (int j = 0; j < rows.size(); j++) {
Row row = rows.get(j);
List<Column> columns = row.getColumns();
if (!table.needReplicate(columns, helper)) continue;
switch (table.getType()) {
case String:
dbhelper.delete(table.getKey(columns));
dbhelper.setBinlogKey(
getBinLogName(helper), Long.toString(helper.position), helper.tableName);
break;
case Set:
score = -1.00;
value =
helper.getColStr(
valueIndex,
columns.get(valueIndex),
(byte) 1,
table.masterfields.get(valueIndex));
dbhelper.zincrby(table.getKey(columns), score, value);
dbhelper.setBinlogKey(
getBinLogName(helper), Long.toString(helper.position), helper.tableName);
break;
case SortedSet:
score =
Double.parseDouble(
helper.getColStr(
scoreIndex,
columns.get(scoreIndex),
(byte) 1,
table.masterfields.get(scoreIndex)))
* table.scorefield.multiplier;
member =
helper.getColStr(
memberIndex,
columns.get(memberIndex),
(byte) 1,
table.masterfields.get(memberIndex));
dbhelper.zincrby(table.getKey(columns), -1 * score, member);
dbhelper.setBinlogKey(
getBinLogName(helper), Long.toString(helper.position), helper.tableName);
break;
case List:
ErrorHelper.errExit(
"innerDelete====" + Infos.Illegal + "RedisStructure:" + table.getType().toString());
break;
default:
ErrorHelper.errExit(
"innerDelete====" + Infos.Illegal + "RedisStructure:" + table.getType().toString());
break;
}
}
}
@Override
public List<SourceRecord> poll() throws InterruptedException {
try {
MaxwellAbstractRowsEvent event = replicator.getEvent();
this.maxwellContext.ensurePositionThread();
if (event == null) {
return null;
}
if (event.getTable().getDatabase().getName().equals("maxwell")) {
return null;
}
String databaseName = event.getDatabase().getName();
String tableName = event.getTable().getName();
String topicName = databaseName + "." + tableName;
ArrayList<SourceRecord> records = new ArrayList<>();
Table table = event.getTable();
List<Row> rows = event.filteredRows();
// databaseName.tableName
// create schema for primary key
Schema pkSchema = DataConverter.convertPrimaryKeySchema(table);
List<Struct> primaryKeys = new ArrayList<Struct>();
for (Row row : rows) {
// make primary key schema
Struct pkStruct = new Struct(pkSchema);
for (String pk : table.getPKList()) {
int idx = table.findColumnIndex(pk);
Column column = row.getColumns().get(idx);
ColumnDef def = table.getColumnList().get(idx);
switch (def.getType()) {
case "int":
IntColumnDef intDef = (IntColumnDef) def;
Long l = intDef.toLong(column.getValue());
pkStruct.put(pk, l.intValue());
break;
default:
throw new RuntimeException("unsupported type");
}
}
primaryKeys.add(pkStruct);
}
Iterator<String> jsonIter = event.toJSONStrings().iterator();
// Iterator<String> keysIter = event.getPKStrings().iterator();
Iterator<Struct> pkIter = primaryKeys.iterator();
while (jsonIter.hasNext() && pkIter.hasNext()) {
String json = jsonIter.next();
Struct key = pkIter.next();
System.out.print("got a maxwell event!");
System.out.println(json);
SourceRecord rec =
new SourceRecord(
sourcePartition(),
sourceOffset(event),
topicName,
null, // partition
pkSchema,
key,
VALUE_SCHEMA,
json);
records.add(rec);
}
return records;
// return records;
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
@Override
public boolean doUpdate(ColumnTypeHelper helper, UpdateRowsEvent event) {
String preSql = "";
String aftSql = "";
boolean doOne = false;
String[] sqls = null;
int sqlsIndex = 0;
doBeforeUpdate(helper, event);
for (int i = 0; i < tables.size(); i++) {
RDBSchnauzer rep = (RDBSchnauzer) tables.get(i);
if (notEqualS(helper.databaseName, this.masterDb.dbname)) continue;
if (notEqualS(helper.tableName, rep.getMasterTableName())) continue;
List<Pair<Row>> rows = event.getRows();
String[] tableSqls = new String[rows.size() + 1];
int pos = 1;
byte[] unsigntags = rep.getUnsignedTags();
for (int j = 0; j < rows.size(); j++) {
String sql = "Update " + rep.getSlaveTableName() + " set ";
Pair<Row> pRow = rows.get(j);
Row row = pRow.getAfter();
Row preRow = pRow.getBefore();
List<Column> columns = row.getColumns();
List<Column> pcolumns = preRow.getColumns();
if (!rep.needReplicate(columns)) continue;
try {
preSql =
helper.getUpdataStr(pcolumns, rep.getColumnNames(), unsigntags, rep.getFullFields());
aftSql =
helper.getUpdataStr(columns, rep.getColumnNames(), unsigntags, rep.getFullFields());
} catch (Exception e) {
mailAndLog(Infos.Create + Infos.DoUpdate + Infos.Failed, e.getMessage());
return false;
}
if (isEqualS(preSql, aftSql)) continue;
sql += aftSql;
try {
sql += " where " + getWhereStr(rep, columns, helper);
} catch (Exception e) {
mailAndLog(Infos.Create + Infos.DoUpdate + Infos.Failed, e.getMessage());
return false;
}
tableSqls[pos++] = sql;
doOne = true;
}
if (sqls == null) {
sqls = new String[tables.size() * rows.size() + 1];
for (int k = 0; k < sqls.length; k++) sqls[k] = "";
sqls[0] = getStatusUpdateSql(helper);
}
int sIndex = 0;
for (int j = 0; j < tableSqls.length; j++) {
sIndex = (sqlsIndex + 1);
if ((tableSqls[j] == null) || (tableSqls[j].isEmpty())) continue;
if ((sqls[sIndex] != null) && (!sqls[sIndex].isEmpty())) sqls[i + 1] += ";";
sqls[++sqlsIndex] += tableSqls[j];
}
}
String[] errInfo = new String[1];
if (!doOne || !logSql(sqls)) {
autoSetPosition(helper);
return true;
}
if (dbhelper.excuteSqlByTransaction(sqls, errInfo)) {
this.slaveDb.binlog = this.getBinLogName(helper);
this.slaveDb.pos = helper.position;
this.slaveDb.tablepos = helper.tableMapPos;
LOGGER.info(Infos.DoUpdate + Infos.OK);
return true;
}
String sInfo = getWarning(sqls, errInfo);
mailAndLog(Infos.DoUpdate + Infos.Failed, sInfo);
this.slaveDb.errorMsg = "Update command faild";
HeartBeatInfo hinfo = new HeartBeatInfo();
ConfigGetHelper conf = new ConfigGetHelper(Integer.toString(this.slaveDb.masterID));
conf.getHeartBeatSet(hinfo);
if (hinfo.port > 0 && !hinfo.host.isEmpty()) {
LOGGER.info(Infos.SendErrorInfo);
LocalInfo info = LocalInfoGetter.getLocalInfo();
hinfo.SerialNo = info.getSNStr();
HeartBeatSender beatSender = new HeartBeatSender(this.masterDb, this.slaveDb, hinfo);
beatSender.sendError();
}
rollBackToTableEvent(helper);
return false;
}