/** 根据视图同步定义的columnPair，获取需要反查的字段列表，不包括主键 */
private List<String> buildMaxColumnsFromColumnPairs(
List<DataMediaPair> mediaPairs, List<EventColumn> pks) {
Set<String> allColumns = new HashSet<String>();
Map<String, EventColumn> pkMap = new HashMap<String, EventColumn>(pks.size(), 1f);
for (EventColumn pk : pks) {
pkMap.put(StringUtils.lowerCase(pk.getColumnName()), pk);
}
for (DataMediaPair mediaPair : mediaPairs) { // 一个源库可以对应多个目标，多路复制
List<ColumnPair> columnPairs = mediaPair.getColumnPairs();
if (CollectionUtils.isEmpty(columnPairs) || mediaPair.getColumnPairMode().isExclude()) {
// 1. 如果有一个没有视图定义，说明需要所有字段
// 2. 如果有一个表存在exclude模式，简单处理，直接反查所有字段，到后面进行过滤
return new ArrayList<String>(); // 返回空集合，代表没有view
// filter，需要所有字段
} else {
for (ColumnPair columnPair : columnPairs) {
String columnName = columnPair.getSourceColumn().getName();
if (!pkMap.containsKey(StringUtils.lowerCase(columnName))) {
allColumns.add(columnPair.getSourceColumn().getName()); // 加入的为非主键
}
}
}
}
return new ArrayList<String>(allColumns);
}
private String dumpEventColumn(List<EventColumn> columns) {
StringBuilder builder = new StringBuilder(event_default_capacity);
int size = columns.size();
for (int i = 0; i < size; i++) {
EventColumn column = columns.get(i);
builder.append("\t").append(column.toString());
if (i < columns.size() - 1) {
builder.append(SEP);
}
}
return builder.toString();
}
private EventColumn buildColumn(BatchProto.Column columnProto) {
EventColumn column = new EventColumn();
column.setColumnName(columnProto.getName());
column.setNull(columnProto.getIsNull());
column.setColumnType(columnProto.getType());
column.setColumnValue(columnProto.getValue());
column.setKey(columnProto.getIsPrimaryKey());
column.setIndex(columnProto.getIndex());
column.setUpdate(columnProto.getIsUpdate()); // add by ljh 2012-08-30，标记变更字段
return column;
}
private BatchProto.Column buildColumn(EventColumn keyColumn) {
BatchProto.Column.Builder columnBuilder = BatchProto.Column.newBuilder();
columnBuilder.setName(keyColumn.getColumnName());
columnBuilder.setType(keyColumn.getColumnType());
columnBuilder.setIsNull(keyColumn.isNull());
columnBuilder.setIsPrimaryKey(keyColumn.isKey());
columnBuilder.setIndex(keyColumn.getIndex());
if (keyColumn.getColumnValue() != null) {
columnBuilder.setValue(keyColumn.getColumnValue());
}
columnBuilder.setIsUpdate(keyColumn.isUpdate()); // add by ljh 2012-08-30，标记变更字段
return columnBuilder.build();
}
/** 构建数据库主键字段的信息 */
private TableData buildTableData(Table table, List<EventColumn> keys) {
Column[] tableColumns = table.getColumns();
TableData data = new TableData();
data.indexs = new int[keys.size()];
data.columnNames = new String[keys.size()];
data.columnTypes = new int[keys.size()];
data.columnValues = new Object[keys.size()];
int i = 0;
int index = 0;
for (EventColumn keyColumn : keys) {
for (Column tableColumn : tableColumns) {
if (StringUtils.equalsIgnoreCase(keyColumn.getColumnName(), tableColumn.getName())) {
data.indexs[i] = index;
data.columnNames[i] = tableColumn.getName();
data.columnTypes[i] = tableColumn.getTypeCode();
data.columnValues[i] =
SqlUtils.stringToSqlValue(
keyColumn.getColumnValue(),
tableColumn.getTypeCode(),
tableColumn.isRequired(),
false);
i++;
break;
}
index++;
}
}
if (i != keys.size()) {
throw new ExtractException(
"keys is not found in table " + table.toString() + " keys : " + dumpEventColumn(keys));
}
return data;
}
@SuppressWarnings("unused")
private boolean checkNeedDbForRowMode(
Table table, List<String> viewColumns, EventData eventData) {
if (viewColumns.size() != 0) { // 说明有视图
if (viewColumns.size() != eventData.getColumns().size()) {
return true;
}
// 检查一下当前是否所有字段都在view字段列表里
for (EventColumn column : eventData.getColumns()) {
if (!viewColumns.contains(column.getColumnName())) {
return true;
}
}
return false;
} else {
if (table.getColumnCount() == eventData.getColumns().size() + eventData.getKeys().size()) {
return false;
} else {
return true;
}
}
}
private EventColumn buildColumn(
String name, int type, String value, boolean isKey, boolean isNull) {
EventColumn column = new EventColumn();
column.setColumnName(name);
column.setColumnType(type);
column.setColumnValue(value);
column.setKey(isKey);
column.setNull(isNull);
return column;
}
/** 构建数据库非主键字段的信息 */
private TableData buildTableData(
Table table, List<EventColumn> columns, boolean needAll, List<String> viewColumnNames) {
Column[] tableColumns = table.getColumns();
List<Column> noPkcolumns = new ArrayList<Column>();
for (Column tableColumn : tableColumns) {
if (!tableColumn.isPrimaryKey()) {
noPkcolumns.add(tableColumn);
}
}
TableData data = new TableData();
int size = columns.size();
if (needAll) {
size =
viewColumnNames.size() != 0
? viewColumnNames.size()
: noPkcolumns.size(); // 如果view不为空就使用view作为反查字段
}
data.indexs = new int[size];
data.columnNames = new String[size];
data.columnTypes = new int[size];
data.columnValues = new Object[size];
int i = 0;
if (needAll) {
int index = 0;
if (viewColumnNames.size() != 0) { // 存在视图定义
for (Column tableColumn : tableColumns) {
if (viewColumnNames.contains(tableColumn.getName())) { // 只放入在view中定义的
data.indexs[i] = index; // 计算下下标
data.columnNames[i] = tableColumn.getName();
data.columnTypes[i] = tableColumn.getTypeCode();
i++;
}
index++;
}
} else {
for (Column tableColumn : tableColumns) {
if (!tableColumn.isPrimaryKey()) {
data.indexs[i] = index; // 计算下下标
data.columnNames[i] = tableColumn.getName();
data.columnTypes[i] = tableColumn.getTypeCode();
i++;
}
index++;
}
}
} else {
for (EventColumn column : columns) {
int index = 0;
for (Column tableColumn : tableColumns) {
if (StringUtils.equalsIgnoreCase(column.getColumnName(), tableColumn.getName())) {
data.indexs[i] = index; // 计算下下标
data.columnNames[i] = tableColumn.getName();
data.columnTypes[i] = tableColumn.getTypeCode();
i++;
break;
}
index++;
}
}
if (i != columns.size()) {
throw new ExtractException(
"columns is not found in table "
+ table.toString()
+ " columns : "
+ dumpEventColumn(columns));
}
}
return data;
}
public void run() {
try {
MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipeline.getId()));
Thread.currentThread()
.setName(String.format(WORKER_NAME_FORMAT, pipeline.getId(), pipeline.getName()));
// 获取数据表信息
DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, eventData.getTableId());
DbDialect dbDialect =
dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
Table table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
TableData keyTableData = buildTableData(table, eventData.getKeys());
// oracle类型特殊处理下
if (dbDialect instanceof OracleDialect) {
keyTableData.columnTypes = getOraclePkTypes(table, keyTableData.columnNames);
}
boolean needAll =
pipeline.getParameters().getSyncMode().isRow()
|| (eventData.getSyncMode() != null && eventData.getSyncMode().isRow());
// 增加一种case, 针对oracle erosa有时侯结果记录只有主键，没有变更字段，需要做一次反查，获取所有字段
needAll |=
CollectionUtils.isEmpty(eventData.getUpdatedColumns())
&& dataMedia.getSource().getType().isOracle();
List<DataMediaPair> mediaParis =
ConfigHelper.findDataMediaPairByMediaId(pipeline, dataMedia.getId());
List<String> viewColumnNames =
buildMaxColumnsFromColumnPairs(mediaParis, eventData.getKeys());
// TODO 后续版本测试下
// if (needAll) {
// boolean needDb = checkNeedDbForRowMode(table,
// viewColumnNames, eventData);
// if (needAll && !needDb) {// 不需要进行反查
// item.setFilter(false);
// return;
// }
// }
// modified by ljh at 2012-11-04
// 反查数据时只反查带update=true标识的数据，因为update=false的记录可能只是进行filter需要用到的数据，不需要反查
TableData columnTableData =
buildTableData(table, eventData.getUpdatedColumns(), needAll, viewColumnNames);
if (columnTableData.columnNames.length == 0) {
// 全主键，不需要进行反查
} else {
List<String> newColumnValues =
select(
dbDialect,
eventData.getSchemaName(),
eventData.getTableName(),
keyTableData,
columnTableData);
if (newColumnValues == null) {
// miss from db
// 设置为filter=true，可能存在丢数据的风险.
// 比如针对源库发生主备切换，otter反查的是备库，查询不到对应的记录
// item.setFilter(true);
// 针对需要自定义反查数据库的，允许忽略
// a. 自由门触发的数据，不存在时可以忽略
// b. 回环补救算法触发的数据，不存在时可以忽略
boolean needFilter = eventData.isRemedy() || pipeline.getParameters().getSkipNoRow();
item.setFilter(needFilter);
// 判断主键是否有变更，如果变更了，就原样返回item
int index = 0;
for (EventColumn oldKey : eventData.getOldKeys()) {
if (!oldKey.equals(eventData.getKeys().get(index))) {
item.setFilter(false);
break;
}
}
} else {
// 构造反查的返回结果
List<EventColumn> newEventColumns = new ArrayList<EventColumn>();
for (int i = 0; i < newColumnValues.size(); i++) {
EventColumn column = new EventColumn();
column.setIndex(columnTableData.indexs[i]);
column.setColumnName(columnTableData.columnNames[i]);
column.setColumnType(columnTableData.columnTypes[i]);
column.setNull(newColumnValues.get(i) == null);
column.setColumnValue(newColumnValues.get(i));
column.setUpdate(true);
newEventColumns.add(column);
}
// 处理下columns中不在反查字段内的字段列表
for (EventColumn column : eventData.getColumns()) {
boolean override = false;
for (EventColumn newEventColumn : newEventColumns) {
if (StringUtils.equalsIgnoreCase(
newEventColumn.getColumnName(), column.getColumnName())) {
override = true;
break;
}
}
if (!override) { // 针对newcolumns不存在的记录进行添加
newEventColumns.add(column);
}
}
Collections.sort(newEventColumns, new EventColumnIndexComparable()); // 重新排个序
eventData.setColumns(newEventColumns);
}
}
} catch (InterruptedException e) {
// ignore
} finally {
Thread.currentThread().setName(WORKER_NAME);
MDC.remove(OtterConstants.splitPipelineLogFileKey);
}
}
