protected void createDatabase() {
if (databasePlatform == null) {
ResettableBasicDataSource ds = new ResettableBasicDataSource();
ds.setDriverClassName(Driver.class.getName());
ds.setMaxActive(1);
ds.setInitialSize(1);
ds.setMinIdle(1);
ds.setMaxIdle(1);
databaseName = UUID.randomUUID().toString();
if (inMemoryCompare) {
ds.setUrl("jdbc:h2:mem:" + databaseName);
} else {
ds.setUrl("jdbc:h2:file:./" + databaseName);
}
databasePlatform =
JdbcDatabasePlatformFactory.createNewPlatformInstance(
ds, new SqlTemplateSettings(), true, false);
Model inputModel = context.getFlowStep().getComponent().getInputModel();
List<ModelEntity> entities = inputModel.getModelEntities();
for (ModelEntity entity : entities) {
Table table = new Table();
table.setName(entity.getName() + "_1");
List<ModelAttribute> attributes = entity.getModelAttributes();
for (ModelAttribute attribute : attributes) {
DataType dataType = attribute.getDataType();
Column column = new Column(attribute.getName());
if (dataType.isNumeric()) {
column.setTypeCode(Types.DECIMAL);
} else if (dataType.isBoolean()) {
column.setTypeCode(Types.BOOLEAN);
} else if (dataType.isTimestamp()) {
column.setTypeCode(Types.TIMESTAMP);
} else if (dataType.isBinary()) {
column.setTypeCode(Types.BLOB);
} else {
column.setTypeCode(Types.LONGVARCHAR);
}
column.setPrimaryKey(attribute.isPk());
table.addColumn(column);
}
alterCaseToMatchLogicalCase(table);
databasePlatform.createTables(false, false, table);
table.setName(entity.getName().toUpperCase() + "_2");
databasePlatform.createTables(false, false, table);
}
log(LogLevel.INFO, "Creating databasePlatform with the following url: %s", ds.getUrl());
}
}
public boolean start(Table table) {
/*
* in the case when the target schema or catalog is set then we need to
* use the previous schema or catalog to look up the table locally.
*/
this.currentTable =
platform.getTableFromCache(
table.getOldCatalog(), table.getOldSchema(), table.getName(), false);
if (this.currentTable != null) {
this.currentTable =
currentTable.copyAndFilterColumns(
table.getColumnNames(), table.getPrimaryKeyColumnNames(), true);
/*
* restore the schema and catalog from the passed in table because
* they might have not been originally set, but were set when
* looking up the table locally
*/
this.currentTable.setSchema(table.getSchema());
this.currentTable.setCatalog(table.getCatalog());
this.currentTable.setName(table.getName());
return true;
} else {
return false;
}
}
protected String buildSql(DmlType dmlType, String[] values, Column[] columns) {
// TODO we should try to reuse statements
// TODO support primary key updates
DmlStatement statement =
DmlStatementFactory.createDmlStatement(
targetDatabaseName, dmlType, currentTable, useQuotedIdentifiers);
Object[] objects = platform.getObjectValues(binaryEncoding, values, columns, false, false);
Row row = new Row(columns.length);
for (int i = 0; i < columns.length; i++) {
row.put(columns[i].getName(), objects[i]);
}
return statement.buildDynamicSql(binaryEncoding, row, false, useJdbcTimestampFormat);
}
protected void calculateDiff(ISendMessageCallback callback) {
Map<ModelEntity, String> changeSqls = new HashMap<>();
Map<ModelEntity, String> addSqls = new HashMap<>();
Map<ModelEntity, String> delSqls = new HashMap<>();
Component component = context.getFlowStep().getComponent();
for (ModelEntity entity : entities) {
StringBuilder addSql = new StringBuilder("select ");
StringBuilder chgSql = new StringBuilder(addSql);
StringBuilder delSql = new StringBuilder(addSql);
appendColumns(addSql, "curr.", entity);
appendColumns(delSql, "orig.", entity);
appendColumns(chgSql, "curr.", entity);
addSql.append(
" from " + entity.getName() + "_2 curr left join " + entity.getName() + "_1 orig on ");
delSql.append(
" from " + entity.getName() + "_1 orig left join " + entity.getName() + "_2 curr on ");
chgSql.append(
" from " + entity.getName() + "_1 orig join " + entity.getName() + "_2 curr on ");
boolean secondPk = false;
for (ModelAttribute attribute : entity.getModelAttributes()) {
if (attribute.isPk()) {
if (secondPk) {
addSql.append(" and ");
delSql.append(" and ");
chgSql.append(" and ");
}
addSql
.append("curr.")
.append(attribute.getName())
.append("=")
.append("orig.")
.append(attribute.getName());
delSql
.append("curr.")
.append(attribute.getName())
.append("=")
.append("orig.")
.append(attribute.getName());
chgSql
.append("curr.")
.append(attribute.getName())
.append("=")
.append("orig.")
.append(attribute.getName());
secondPk = true;
}
}
addSql.append(" where ");
delSql.append(" where ");
chgSql.append(" where ");
secondPk = false;
boolean secondCol = false;
for (ModelAttribute attribute : entity.getModelAttributes()) {
if (attribute.isPk()) {
if (secondPk) {
addSql.append(" or ");
delSql.append(" or ");
}
addSql.append("orig.").append(attribute.getName()).append(" is null");
delSql.append("curr.").append(attribute.getName()).append(" is null");
secondPk = true;
} else {
ComponentAttributeSetting matchColumnSetting =
component.getSingleAttributeSetting(
attribute.getId(), DataDiff.ATTRIBUTE_COMPARE_ENABLED);
boolean matchColumn =
matchColumnSetting != null
? Boolean.parseBoolean(matchColumnSetting.getValue())
: true;
if (matchColumn) {
if (secondCol) {
chgSql.append(" or ");
}
chgSql
.append("curr.")
.append(attribute.getName())
.append(" != ")
.append("orig.")
.append(attribute.getName());
chgSql.append(" or ");
chgSql
.append("curr.")
.append(attribute.getName())
.append(" is null and ")
.append("orig.")
.append(attribute.getName())
.append(" is not null ");
chgSql.append(" or ");
chgSql
.append("curr.")
.append(attribute.getName())
.append(" is not null and ")
.append("orig.")
.append(attribute.getName())
.append(" is null ");
secondCol = true;
}
}
}
// we only want to do a change compare if this entity has
// cols to compare other than the primary key.
if (!entity.hasOnlyPrimaryKeys() && secondCol) {
changeSqls.put(entity, chgSql.toString());
log(LogLevel.INFO, "Generated diff sql for CHG: %s", chgSql);
}
log(LogLevel.INFO, "Generated diff sql for ADD: %s", addSql);
log(LogLevel.INFO, "Generated diff sql for DEL: %s", delSql);
addSqls.put(entity, addSql.toString());
delSqls.put(entity, delSql.toString());
}
RdbmsReader reader = new RdbmsReader();
reader.setDataSource(databasePlatform.getDataSource());
reader.setContext(context);
reader.setComponentDefinition(componentDefinition);
reader.setRowsPerMessage(rowsPerMessage);
reader.setThreadNumber(threadNumber);
for (ModelEntity entity : entities) {
ComponentEntitySetting add =
component.getSingleEntitySetting(entity.getId(), DataDiff.ENTITY_ADD_ENABLED);
ComponentEntitySetting chg =
component.getSingleEntitySetting(entity.getId(), DataDiff.ENTITY_CHG_ENABLED);
boolean addEnabled = add != null ? Boolean.parseBoolean(add.getValue()) : true;
boolean chgEnabled = chg != null ? Boolean.parseBoolean(chg.getValue()) : true;
if (addEnabled) {
reader.setSql(addSqls.get(entity));
reader.setEntityChangeType(ChangeType.ADD);
reader.handle(new ControlMessage(this.context.getFlowStep().getId()), callback, false);
info("Sent %d ADD records for %s", reader.getRowReadDuringHandle(), entity.getName());
}
if (chgEnabled && changeSqls.get(entity) != null) {
reader.setSql(changeSqls.get(entity));
reader.setEntityChangeType(ChangeType.CHG);
reader.handle(new ControlMessage(this.context.getFlowStep().getId()), callback, false);
info("Sent %d CHG records for %s", reader.getRowReadDuringHandle(), entity.getName());
}
}
for (int i = entities.size() - 1; i >= 0; i--) {
ModelEntity entity = entities.get(i);
ComponentEntitySetting del =
component.getSingleEntitySetting(entity.getId(), DataDiff.ENTITY_DEL_ENABLED);
boolean delEnabled = del != null ? Boolean.parseBoolean(del.getValue()) : true;
if (delEnabled) {
reader.setSql(delSqls.get(entity));
reader.setEntityChangeType(ChangeType.DEL);
reader.handle(new ControlMessage(this.context.getFlowStep().getId()), callback, false);
info("Sent %d DEL records for %s", reader.getRowReadDuringHandle(), entity.getName());
}
}
ResettableBasicDataSource ds = databasePlatform.getDataSource();
ds.close();
if (!inMemoryCompare) {
try {
Files.list(Paths.get(System.getProperty("h2.baseDir")))
.filter(path -> path.toFile().getName().startsWith(databaseName))
.forEach(path -> deleteDatabaseFile(path.toFile()));
} catch (IOException e) {
log.warn("Failed to delete file", e);
}
}
databasePlatform = null;
databaseName = null;
databaseWriter = null;
}
public boolean isPlatformSupported(IDatabasePlatform platform) {
return (DatabaseNamesConstants.MSSQL2000.equals(platform.getName())
|| DatabaseNamesConstants.MSSQL2005.equals(platform.getName())
|| DatabaseNamesConstants.MSSQL2008.equals(platform.getName()));
}