Beispiel #1
0
  protected void createDatabase() {
    if (databasePlatform == null) {
      ResettableBasicDataSource ds = new ResettableBasicDataSource();
      ds.setDriverClassName(Driver.class.getName());
      ds.setMaxActive(1);
      ds.setInitialSize(1);
      ds.setMinIdle(1);
      ds.setMaxIdle(1);
      databaseName = UUID.randomUUID().toString();
      if (inMemoryCompare) {
        ds.setUrl("jdbc:h2:mem:" + databaseName);
      } else {
        ds.setUrl("jdbc:h2:file:./" + databaseName);
      }
      databasePlatform =
          JdbcDatabasePlatformFactory.createNewPlatformInstance(
              ds, new SqlTemplateSettings(), true, false);

      Model inputModel = context.getFlowStep().getComponent().getInputModel();
      List<ModelEntity> entities = inputModel.getModelEntities();
      for (ModelEntity entity : entities) {
        Table table = new Table();
        table.setName(entity.getName() + "_1");
        List<ModelAttribute> attributes = entity.getModelAttributes();
        for (ModelAttribute attribute : attributes) {
          DataType dataType = attribute.getDataType();
          Column column = new Column(attribute.getName());
          if (dataType.isNumeric()) {
            column.setTypeCode(Types.DECIMAL);
          } else if (dataType.isBoolean()) {
            column.setTypeCode(Types.BOOLEAN);
          } else if (dataType.isTimestamp()) {
            column.setTypeCode(Types.TIMESTAMP);
          } else if (dataType.isBinary()) {
            column.setTypeCode(Types.BLOB);
          } else {
            column.setTypeCode(Types.LONGVARCHAR);
          }

          column.setPrimaryKey(attribute.isPk());
          table.addColumn(column);
        }
        alterCaseToMatchLogicalCase(table);
        databasePlatform.createTables(false, false, table);

        table.setName(entity.getName().toUpperCase() + "_2");
        databasePlatform.createTables(false, false, table);
      }

      log(LogLevel.INFO, "Creating databasePlatform with the following url: %s", ds.getUrl());
    }
  }
Beispiel #2
0
  protected void calculateDiff(ISendMessageCallback callback) {
    Map<ModelEntity, String> changeSqls = new HashMap<>();
    Map<ModelEntity, String> addSqls = new HashMap<>();
    Map<ModelEntity, String> delSqls = new HashMap<>();
    Component component = context.getFlowStep().getComponent();
    for (ModelEntity entity : entities) {
      StringBuilder addSql = new StringBuilder("select ");
      StringBuilder chgSql = new StringBuilder(addSql);
      StringBuilder delSql = new StringBuilder(addSql);
      appendColumns(addSql, "curr.", entity);
      appendColumns(delSql, "orig.", entity);
      appendColumns(chgSql, "curr.", entity);

      addSql.append(
          " from " + entity.getName() + "_2 curr left join " + entity.getName() + "_1 orig on ");
      delSql.append(
          " from " + entity.getName() + "_1 orig left join " + entity.getName() + "_2 curr on ");
      chgSql.append(
          " from " + entity.getName() + "_1 orig join " + entity.getName() + "_2 curr on ");
      boolean secondPk = false;
      for (ModelAttribute attribute : entity.getModelAttributes()) {
        if (attribute.isPk()) {
          if (secondPk) {
            addSql.append(" and ");
            delSql.append(" and ");
            chgSql.append(" and ");
          }
          addSql
              .append("curr.")
              .append(attribute.getName())
              .append("=")
              .append("orig.")
              .append(attribute.getName());
          delSql
              .append("curr.")
              .append(attribute.getName())
              .append("=")
              .append("orig.")
              .append(attribute.getName());
          chgSql
              .append("curr.")
              .append(attribute.getName())
              .append("=")
              .append("orig.")
              .append(attribute.getName());
          secondPk = true;
        }
      }

      addSql.append(" where ");
      delSql.append(" where ");
      chgSql.append(" where ");
      secondPk = false;
      boolean secondCol = false;
      for (ModelAttribute attribute : entity.getModelAttributes()) {
        if (attribute.isPk()) {
          if (secondPk) {
            addSql.append(" or ");
            delSql.append(" or ");
          }
          addSql.append("orig.").append(attribute.getName()).append(" is null");
          delSql.append("curr.").append(attribute.getName()).append(" is null");
          secondPk = true;
        } else {
          ComponentAttributeSetting matchColumnSetting =
              component.getSingleAttributeSetting(
                  attribute.getId(), DataDiff.ATTRIBUTE_COMPARE_ENABLED);
          boolean matchColumn =
              matchColumnSetting != null
                  ? Boolean.parseBoolean(matchColumnSetting.getValue())
                  : true;
          if (matchColumn) {
            if (secondCol) {
              chgSql.append(" or ");
            }
            chgSql
                .append("curr.")
                .append(attribute.getName())
                .append(" != ")
                .append("orig.")
                .append(attribute.getName());
            chgSql.append(" or ");
            chgSql
                .append("curr.")
                .append(attribute.getName())
                .append(" is null and ")
                .append("orig.")
                .append(attribute.getName())
                .append(" is not null ");
            chgSql.append(" or ");
            chgSql
                .append("curr.")
                .append(attribute.getName())
                .append(" is not null and ")
                .append("orig.")
                .append(attribute.getName())
                .append(" is null ");
            secondCol = true;
          }
        }
      }

      // we only want to do a change compare if this entity has
      // cols to compare other than the primary key.
      if (!entity.hasOnlyPrimaryKeys() && secondCol) {
        changeSqls.put(entity, chgSql.toString());
        log(LogLevel.INFO, "Generated diff sql for CHG: %s", chgSql);
      }
      log(LogLevel.INFO, "Generated diff sql for ADD: %s", addSql);
      log(LogLevel.INFO, "Generated diff sql for DEL: %s", delSql);
      addSqls.put(entity, addSql.toString());
      delSqls.put(entity, delSql.toString());
    }

    RdbmsReader reader = new RdbmsReader();
    reader.setDataSource(databasePlatform.getDataSource());
    reader.setContext(context);
    reader.setComponentDefinition(componentDefinition);
    reader.setRowsPerMessage(rowsPerMessage);
    reader.setThreadNumber(threadNumber);

    for (ModelEntity entity : entities) {
      ComponentEntitySetting add =
          component.getSingleEntitySetting(entity.getId(), DataDiff.ENTITY_ADD_ENABLED);
      ComponentEntitySetting chg =
          component.getSingleEntitySetting(entity.getId(), DataDiff.ENTITY_CHG_ENABLED);
      boolean addEnabled = add != null ? Boolean.parseBoolean(add.getValue()) : true;
      boolean chgEnabled = chg != null ? Boolean.parseBoolean(chg.getValue()) : true;
      if (addEnabled) {
        reader.setSql(addSqls.get(entity));
        reader.setEntityChangeType(ChangeType.ADD);
        reader.handle(new ControlMessage(this.context.getFlowStep().getId()), callback, false);
        info("Sent %d ADD records for %s", reader.getRowReadDuringHandle(), entity.getName());
      }

      if (chgEnabled && changeSqls.get(entity) != null) {
        reader.setSql(changeSqls.get(entity));
        reader.setEntityChangeType(ChangeType.CHG);
        reader.handle(new ControlMessage(this.context.getFlowStep().getId()), callback, false);
        info("Sent %d CHG records for %s", reader.getRowReadDuringHandle(), entity.getName());
      }
    }

    for (int i = entities.size() - 1; i >= 0; i--) {
      ModelEntity entity = entities.get(i);
      ComponentEntitySetting del =
          component.getSingleEntitySetting(entity.getId(), DataDiff.ENTITY_DEL_ENABLED);
      boolean delEnabled = del != null ? Boolean.parseBoolean(del.getValue()) : true;

      if (delEnabled) {
        reader.setSql(delSqls.get(entity));
        reader.setEntityChangeType(ChangeType.DEL);
        reader.handle(new ControlMessage(this.context.getFlowStep().getId()), callback, false);
        info("Sent %d DEL records for %s", reader.getRowReadDuringHandle(), entity.getName());
      }
    }

    ResettableBasicDataSource ds = databasePlatform.getDataSource();

    ds.close();

    if (!inMemoryCompare) {
      try {
        Files.list(Paths.get(System.getProperty("h2.baseDir")))
            .filter(path -> path.toFile().getName().startsWith(databaseName))
            .forEach(path -> deleteDatabaseFile(path.toFile()));
      } catch (IOException e) {
        log.warn("Failed to delete file", e);
      }
    }

    databasePlatform = null;
    databaseName = null;
    databaseWriter = null;
  }