@Test
public void dataGeneration() throws Exception {
setUp(RecipeExecLocation.SourceCluster);
runSql(connection, "use hdr_sdb1");
createVanillaTable(connection, "store_sales");
createSerDeTable(connection);
createPartitionedTable(connection);
createExternalTable(connection, clusterFS, baseTestHDFSDir + "click_data/", "click_data");
createExternalPartitionedTable(
connection, clusterFS, baseTestHDFSDir + "click_data2/", "click_data2");
runSql(connection2, "use hdr_sdb1");
createVanillaTable(connection2, "store_sales");
createSerDeTable(connection2);
createPartitionedTable(connection2);
createExternalTable(connection2, clusterFS2, baseTestHDFSDir + "click_data/", "click_data");
createExternalPartitionedTable(
connection2, clusterFS2, baseTestHDFSDir + "click_data2/", "click_data2");
final NotifyingAssert anAssert = new NotifyingAssert(true);
HiveAssert.assertDbEqual(
cluster,
clusterHC.getDatabase("hdr_sdb1"),
cluster2,
clusterHC2.getDatabase("hdr_sdb1"),
anAssert);
HiveAssert.assertTableEqual(
cluster,
clusterHC.getTable("hdr_sdb1", "click_data"),
cluster2,
clusterHC2.getTable("hdr_sdb1", "click_data"),
anAssert);
anAssert.assertAll();
}
@Test
public void drTwoTablesOneRequest() throws Exception {
final RecipeExecLocation recipeExecLocation = RecipeExecLocation.TargetCluster;
setUp(recipeExecLocation);
final String tblName = "firstTableDR";
final String tbl2Name = "secondTableDR";
recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName + ',' + tbl2Name);
final List<String> command = recipeMerlin.getSubmissionCommand();
runSql(connection, "create table " + tblName + "(comment string)");
runSql(connection, "create table " + tbl2Name + "(comment string)");
bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName);
bootstrapCopy(connection, clusterFS, tbl2Name, connection2, clusterFS2, tbl2Name);
runSql(
connection,
"insert into table "
+ tblName
+ " values"
+ "('this string has been added post bootstrap - should appear after dr')");
runSql(
connection,
"insert into table "
+ tbl2Name
+ " values"
+ "('this string has been added post bootstrap - should appear after dr')");
Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed.");
InstanceUtil.waitTillInstanceReachState(
recipeExecLocation.getRecipeOC(clusterOC, clusterOC2),
recipeMerlin.getName(),
1,
CoordinatorAction.Status.SUCCEEDED,
EntityType.PROCESS);
final NotifyingAssert anAssert = new NotifyingAssert(true);
HiveAssert.assertTableEqual(
cluster,
clusterHC.getTable(DB_NAME, tblName),
cluster2,
clusterHC2.getTable(DB_NAME, tblName),
anAssert);
HiveAssert.assertTableEqual(
cluster,
clusterHC.getTable(DB_NAME, tbl2Name),
cluster2,
clusterHC2.getTable(DB_NAME, tbl2Name),
anAssert);
anAssert.assertAll();
}
@Test
public void drChangeColumn() throws Exception {
final RecipeExecLocation recipeExecLocation = RecipeExecLocation.SourceCluster;
setUp(recipeExecLocation);
final String tblName = "tableForColumnChange";
recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName);
final List<String> command1 = recipeMerlin.getSubmissionCommand();
final String recipe1Name = recipeMerlin.getName();
runSql(connection, "create table " + tblName + "(id int)");
bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName);
Assert.assertEquals(Bundle.runFalconCLI(command1), 0, "Recipe submission failed.");
runSql(connection, "ALTER TABLE " + tblName + " CHANGE id id STRING COMMENT 'some_comment'");
InstanceUtil.waitTillInstanceReachState(
recipeExecLocation.getRecipeOC(clusterOC, clusterOC2),
recipe1Name,
1,
CoordinatorAction.Status.SUCCEEDED,
EntityType.PROCESS);
HiveAssert.assertTableEqual(
cluster,
clusterHC.getTable(DB_NAME, tblName),
cluster2,
clusterHC2.getTable(DB_NAME, tblName),
new NotifyingAssert(true))
.assertAll();
}
@Test(enabled = false)
public void assertionTest() throws Exception {
setUp(RecipeExecLocation.SourceCluster);
final SoftAssert anAssert = new SoftAssert();
HiveAssert.assertTableEqual(
cluster,
clusterHC.getTable("default", "hcatsmoke10546"),
cluster2,
clusterHC2.getTable("default", "hcatsmoke10548"),
anAssert);
HiveAssert.assertDbEqual(
cluster,
clusterHC.getDatabase("default"),
cluster2,
clusterHC2.getDatabase("default"),
anAssert);
anAssert.assertAll();
}
/**
* 1 src tbl 1 dst tbl. Change table properties and comment at the source. Changes should get
* reflected at destination.
*/
@Test
public void drChangeCommentAndPropertyTest() throws Exception {
final RecipeExecLocation recipeExecLocation = RecipeExecLocation.SourceCluster;
setUp(recipeExecLocation);
final String tblName = "myTable";
recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName);
final List<String> command = recipeMerlin.getSubmissionCommand();
runSql(connection, "create table " + tblName + "(field string)");
// add new table property
runSql(
connection,
"ALTER TABLE " + tblName + " SET TBLPROPERTIES('someProperty' = 'initialValue')");
// set comment
runSql(
connection,
"ALTER TABLE "
+ tblName
+ " SET TBLPROPERTIES('comment' = 'this comment will be "
+ "changed, SHOULD NOT appear')");
LOGGER.info(tblName + " before bootstrap copy: ");
runSql(connection, "describe extended " + tblName);
bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName);
// change table property and comment
runSql(
connection,
"ALTER TABLE " + tblName + " SET TBLPROPERTIES('someProperty' = 'anotherValue')");
runSql(
connection,
"ALTER TABLE "
+ tblName
+ " SET TBLPROPERTIES('comment' = 'this comment should "
+ "appear after replication done')");
LOGGER.info(tblName + " after modifications, before replication: ");
runSql(connection, "describe extended " + tblName);
Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed.");
InstanceUtil.waitTillInstanceReachState(
recipeExecLocation.getRecipeOC(clusterOC, clusterOC2),
recipeMerlin.getName(),
1,
CoordinatorAction.Status.SUCCEEDED,
EntityType.PROCESS);
HiveAssert.assertTableEqual(
cluster,
clusterHC.getTable(DB_NAME, tblName),
cluster2,
clusterHC2.getTable(DB_NAME, tblName),
new NotifyingAssert(true))
.assertAll();
}
/**
* Test creates a table on first cluster using static partitioning. Then it creates the same table
* on the second cluster using dynamic partitioning. Finally it checks the equality of these
* tables.
*
* @throws SQLException
* @throws IOException
*/
@Test
public void dynamicPartitionsTest() throws Exception {
setUp(RecipeExecLocation.SourceCluster);
// create table with static partitions on first cluster
createPartitionedTable(connection, false);
// create table with dynamic partitions on second cluster
createPartitionedTable(connection2, true);
// check that both tables are equal
HiveAssert.assertTableEqual(
cluster,
clusterHC.getTable("hdr_sdb1", "global_store_sales"),
cluster2,
clusterHC2.getTable("hdr_sdb1", "global_store_sales"),
new SoftAssert())
.assertAll();
}
@Test
public void drSerDeWithProperties() throws Exception {
final RecipeExecLocation recipeExecLocation = RecipeExecLocation.SourceCluster;
setUp(recipeExecLocation);
final String tblName = "serdeTable";
recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName);
final List<String> command = recipeMerlin.getSubmissionCommand();
runSql(
connection,
"create table "
+ tblName
+ "(comment string) "
+ "row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'");
bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName);
runSql(
connection,
"insert into table "
+ tblName
+ " values"
+ "('this string has been added post bootstrap - should appear after dr')");
runSql(
connection, "ALTER TABLE " + tblName + " SET SERDEPROPERTIES ('someProperty' = 'value')");
Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed.");
InstanceUtil.waitTillInstanceReachState(
recipeExecLocation.getRecipeOC(clusterOC, clusterOC2),
recipeMerlin.getName(),
1,
CoordinatorAction.Status.SUCCEEDED,
EntityType.PROCESS);
HiveAssert.assertTableEqual(
cluster,
clusterHC.getTable(DB_NAME, tblName),
cluster2,
clusterHC2.getTable(DB_NAME, tblName),
new NotifyingAssert(true))
.assertAll();
}
@Test
public void drExternalToNonExternal() throws Exception {
final RecipeExecLocation recipeExecLocation = RecipeExecLocation.SourceCluster;
setUp(recipeExecLocation);
final String tblName = "externalToNonExternal";
recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName);
final List<String> command = recipeMerlin.getSubmissionCommand();
createExternalTable(connection, clusterFS, baseTestHDFSDir + "click_data/", tblName);
bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName);
// change column name
runSql(connection, "alter table " + tblName + " change column data data_new string");
Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed.");
InstanceUtil.waitTillInstanceReachState(
recipeExecLocation.getRecipeOC(clusterOC, clusterOC2),
recipeMerlin.getName(),
1,
CoordinatorAction.Status.SUCCEEDED,
EntityType.PROCESS);
final NotifyingAssert anAssert = new NotifyingAssert(true);
HiveAssert.assertTableEqual(
cluster,
clusterHC.getTable(DB_NAME, tblName),
cluster2,
clusterHC2.getTable(DB_NAME, tblName),
anAssert,
false);
anAssert.assertNotEquals(
clusterHC2.getTable(DB_NAME, tblName).getTabletype(),
clusterHC.getTable(DB_NAME, tblName).getTableName(),
"Source and destination tables should have different Tabletype");
anAssert.assertNotEquals(
clusterHC2.getTable(DB_NAME, tblName).getTblProps().get("EXTERNAL"),
clusterHC.getTable(DB_NAME, tblName).getTblProps().get("EXTERNAL"),
"Source and destination tables should have different value of property EXTERNAL");
anAssert.assertAll();
}
/**
* 1 src tbl 1 dst tbl replication. Insert/overwrite partitions using dynamic partitions queries.
* The changes should get reflected at destination.
*
* @throws Exception
*/
@Test
public void drInsertOverwriteDynamicPartition() throws Exception {
final RecipeExecLocation recipeExecLocation = RecipeExecLocation.SourceCluster;
setUp(recipeExecLocation);
final String tblName = "drInsertOverwritePartition";
final String hlpTblName = "drInsertOverwritePartitionHelperTbl";
recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName);
final List<String> command = recipeMerlin.getSubmissionCommand();
// disable strict mode to use only dynamic partition
runSql(connection, "set hive.exec.dynamic.partition.mode=nonstrict");
runSql(
connection,
"create table " + hlpTblName + "(comment string) partitioned by (pname string)");
runSql(
connection,
"insert into table "
+ hlpTblName
+ " partition (pname)"
+ " values('overwrite data - should appear after dr', 'OVERWRITE_PART')");
runSql(
connection,
"insert into table "
+ hlpTblName
+ " partition (pname)"
+ " values('newdata row2 - should appear after dr', 'NEW_DATA')");
runSql(
connection,
"insert into table "
+ hlpTblName
+ " partition (pname)"
+ " values('newdata row1 - should appear after dr', 'NEW_DATA')");
runSql(
connection, "create table " + tblName + "(comment string) partitioned by (pname string)");
runSql(
connection,
"insert into table "
+ tblName
+ " partition (pname) values"
+ "('this data should be retained - should appear after dr', 'OLD_PART')");
runSql(
connection,
"insert into table "
+ tblName
+ " partition (pname) values"
+ "('this data should get overwritten - should NOT appear after dr', 'OVERWRITE_PART')");
LOGGER.info(tblName + " before bootstrap copying: ");
runSql(connection, "select * from " + tblName);
bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName);
runSql(
connection,
"insert overwrite table "
+ tblName
+ " partition (pname) "
+ "select comment, pname from "
+ hlpTblName
+ " where comment REGEXP '^overwrite'");
runSql(
connection,
"insert overwrite table "
+ tblName
+ " partition (pname) "
+ "select comment, pname from "
+ hlpTblName
+ " where comment REGEXP '^newdata'");
LOGGER.info(tblName + " after modifications, before replication: ");
runSql(connection, "select * from " + tblName);
Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed.");
InstanceUtil.waitTillInstanceReachState(
recipeExecLocation.getRecipeOC(clusterOC, clusterOC2),
recipeMerlin.getName(),
1,
CoordinatorAction.Status.SUCCEEDED,
EntityType.PROCESS);
HiveAssert.assertTableEqual(
cluster,
clusterHC.getTable(DB_NAME, tblName),
cluster2,
clusterHC2.getTable(DB_NAME, tblName),
new NotifyingAssert(true))
.assertAll();
}
/**
* 1 src tbl 1 dst tbl replication. Insert/delete/replace partitions using dynamic partition
* queries. The changes should get reflected at destination.
*/
@Test
public void drInsertDropReplaceDynamicPartition() throws Exception {
final RecipeExecLocation recipeExecLocation = RecipeExecLocation.SourceCluster;
setUp(recipeExecLocation);
final String tblName = "dynamicPartitionDR";
recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName);
final List<String> command = recipeMerlin.getSubmissionCommand();
// disable strict mode to use only dynamic partition
runSql(connection, "set hive.exec.dynamic.partition.mode=nonstrict");
runSql(
connection, "create table " + tblName + "(comment string) partitioned by (pname string)");
runSql(
connection,
"insert into table "
+ tblName
+ " partition (pname) values"
+ "('this partition is going to be deleted - should NOT appear after dr', 'DELETE')");
runSql(
connection,
"insert into table "
+ tblName
+ " partition (pname) values"
+ "('this partition is going to be replaced - should NOT appear after dr', 'REPLACE')");
runSql(
connection,
"insert into table "
+ tblName
+ " partition (pname) values"
+ "('this partition will have more data - should appear after dr', 'ADD_DATA')");
LOGGER.info(tblName + " before bootstrap copying: ");
runSql(connection, "select * from " + tblName);
bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName);
runSql(
connection,
"insert into table "
+ tblName
+ " partition (pname) values"
+ "('this partition has been added post bootstrap - should appear after dr', 'NEW_PART')");
runSql(
connection,
"insert into table "
+ tblName
+ " partition (pname) values"
+ "('more data has been added post bootstrap - should appear after dr', 'ADD_DATA')");
runSql(connection, "alter table " + tblName + " drop partition(pname = 'DELETE')");
runSql(connection, "alter table " + tblName + " drop partition(pname = 'REPLACE')");
runSql(
connection,
"insert into table "
+ tblName
+ " partition (pname) values"
+ "('this partition has been replaced - should appear after dr', 'REPLACE')");
LOGGER.info(tblName + " after modifications, before replication: ");
runSql(connection, "select * from " + tblName);
Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed.");
InstanceUtil.waitTillInstanceReachState(
recipeExecLocation.getRecipeOC(clusterOC, clusterOC2),
recipeMerlin.getName(),
1,
CoordinatorAction.Status.SUCCEEDED,
EntityType.PROCESS);
HiveAssert.assertTableEqual(
cluster,
clusterHC.getTable(DB_NAME, tblName),
cluster2,
clusterHC2.getTable(DB_NAME, tblName),
new NotifyingAssert(true))
.assertAll();
}
@Test
public void drTwoDstTablesTwoRequests() throws Exception {
final RecipeExecLocation recipeExecLocation = RecipeExecLocation.TargetCluster;
setUp(recipeExecLocation);
final HCatClient clusterHC3 = cluster3.getClusterHelper().getHCatClient();
final Connection connection3 = cluster3.getClusterHelper().getHiveJdbcConnection();
runSql(connection3, "drop database if exists hdr_sdb1 cascade");
runSql(connection3, "create database hdr_sdb1");
runSql(connection3, "use hdr_sdb1");
final String tblName = "vanillaTable";
recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName);
final String recipe1Name = recipeMerlin.getName();
final List<String> command1 = recipeMerlin.getSubmissionCommand();
final Bundle bundle3 = new Bundle(BundleUtil.readHCatBundle(), cluster3);
bundle3.generateUniqueBundle(this);
bundle3.submitClusters(prism);
recipeMerlin
.withTargetCluster(bundle3.getClusterElement())
.withRecipeCluster(
recipeExecLocation.getRecipeCluster(
bundles[0].getClusterElement(), bundle3.getClusterElement()));
recipeMerlin.setUniqueName(this.getClass().getSimpleName());
final List<String> command2 = recipeMerlin.getSubmissionCommand();
final String recipe2Name = recipeMerlin.getName();
runSql(connection, "create table " + tblName + "(comment string)");
bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName);
bootstrapCopy(connection, clusterFS, tblName, connection3, clusterFS3, tblName);
runSql(
connection,
"insert into table "
+ tblName
+ " values"
+ "('this string has been added post bootstrap - should appear after dr')");
Assert.assertEquals(Bundle.runFalconCLI(command1), 0, "Recipe submission failed.");
Assert.assertEquals(Bundle.runFalconCLI(command2), 0, "Recipe submission failed.");
InstanceUtil.waitTillInstanceReachState(
recipeExecLocation.getRecipeOC(clusterOC, clusterOC2),
recipe1Name,
1,
CoordinatorAction.Status.SUCCEEDED,
EntityType.PROCESS);
InstanceUtil.waitTillInstanceReachState(
recipeExecLocation.getRecipeOC(clusterOC, clusterOC3),
recipe2Name,
1,
CoordinatorAction.Status.SUCCEEDED,
EntityType.PROCESS);
final NotifyingAssert anAssert = new NotifyingAssert(true);
HiveAssert.assertTableEqual(
cluster,
clusterHC.getTable(DB_NAME, tblName),
cluster2,
clusterHC2.getTable(DB_NAME, tblName),
anAssert);
HiveAssert.assertTableEqual(
cluster,
clusterHC.getTable(DB_NAME, tblName),
cluster3,
clusterHC3.getTable(DB_NAME, tblName),
anAssert);
anAssert.assertAll();
}
@Test(dataProvider = "getRecipeLocation")
public void drPartition(final RecipeExecLocation recipeExecLocation) throws Exception {
setUp(recipeExecLocation);
final String tblName = "partitionDR";
recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName);
final List<String> command = recipeMerlin.getSubmissionCommand();
runSql(
connection, "create table " + tblName + "(comment string) partitioned by (pname string)");
runSql(
connection,
"insert into table "
+ tblName
+ " partition (pname = 'DELETE') values"
+ "('this partition is going to be deleted - should NOT appear after dr')");
runSql(
connection,
"insert into table "
+ tblName
+ " partition (pname = 'REPLACE') values"
+ "('this partition is going to be replaced - should NOT appear after dr')");
runSql(
connection,
"insert into table "
+ tblName
+ " partition (pname = 'ADD_DATA') values"
+ "('this partition will have more data - should appear after dr')");
bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName);
runSql(
connection,
"insert into table "
+ tblName
+ " partition (pname = 'NEW_PART') values"
+ "('this partition has been added post bootstrap - should appear after dr')");
runSql(
connection,
"insert into table "
+ tblName
+ " partition (pname = 'ADD_DATA') values"
+ "('more data has been added post bootstrap - should appear after dr')");
runSql(connection, "alter table " + tblName + " drop partition(pname = 'DELETE')");
runSql(connection, "alter table " + tblName + " drop partition(pname = 'REPLACE')");
runSql(
connection,
"insert into table "
+ tblName
+ " partition (pname = 'REPLACE') values"
+ "('this partition has been replaced - should appear after dr')");
Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed.");
InstanceUtil.waitTillInstanceReachState(
recipeExecLocation.getRecipeOC(clusterOC, clusterOC2),
recipeMerlin.getName(),
1,
CoordinatorAction.Status.SUCCEEDED,
EntityType.PROCESS);
HiveAssert.assertTableEqual(
cluster,
clusterHC.getTable(DB_NAME, tblName),
cluster2,
clusterHC2.getTable(DB_NAME, tblName),
new NotifyingAssert(true))
.assertAll();
}