@Test public void dataGeneration() throws Exception { setUp(RecipeExecLocation.SourceCluster); runSql(connection, "use hdr_sdb1"); createVanillaTable(connection, "store_sales"); createSerDeTable(connection); createPartitionedTable(connection); createExternalTable(connection, clusterFS, baseTestHDFSDir + "click_data/", "click_data"); createExternalPartitionedTable( connection, clusterFS, baseTestHDFSDir + "click_data2/", "click_data2"); runSql(connection2, "use hdr_sdb1"); createVanillaTable(connection2, "store_sales"); createSerDeTable(connection2); createPartitionedTable(connection2); createExternalTable(connection2, clusterFS2, baseTestHDFSDir + "click_data/", "click_data"); createExternalPartitionedTable( connection2, clusterFS2, baseTestHDFSDir + "click_data2/", "click_data2"); final NotifyingAssert anAssert = new NotifyingAssert(true); HiveAssert.assertDbEqual( cluster, clusterHC.getDatabase("hdr_sdb1"), cluster2, clusterHC2.getDatabase("hdr_sdb1"), anAssert); HiveAssert.assertTableEqual( cluster, clusterHC.getTable("hdr_sdb1", "click_data"), cluster2, clusterHC2.getTable("hdr_sdb1", "click_data"), anAssert); anAssert.assertAll(); }
@Test public void drChangeColumn() throws Exception { final RecipeExecLocation recipeExecLocation = RecipeExecLocation.SourceCluster; setUp(recipeExecLocation); final String tblName = "tableForColumnChange"; recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName); final List<String> command1 = recipeMerlin.getSubmissionCommand(); final String recipe1Name = recipeMerlin.getName(); runSql(connection, "create table " + tblName + "(id int)"); bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName); Assert.assertEquals(Bundle.runFalconCLI(command1), 0, "Recipe submission failed."); runSql(connection, "ALTER TABLE " + tblName + " CHANGE id id STRING COMMENT 'some_comment'"); InstanceUtil.waitTillInstanceReachState( recipeExecLocation.getRecipeOC(clusterOC, clusterOC2), recipe1Name, 1, CoordinatorAction.Status.SUCCEEDED, EntityType.PROCESS); HiveAssert.assertTableEqual( cluster, clusterHC.getTable(DB_NAME, tblName), cluster2, clusterHC2.getTable(DB_NAME, tblName), new NotifyingAssert(true)) .assertAll(); }
/** * 1 src tbl 1 dst tbl. Change table properties and comment at the source. Changes should get * reflected at destination. */ @Test public void drChangeCommentAndPropertyTest() throws Exception { final RecipeExecLocation recipeExecLocation = RecipeExecLocation.SourceCluster; setUp(recipeExecLocation); final String tblName = "myTable"; recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName); final List<String> command = recipeMerlin.getSubmissionCommand(); runSql(connection, "create table " + tblName + "(field string)"); // add new table property runSql( connection, "ALTER TABLE " + tblName + " SET TBLPROPERTIES('someProperty' = 'initialValue')"); // set comment runSql( connection, "ALTER TABLE " + tblName + " SET TBLPROPERTIES('comment' = 'this comment will be " + "changed, SHOULD NOT appear')"); LOGGER.info(tblName + " before bootstrap copy: "); runSql(connection, "describe extended " + tblName); bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName); // change table property and comment runSql( connection, "ALTER TABLE " + tblName + " SET TBLPROPERTIES('someProperty' = 'anotherValue')"); runSql( connection, "ALTER TABLE " + tblName + " SET TBLPROPERTIES('comment' = 'this comment should " + "appear after replication done')"); LOGGER.info(tblName + " after modifications, before replication: "); runSql(connection, "describe extended " + tblName); Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed."); InstanceUtil.waitTillInstanceReachState( recipeExecLocation.getRecipeOC(clusterOC, clusterOC2), recipeMerlin.getName(), 1, CoordinatorAction.Status.SUCCEEDED, EntityType.PROCESS); HiveAssert.assertTableEqual( cluster, clusterHC.getTable(DB_NAME, tblName), cluster2, clusterHC2.getTable(DB_NAME, tblName), new NotifyingAssert(true)) .assertAll(); }
@Test public void drTwoTablesOneRequest() throws Exception { final RecipeExecLocation recipeExecLocation = RecipeExecLocation.TargetCluster; setUp(recipeExecLocation); final String tblName = "firstTableDR"; final String tbl2Name = "secondTableDR"; recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName + ',' + tbl2Name); final List<String> command = recipeMerlin.getSubmissionCommand(); runSql(connection, "create table " + tblName + "(comment string)"); runSql(connection, "create table " + tbl2Name + "(comment string)"); bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName); bootstrapCopy(connection, clusterFS, tbl2Name, connection2, clusterFS2, tbl2Name); runSql( connection, "insert into table " + tblName + " values" + "('this string has been added post bootstrap - should appear after dr')"); runSql( connection, "insert into table " + tbl2Name + " values" + "('this string has been added post bootstrap - should appear after dr')"); Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed."); InstanceUtil.waitTillInstanceReachState( recipeExecLocation.getRecipeOC(clusterOC, clusterOC2), recipeMerlin.getName(), 1, CoordinatorAction.Status.SUCCEEDED, EntityType.PROCESS); final NotifyingAssert anAssert = new NotifyingAssert(true); HiveAssert.assertTableEqual( cluster, clusterHC.getTable(DB_NAME, tblName), cluster2, clusterHC2.getTable(DB_NAME, tblName), anAssert); HiveAssert.assertTableEqual( cluster, clusterHC.getTable(DB_NAME, tbl2Name), cluster2, clusterHC2.getTable(DB_NAME, tbl2Name), anAssert); anAssert.assertAll(); }
/** * Test creates a table on first cluster using static partitioning. Then it creates the same table * on the second cluster using dynamic partitioning. Finally it checks the equality of these * tables. * * @throws SQLException * @throws IOException */ @Test public void dynamicPartitionsTest() throws Exception { setUp(RecipeExecLocation.SourceCluster); // create table with static partitions on first cluster createPartitionedTable(connection, false); // create table with dynamic partitions on second cluster createPartitionedTable(connection2, true); // check that both tables are equal HiveAssert.assertTableEqual( cluster, clusterHC.getTable("hdr_sdb1", "global_store_sales"), cluster2, clusterHC2.getTable("hdr_sdb1", "global_store_sales"), new SoftAssert()) .assertAll(); }
@Test(enabled = false) public void assertionTest() throws Exception { setUp(RecipeExecLocation.SourceCluster); final SoftAssert anAssert = new SoftAssert(); HiveAssert.assertTableEqual( cluster, clusterHC.getTable("default", "hcatsmoke10546"), cluster2, clusterHC2.getTable("default", "hcatsmoke10548"), anAssert); HiveAssert.assertDbEqual( cluster, clusterHC.getDatabase("default"), cluster2, clusterHC2.getDatabase("default"), anAssert); anAssert.assertAll(); }
@Test public void drSerDeWithProperties() throws Exception { final RecipeExecLocation recipeExecLocation = RecipeExecLocation.SourceCluster; setUp(recipeExecLocation); final String tblName = "serdeTable"; recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName); final List<String> command = recipeMerlin.getSubmissionCommand(); runSql( connection, "create table " + tblName + "(comment string) " + "row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'"); bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName); runSql( connection, "insert into table " + tblName + " values" + "('this string has been added post bootstrap - should appear after dr')"); runSql( connection, "ALTER TABLE " + tblName + " SET SERDEPROPERTIES ('someProperty' = 'value')"); Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed."); InstanceUtil.waitTillInstanceReachState( recipeExecLocation.getRecipeOC(clusterOC, clusterOC2), recipeMerlin.getName(), 1, CoordinatorAction.Status.SUCCEEDED, EntityType.PROCESS); HiveAssert.assertTableEqual( cluster, clusterHC.getTable(DB_NAME, tblName), cluster2, clusterHC2.getTable(DB_NAME, tblName), new NotifyingAssert(true)) .assertAll(); }
private static boolean tableExists( HCatClient client, final String database, final String tableName) throws Exception { try { HCatTable table = client.getTable(database, tableName); return table != null; } catch (ObjectNotFoundException e) { System.out.println(e.getMessage()); return false; } catch (HCatException e) { throw new Exception("Exception checking if the table exists:" + e.getMessage(), e); } }
@Test public void drExternalToNonExternal() throws Exception { final RecipeExecLocation recipeExecLocation = RecipeExecLocation.SourceCluster; setUp(recipeExecLocation); final String tblName = "externalToNonExternal"; recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName); final List<String> command = recipeMerlin.getSubmissionCommand(); createExternalTable(connection, clusterFS, baseTestHDFSDir + "click_data/", tblName); bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName); // change column name runSql(connection, "alter table " + tblName + " change column data data_new string"); Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed."); InstanceUtil.waitTillInstanceReachState( recipeExecLocation.getRecipeOC(clusterOC, clusterOC2), recipeMerlin.getName(), 1, CoordinatorAction.Status.SUCCEEDED, EntityType.PROCESS); final NotifyingAssert anAssert = new NotifyingAssert(true); HiveAssert.assertTableEqual( cluster, clusterHC.getTable(DB_NAME, tblName), cluster2, clusterHC2.getTable(DB_NAME, tblName), anAssert, false); anAssert.assertNotEquals( clusterHC2.getTable(DB_NAME, tblName).getTabletype(), clusterHC.getTable(DB_NAME, tblName).getTableName(), "Source and destination tables should have different Tabletype"); anAssert.assertNotEquals( clusterHC2.getTable(DB_NAME, tblName).getTblProps().get("EXTERNAL"), clusterHC.getTable(DB_NAME, tblName).getTblProps().get("EXTERNAL"), "Source and destination tables should have different value of property EXTERNAL"); anAssert.assertAll(); }
/** * 1 src tbl 1 dst tbl replication. Insert/overwrite partitions using dynamic partitions queries. * The changes should get reflected at destination. * * @throws Exception */ @Test public void drInsertOverwriteDynamicPartition() throws Exception { final RecipeExecLocation recipeExecLocation = RecipeExecLocation.SourceCluster; setUp(recipeExecLocation); final String tblName = "drInsertOverwritePartition"; final String hlpTblName = "drInsertOverwritePartitionHelperTbl"; recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName); final List<String> command = recipeMerlin.getSubmissionCommand(); // disable strict mode to use only dynamic partition runSql(connection, "set hive.exec.dynamic.partition.mode=nonstrict"); runSql( connection, "create table " + hlpTblName + "(comment string) partitioned by (pname string)"); runSql( connection, "insert into table " + hlpTblName + " partition (pname)" + " values('overwrite data - should appear after dr', 'OVERWRITE_PART')"); runSql( connection, "insert into table " + hlpTblName + " partition (pname)" + " values('newdata row2 - should appear after dr', 'NEW_DATA')"); runSql( connection, "insert into table " + hlpTblName + " partition (pname)" + " values('newdata row1 - should appear after dr', 'NEW_DATA')"); runSql( connection, "create table " + tblName + "(comment string) partitioned by (pname string)"); runSql( connection, "insert into table " + tblName + " partition (pname) values" + "('this data should be retained - should appear after dr', 'OLD_PART')"); runSql( connection, "insert into table " + tblName + " partition (pname) values" + "('this data should get overwritten - should NOT appear after dr', 'OVERWRITE_PART')"); LOGGER.info(tblName + " before bootstrap copying: "); runSql(connection, "select * from " + tblName); bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName); runSql( connection, "insert overwrite table " + tblName + " partition (pname) " + "select comment, pname from " + hlpTblName + " where comment REGEXP '^overwrite'"); runSql( connection, "insert overwrite table " + tblName + " partition (pname) " + "select comment, pname from " + hlpTblName + " where comment REGEXP '^newdata'"); LOGGER.info(tblName + " after modifications, before replication: "); runSql(connection, "select * from " + tblName); Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed."); InstanceUtil.waitTillInstanceReachState( recipeExecLocation.getRecipeOC(clusterOC, clusterOC2), recipeMerlin.getName(), 1, CoordinatorAction.Status.SUCCEEDED, EntityType.PROCESS); HiveAssert.assertTableEqual( cluster, clusterHC.getTable(DB_NAME, tblName), cluster2, clusterHC2.getTable(DB_NAME, tblName), new NotifyingAssert(true)) .assertAll(); }
/** * 1 src tbl 1 dst tbl replication. Insert/delete/replace partitions using dynamic partition * queries. The changes should get reflected at destination. */ @Test public void drInsertDropReplaceDynamicPartition() throws Exception { final RecipeExecLocation recipeExecLocation = RecipeExecLocation.SourceCluster; setUp(recipeExecLocation); final String tblName = "dynamicPartitionDR"; recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName); final List<String> command = recipeMerlin.getSubmissionCommand(); // disable strict mode to use only dynamic partition runSql(connection, "set hive.exec.dynamic.partition.mode=nonstrict"); runSql( connection, "create table " + tblName + "(comment string) partitioned by (pname string)"); runSql( connection, "insert into table " + tblName + " partition (pname) values" + "('this partition is going to be deleted - should NOT appear after dr', 'DELETE')"); runSql( connection, "insert into table " + tblName + " partition (pname) values" + "('this partition is going to be replaced - should NOT appear after dr', 'REPLACE')"); runSql( connection, "insert into table " + tblName + " partition (pname) values" + "('this partition will have more data - should appear after dr', 'ADD_DATA')"); LOGGER.info(tblName + " before bootstrap copying: "); runSql(connection, "select * from " + tblName); bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName); runSql( connection, "insert into table " + tblName + " partition (pname) values" + "('this partition has been added post bootstrap - should appear after dr', 'NEW_PART')"); runSql( connection, "insert into table " + tblName + " partition (pname) values" + "('more data has been added post bootstrap - should appear after dr', 'ADD_DATA')"); runSql(connection, "alter table " + tblName + " drop partition(pname = 'DELETE')"); runSql(connection, "alter table " + tblName + " drop partition(pname = 'REPLACE')"); runSql( connection, "insert into table " + tblName + " partition (pname) values" + "('this partition has been replaced - should appear after dr', 'REPLACE')"); LOGGER.info(tblName + " after modifications, before replication: "); runSql(connection, "select * from " + tblName); Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed."); InstanceUtil.waitTillInstanceReachState( recipeExecLocation.getRecipeOC(clusterOC, clusterOC2), recipeMerlin.getName(), 1, CoordinatorAction.Status.SUCCEEDED, EntityType.PROCESS); HiveAssert.assertTableEqual( cluster, clusterHC.getTable(DB_NAME, tblName), cluster2, clusterHC2.getTable(DB_NAME, tblName), new NotifyingAssert(true)) .assertAll(); }
@Test public void drTwoDstTablesTwoRequests() throws Exception { final RecipeExecLocation recipeExecLocation = RecipeExecLocation.TargetCluster; setUp(recipeExecLocation); final HCatClient clusterHC3 = cluster3.getClusterHelper().getHCatClient(); final Connection connection3 = cluster3.getClusterHelper().getHiveJdbcConnection(); runSql(connection3, "drop database if exists hdr_sdb1 cascade"); runSql(connection3, "create database hdr_sdb1"); runSql(connection3, "use hdr_sdb1"); final String tblName = "vanillaTable"; recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName); final String recipe1Name = recipeMerlin.getName(); final List<String> command1 = recipeMerlin.getSubmissionCommand(); final Bundle bundle3 = new Bundle(BundleUtil.readHCatBundle(), cluster3); bundle3.generateUniqueBundle(this); bundle3.submitClusters(prism); recipeMerlin .withTargetCluster(bundle3.getClusterElement()) .withRecipeCluster( recipeExecLocation.getRecipeCluster( bundles[0].getClusterElement(), bundle3.getClusterElement())); recipeMerlin.setUniqueName(this.getClass().getSimpleName()); final List<String> command2 = recipeMerlin.getSubmissionCommand(); final String recipe2Name = recipeMerlin.getName(); runSql(connection, "create table " + tblName + "(comment string)"); bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName); bootstrapCopy(connection, clusterFS, tblName, connection3, clusterFS3, tblName); runSql( connection, "insert into table " + tblName + " values" + "('this string has been added post bootstrap - should appear after dr')"); Assert.assertEquals(Bundle.runFalconCLI(command1), 0, "Recipe submission failed."); Assert.assertEquals(Bundle.runFalconCLI(command2), 0, "Recipe submission failed."); InstanceUtil.waitTillInstanceReachState( recipeExecLocation.getRecipeOC(clusterOC, clusterOC2), recipe1Name, 1, CoordinatorAction.Status.SUCCEEDED, EntityType.PROCESS); InstanceUtil.waitTillInstanceReachState( recipeExecLocation.getRecipeOC(clusterOC, clusterOC3), recipe2Name, 1, CoordinatorAction.Status.SUCCEEDED, EntityType.PROCESS); final NotifyingAssert anAssert = new NotifyingAssert(true); HiveAssert.assertTableEqual( cluster, clusterHC.getTable(DB_NAME, tblName), cluster2, clusterHC2.getTable(DB_NAME, tblName), anAssert); HiveAssert.assertTableEqual( cluster, clusterHC.getTable(DB_NAME, tblName), cluster3, clusterHC3.getTable(DB_NAME, tblName), anAssert); anAssert.assertAll(); }
@Test(dataProvider = "getRecipeLocation") public void drPartition(final RecipeExecLocation recipeExecLocation) throws Exception { setUp(recipeExecLocation); final String tblName = "partitionDR"; recipeMerlin.withSourceDb(DB_NAME).withSourceTable(tblName); final List<String> command = recipeMerlin.getSubmissionCommand(); runSql( connection, "create table " + tblName + "(comment string) partitioned by (pname string)"); runSql( connection, "insert into table " + tblName + " partition (pname = 'DELETE') values" + "('this partition is going to be deleted - should NOT appear after dr')"); runSql( connection, "insert into table " + tblName + " partition (pname = 'REPLACE') values" + "('this partition is going to be replaced - should NOT appear after dr')"); runSql( connection, "insert into table " + tblName + " partition (pname = 'ADD_DATA') values" + "('this partition will have more data - should appear after dr')"); bootstrapCopy(connection, clusterFS, tblName, connection2, clusterFS2, tblName); runSql( connection, "insert into table " + tblName + " partition (pname = 'NEW_PART') values" + "('this partition has been added post bootstrap - should appear after dr')"); runSql( connection, "insert into table " + tblName + " partition (pname = 'ADD_DATA') values" + "('more data has been added post bootstrap - should appear after dr')"); runSql(connection, "alter table " + tblName + " drop partition(pname = 'DELETE')"); runSql(connection, "alter table " + tblName + " drop partition(pname = 'REPLACE')"); runSql( connection, "insert into table " + tblName + " partition (pname = 'REPLACE') values" + "('this partition has been replaced - should appear after dr')"); Assert.assertEquals(Bundle.runFalconCLI(command), 0, "Recipe submission failed."); InstanceUtil.waitTillInstanceReachState( recipeExecLocation.getRecipeOC(clusterOC, clusterOC2), recipeMerlin.getName(), 1, CoordinatorAction.Status.SUCCEEDED, EntityType.PROCESS); HiveAssert.assertTableEqual( cluster, clusterHC.getTable(DB_NAME, tblName), cluster2, clusterHC2.getTable(DB_NAME, tblName), new NotifyingAssert(true)) .assertAll(); }