@Test public void testFindPotentialCompactions() throws Exception { // Test that committing unlocks long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb"); comp.setTablename("mytable"); comp.setOperationType(DataOperationType.UPDATE); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb"); comp.setTablename("yourtable"); comp.setPartitionname("mypartition"); comp.setOperationType(DataOperationType.UPDATE); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); assertTrue(res.getState() == LockState.ACQUIRED); txnHandler.commitTxn(new CommitTxnRequest(txnid)); assertEquals(0, txnHandler.numLocksInLockTable()); Set<CompactionInfo> potentials = txnHandler.findPotentialCompactions(100); assertEquals(2, potentials.size()); boolean sawMyTable = false, sawYourTable = false; for (CompactionInfo ci : potentials) { sawMyTable |= (ci.dbname.equals("mydb") && ci.tableName.equals("mytable") && ci.partName == null); sawYourTable |= (ci.dbname.equals("mydb") && ci.tableName.equals("yourtable") && ci.partName.equals("mypartition")); } assertTrue(sawMyTable); assertTrue(sawYourTable); }
@Test public void dropPartition() throws Exception { Table t = newTable("default", "dp", true); Partition p = newPartition(t, "today"); addBaseFile(t, p, 20L, 20); addDeltaFile(t, p, 21L, 22L, 2); addDeltaFile(t, p, 23L, 24L, 2); burnThroughTransactions(23); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.PARTITION, "default"); comp.setTablename("dp"); comp.setPartitionname("ds=today"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); ms.dropPartition("default", "dp", Collections.singletonList("today"), true); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(0, compacts.size()); }
@Test public void noCompactOnManyDifferentPartitionAborts() throws Exception { Table t = newTable("default", "ncomdpa", true); for (int i = 0; i < 11; i++) { Partition p = newPartition(t, "day-" + i); } HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("ncomdpa"); comp.setPartitionname("ds=day-" + i); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.abortTxn(new AbortTxnRequest(txnid)); } startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); }
@Test public void majorCompactOnPartitionTooManyAborts() throws Exception { Table t = newTable("default", "mcoptma", true); Partition p = newPartition(t, "today"); HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("mcoptma"); comp.setPartitionname("ds=today"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.abortTxn(new AbortTxnRequest(txnid)); } startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(1, compacts.size()); Assert.assertEquals("initiated", compacts.get(0).getState()); Assert.assertEquals("mcoptma", compacts.get(0).getTablename()); Assert.assertEquals("ds=today", compacts.get(0).getPartitionname()); Assert.assertEquals(CompactionType.MAJOR, compacts.get(0).getType()); }
@Test public void twoTxnsOnSamePartitionGenerateOneCompactionRequest() throws Exception { Table t = newTable("default", "ttospgocr", true); Partition p = newPartition(t, "today"); addBaseFile(t, p, 20L, 20); addDeltaFile(t, p, 21L, 22L, 2); addDeltaFile(t, p, 23L, 24L, 2); burnThroughTransactions(23); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.PARTITION, "default"); comp.setTablename("ttospgocr"); comp.setPartitionname("ds=today"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); txnid = openTxn(); comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.PARTITION, "default"); comp.setTablename("ttospgocr"); comp.setPartitionname("ds=today"); components = new ArrayList<LockComponent>(1); components.add(comp); req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(1, compacts.size()); Assert.assertEquals("initiated", compacts.get(0).getState()); Assert.assertEquals("ttospgocr", compacts.get(0).getTablename()); Assert.assertEquals("ds=today", compacts.get(0).getPartitionname()); Assert.assertEquals(CompactionType.MAJOR, compacts.get(0).getType()); }
@Test public void testMarkCleanedCleansTxnsAndTxnComponents() throws Exception { long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb"); comp.setTablename("mytable"); comp.setOperationType(DataOperationType.INSERT); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); assertTrue(res.getState() == LockState.ACQUIRED); txnHandler.abortTxn(new AbortTxnRequest(txnid)); txnid = openTxn(); comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb"); comp.setTablename("yourtable"); comp.setOperationType(DataOperationType.DELETE); components = new ArrayList<LockComponent>(1); components.add(comp); req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); res = txnHandler.lock(req); assertTrue(res.getState() == LockState.ACQUIRED); txnHandler.abortTxn(new AbortTxnRequest(txnid)); txnid = openTxn(); comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb"); comp.setTablename("foo"); comp.setPartitionname("bar"); comp.setOperationType(DataOperationType.UPDATE); components = new ArrayList<LockComponent>(1); components.add(comp); req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); res = txnHandler.lock(req); assertTrue(res.getState() == LockState.ACQUIRED); comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb"); comp.setTablename("foo"); comp.setPartitionname("baz"); comp.setOperationType(DataOperationType.UPDATE); components = new ArrayList<LockComponent>(1); components.add(comp); req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); res = txnHandler.lock(req); assertTrue(res.getState() == LockState.ACQUIRED); txnHandler.abortTxn(new AbortTxnRequest(txnid)); CompactionInfo ci = new CompactionInfo(); // Now clean them and check that they are removed from the count. CompactionRequest rqst = new CompactionRequest("mydb", "mytable", CompactionType.MAJOR); txnHandler.compact(rqst); assertEquals(0, txnHandler.findReadyToClean().size()); ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); txnHandler.markCompacted(ci); List<CompactionInfo> toClean = txnHandler.findReadyToClean(); assertEquals(1, toClean.size()); txnHandler.markCleaned(ci); // Check that we are cleaning up the empty aborted transactions GetOpenTxnsResponse txnList = txnHandler.getOpenTxns(); assertEquals(3, txnList.getOpen_txnsSize()); txnHandler.cleanEmptyAbortedTxns(); txnList = txnHandler.getOpenTxns(); assertEquals(2, txnList.getOpen_txnsSize()); rqst = new CompactionRequest("mydb", "foo", CompactionType.MAJOR); rqst.setPartitionname("bar"); txnHandler.compact(rqst); assertEquals(0, txnHandler.findReadyToClean().size()); ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); txnHandler.markCompacted(ci); toClean = txnHandler.findReadyToClean(); assertEquals(1, toClean.size()); txnHandler.markCleaned(ci); txnHandler.openTxns(new OpenTxnRequest(1, "me", "localhost")); txnHandler.cleanEmptyAbortedTxns(); txnList = txnHandler.getOpenTxns(); assertEquals(3, txnList.getOpen_txnsSize()); }