@Test public void testFindPotentialCompactions() throws Exception { // Test that committing unlocks long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb"); comp.setTablename("mytable"); comp.setOperationType(DataOperationType.UPDATE); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb"); comp.setTablename("yourtable"); comp.setPartitionname("mypartition"); comp.setOperationType(DataOperationType.UPDATE); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); assertTrue(res.getState() == LockState.ACQUIRED); txnHandler.commitTxn(new CommitTxnRequest(txnid)); assertEquals(0, txnHandler.numLocksInLockTable()); Set<CompactionInfo> potentials = txnHandler.findPotentialCompactions(100); assertEquals(2, potentials.size()); boolean sawMyTable = false, sawYourTable = false; for (CompactionInfo ci : potentials) { sawMyTable |= (ci.dbname.equals("mydb") && ci.tableName.equals("mytable") && ci.partName == null); sawYourTable |= (ci.dbname.equals("mydb") && ci.tableName.equals("yourtable") && ci.partName.equals("mypartition")); } assertTrue(sawMyTable); assertTrue(sawYourTable); }
@Test public void dropPartition() throws Exception { Table t = newTable("default", "dp", true); Partition p = newPartition(t, "today"); addBaseFile(t, p, 20L, 20); addDeltaFile(t, p, 21L, 22L, 2); addDeltaFile(t, p, 23L, 24L, 2); burnThroughTransactions(23); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.PARTITION, "default"); comp.setTablename("dp"); comp.setPartitionname("ds=today"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); ms.dropPartition("default", "dp", Collections.singletonList("today"), true); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(0, compacts.size()); }
@Test public void majorCompactOnTableTooManyAborts() throws Exception { Table t = newTable("default", "mcottma", false); HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("mcottma"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.abortTxn(new AbortTxnRequest(txnid)); } startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(1, compacts.size()); Assert.assertEquals("initiated", compacts.get(0).getState()); Assert.assertEquals("mcottma", compacts.get(0).getTablename()); Assert.assertEquals(CompactionType.MAJOR, compacts.get(0).getType()); }
@Test public void noCompactTableNotEnoughDeltas() throws Exception { Table t = newTable("default", "nctned", false); addBaseFile(t, null, 200L, 200); addDeltaFile(t, null, 201L, 205L, 5); addDeltaFile(t, null, 206L, 211L, 6); burnThroughTransactions(210); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("nctned"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); }
@Test public void compactTableHighDeltaPct() throws Exception { Table t = newTable("default", "cthdp", false); addBaseFile(t, null, 20L, 20); addDeltaFile(t, null, 21L, 22L, 2); addDeltaFile(t, null, 23L, 24L, 2); burnThroughTransactions(23); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("cthdp"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(1, compacts.size()); Assert.assertEquals("initiated", compacts.get(0).getState()); Assert.assertEquals("cthdp", compacts.get(0).getTablename()); Assert.assertEquals(CompactionType.MAJOR, compacts.get(0).getType()); }
@Test public void noCompactWhenNoCompactSetLowerCase() throws Exception { Map<String, String> parameters = new HashMap<String, String>(1); parameters.put("no_auto_compaction", "true"); Table t = newTable("default", "ncwncs", false, parameters); HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("ncwncs"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.abortTxn(new AbortTxnRequest(txnid)); } startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); }
@Test public void cleanEmptyAbortedTxns() throws Exception { // Test that we are cleaning aborted transactions with no components left in txn_components. // Put one aborted transaction with an entry in txn_components to make sure we don't // accidently clean it too. Table t = newTable("default", "ceat", false); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("ceat"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.abortTxn(new AbortTxnRequest(txnid)); for (int i = 0; i < 100; i++) { txnid = openTxn(); txnHandler.abortTxn(new AbortTxnRequest(txnid)); } GetOpenTxnsResponse openTxns = txnHandler.getOpenTxns(); Assert.assertEquals(101, openTxns.getOpen_txnsSize()); startInitiator(); openTxns = txnHandler.getOpenTxns(); Assert.assertEquals(1, openTxns.getOpen_txnsSize()); }
@Test public void noCompactOnManyDifferentPartitionAborts() throws Exception { Table t = newTable("default", "ncomdpa", true); for (int i = 0; i < 11; i++) { Partition p = newPartition(t, "day-" + i); } HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("ncomdpa"); comp.setPartitionname("ds=day-" + i); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.abortTxn(new AbortTxnRequest(txnid)); } startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); Assert.assertEquals(0, rsp.getCompactsSize()); }
@Test public void twoTxnsOnSamePartitionGenerateOneCompactionRequest() throws Exception { Table t = newTable("default", "ttospgocr", true); Partition p = newPartition(t, "today"); addBaseFile(t, p, 20L, 20); addDeltaFile(t, p, 21L, 22L, 2); addDeltaFile(t, p, 23L, 24L, 2); burnThroughTransactions(23); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.PARTITION, "default"); comp.setTablename("ttospgocr"); comp.setPartitionname("ds=today"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); txnid = openTxn(); comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.PARTITION, "default"); comp.setTablename("ttospgocr"); comp.setPartitionname("ds=today"); components = new ArrayList<LockComponent>(1); components.add(comp); req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(1, compacts.size()); Assert.assertEquals("initiated", compacts.get(0).getState()); Assert.assertEquals("ttospgocr", compacts.get(0).getTablename()); Assert.assertEquals("ds=today", compacts.get(0).getPartitionname()); Assert.assertEquals(CompactionType.MAJOR, compacts.get(0).getType()); }
@Test public void addDynamicPartitions() throws Exception { String dbName = "default"; String tableName = "adp_table"; OpenTxnsResponse openTxns = txnHandler.openTxns(new OpenTxnRequest(1, "me", "localhost")); long txnId = openTxns.getTxn_ids().get(0); // lock a table, as in dynamic partitions LockComponent lc = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, dbName); lc.setTablename(tableName); DataOperationType dop = DataOperationType.UPDATE; lc.setOperationType(dop); LockRequest lr = new LockRequest(Arrays.asList(lc), "me", "localhost"); lr.setTxnid(txnId); LockResponse lock = txnHandler.lock(lr); assertEquals(LockState.ACQUIRED, lock.getState()); AddDynamicPartitions adp = new AddDynamicPartitions( txnId, dbName, tableName, Arrays.asList("ds=yesterday", "ds=today")); adp.setOperationType(dop); txnHandler.addDynamicPartitions(adp); txnHandler.commitTxn(new CommitTxnRequest(txnId)); Set<CompactionInfo> potentials = txnHandler.findPotentialCompactions(1000); assertEquals(2, potentials.size()); SortedSet<CompactionInfo> sorted = new TreeSet<CompactionInfo>(potentials); int i = 0; for (CompactionInfo ci : sorted) { assertEquals(dbName, ci.dbname); assertEquals(tableName, ci.tableName); switch (i++) { case 0: assertEquals("ds=today", ci.partName); break; case 1: assertEquals("ds=yesterday", ci.partName); break; default: throw new RuntimeException("What?"); } } }
@Test public void chooseMajorOverMinorWhenBothValid() throws Exception { Table t = newTable("default", "cmomwbv", false); addBaseFile(t, null, 200L, 200); addDeltaFile(t, null, 201L, 211L, 11); addDeltaFile(t, null, 212L, 222L, 11); addDeltaFile(t, null, 223L, 233L, 11); addDeltaFile(t, null, 234L, 244L, 11); addDeltaFile(t, null, 245L, 255L, 11); addDeltaFile(t, null, 256L, 266L, 11); addDeltaFile(t, null, 267L, 277L, 11); addDeltaFile(t, null, 278L, 288L, 11); addDeltaFile(t, null, 289L, 299L, 11); addDeltaFile(t, null, 300L, 310L, 11); addDeltaFile(t, null, 311L, 321L, 11); burnThroughTransactions(320); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("cmomwbv"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(1, compacts.size()); Assert.assertEquals("initiated", compacts.get(0).getState()); Assert.assertEquals("cmomwbv", compacts.get(0).getTablename()); Assert.assertEquals(CompactionType.MAJOR, compacts.get(0).getType()); }
@Test public void testMarkCleanedCleansTxnsAndTxnComponents() throws Exception { long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb"); comp.setTablename("mytable"); comp.setOperationType(DataOperationType.INSERT); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); assertTrue(res.getState() == LockState.ACQUIRED); txnHandler.abortTxn(new AbortTxnRequest(txnid)); txnid = openTxn(); comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb"); comp.setTablename("yourtable"); comp.setOperationType(DataOperationType.DELETE); components = new ArrayList<LockComponent>(1); components.add(comp); req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); res = txnHandler.lock(req); assertTrue(res.getState() == LockState.ACQUIRED); txnHandler.abortTxn(new AbortTxnRequest(txnid)); txnid = openTxn(); comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb"); comp.setTablename("foo"); comp.setPartitionname("bar"); comp.setOperationType(DataOperationType.UPDATE); components = new ArrayList<LockComponent>(1); components.add(comp); req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); res = txnHandler.lock(req); assertTrue(res.getState() == LockState.ACQUIRED); comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.DB, "mydb"); comp.setTablename("foo"); comp.setPartitionname("baz"); comp.setOperationType(DataOperationType.UPDATE); components = new ArrayList<LockComponent>(1); components.add(comp); req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); res = txnHandler.lock(req); assertTrue(res.getState() == LockState.ACQUIRED); txnHandler.abortTxn(new AbortTxnRequest(txnid)); CompactionInfo ci = new CompactionInfo(); // Now clean them and check that they are removed from the count. CompactionRequest rqst = new CompactionRequest("mydb", "mytable", CompactionType.MAJOR); txnHandler.compact(rqst); assertEquals(0, txnHandler.findReadyToClean().size()); ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); txnHandler.markCompacted(ci); List<CompactionInfo> toClean = txnHandler.findReadyToClean(); assertEquals(1, toClean.size()); txnHandler.markCleaned(ci); // Check that we are cleaning up the empty aborted transactions GetOpenTxnsResponse txnList = txnHandler.getOpenTxns(); assertEquals(3, txnList.getOpen_txnsSize()); txnHandler.cleanEmptyAbortedTxns(); txnList = txnHandler.getOpenTxns(); assertEquals(2, txnList.getOpen_txnsSize()); rqst = new CompactionRequest("mydb", "foo", CompactionType.MAJOR); rqst.setPartitionname("bar"); txnHandler.compact(rqst); assertEquals(0, txnHandler.findReadyToClean().size()); ci = txnHandler.findNextToCompact("fred"); assertNotNull(ci); txnHandler.markCompacted(ci); toClean = txnHandler.findReadyToClean(); assertEquals(1, toClean.size()); txnHandler.markCleaned(ci); txnHandler.openTxns(new OpenTxnRequest(1, "me", "localhost")); txnHandler.cleanEmptyAbortedTxns(); txnList = txnHandler.getOpenTxns(); assertEquals(3, txnList.getOpen_txnsSize()); }