@Test public void dropPartition() throws Exception { Table t = newTable("default", "dp", true); Partition p = newPartition(t, "today"); addBaseFile(t, p, 20L, 20); addDeltaFile(t, p, 21L, 22L, 2); addDeltaFile(t, p, 23L, 24L, 2); burnThroughTransactions(23); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.PARTITION, "default"); comp.setTablename("dp"); comp.setPartitionname("ds=today"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); ms.dropPartition("default", "dp", Collections.singletonList("today"), true); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(0, compacts.size()); }
@Test public void majorCompactOnTableTooManyAborts() throws Exception { Table t = newTable("default", "mcottma", false); HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10); for (int i = 0; i < 11; i++) { long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("mcottma"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.abortTxn(new AbortTxnRequest(txnid)); } startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(1, compacts.size()); Assert.assertEquals("initiated", compacts.get(0).getState()); Assert.assertEquals("mcottma", compacts.get(0).getTablename()); Assert.assertEquals(CompactionType.MAJOR, compacts.get(0).getType()); }
@Test public void recoverFailedLocalWorkers() throws Exception { Table t = newTable("default", "rflw1", false); CompactionRequest rqst = new CompactionRequest("default", "rflw1", CompactionType.MINOR); txnHandler.compact(rqst); t = newTable("default", "rflw2", false); rqst = new CompactionRequest("default", "rflw2", CompactionType.MINOR); txnHandler.compact(rqst); txnHandler.findNextToCompact(Worker.hostname() + "-193892"); txnHandler.findNextToCompact("nosuchhost-193892"); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(2, compacts.size()); boolean sawInitiated = false; for (ShowCompactResponseElement c : compacts) { if (c.getState().equals("working")) { Assert.assertEquals("nosuchhost-193892", c.getWorkerid()); } else if (c.getState().equals("initiated")) { sawInitiated = true; } else { Assert.fail("Unexpected state"); } } Assert.assertTrue(sawInitiated); }
@Test public void compactTableHighDeltaPct() throws Exception { Table t = newTable("default", "cthdp", false); addBaseFile(t, null, 20L, 20); addDeltaFile(t, null, 21L, 22L, 2); addDeltaFile(t, null, 23L, 24L, 2); burnThroughTransactions(23); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("cthdp"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(1, compacts.size()); Assert.assertEquals("initiated", compacts.get(0).getState()); Assert.assertEquals("cthdp", compacts.get(0).getTablename()); Assert.assertEquals(CompactionType.MAJOR, compacts.get(0).getType()); }
@Test public void recoverFailedRemoteWorkers() throws Exception { Table t = newTable("default", "rfrw1", false); CompactionRequest rqst = new CompactionRequest("default", "rfrw1", CompactionType.MINOR); txnHandler.compact(rqst); txnHandler.findNextToCompact("nosuchhost-193892"); conf.setTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_WORKER_TIMEOUT, 1L, TimeUnit.MILLISECONDS); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(1, compacts.size()); Assert.assertEquals("initiated", compacts.get(0).getState()); }
@Test public void twoTxnsOnSamePartitionGenerateOneCompactionRequest() throws Exception { Table t = newTable("default", "ttospgocr", true); Partition p = newPartition(t, "today"); addBaseFile(t, p, 20L, 20); addDeltaFile(t, p, 21L, 22L, 2); addDeltaFile(t, p, 23L, 24L, 2); burnThroughTransactions(23); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.PARTITION, "default"); comp.setTablename("ttospgocr"); comp.setPartitionname("ds=today"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); txnid = openTxn(); comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.PARTITION, "default"); comp.setTablename("ttospgocr"); comp.setPartitionname("ds=today"); components = new ArrayList<LockComponent>(1); components.add(comp); req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(1, compacts.size()); Assert.assertEquals("initiated", compacts.get(0).getState()); Assert.assertEquals("ttospgocr", compacts.get(0).getTablename()); Assert.assertEquals("ds=today", compacts.get(0).getPartitionname()); Assert.assertEquals(CompactionType.MAJOR, compacts.get(0).getType()); }
@Test public void chooseMajorOverMinorWhenBothValid() throws Exception { Table t = newTable("default", "cmomwbv", false); addBaseFile(t, null, 200L, 200); addDeltaFile(t, null, 201L, 211L, 11); addDeltaFile(t, null, 212L, 222L, 11); addDeltaFile(t, null, 223L, 233L, 11); addDeltaFile(t, null, 234L, 244L, 11); addDeltaFile(t, null, 245L, 255L, 11); addDeltaFile(t, null, 256L, 266L, 11); addDeltaFile(t, null, 267L, 277L, 11); addDeltaFile(t, null, 278L, 288L, 11); addDeltaFile(t, null, 289L, 299L, 11); addDeltaFile(t, null, 300L, 310L, 11); addDeltaFile(t, null, 311L, 321L, 11); burnThroughTransactions(320); long txnid = openTxn(); LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default"); comp.setTablename("cmomwbv"); List<LockComponent> components = new ArrayList<LockComponent>(1); components.add(comp); LockRequest req = new LockRequest(components, "me", "localhost"); req.setTxnid(txnid); LockResponse res = txnHandler.lock(req); txnHandler.commitTxn(new CommitTxnRequest(txnid)); startInitiator(); ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); List<ShowCompactResponseElement> compacts = rsp.getCompacts(); Assert.assertEquals(1, compacts.size()); Assert.assertEquals("initiated", compacts.get(0).getState()); Assert.assertEquals("cmomwbv", compacts.get(0).getTablename()); Assert.assertEquals(CompactionType.MAJOR, compacts.get(0).getType()); }