@Test(timeout = 10000) public void testConfigs() throws IOException { long maxTaskMem = 8192 * 1024 * 1024l; // Test Shuffle fetch buffer and post merge buffer percentage Configuration conf = new TezConfiguration(defaultConf); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.8f); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.5f); Assert.assertTrue(MergeManager.getInitialMemoryRequirement(conf, maxTaskMem) == 6871947776l); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.5f); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.5f); Assert.assertTrue( MergeManager.getInitialMemoryRequirement(conf, maxTaskMem) > Integer.MAX_VALUE); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.9f); Assert.assertTrue( MergeManager.getInitialMemoryRequirement(conf, maxTaskMem) > Integer.MAX_VALUE); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.1f); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.1f); Assert.assertTrue( MergeManager.getInitialMemoryRequirement(conf, maxTaskMem) < Integer.MAX_VALUE); try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 2.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong buffer percent configuration exception"); } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, -2.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong buffer percent configuration exception"); } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 1.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong post merge buffer percent configuration exception"); } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, -1.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong post merge buffer percent configuration exception"); } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 1.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong shuffle fetch buffer percent configuration exception"); } catch (IllegalArgumentException ie) { } try { conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, -1.4f); MergeManager.getInitialMemoryRequirement(conf, maxTaskMem); Assert.fail("Should have thrown wrong shuffle fetch buffer percent configuration exception"); } catch (IllegalArgumentException ie) { } // test post merge mem limit conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f); conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.8f); FileSystem localFs = FileSystem.getLocal(conf); LocalDirAllocator localDirAllocator = new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS); InputContext t0inputContext = createMockInputContext(UUID.randomUUID().toString(), maxTaskMem); ExceptionReporter t0exceptionReporter = mock(ExceptionReporter.class); long initialMemoryAvailable = (long) (maxTaskMem * 0.8); MergeManager mergeManager = new MergeManager( conf, localFs, localDirAllocator, t0inputContext, null, null, null, null, t0exceptionReporter, initialMemoryAvailable, null, false, -1); Assert.assertTrue(mergeManager.postMergeMemLimit > Integer.MAX_VALUE); initialMemoryAvailable = 200 * 1024 * 1024l; // initial mem < memlimit mergeManager = new MergeManager( conf, localFs, localDirAllocator, t0inputContext, null, null, null, null, t0exceptionReporter, initialMemoryAvailable, null, false, -1); Assert.assertTrue(mergeManager.postMergeMemLimit == initialMemoryAvailable); }