@Test(timeout = 10000)
public void testConfigs() throws IOException {
long maxTaskMem = 8192 * 1024 * 1024l;
// Test Shuffle fetch buffer and post merge buffer percentage
Configuration conf = new TezConfiguration(defaultConf);
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.8f);
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.5f);
Assert.assertTrue(MergeManager.getInitialMemoryRequirement(conf, maxTaskMem) == 6871947776l);
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.5f);
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.5f);
Assert.assertTrue(
MergeManager.getInitialMemoryRequirement(conf, maxTaskMem) > Integer.MAX_VALUE);
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f);
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.9f);
Assert.assertTrue(
MergeManager.getInitialMemoryRequirement(conf, maxTaskMem) > Integer.MAX_VALUE);
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.1f);
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.1f);
Assert.assertTrue(
MergeManager.getInitialMemoryRequirement(conf, maxTaskMem) < Integer.MAX_VALUE);
try {
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 2.4f);
MergeManager.getInitialMemoryRequirement(conf, maxTaskMem);
Assert.fail("Should have thrown wrong buffer percent configuration exception");
} catch (IllegalArgumentException ie) {
}
try {
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, -2.4f);
MergeManager.getInitialMemoryRequirement(conf, maxTaskMem);
Assert.fail("Should have thrown wrong buffer percent configuration exception");
} catch (IllegalArgumentException ie) {
}
try {
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 1.4f);
MergeManager.getInitialMemoryRequirement(conf, maxTaskMem);
Assert.fail("Should have thrown wrong post merge buffer percent configuration exception");
} catch (IllegalArgumentException ie) {
}
try {
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, -1.4f);
MergeManager.getInitialMemoryRequirement(conf, maxTaskMem);
Assert.fail("Should have thrown wrong post merge buffer percent configuration exception");
} catch (IllegalArgumentException ie) {
}
try {
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 1.4f);
MergeManager.getInitialMemoryRequirement(conf, maxTaskMem);
Assert.fail("Should have thrown wrong shuffle fetch buffer percent configuration exception");
} catch (IllegalArgumentException ie) {
}
try {
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, -1.4f);
MergeManager.getInitialMemoryRequirement(conf, maxTaskMem);
Assert.fail("Should have thrown wrong shuffle fetch buffer percent configuration exception");
} catch (IllegalArgumentException ie) {
}
// test post merge mem limit
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f);
conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0.8f);
FileSystem localFs = FileSystem.getLocal(conf);
LocalDirAllocator localDirAllocator =
new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS);
InputContext t0inputContext = createMockInputContext(UUID.randomUUID().toString(), maxTaskMem);
ExceptionReporter t0exceptionReporter = mock(ExceptionReporter.class);
long initialMemoryAvailable = (long) (maxTaskMem * 0.8);
MergeManager mergeManager =
new MergeManager(
conf,
localFs,
localDirAllocator,
t0inputContext,
null,
null,
null,
null,
t0exceptionReporter,
initialMemoryAvailable,
null,
false,
-1);
Assert.assertTrue(mergeManager.postMergeMemLimit > Integer.MAX_VALUE);
initialMemoryAvailable = 200 * 1024 * 1024l; // initial mem < memlimit
mergeManager =
new MergeManager(
conf,
localFs,
localDirAllocator,
t0inputContext,
null,
null,
null,
null,
t0exceptionReporter,
initialMemoryAvailable,
null,
false,
-1);
Assert.assertTrue(mergeManager.postMergeMemLimit == initialMemoryAvailable);
}
