/**
* Adjusts the domain of the <tt>mapred.reduce.tasks</tt> parameter descriptor based on the
* virtual job profile.
*
* @param paramDescr the current parameter descriptor
* @param jobProfile the job profile
* @param taskMemory the task memory (in bytes)
* @param numRedSlots the total number of reduce slots in the cluster
*/
public static void adjustParamDescrRedTasks(
IntegerParamDescriptor paramDescr,
MRJobProfile jobProfile,
long taskMemory,
int numRedSlots) {
// Get the reduce profile
MRReduceProfile redProfile = jobProfile.getAvgReduceProfile();
if (redProfile == null || redProfile.isEmpty()) return;
// Calculate the (uncompressed) reduce input size
double shuffleSize =
redProfile.getNumTasks()
* redProfile.getCounter(MRCounter.REDUCE_SHUFFLE_BYTES)
/ redProfile.getStatistic(MRStatistics.INTERM_COMPRESS_RATIO, 1d);
// Calculate the number of reduce groups
long numGroups =
redProfile.getNumTasks() * redProfile.getCounter(MRCounter.REDUCE_INPUT_GROUPS, 1l);
// Calculate the min and max number of reducers
double min = Math.ceil(shuffleSize / (2 * taskMemory));
double max = Math.ceil(4 * shuffleSize / taskMemory);
max = Math.min(max, numGroups);
max = Math.max(max, numRedSlots);
if (max < min) max = min;
// Set the min and max number of reducers
paramDescr.setMinMaxValue((int) min, (int) max);
}
/**
* Adjusts the domain of the <tt>io.sort.mb</tt> parameter descriptor based on the virtual job
* profile.
*
* @param paramDescr the current parameter descriptor
* @param jobProfile the job profile
* @param taskMemory the task memory (in bytes)
*/
public static void adjustParamDescrSortMB(
IntegerParamDescriptor paramDescr, MRJobProfile jobProfile, long taskMemory) {
// Find the memory required by the map tasks
long mapMemory = 0l;
for (MRMapProfile mapProfile : jobProfile.getAvgMapProfiles()) {
mapMemory += WhatIfUtils.getMapMemoryRequired(mapProfile);
}
mapMemory /= jobProfile.getAvgMapProfiles().size();
// Set the memory left for io.sort.mb
long ioSortMem = taskMemory - mapMemory;
if (ioSortMem > (long) (MAX_MEM_RATIO * taskMemory))
ioSortMem = (long) (MAX_MEM_RATIO * taskMemory);
if (ioSortMem < (long) (MIN_MEM_RATIO * taskMemory))
ioSortMem = (long) (MIN_MEM_RATIO * taskMemory);
paramDescr.setMaxValue((int) (ioSortMem >> 20));
}