/** * Adjusts the domain of the <tt>mapred.reduce.tasks</tt> parameter descriptor based on the * virtual job profile. * * @param paramDescr the current parameter descriptor * @param jobProfile the job profile * @param taskMemory the task memory (in bytes) * @param numRedSlots the total number of reduce slots in the cluster */ public static void adjustParamDescrRedTasks( IntegerParamDescriptor paramDescr, MRJobProfile jobProfile, long taskMemory, int numRedSlots) { // Get the reduce profile MRReduceProfile redProfile = jobProfile.getAvgReduceProfile(); if (redProfile == null || redProfile.isEmpty()) return; // Calculate the (uncompressed) reduce input size double shuffleSize = redProfile.getNumTasks() * redProfile.getCounter(MRCounter.REDUCE_SHUFFLE_BYTES) / redProfile.getStatistic(MRStatistics.INTERM_COMPRESS_RATIO, 1d); // Calculate the number of reduce groups long numGroups = redProfile.getNumTasks() * redProfile.getCounter(MRCounter.REDUCE_INPUT_GROUPS, 1l); // Calculate the min and max number of reducers double min = Math.ceil(shuffleSize / (2 * taskMemory)); double max = Math.ceil(4 * shuffleSize / taskMemory); max = Math.min(max, numGroups); max = Math.max(max, numRedSlots); if (max < min) max = min; // Set the min and max number of reducers paramDescr.setMinMaxValue((int) min, (int) max); }
/** * Adjusts the domain of the <tt>io.sort.mb</tt> parameter descriptor based on the virtual job * profile. * * @param paramDescr the current parameter descriptor * @param jobProfile the job profile * @param taskMemory the task memory (in bytes) */ public static void adjustParamDescrSortMB( IntegerParamDescriptor paramDescr, MRJobProfile jobProfile, long taskMemory) { // Find the memory required by the map tasks long mapMemory = 0l; for (MRMapProfile mapProfile : jobProfile.getAvgMapProfiles()) { mapMemory += WhatIfUtils.getMapMemoryRequired(mapProfile); } mapMemory /= jobProfile.getAvgMapProfiles().size(); // Set the memory left for io.sort.mb long ioSortMem = taskMemory - mapMemory; if (ioSortMem > (long) (MAX_MEM_RATIO * taskMemory)) ioSortMem = (long) (MAX_MEM_RATIO * taskMemory); if (ioSortMem < (long) (MIN_MEM_RATIO * taskMemory)) ioSortMem = (long) (MIN_MEM_RATIO * taskMemory); paramDescr.setMaxValue((int) (ioSortMem >> 20)); }