/** Get a pool by name, creating it if necessary */
public synchronized Pool getPool(String name) {
Pool pool = pools.get(name);
if (pool == null) {
pool = new Pool(scheduler, name);
pool.setSchedulingMode(defaultSchedulingMode);
pools.put(name, pool);
}
return pool;
}
/**
* Updates the allocation list from the allocation config file. This file is expected to be in the
* following whitespace-separated format: <code>
* poolName1 mapAlloc reduceAlloc
* poolName2 mapAlloc reduceAlloc
* ...
* </code> Blank lines and lines starting with # are ignored.
*
* @throws IOException if the config file cannot be read.
* @throws AllocationConfigurationException if allocations are invalid.
* @throws ParserConfigurationException if XML parser is misconfigured.
* @throws SAXException if config file is malformed.
*/
public void reloadAllocs()
throws IOException, ParserConfigurationException, SAXException,
AllocationConfigurationException {
if (allocFile == null) return;
// Create some temporary hashmaps to hold the new allocs, and we only save
// them in our fields if we have parsed the entire allocs file successfully.
Map<String, Integer> mapAllocs = new HashMap<String, Integer>();
Map<String, Integer> reduceAllocs = new HashMap<String, Integer>();
Map<String, Integer> poolMaxJobs = new HashMap<String, Integer>();
Map<String, Integer> userMaxJobs = new HashMap<String, Integer>();
Map<String, Integer> poolMaxMaps = new HashMap<String, Integer>();
Map<String, Integer> poolMaxReduces = new HashMap<String, Integer>();
Map<String, Double> poolWeights = new HashMap<String, Double>();
Map<String, SchedulingMode> poolModes = new HashMap<String, SchedulingMode>();
Map<String, Long> minSharePreemptionTimeouts = new HashMap<String, Long>();
int userMaxJobsDefault = Integer.MAX_VALUE;
int poolMaxJobsDefault = Integer.MAX_VALUE;
// Remember all pool names so we can display them on web UI, etc.
List<String> poolNamesInAllocFile = new ArrayList<String>();
long fairSharePreemptionTimeout = Long.MAX_VALUE;
long defaultMinSharePreemptionTimeout = Long.MAX_VALUE;
SchedulingMode defaultSchedulingMode = SchedulingMode.FAIR;
// Read and parse the allocations file.
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
docBuilderFactory.setIgnoringComments(true);
DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
Document doc = builder.parse(new File(allocFile));
Element root = doc.getDocumentElement();
if (!"allocations".equals(root.getTagName()))
throw new AllocationConfigurationException(
"Bad fair scheduler config " + "file: top-level element not <allocations>");
NodeList elements = root.getChildNodes();
for (int i = 0; i < elements.getLength(); i++) {
Node node = elements.item(i);
if (!(node instanceof Element)) continue;
Element element = (Element) node;
if ("pool".equals(element.getTagName())) {
String poolName = element.getAttribute("name");
poolNamesInAllocFile.add(poolName);
NodeList fields = element.getChildNodes();
for (int j = 0; j < fields.getLength(); j++) {
Node fieldNode = fields.item(j);
if (!(fieldNode instanceof Element)) continue;
Element field = (Element) fieldNode;
if ("minMaps".equals(field.getTagName())) {
String text = ((Text) field.getFirstChild()).getData().trim();
int val = Integer.parseInt(text);
mapAllocs.put(poolName, val);
} else if ("minReduces".equals(field.getTagName())) {
String text = ((Text) field.getFirstChild()).getData().trim();
int val = Integer.parseInt(text);
reduceAllocs.put(poolName, val);
} else if ("maxMaps".equals(field.getTagName())) {
String text = ((Text) field.getFirstChild()).getData().trim();
int val = Integer.parseInt(text);
poolMaxMaps.put(poolName, val);
} else if ("maxReduces".equals(field.getTagName())) {
String text = ((Text) field.getFirstChild()).getData().trim();
int val = Integer.parseInt(text);
poolMaxReduces.put(poolName, val);
} else if ("maxRunningJobs".equals(field.getTagName())) {
String text = ((Text) field.getFirstChild()).getData().trim();
int val = Integer.parseInt(text);
poolMaxJobs.put(poolName, val);
} else if ("weight".equals(field.getTagName())) {
String text = ((Text) field.getFirstChild()).getData().trim();
double val = Double.parseDouble(text);
poolWeights.put(poolName, val);
} else if ("minSharePreemptionTimeout".equals(field.getTagName())) {
String text = ((Text) field.getFirstChild()).getData().trim();
long val = Long.parseLong(text) * 1000L;
minSharePreemptionTimeouts.put(poolName, val);
} else if ("schedulingMode".equals(field.getTagName())) {
String text = ((Text) field.getFirstChild()).getData().trim();
poolModes.put(poolName, parseSchedulingMode(text));
}
}
if (poolMaxMaps.containsKey(poolName)
&& mapAllocs.containsKey(poolName)
&& poolMaxMaps.get(poolName) < mapAllocs.get(poolName)) {
LOG.warn(
String.format(
"Pool %s has max maps %d less than min maps %d",
poolName, poolMaxMaps.get(poolName), mapAllocs.get(poolName)));
}
if (poolMaxReduces.containsKey(poolName)
&& reduceAllocs.containsKey(poolName)
&& poolMaxReduces.get(poolName) < reduceAllocs.get(poolName)) {
LOG.warn(
String.format(
"Pool %s has max reduces %d less than min reduces %d",
poolName, poolMaxReduces.get(poolName), reduceAllocs.get(poolName)));
}
} else if ("user".equals(element.getTagName())) {
String userName = element.getAttribute("name");
NodeList fields = element.getChildNodes();
for (int j = 0; j < fields.getLength(); j++) {
Node fieldNode = fields.item(j);
if (!(fieldNode instanceof Element)) continue;
Element field = (Element) fieldNode;
if ("maxRunningJobs".equals(field.getTagName())) {
String text = ((Text) field.getFirstChild()).getData().trim();
int val = Integer.parseInt(text);
userMaxJobs.put(userName, val);
}
}
} else if ("userMaxJobsDefault".equals(element.getTagName())) {
String text = ((Text) element.getFirstChild()).getData().trim();
int val = Integer.parseInt(text);
userMaxJobsDefault = val;
} else if ("poolMaxJobsDefault".equals(element.getTagName())) {
String text = ((Text) element.getFirstChild()).getData().trim();
int val = Integer.parseInt(text);
poolMaxJobsDefault = val;
} else if ("fairSharePreemptionTimeout".equals(element.getTagName())) {
String text = ((Text) element.getFirstChild()).getData().trim();
long val = Long.parseLong(text) * 1000L;
fairSharePreemptionTimeout = val;
} else if ("defaultMinSharePreemptionTimeout".equals(element.getTagName())) {
String text = ((Text) element.getFirstChild()).getData().trim();
long val = Long.parseLong(text) * 1000L;
defaultMinSharePreemptionTimeout = val;
} else if ("defaultPoolSchedulingMode".equals(element.getTagName())) {
String text = ((Text) element.getFirstChild()).getData().trim();
defaultSchedulingMode = parseSchedulingMode(text);
} else {
LOG.warn("Bad element in allocations file: " + element.getTagName());
}
}
// Commit the reload; also create any pool defined in the alloc file
// if it does not already exist, so it can be displayed on the web UI.
synchronized (this) {
this.mapAllocs = mapAllocs;
this.reduceAllocs = reduceAllocs;
this.poolMaxMaps = poolMaxMaps;
this.poolMaxReduces = poolMaxReduces;
this.poolMaxJobs = poolMaxJobs;
this.userMaxJobs = userMaxJobs;
this.userMaxJobsDefault = userMaxJobsDefault;
this.poolMaxJobsDefault = poolMaxJobsDefault;
this.poolWeights = poolWeights;
this.minSharePreemptionTimeouts = minSharePreemptionTimeouts;
this.fairSharePreemptionTimeout = fairSharePreemptionTimeout;
this.defaultMinSharePreemptionTimeout = defaultMinSharePreemptionTimeout;
this.defaultSchedulingMode = defaultSchedulingMode;
for (String name : poolNamesInAllocFile) {
Pool pool = getPool(name);
if (poolModes.containsKey(name)) {
pool.setSchedulingMode(poolModes.get(name));
} else {
pool.setSchedulingMode(defaultSchedulingMode);
}
}
}
}
