/** Get a pool by name, creating it if necessary */ public synchronized Pool getPool(String name) { Pool pool = pools.get(name); if (pool == null) { pool = new Pool(scheduler, name); pool.setSchedulingMode(defaultSchedulingMode); pools.put(name, pool); } return pool; }
/** * Updates the allocation list from the allocation config file. This file is expected to be in the * following whitespace-separated format: <code> * poolName1 mapAlloc reduceAlloc * poolName2 mapAlloc reduceAlloc * ... * </code> Blank lines and lines starting with # are ignored. * * @throws IOException if the config file cannot be read. * @throws AllocationConfigurationException if allocations are invalid. * @throws ParserConfigurationException if XML parser is misconfigured. * @throws SAXException if config file is malformed. */ public void reloadAllocs() throws IOException, ParserConfigurationException, SAXException, AllocationConfigurationException { if (allocFile == null) return; // Create some temporary hashmaps to hold the new allocs, and we only save // them in our fields if we have parsed the entire allocs file successfully. Map<String, Integer> mapAllocs = new HashMap<String, Integer>(); Map<String, Integer> reduceAllocs = new HashMap<String, Integer>(); Map<String, Integer> poolMaxJobs = new HashMap<String, Integer>(); Map<String, Integer> userMaxJobs = new HashMap<String, Integer>(); Map<String, Integer> poolMaxMaps = new HashMap<String, Integer>(); Map<String, Integer> poolMaxReduces = new HashMap<String, Integer>(); Map<String, Double> poolWeights = new HashMap<String, Double>(); Map<String, SchedulingMode> poolModes = new HashMap<String, SchedulingMode>(); Map<String, Long> minSharePreemptionTimeouts = new HashMap<String, Long>(); int userMaxJobsDefault = Integer.MAX_VALUE; int poolMaxJobsDefault = Integer.MAX_VALUE; // Remember all pool names so we can display them on web UI, etc. List<String> poolNamesInAllocFile = new ArrayList<String>(); long fairSharePreemptionTimeout = Long.MAX_VALUE; long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; SchedulingMode defaultSchedulingMode = SchedulingMode.FAIR; // Read and parse the allocations file. DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); docBuilderFactory.setIgnoringComments(true); DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); Document doc = builder.parse(new File(allocFile)); Element root = doc.getDocumentElement(); if (!"allocations".equals(root.getTagName())) throw new AllocationConfigurationException( "Bad fair scheduler config " + "file: top-level element not <allocations>"); NodeList elements = root.getChildNodes(); for (int i = 0; i < elements.getLength(); i++) { Node node = elements.item(i); if (!(node instanceof Element)) continue; Element element = (Element) node; if ("pool".equals(element.getTagName())) { String poolName = element.getAttribute("name"); poolNamesInAllocFile.add(poolName); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("minMaps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); mapAllocs.put(poolName, val); } else if ("minReduces".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); reduceAllocs.put(poolName, val); } else if ("maxMaps".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxMaps.put(poolName, val); } else if ("maxReduces".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxReduces.put(poolName, val); } else if ("maxRunningJobs".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxJobs.put(poolName, val); } else if ("weight".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); double val = Double.parseDouble(text); poolWeights.put(poolName, val); } else if ("minSharePreemptionTimeout".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; minSharePreemptionTimeouts.put(poolName, val); } else if ("schedulingMode".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); poolModes.put(poolName, parseSchedulingMode(text)); } } if (poolMaxMaps.containsKey(poolName) && mapAllocs.containsKey(poolName) && poolMaxMaps.get(poolName) < mapAllocs.get(poolName)) { LOG.warn( String.format( "Pool %s has max maps %d less than min maps %d", poolName, poolMaxMaps.get(poolName), mapAllocs.get(poolName))); } if (poolMaxReduces.containsKey(poolName) && reduceAllocs.containsKey(poolName) && poolMaxReduces.get(poolName) < reduceAllocs.get(poolName)) { LOG.warn( String.format( "Pool %s has max reduces %d less than min reduces %d", poolName, poolMaxReduces.get(poolName), reduceAllocs.get(poolName))); } } else if ("user".equals(element.getTagName())) { String userName = element.getAttribute("name"); NodeList fields = element.getChildNodes(); for (int j = 0; j < fields.getLength(); j++) { Node fieldNode = fields.item(j); if (!(fieldNode instanceof Element)) continue; Element field = (Element) fieldNode; if ("maxRunningJobs".equals(field.getTagName())) { String text = ((Text) field.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxJobs.put(userName, val); } } } else if ("userMaxJobsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); userMaxJobsDefault = val; } else if ("poolMaxJobsDefault".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); int val = Integer.parseInt(text); poolMaxJobsDefault = val; } else if ("fairSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; fairSharePreemptionTimeout = val; } else if ("defaultMinSharePreemptionTimeout".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); long val = Long.parseLong(text) * 1000L; defaultMinSharePreemptionTimeout = val; } else if ("defaultPoolSchedulingMode".equals(element.getTagName())) { String text = ((Text) element.getFirstChild()).getData().trim(); defaultSchedulingMode = parseSchedulingMode(text); } else { LOG.warn("Bad element in allocations file: " + element.getTagName()); } } // Commit the reload; also create any pool defined in the alloc file // if it does not already exist, so it can be displayed on the web UI. synchronized (this) { this.mapAllocs = mapAllocs; this.reduceAllocs = reduceAllocs; this.poolMaxMaps = poolMaxMaps; this.poolMaxReduces = poolMaxReduces; this.poolMaxJobs = poolMaxJobs; this.userMaxJobs = userMaxJobs; this.userMaxJobsDefault = userMaxJobsDefault; this.poolMaxJobsDefault = poolMaxJobsDefault; this.poolWeights = poolWeights; this.minSharePreemptionTimeouts = minSharePreemptionTimeouts; this.fairSharePreemptionTimeout = fairSharePreemptionTimeout; this.defaultMinSharePreemptionTimeout = defaultMinSharePreemptionTimeout; this.defaultSchedulingMode = defaultSchedulingMode; for (String name : poolNamesInAllocFile) { Pool pool = getPool(name); if (poolModes.containsKey(name)) { pool.setSchedulingMode(poolModes.get(name)); } else { pool.setSchedulingMode(defaultSchedulingMode); } } } }