@SuppressWarnings("unchecked") private void initMetrics() throws Exception { metrics = new MetricRegistry(); // configuration metricsOutputDir = conf.get(SLSConfiguration.METRICS_OUTPUT_DIR); int metricsWebAddressPort = conf.getInt( SLSConfiguration.METRICS_WEB_ADDRESS_PORT, SLSConfiguration.METRICS_WEB_ADDRESS_PORT_DEFAULT); // create SchedulerMetrics for current scheduler String schedulerMetricsType = conf.get(scheduler.getClass().getName()); Class schedulerMetricsClass = schedulerMetricsType == null ? defaultSchedulerMetricsMap.get(scheduler.getClass()) : Class.forName(schedulerMetricsType); schedulerMetrics = (SchedulerMetrics) ReflectionUtils.newInstance(schedulerMetricsClass, new Configuration()); schedulerMetrics.init(scheduler, metrics); // register various metrics registerJvmMetrics(); registerClusterResourceMetrics(); registerContainerAppNumMetrics(); registerSchedulerMetrics(); // .csv output initMetricsCSVOutput(); // start web app to provide real-time tracking web = new SLSWebApp(this, metricsWebAddressPort); web.start(); // a thread to update histogram timer pool = new ScheduledThreadPoolExecutor(2); pool.scheduleAtFixedRate(new HistogramsRunnable(), 0, 1000, TimeUnit.MILLISECONDS); // a thread to output metrics for real-tiem tracking pool.scheduleAtFixedRate(new MetricsLogRunnable(), 0, 1000, TimeUnit.MILLISECONDS); // application running information jobRuntimeLogBW = new BufferedWriter( new OutputStreamWriter( new FileOutputStream(metricsOutputDir + "/jobruntime.csv"), "UTF-8")); jobRuntimeLogBW.write( "JobID,real_start_time,real_end_time," + "simulate_start_time,simulate_end_time" + EOL); jobRuntimeLogBW.flush(); }
public void removeTrackedApp(ApplicationAttemptId appAttemptId, String oldAppId) { if (metricsON) { schedulerMetrics.untrackApp(appAttemptId, oldAppId); } }
private void updateQueueWithAllocateRequest( Allocation allocation, ApplicationAttemptId attemptId, List<ResourceRequest> resourceRequests, List<ContainerId> containerIds) throws IOException { // update queue information Resource pendingResource = Resources.createResource(0, 0); Resource allocatedResource = Resources.createResource(0, 0); String queueName = appQueueMap.get(attemptId.getApplicationId()); // container requested for (ResourceRequest request : resourceRequests) { if (request.getResourceName().equals(ResourceRequest.ANY)) { Resources.addTo( pendingResource, Resources.multiply(request.getCapability(), request.getNumContainers())); } } // container allocated for (Container container : allocation.getContainers()) { Resources.addTo(allocatedResource, container.getResource()); Resources.subtractFrom(pendingResource, container.getResource()); } // container released from AM SchedulerAppReport report = scheduler.getSchedulerAppInfo(attemptId); for (ContainerId containerId : containerIds) { Container container = null; for (RMContainer c : report.getLiveContainers()) { if (c.getContainerId().equals(containerId)) { container = c.getContainer(); break; } } if (container != null) { // released allocated containers Resources.subtractFrom(allocatedResource, container.getResource()); } else { for (RMContainer c : report.getReservedContainers()) { if (c.getContainerId().equals(containerId)) { container = c.getContainer(); break; } } if (container != null) { // released reserved containers Resources.subtractFrom(pendingResource, container.getResource()); } } } // containers released/preemption from scheduler Set<ContainerId> preemptionContainers = new HashSet<ContainerId>(); if (allocation.getContainerPreemptions() != null) { preemptionContainers.addAll(allocation.getContainerPreemptions()); } if (allocation.getStrictContainerPreemptions() != null) { preemptionContainers.addAll(allocation.getStrictContainerPreemptions()); } if (!preemptionContainers.isEmpty()) { for (ContainerId containerId : preemptionContainers) { if (!preemptionContainerMap.containsKey(containerId)) { Container container = null; for (RMContainer c : report.getLiveContainers()) { if (c.getContainerId().equals(containerId)) { container = c.getContainer(); break; } } if (container != null) { preemptionContainerMap.put(containerId, container.getResource()); } } } } // update metrics SortedMap<String, Counter> counterMap = metrics.getCounters(); String names[] = new String[] { "counter.queue." + queueName + ".pending.memory", "counter.queue." + queueName + ".pending.cores", "counter.queue." + queueName + ".allocated.memory", "counter.queue." + queueName + ".allocated.cores" }; int values[] = new int[] { pendingResource.getMemory(), pendingResource.getVirtualCores(), allocatedResource.getMemory(), allocatedResource.getVirtualCores() }; for (int i = names.length - 1; i >= 0; i--) { if (!counterMap.containsKey(names[i])) { metrics.counter(names[i]); counterMap = metrics.getCounters(); } counterMap.get(names[i]).inc(values[i]); } queueLock.lock(); try { if (!schedulerMetrics.isTracked(queueName)) { schedulerMetrics.trackQueue(queueName); } } finally { queueLock.unlock(); } }