/** Exercise 4 - Read a list of people from Cassandra */ private static void readPeopleFromCassandra(JavaSparkContext javaSparkContext) { SparkContextJavaFunctions sparkContextJavaFunctions = CassandraJavaUtil.javaFunctions(javaSparkContext); CassandraJavaRDD<Person> personCassandraJavaRDD = sparkContextJavaFunctions.cassandraTable( "test", "people", CassandraJavaUtil.mapRowTo(Person.class)); JavaRDD<String> readPeopleRDD = personCassandraJavaRDD.map(Person::toString); System.out.println( "Data as Person beans: \n" + StringUtils.join("\n", readPeopleRDD.collect())); // select id from test.people where id=45; JavaRDD<String> rdd4 = sparkContextJavaFunctions .cassandraTable("test", "people") .select("id") .where("id = 45") .map(CassandraRow::toString); System.out.println( "Data with only 'id' column fetched: \n" + StringUtils.join("\n", rdd4.collect())); }
/** * Verify if the NodeManager could identify disk failures. * * @param localORLogDirs <em>true</em> represent nm-local-dirs and <em>false </em> means * nm-log-dirs * @param expectedDirs expected nm-local-dirs/nm-log-dirs as a string * @param isHealthy <em>true</em> if the overall node should be healthy */ private void verifyDisksHealth(boolean localORLogDirs, String expectedDirs, boolean isHealthy) { // Wait for the NodeManager to identify disk failures. waitForDiskHealthCheck(); List<String> list = localORLogDirs ? dirsHandler.getLocalDirs() : dirsHandler.getLogDirs(); String seenDirs = StringUtils.join(",", list); LOG.info("ExpectedDirs=" + expectedDirs); LOG.info("SeenDirs=" + seenDirs); Assert.assertTrue( "NodeManager could not identify disk failure.", expectedDirs.equals(seenDirs)); Assert.assertEquals( "Node's health in terms of disks is wrong", isHealthy, dirsHandler.areDisksHealthy()); for (int i = 0; i < 10; i++) { Iterator<RMNode> iter = yarnCluster.getResourceManager().getRMContext().getRMNodes().values().iterator(); if (iter.next().getNodeHealthStatus().getIsNodeHealthy() == isHealthy) { break; } // wait for the node health info to go to RM try { Thread.sleep(1000); } catch (InterruptedException e) { LOG.error("Interrupted while waiting for NM->RM heartbeat."); } } Iterator<RMNode> iter = yarnCluster.getResourceManager().getRMContext().getRMNodes().values().iterator(); Assert.assertEquals( "RM is not updated with the health status of a node", isHealthy, iter.next().getNodeHealthStatus().getIsNodeHealthy()); }
public JavaApiDemo() { DemoApp demoApp = DemoApp$.MODULE$.apply(); JavaSparkContext sc = new JavaSparkContext(demoApp.sc()); // Here we are going to save some data to Cassandra... List<Person> people = Arrays.asList( Person.newInstance(1, "John", new Date()), Person.newInstance(2, "Anna", new Date()), Person.newInstance(3, "Andrew", new Date())); JavaRDD<Person> rdd = sc.parallelize(people); javaFunctions(rdd, Person.class).saveToCassandra("test", "people"); // then, we want to read that data as an RDD of CassandraRows and convert them to strings... JavaRDD<String> cassandraRowsRDD = javaFunctions(sc) .cassandraTable("test", "people") .toJavaRDD() .map( new Function<CassandraRow, String>() { @Override public String call(CassandraRow cassandraRow) throws Exception { return cassandraRow.toString(); } }); System.out.println( "Data as CassandraRows: \n" + StringUtils.join("\n", cassandraRowsRDD.toArray())); // finally, we want to read that data as an RDD of Person beans and also convert them to // strings... JavaRDD<String> rdd2 = javaFunctions(sc) .cassandraTable("test", "people", Person.class) .toJavaRDD() .map( new Function<Person, String>() { @Override public String call(Person person) throws Exception { return person.toString(); } }); System.out.println("Data as Person beans: \n" + StringUtils.join("\n", rdd2.toArray())); sc.stop(); }
private void testDirsFailures(boolean localORLogDirs) throws IOException { String dirType = localORLogDirs ? "local" : "log"; String dirsProperty = localORLogDirs ? YarnConfiguration.NM_LOCAL_DIRS : YarnConfiguration.NM_LOG_DIRS; Configuration conf = new Configuration(); // set disk health check interval to a small value (say 1 sec). conf.setLong(YarnConfiguration.NM_DISK_HEALTH_CHECK_INTERVAL_MS, DISK_HEALTH_CHECK_INTERVAL); // If 2 out of the total 4 local-dirs fail OR if 2 Out of the total 4 // log-dirs fail, then the node's health status should become unhealthy. conf.setFloat(YarnConfiguration.NM_MIN_HEALTHY_DISKS_FRACTION, 0.60F); if (yarnCluster != null) { yarnCluster.stop(); FileUtil.fullyDelete(localFSDirBase); localFSDirBase.mkdirs(); } LOG.info("Starting up YARN cluster"); yarnCluster = new MiniYARNCluster(TestDiskFailures.class.getName(), 1, numLocalDirs, numLogDirs); yarnCluster.init(conf); yarnCluster.start(); NodeManager nm = yarnCluster.getNodeManager(0); LOG.info("Configured nm-" + dirType + "-dirs=" + nm.getConfig().get(dirsProperty)); dirsHandler = nm.getNodeHealthChecker().getDiskHandler(); List<String> list = localORLogDirs ? dirsHandler.getLocalDirs() : dirsHandler.getLogDirs(); String[] dirs = list.toArray(new String[list.size()]); Assert.assertEquals("Number of nm-" + dirType + "-dirs is wrong.", numLocalDirs, dirs.length); String expectedDirs = StringUtils.join(",", list); // validate the health of disks initially verifyDisksHealth(localORLogDirs, expectedDirs, true); // Make 1 nm-local-dir fail and verify if "the nodemanager can identify // the disk failure(s) and can update the list of good nm-local-dirs. prepareDirToFail(dirs[2]); expectedDirs = dirs[0] + "," + dirs[1] + "," + dirs[3]; verifyDisksHealth(localORLogDirs, expectedDirs, true); // Now, make 1 more nm-local-dir/nm-log-dir fail and verify if "the // nodemanager can identify the disk failures and can update the list of // good nm-local-dirs/nm-log-dirs and can update the overall health status // of the node to unhealthy". prepareDirToFail(dirs[0]); expectedDirs = dirs[1] + "," + dirs[3]; verifyDisksHealth(localORLogDirs, expectedDirs, false); // Fail the remaining 2 local-dirs/log-dirs and verify if NM remains with // empty list of local-dirs/log-dirs and the overall health status is // unhealthy. prepareDirToFail(dirs[1]); prepareDirToFail(dirs[3]); expectedDirs = ""; verifyDisksHealth(localORLogDirs, expectedDirs, false); }
@Test @TestException(exception = ServerException.class, msgRegExp = "S10.*") @TestDir public void serviceWithMissingDependency() throws Exception { String dir = TestDirHelper.getTestDir().getAbsolutePath(); Configuration conf = new Configuration(false); String services = StringUtils.join( ",", Arrays.asList(MyService3.class.getName(), MyService6.class.getName())); conf.set("server.services", services); Server server = new Server("server", dir, dir, dir, dir, conf); server.init(); }
private void uploadLogsForContainer(ContainerId containerId) { if (this.logAggregationDisabled) { return; } // Lazy creation of the writer if (this.writer == null) { LOG.info( "Starting aggregate log-file for app " + this.applicationId + " at " + this.remoteNodeTmpLogFileForApp); try { this.writer = new LogWriter(this.conf, this.remoteNodeTmpLogFileForApp, this.userUgi); // Write ACLs once when and if the writer is created. this.writer.writeApplicationACLs(appAcls); this.writer.writeApplicationOwner(this.userUgi.getShortUserName()); } catch (IOException e) { LOG.error( "Cannot create writer for app " + this.applicationId + ". Disabling log-aggregation for this app.", e); this.logAggregationDisabled = true; return; } } LOG.info( "Uploading logs for container " + containerId + ". Current good log dirs are " + StringUtils.join(",", dirsHandler.getLogDirs())); LogKey logKey = new LogKey(containerId); LogValue logValue = new LogValue(dirsHandler.getLogDirs(), containerId, userUgi.getShortUserName()); try { this.writer.append(logKey, logValue); } catch (IOException e) { LOG.error("Couldn't upload logs for " + containerId + ". Skipping this container."); } }
@Test @TestDir public void services() throws Exception { String dir = TestDirHelper.getTestDir().getAbsolutePath(); Configuration conf; Server server; // no services ORDER.clear(); conf = new Configuration(false); server = new Server("server", dir, dir, dir, dir, conf); server.init(); assertEquals(ORDER.size(), 0); // 2 services init/destroy ORDER.clear(); String services = StringUtils.join( ",", Arrays.asList(MyService1.class.getName(), MyService3.class.getName())); conf = new Configuration(false); conf.set("server.services", services); server = new Server("server", dir, dir, dir, dir, conf); server.init(); assertEquals(server.get(MyService1.class).getInterface(), MyService1.class); assertEquals(server.get(MyService3.class).getInterface(), MyService3.class); assertEquals(ORDER.size(), 4); assertEquals(ORDER.get(0), "s1.init"); assertEquals(ORDER.get(1), "s3.init"); assertEquals(ORDER.get(2), "s1.postInit"); assertEquals(ORDER.get(3), "s3.postInit"); server.destroy(); assertEquals(ORDER.size(), 6); assertEquals(ORDER.get(4), "s3.destroy"); assertEquals(ORDER.get(5), "s1.destroy"); // 3 services, 2nd one fails on init ORDER.clear(); services = StringUtils.join( ",", Arrays.asList( MyService1.class.getName(), MyService2.class.getName(), MyService3.class.getName())); conf = new Configuration(false); conf.set("server.services", services); server = new Server("server", dir, dir, dir, dir, conf); try { server.init(); fail(); } catch (ServerException ex) { assertEquals(MyService2.class, ex.getError().getClass()); } catch (Exception ex) { fail(); } assertEquals(ORDER.size(), 3); assertEquals(ORDER.get(0), "s1.init"); assertEquals(ORDER.get(1), "s2.init"); assertEquals(ORDER.get(2), "s1.destroy"); // 2 services one fails on destroy ORDER.clear(); services = StringUtils.join( ",", Arrays.asList(MyService1.class.getName(), MyService5.class.getName())); conf = new Configuration(false); conf.set("server.services", services); server = new Server("server", dir, dir, dir, dir, conf); server.init(); assertEquals(ORDER.size(), 4); assertEquals(ORDER.get(0), "s1.init"); assertEquals(ORDER.get(1), "s5.init"); assertEquals(ORDER.get(2), "s1.postInit"); assertEquals(ORDER.get(3), "s5.postInit"); server.destroy(); assertEquals(ORDER.size(), 6); assertEquals(ORDER.get(4), "s5.destroy"); assertEquals(ORDER.get(5), "s1.destroy"); // service override via ext ORDER.clear(); services = StringUtils.join( ",", Arrays.asList(MyService1.class.getName(), MyService3.class.getName())); String servicesExt = StringUtils.join(",", Arrays.asList(MyService1a.class.getName())); conf = new Configuration(false); conf.set("server.services", services); conf.set("server.services.ext", servicesExt); server = new Server("server", dir, dir, dir, dir, conf); server.init(); assertEquals(server.get(MyService1.class).getClass(), MyService1a.class); assertEquals(ORDER.size(), 4); assertEquals(ORDER.get(0), "s1a.init"); assertEquals(ORDER.get(1), "s3.init"); assertEquals(ORDER.get(2), "s1a.postInit"); assertEquals(ORDER.get(3), "s3.postInit"); server.destroy(); assertEquals(ORDER.size(), 6); assertEquals(ORDER.get(4), "s3.destroy"); assertEquals(ORDER.get(5), "s1a.destroy"); // service override via setService ORDER.clear(); services = StringUtils.join( ",", Arrays.asList(MyService1.class.getName(), MyService3.class.getName())); conf = new Configuration(false); conf.set("server.services", services); server = new Server("server", dir, dir, dir, dir, conf); server.init(); server.setService(MyService1a.class); assertEquals(ORDER.size(), 6); assertEquals(ORDER.get(4), "s1.destroy"); assertEquals(ORDER.get(5), "s1a.init"); assertEquals(server.get(MyService1.class).getClass(), MyService1a.class); server.destroy(); assertEquals(ORDER.size(), 8); assertEquals(ORDER.get(6), "s3.destroy"); assertEquals(ORDER.get(7), "s1a.destroy"); // service add via setService ORDER.clear(); services = StringUtils.join( ",", Arrays.asList(MyService1.class.getName(), MyService3.class.getName())); conf = new Configuration(false); conf.set("server.services", services); server = new Server("server", dir, dir, dir, dir, conf); server.init(); server.setService(MyService5.class); assertEquals(ORDER.size(), 5); assertEquals(ORDER.get(4), "s5.init"); assertEquals(server.get(MyService5.class).getClass(), MyService5.class); server.destroy(); assertEquals(ORDER.size(), 8); assertEquals(ORDER.get(5), "s5.destroy"); assertEquals(ORDER.get(6), "s3.destroy"); assertEquals(ORDER.get(7), "s1.destroy"); // service add via setService exception ORDER.clear(); services = StringUtils.join( ",", Arrays.asList(MyService1.class.getName(), MyService3.class.getName())); conf = new Configuration(false); conf.set("server.services", services); server = new Server("server", dir, dir, dir, dir, conf); server.init(); try { server.setService(MyService7.class); fail(); } catch (ServerException ex) { assertEquals(ServerException.ERROR.S09, ex.getError()); } catch (Exception ex) { fail(); } assertEquals(ORDER.size(), 6); assertEquals(ORDER.get(4), "s3.destroy"); assertEquals(ORDER.get(5), "s1.destroy"); // service with dependency ORDER.clear(); services = StringUtils.join( ",", Arrays.asList(MyService1.class.getName(), MyService6.class.getName())); conf = new Configuration(false); conf.set("server.services", services); server = new Server("server", dir, dir, dir, dir, conf); server.init(); assertEquals(server.get(MyService1.class).getInterface(), MyService1.class); assertEquals(server.get(MyService6.class).getInterface(), MyService6.class); server.destroy(); }
/** * @param taskid * @param workDir * @param classPaths * @param logSize * @return * @throws IOException */ private Vector<String> getVMArgs( TaskAttemptID taskid, File workDir, List<String> classPaths, long logSize) throws IOException { Vector<String> vargs = new Vector<String>(8); File jvm = // use same jvm as parent new File(new File(System.getProperty("java.home"), "bin"), "java"); vargs.add(jvm.toString()); // Add child (task) java-vm options. // // The following symbols if present in mapred.{map|reduce}.child.java.opts // value are replaced: // + @taskid@ is interpolated with value of TaskID. // Other occurrences of @ will not be altered. // // Example with multiple arguments and substitutions, showing // jvm GC logging, and start of a passwordless JVM JMX agent so can // connect with jconsole and the likes to watch child memory, threads // and get thread dumps. // // <property> // <name>mapred.map.child.java.opts</name> // <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@[email protected] \ // -Dcom.sun.management.jmxremote.authenticate=false \ // -Dcom.sun.management.jmxremote.ssl=false \ // </value> // </property> // // <property> // <name>mapred.reduce.child.java.opts</name> // <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@[email protected] \ // -Dcom.sun.management.jmxremote.authenticate=false \ // -Dcom.sun.management.jmxremote.ssl=false \ // </value> // </property> // String[] javaOptsSplit = parseChildJavaOpts(getChildJavaOpts(conf, JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS), taskid); // Add java.library.path; necessary for loading native libraries. // // 1. To support native-hadoop library i.e. libhadoop.so, we add the // parent processes' java.library.path to the child. // 2. We also add the 'cwd' of the task to it's java.library.path to help // users distribute native libraries via the DistributedCache. // 3. The user can also specify extra paths to be added to the // java.library.path via mapred.{map|reduce}.child.java.opts. // String libraryPath = System.getProperty("java.library.path"); if (libraryPath == null) { libraryPath = workDir.getAbsolutePath(); } else { libraryPath += SYSTEM_PATH_SEPARATOR + workDir; } boolean hasUserLDPath = false; for (int i = 0; i < javaOptsSplit.length; i++) { if (javaOptsSplit[i].startsWith("-Djava.library.path=")) { javaOptsSplit[i] += SYSTEM_PATH_SEPARATOR + libraryPath; hasUserLDPath = true; break; } } if (!hasUserLDPath) { vargs.add("-Djava.library.path=" + libraryPath); } for (int i = 0; i < javaOptsSplit.length; i++) { vargs.add(javaOptsSplit[i]); } Path childTmpDir = createChildTmpDir(workDir, conf, false); vargs.add("-Djava.io.tmpdir=" + childTmpDir); // Add classpath. vargs.add("-classpath"); String classPath = StringUtils.join(SYSTEM_PATH_SEPARATOR, classPaths); vargs.add(classPath); // Setup the log4j prop setupLog4jProperties(vargs, taskid, logSize); if (conf.getProfileEnabled()) { if (conf.getProfileTaskRange(t.isMapTask()).isIncluded(t.getPartition())) { File prof = TaskLog.getTaskLogFile(taskid, t.isTaskCleanupTask(), TaskLog.LogName.PROFILE); vargs.add(String.format(conf.getProfileParams(), prof.toString())); } } // Add main class and its arguments vargs.add(Child.class.getName()); // main of Child // pass umbilical address InetSocketAddress address = tracker.getTaskTrackerReportAddress(); vargs.add(address.getAddress().getHostAddress()); vargs.add(Integer.toString(address.getPort())); vargs.add(taskid.toString()); // pass task identifier // pass task log location vargs.add(TaskLog.getAttemptDir(taskid, t.isTaskCleanupTask()).toString()); return vargs; }
@Test(timeout = 30000) public void testCreateJarWithClassPath() throws Exception { // setup test directory for files Assert.assertFalse(tmp.exists()); Assert.assertTrue(tmp.mkdirs()); // create files expected to match a wildcard List<File> wildcardMatches = Arrays.asList( new File(tmp, "wildcard1.jar"), new File(tmp, "wildcard2.jar"), new File(tmp, "wildcard3.JAR"), new File(tmp, "wildcard4.JAR")); for (File wildcardMatch : wildcardMatches) { Assert.assertTrue("failure creating file: " + wildcardMatch, wildcardMatch.createNewFile()); } // create non-jar files, which we expect to not be included in the classpath Assert.assertTrue(new File(tmp, "text.txt").createNewFile()); Assert.assertTrue(new File(tmp, "executable.exe").createNewFile()); Assert.assertTrue(new File(tmp, "README").createNewFile()); // create classpath jar String wildcardPath = tmp.getCanonicalPath() + File.separator + "*"; String nonExistentSubdir = tmp.getCanonicalPath() + Path.SEPARATOR + "subdir" + Path.SEPARATOR; List<String> classPaths = Arrays.asList("", "cp1.jar", "cp2.jar", wildcardPath, "cp3.jar", nonExistentSubdir); String inputClassPath = StringUtils.join(File.pathSeparator, classPaths); String[] jarCp = FileUtil.createJarWithClassPath( inputClassPath + File.pathSeparator + "unexpandedwildcard/*", new Path(tmp.getCanonicalPath()), System.getenv()); String classPathJar = jarCp[0]; assertNotEquals( "Unexpanded wildcard was not placed in extra classpath", jarCp[1].indexOf("unexpanded"), -1); // verify classpath by reading manifest from jar file JarFile jarFile = null; try { jarFile = new JarFile(classPathJar); Manifest jarManifest = jarFile.getManifest(); Assert.assertNotNull(jarManifest); Attributes mainAttributes = jarManifest.getMainAttributes(); Assert.assertNotNull(mainAttributes); Assert.assertTrue(mainAttributes.containsKey(Attributes.Name.CLASS_PATH)); String classPathAttr = mainAttributes.getValue(Attributes.Name.CLASS_PATH); Assert.assertNotNull(classPathAttr); List<String> expectedClassPaths = new ArrayList<String>(); for (String classPath : classPaths) { if (classPath.length() == 0) { continue; } if (wildcardPath.equals(classPath)) { // add wildcard matches for (File wildcardMatch : wildcardMatches) { expectedClassPaths.add(wildcardMatch.toURI().toURL().toExternalForm()); } } else { File fileCp = null; if (!new Path(classPath).isAbsolute()) { fileCp = new File(tmp, classPath); } else { fileCp = new File(classPath); } if (nonExistentSubdir.equals(classPath)) { // expect to maintain trailing path separator if present in input, even // if directory doesn't exist yet expectedClassPaths.add(fileCp.toURI().toURL().toExternalForm() + Path.SEPARATOR); } else { expectedClassPaths.add(fileCp.toURI().toURL().toExternalForm()); } } } List<String> actualClassPaths = Arrays.asList(classPathAttr.split(" ")); Collections.sort(expectedClassPaths); Collections.sort(actualClassPaths); Assert.assertEquals(expectedClassPaths, actualClassPaths); } finally { if (jarFile != null) { try { jarFile.close(); } catch (IOException e) { LOG.warn("exception closing jarFile: " + classPathJar, e); } } } }
@Public @Evolving public class YarnConfiguration extends Configuration { private static final Joiner JOINER = Joiner.on(""); private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml"; private static final String YARN_SITE_XML_FILE = "yarn-site.xml"; static { Configuration.addDefaultResource(YARN_DEFAULT_XML_FILE); Configuration.addDefaultResource(YARN_SITE_XML_FILE); } // Configurations public static final String YARN_PREFIX = "yarn."; /** Delay before deleting resource to ease debugging of NM issues */ public static final String DEBUG_NM_DELETE_DELAY_SEC = YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec"; //////////////////////////////// // IPC Configs //////////////////////////////// public static final String IPC_PREFIX = YARN_PREFIX + "ipc."; /** Factory to create client IPC classes. */ public static final String IPC_CLIENT_FACTORY_CLASS = IPC_PREFIX + "client.factory.class"; public static final String DEFAULT_IPC_CLIENT_FACTORY_CLASS = "org.apache.hadoop.yarn.factories.impl.pb.RpcClientFactoryPBImpl"; /** Factory to create server IPC classes. */ public static final String IPC_SERVER_FACTORY_CLASS = IPC_PREFIX + "server.factory.class"; public static final String DEFAULT_IPC_SERVER_FACTORY_CLASS = "org.apache.hadoop.yarn.factories.impl.pb.RpcServerFactoryPBImpl"; /** Factory to create serializeable records. */ public static final String IPC_RECORD_FACTORY_CLASS = IPC_PREFIX + "record.factory.class"; public static final String DEFAULT_IPC_RECORD_FACTORY_CLASS = "org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl"; /** RPC class implementation */ public static final String IPC_RPC_IMPL = IPC_PREFIX + "rpc.class"; public static final String DEFAULT_IPC_RPC_IMPL = "org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC"; //////////////////////////////// // Resource Manager Configs //////////////////////////////// public static final String RM_PREFIX = "yarn.resourcemanager."; /** The address of the applications manager interface in the RM. */ public static final String RM_ADDRESS = RM_PREFIX + "address"; public static final int DEFAULT_RM_PORT = 8032; public static final String DEFAULT_RM_ADDRESS = "0.0.0.0:" + DEFAULT_RM_PORT; /** The number of threads used to handle applications manager requests. */ public static final String RM_CLIENT_THREAD_COUNT = RM_PREFIX + "client.thread-count"; public static final int DEFAULT_RM_CLIENT_THREAD_COUNT = 50; /** The Kerberos principal for the resource manager. */ public static final String RM_PRINCIPAL = RM_PREFIX + "principal"; /** The address of the scheduler interface. */ public static final String RM_SCHEDULER_ADDRESS = RM_PREFIX + "scheduler.address"; public static final int DEFAULT_RM_SCHEDULER_PORT = 8030; public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" + DEFAULT_RM_SCHEDULER_PORT; /** Miniumum request grant-able by the RM scheduler. */ public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_MB = YARN_PREFIX + "scheduler.minimum-allocation-mb"; public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024; public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = YARN_PREFIX + "scheduler.minimum-allocation-vcores"; public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 1; // add by lism public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_GPU_CORES = YARN_PREFIX + "scheduler.minimum-allocation-gpu-cores"; public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_GPU_CORES = 1; // end add /** Maximum request grant-able by the RM scheduler. */ public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = YARN_PREFIX + "scheduler.maximum-allocation-mb"; public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 8192; public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = YARN_PREFIX + "scheduler.maximum-allocation-vcores"; public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES = 4; // add by lism public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_GPU_CORES = YARN_PREFIX + "scheduler.maximum-allocation-gpu-cores"; public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_GPU_CORES = 4; // end add /** Number of threads to handle scheduler interface. */ public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT = RM_PREFIX + "scheduler.client.thread-count"; public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50; /** * Enable periodic monitor threads. * * @see #RM_SCHEDULER_MONITOR_POLICIES */ public static final String RM_SCHEDULER_ENABLE_MONITORS = RM_PREFIX + "scheduler.monitor.enable"; public static final boolean DEFAULT_RM_SCHEDULER_ENABLE_MONITORS = false; /** List of SchedulingEditPolicy classes affecting the scheduler. */ public static final String RM_SCHEDULER_MONITOR_POLICIES = RM_PREFIX + "scheduler.monitor.policies"; /** The address of the RM web application. */ public static final String RM_WEBAPP_ADDRESS = RM_PREFIX + "webapp.address"; public static final int DEFAULT_RM_WEBAPP_PORT = 8088; public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:" + DEFAULT_RM_WEBAPP_PORT; public static final String RM_RESOURCE_TRACKER_ADDRESS = RM_PREFIX + "resource-tracker.address"; public static final int DEFAULT_RM_RESOURCE_TRACKER_PORT = 8031; public static final String DEFAULT_RM_RESOURCE_TRACKER_ADDRESS = "0.0.0.0:" + DEFAULT_RM_RESOURCE_TRACKER_PORT; /** The expiry interval for application master reporting. */ public static final String RM_AM_EXPIRY_INTERVAL_MS = YARN_PREFIX + "am.liveness-monitor.expiry-interval-ms"; public static final int DEFAULT_RM_AM_EXPIRY_INTERVAL_MS = 600000; /** How long to wait until a node manager is considered dead. */ public static final String RM_NM_EXPIRY_INTERVAL_MS = YARN_PREFIX + "nm.liveness-monitor.expiry-interval-ms"; public static final int DEFAULT_RM_NM_EXPIRY_INTERVAL_MS = 600000; /** Are acls enabled. */ public static final String YARN_ACL_ENABLE = YARN_PREFIX + "acl.enable"; public static final boolean DEFAULT_YARN_ACL_ENABLE = false; /** ACL of who can be admin of YARN cluster. */ public static final String YARN_ADMIN_ACL = YARN_PREFIX + "admin.acl"; public static final String DEFAULT_YARN_ADMIN_ACL = "*"; /** ACL used in case none is found. Allows nothing. */ public static final String DEFAULT_YARN_APP_ACL = " "; /** The address of the RM admin interface. */ public static final String RM_ADMIN_ADDRESS = RM_PREFIX + "admin.address"; public static final int DEFAULT_RM_ADMIN_PORT = 8033; public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:" + DEFAULT_RM_ADMIN_PORT; /** Number of threads used to handle RM admin interface. */ public static final String RM_ADMIN_CLIENT_THREAD_COUNT = RM_PREFIX + "admin.client.thread-count"; public static final int DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT = 1; /** * The maximum number of application attempts. It's a global setting for all application masters. */ public static final String RM_AM_MAX_ATTEMPTS = RM_PREFIX + "am.max-attempts"; public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 2; /** The keytab for the resource manager. */ public static final String RM_KEYTAB = RM_PREFIX + "keytab"; /** How long to wait until a container is considered dead. */ public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = RM_PREFIX + "rm.container-allocation.expiry-interval-ms"; public static final int DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = 600000; /** Path to file with nodes to include. */ public static final String RM_NODES_INCLUDE_FILE_PATH = RM_PREFIX + "nodes.include-path"; public static final String DEFAULT_RM_NODES_INCLUDE_FILE_PATH = ""; /** Path to file with nodes to exclude. */ public static final String RM_NODES_EXCLUDE_FILE_PATH = RM_PREFIX + "nodes.exclude-path"; public static final String DEFAULT_RM_NODES_EXCLUDE_FILE_PATH = ""; /** Number of threads to handle resource tracker calls. */ public static final String RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = RM_PREFIX + "resource-tracker.client.thread-count"; public static final int DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT = 50; /** The class to use as the resource scheduler. */ public static final String RM_SCHEDULER = RM_PREFIX + "scheduler.class"; public static final String DEFAULT_RM_SCHEDULER = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"; /** RM set next Heartbeat interval for NM */ public static final String RM_NM_HEARTBEAT_INTERVAL_MS = RM_PREFIX + "nodemanagers.heartbeat-interval-ms"; public static final long DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS = 1000; // Delegation token related keys public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY = RM_PREFIX + "delegation.key.update-interval"; public static final long DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 24 * 60 * 60 * 1000; // 1 day public static final String DELEGATION_TOKEN_RENEW_INTERVAL_KEY = RM_PREFIX + "delegation.token.renew-interval"; public static final long DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24 * 60 * 60 * 1000; // 1 day public static final String DELEGATION_TOKEN_MAX_LIFETIME_KEY = RM_PREFIX + "delegation.token.max-lifetime"; public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60 * 1000; // 7 days public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled"; public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false; /** The class to use as the persistent store. */ public static final String RM_STORE = RM_PREFIX + "store.class"; /** URI for FileSystemRMStateStore */ public static final String FS_RM_STATE_STORE_URI = RM_PREFIX + "fs.state-store.uri"; /** The maximum number of completed applications RM keeps. */ public static final String RM_MAX_COMPLETED_APPLICATIONS = RM_PREFIX + "max-completed-applications"; public static final int DEFAULT_RM_MAX_COMPLETED_APPLICATIONS = 10000; /** Default application name */ public static final String DEFAULT_APPLICATION_NAME = "N/A"; /** Default application type */ public static final String DEFAULT_APPLICATION_TYPE = "YARN"; /** Default application type length */ public static final int APPLICATION_TYPE_LENGTH = 20; /** Default queue name */ public static final String DEFAULT_QUEUE_NAME = "default"; /** Buckets (in minutes) for the number of apps running in each queue. */ public static final String RM_METRICS_RUNTIME_BUCKETS = RM_PREFIX + "metrics.runtime.buckets"; /** Default sizes of the runtime metric buckets in minutes. */ public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS = "60,300,1440"; public static final String RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX + "am-rm-tokens.master-key-rolling-interval-secs"; public static final long DEFAULT_RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 24 * 60 * 60; public static final String RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX + "container-tokens.master-key-rolling-interval-secs"; public static final long DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 24 * 60 * 60; public static final String RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = RM_PREFIX + "nm-tokens.master-key-rolling-interval-secs"; public static final long DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS = 24 * 60 * 60; //////////////////////////////// // Node Manager Configs //////////////////////////////// /** Prefix for all node manager configs. */ public static final String NM_PREFIX = "yarn.nodemanager."; /** Environment variables that will be sent to containers. */ public static final String NM_ADMIN_USER_ENV = NM_PREFIX + "admin-env"; public static final String DEFAULT_NM_ADMIN_USER_ENV = "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX"; /** Environment variables that containers may override rather than use NodeManager's default. */ public static final String NM_ENV_WHITELIST = NM_PREFIX + "env-whitelist"; public static final String DEFAULT_NM_ENV_WHITELIST = StringUtils.join( ",", Arrays.asList( ApplicationConstants.Environment.JAVA_HOME.key(), ApplicationConstants.Environment.HADOOP_COMMON_HOME.key(), ApplicationConstants.Environment.HADOOP_HDFS_HOME.key(), ApplicationConstants.Environment.HADOOP_CONF_DIR.key(), ApplicationConstants.Environment.HADOOP_YARN_HOME.key())); /** address of node manager IPC. */ public static final String NM_ADDRESS = NM_PREFIX + "address"; public static final int DEFAULT_NM_PORT = 0; public static final String DEFAULT_NM_ADDRESS = "0.0.0.0:" + DEFAULT_NM_PORT; /** who will execute(launch) the containers. */ public static final String NM_CONTAINER_EXECUTOR = NM_PREFIX + "container-executor.class"; /** * Adjustment to make to the container os scheduling priority. The valid values for this could * vary depending on the platform. On Linux, higher values mean run the containers at a less * favorable priority than the NM. The value specified is an int. */ public static final String NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = NM_PREFIX + "container-executor.os.sched.priority.adjustment"; public static final int DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY = 0; /** Number of threads container manager uses. */ public static final String NM_CONTAINER_MGR_THREAD_COUNT = NM_PREFIX + "container-manager.thread-count"; public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20; /** Number of threads used in cleanup. */ public static final String NM_DELETE_THREAD_COUNT = NM_PREFIX + "delete.thread-count"; public static final int DEFAULT_NM_DELETE_THREAD_COUNT = 4; /** Keytab for NM. */ public static final String NM_KEYTAB = NM_PREFIX + "keytab"; /** List of directories to store localized files in. */ public static final String NM_LOCAL_DIRS = NM_PREFIX + "local-dirs"; public static final String DEFAULT_NM_LOCAL_DIRS = "/tmp/nm-local-dir"; /** Number of files in each localized directories Avoid tuning this too low. */ public static final String NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = NM_PREFIX + "local-cache.max-files-per-directory"; public static final int DEFAULT_NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY = 8192; /** Address where the localizer IPC is. */ public static final String NM_LOCALIZER_ADDRESS = NM_PREFIX + "localizer.address"; public static final int DEFAULT_NM_LOCALIZER_PORT = 8040; public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" + DEFAULT_NM_LOCALIZER_PORT; /** Interval in between cache cleanups. */ public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = NM_PREFIX + "localizer.cache.cleanup.interval-ms"; public static final long DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS = 10 * 60 * 1000; /** Target size of localizer cache in MB, per local directory. */ public static final String NM_LOCALIZER_CACHE_TARGET_SIZE_MB = NM_PREFIX + "localizer.cache.target-size-mb"; public static final long DEFAULT_NM_LOCALIZER_CACHE_TARGET_SIZE_MB = 10 * 1024; /** Number of threads to handle localization requests. */ public static final String NM_LOCALIZER_CLIENT_THREAD_COUNT = NM_PREFIX + "localizer.client.thread-count"; public static final int DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT = 5; /** Number of threads to use for localization fetching. */ public static final String NM_LOCALIZER_FETCH_THREAD_COUNT = NM_PREFIX + "localizer.fetch.thread-count"; public static final int DEFAULT_NM_LOCALIZER_FETCH_THREAD_COUNT = 4; /** Where to store container logs. */ public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs"; public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs"; /** Interval at which the delayed token removal thread runs */ public static final String RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS = RM_PREFIX + "delayed.delegation-token.removal-interval-ms"; public static final long DEFAULT_RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS = 30000l; /** Whether to enable log aggregation */ public static final String LOG_AGGREGATION_ENABLED = YARN_PREFIX + "log-aggregation-enable"; public static final boolean DEFAULT_LOG_AGGREGATION_ENABLED = false; /** * How long to wait before deleting aggregated logs, -1 disables. Be careful set this too small * and you will spam the name node. */ public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX + "log-aggregation.retain-seconds"; public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1; /** * How long to wait between aggregated log retention checks. If set to a value <= 0 then the value * is computed as one-tenth of the log retention setting. Be careful set this too small and you * will spam the name node. */ public static final String LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = YARN_PREFIX + "log-aggregation.retain-check-interval-seconds"; public static final long DEFAULT_LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS = -1; /** * Number of seconds to retain logs on the NodeManager. Only applicable if Log aggregation is * disabled */ public static final String NM_LOG_RETAIN_SECONDS = NM_PREFIX + "log.retain-seconds"; /** Number of threads used in log cleanup. Only applicable if Log aggregation is disabled */ public static final String NM_LOG_DELETION_THREADS_COUNT = NM_PREFIX + "log.deletion-threads-count"; public static final int DEFAULT_NM_LOG_DELETE_THREAD_COUNT = 4; /** Where to aggregate logs to. */ public static final String NM_REMOTE_APP_LOG_DIR = NM_PREFIX + "remote-app-log-dir"; public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR = "/tmp/logs"; /** * The remote log dir will be created at * NM_REMOTE_APP_LOG_DIR/${user}/NM_REMOTE_APP_LOG_DIR_SUFFIX/${appId} */ public static final String NM_REMOTE_APP_LOG_DIR_SUFFIX = NM_PREFIX + "remote-app-log-dir-suffix"; public static final String DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX = "logs"; public static final String YARN_LOG_SERVER_URL = YARN_PREFIX + "log.server.url"; public static final String YARN_TRACKING_URL_GENERATOR = YARN_PREFIX + "tracking.url.generator"; /** Amount of memory in GB that can be allocated for containers. */ public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb"; public static final int DEFAULT_NM_PMEM_MB = 8 * 1024; /** Specifies whether physical memory check is enabled. */ public static final String NM_PMEM_CHECK_ENABLED = NM_PREFIX + "pmem-check-enabled"; public static final boolean DEFAULT_NM_PMEM_CHECK_ENABLED = true; /** Specifies whether physical memory check is enabled. */ public static final String NM_VMEM_CHECK_ENABLED = NM_PREFIX + "vmem-check-enabled"; public static final boolean DEFAULT_NM_VMEM_CHECK_ENABLED = true; /** Conversion ratio for physical memory to virtual memory. */ public static final String NM_VMEM_PMEM_RATIO = NM_PREFIX + "vmem-pmem-ratio"; public static final float DEFAULT_NM_VMEM_PMEM_RATIO = 2.1f; /** Number of Virtual CPU Cores which can be allocated for containers. */ public static final String NM_VCORES = NM_PREFIX + "resource.cpu-vcores"; public static final int DEFAULT_NM_VCORES = 8; // add by lism public static final String NM_GPU_CORES = NM_PREFIX + "resource.gpu-cores"; public static final int DEFAULT_NM_GPU_CORES = 0; // end lism /** NM Webapp address.* */ public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address"; public static final int DEFAULT_NM_WEBAPP_PORT = 8042; public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:" + DEFAULT_NM_WEBAPP_PORT; /** How often to monitor containers. */ public static final String NM_CONTAINER_MON_INTERVAL_MS = NM_PREFIX + "container-monitor.interval-ms"; public static final int DEFAULT_NM_CONTAINER_MON_INTERVAL_MS = 3000; /** Class that calculates containers current resource utilization. */ public static final String NM_CONTAINER_MON_RESOURCE_CALCULATOR = NM_PREFIX + "container-monitor.resource-calculator.class"; /** Class that calculates process tree resource utilization. */ public static final String NM_CONTAINER_MON_PROCESS_TREE = NM_PREFIX + "container-monitor.process-tree.class"; /** * Enable/Disable disks' health checker. Default is true. An expert level configuration property. */ public static final String NM_DISK_HEALTH_CHECK_ENABLE = NM_PREFIX + "disk-health-checker.enable"; /** Frequency of running disks' health checker. */ public static final String NM_DISK_HEALTH_CHECK_INTERVAL_MS = NM_PREFIX + "disk-health-checker.interval-ms"; /** By default, disks' health is checked every 2 minutes. */ public static final long DEFAULT_NM_DISK_HEALTH_CHECK_INTERVAL_MS = 2 * 60 * 1000; /** * The minimum fraction of number of disks to be healthy for the nodemanager to launch new * containers. This applies to nm-local-dirs and nm-log-dirs. */ public static final String NM_MIN_HEALTHY_DISKS_FRACTION = NM_PREFIX + "disk-health-checker.min-healthy-disks"; /** * By default, at least 5% of disks are to be healthy to say that the node is healthy in terms of * disks. */ public static final float DEFAULT_NM_MIN_HEALTHY_DISKS_FRACTION = 0.25F; /** Frequency of running node health script. */ public static final String NM_HEALTH_CHECK_INTERVAL_MS = NM_PREFIX + "health-checker.interval-ms"; public static final long DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS = 10 * 60 * 1000; /** Health check script time out period. */ public static final String NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = NM_PREFIX + "health-checker.script.timeout-ms"; public static final long DEFAULT_NM_HEALTH_CHECK_SCRIPT_TIMEOUT_MS = 2 * DEFAULT_NM_HEALTH_CHECK_INTERVAL_MS; /** The health check script to run. */ public static final String NM_HEALTH_CHECK_SCRIPT_PATH = NM_PREFIX + "health-checker.script.path"; /** The arguments to pass to the health check script. */ public static final String NM_HEALTH_CHECK_SCRIPT_OPTS = NM_PREFIX + "health-checker.script.opts"; /** The path to the Linux container executor. */ public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH = NM_PREFIX + "linux-container-executor.path"; /** * The UNIX group that the linux-container-executor should run as. This is intended to be set as * part of container-executor.cfg. */ public static final String NM_LINUX_CONTAINER_GROUP = NM_PREFIX + "linux-container-executor.group"; /** The type of resource enforcement to use with the linux container executor. */ public static final String NM_LINUX_CONTAINER_RESOURCES_HANDLER = NM_PREFIX + "linux-container-executor.resources-handler.class"; /** The path the linux container executor should use for cgroups */ public static final String NM_LINUX_CONTAINER_CGROUPS_HIERARCHY = NM_PREFIX + "linux-container-executor.cgroups.hierarchy"; /** Whether the linux container executor should mount cgroups if not found */ public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT = NM_PREFIX + "linux-container-executor.cgroups.mount"; /** Where the linux container executor should mount cgroups if not found */ public static final String NM_LINUX_CONTAINER_CGROUPS_MOUNT_PATH = NM_PREFIX + "linux-container-executor.cgroups.mount-path"; /** T-file compression types used to compress aggregated logs. */ public static final String NM_LOG_AGG_COMPRESSION_TYPE = NM_PREFIX + "log-aggregation.compression-type"; public static final String DEFAULT_NM_LOG_AGG_COMPRESSION_TYPE = "none"; /** The kerberos principal for the node manager. */ public static final String NM_PRINCIPAL = NM_PREFIX + "principal"; public static final String NM_AUX_SERVICES = NM_PREFIX + "aux-services"; public static final String NM_AUX_SERVICE_FMT = NM_PREFIX + "aux-services.%s.class"; public static final String NM_USER_HOME_DIR = NM_PREFIX + "user-home-dir"; public static final String DEFAULT_NM_USER_HOME_DIR = "/home/"; //////////////////////////////// // Web Proxy Configs //////////////////////////////// public static final String PROXY_PREFIX = "yarn.web-proxy."; /** The kerberos principal for the proxy. */ public static final String PROXY_PRINCIPAL = PROXY_PREFIX + "principal"; /** Keytab for Proxy. */ public static final String PROXY_KEYTAB = PROXY_PREFIX + "keytab"; /** The address for the web proxy. */ public static final String PROXY_ADDRESS = PROXY_PREFIX + "address"; /** YARN Service Level Authorization */ public static final String YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL = "security.resourcetracker.protocol.acl"; public static final String YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONCLIENT_PROTOCOL = "security.applicationclient.protocol.acl"; public static final String YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL = "security.resourcemanager-administration.protocol.acl"; public static final String YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL = "security.applicationmaster.protocol.acl"; public static final String YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL = "security.containermanagement.protocol.acl"; public static final String YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCE_LOCALIZER = "security.resourcelocalizer.protocol.acl"; /** No. of milliseconds to wait between sending a SIGTERM and SIGKILL to a running container */ public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = NM_PREFIX + "sleep-delay-before-sigkill.ms"; public static final long DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS = 250; /** Max time to wait for a process to come up when trying to cleanup container resources */ public static final String NM_PROCESS_KILL_WAIT_MS = NM_PREFIX + "process-kill-wait.ms"; public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS = 2000; /** Max time to wait to establish a connection to RM */ public static final String RESOURCEMANAGER_CONNECT_MAX_WAIT_MS = RM_PREFIX + "connect.max-wait.ms"; public static final int DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS = 15 * 60 * 1000; /** Time interval between each attempt to connect to RM */ public static final String RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS = RM_PREFIX + "connect.retry-interval.ms"; public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS = 30 * 1000; /** CLASSPATH for YARN applications. A comma-separated list of CLASSPATH entries */ public static final String YARN_APPLICATION_CLASSPATH = YARN_PREFIX + "application.classpath"; /** Default CLASSPATH for YARN applications. A comma-separated list of CLASSPATH entries */ public static final String[] DEFAULT_YARN_APPLICATION_CLASSPATH = { ApplicationConstants.Environment.HADOOP_CONF_DIR.$(), ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() + "/share/hadoop/common/*", ApplicationConstants.Environment.HADOOP_COMMON_HOME.$() + "/share/hadoop/common/lib/*", ApplicationConstants.Environment.HADOOP_HDFS_HOME.$() + "/share/hadoop/hdfs/*", ApplicationConstants.Environment.HADOOP_HDFS_HOME.$() + "/share/hadoop/hdfs/lib/*", ApplicationConstants.Environment.HADOOP_YARN_HOME.$() + "/share/hadoop/yarn/*", ApplicationConstants.Environment.HADOOP_YARN_HOME.$() + "/share/hadoop/yarn/lib/*" }; /** Container temp directory */ public static final String DEFAULT_CONTAINER_TEMP_DIR = "./tmp"; public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX + "is.minicluster"; /** Whether to use fixed ports with the minicluster. */ public static final String YARN_MINICLUSTER_FIXED_PORTS = YARN_PREFIX + "minicluster.fixed.ports"; /** Default is false to be able to run tests concurrently without port conflicts. */ public static boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false; /** * Whether users are explicitly trying to control resource monitoring configuration for the * MiniYARNCluster. Disabled by default. */ public static final String YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = YARN_PREFIX + "minicluster.control-resource-monitoring"; public static final boolean DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING = false; /** The log directory for the containers */ public static final String YARN_APP_CONTAINER_LOG_DIR = YARN_PREFIX + "app.container.log.dir"; public static final String YARN_APP_CONTAINER_LOG_SIZE = YARN_PREFIX + "app.container.log.filesize"; //////////////////////////////// // Other Configs //////////////////////////////// /** * The interval of the yarn client's querying application state after application submission. The * unit is millisecond. */ public static final String YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS = YARN_PREFIX + "client.app-submission.poll-interval"; public static final long DEFAULT_YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS = 1000; /** Max number of threads in NMClientAsync to process container management events */ public static final String NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE = YARN_PREFIX + "client.nodemanager-client-async.thread-pool-max-size"; public static final int DEFAULT_NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE = 500; /** * Maximum number of proxy connections for node manager. It should always be more than 1. NMClient * and MRAppMaster will use this to cache connection with node manager. There will be at max one * connection per node manager. Ex. configuring it to a value of 5 will make sure that client will * at max have 5 connections cached with 5 different node managers. These connections will be * timed out if idle for more than system wide idle timeout period. The token if used for * authentication then it will be used only at connection creation time. If new token is received * then earlier connection should be closed in order to use newer token. Note: {@link * YarnConfiguration#NM_CLIENT_ASYNC_THREAD_POOL_MAX_SIZE} are related to each other. */ public static final String NM_CLIENT_MAX_NM_PROXIES = YARN_PREFIX + "client.max-nodemanagers-proxies"; public static final int DEFAULT_NM_CLIENT_MAX_NM_PROXIES = 500; public YarnConfiguration() { super(); } public YarnConfiguration(Configuration conf) { super(conf); if (!(conf instanceof YarnConfiguration)) { this.reloadConfiguration(); } } public static String getProxyHostAndPort(Configuration conf) { String addr = conf.get(PROXY_ADDRESS); if (addr == null || addr.isEmpty()) { addr = getRMWebAppHostAndPort(conf); } return addr; } public static String getRMWebAppHostAndPort(Configuration conf) { InetSocketAddress address = conf.getSocketAddr( YarnConfiguration.RM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_PORT); address = NetUtils.getConnectAddress(address); StringBuffer sb = new StringBuffer(); InetAddress resolved = address.getAddress(); if (resolved == null || resolved.isAnyLocalAddress() || resolved.isLoopbackAddress()) { String lh = address.getHostName(); try { lh = InetAddress.getLocalHost().getCanonicalHostName(); } catch (UnknownHostException e) { // Ignore and fallback. } sb.append(lh); } else { sb.append(address.getHostName()); } sb.append(":").append(address.getPort()); return sb.toString(); } public static String getRMWebAppURL(Configuration conf) { return JOINER.join("http://", getRMWebAppHostAndPort(conf)); } }