@Override public String dump_config() throws TException { HiveConf c = new HiveConf(); ByteArrayOutputStream b = new ByteArrayOutputStream(); try { c.writeXml(b); return new String(b.toByteArray(), "UTF-8"); } catch (IOException e) { throw new TException(e); } }
/** * Given a Hive Configuration object - generate a command line fragment for passing such * configuration information to ExecDriver. */ public static String generateCmdLine(HiveConf hconf, Context ctx) throws IOException { HiveConf tempConf = new HiveConf(); Path hConfFilePath = new Path(ctx.getLocalTmpPath(), JOBCONF_FILENAME); OutputStream out = null; Properties deltaP = hconf.getChangedProperties(); boolean hadoopLocalMode = ShimLoader.getHadoopShims().isLocalMode(hconf); String hadoopSysDir = "mapred.system.dir"; String hadoopWorkDir = "mapred.local.dir"; for (Object one : deltaP.keySet()) { String oneProp = (String) one; if (hadoopLocalMode && (oneProp.equals(hadoopSysDir) || oneProp.equals(hadoopWorkDir))) { continue; } tempConf.set(oneProp, hconf.get(oneProp)); } // Multiple concurrent local mode job submissions can cause collisions in // working dirs and system dirs // Workaround is to rename map red working dir to a temp dir in such cases if (hadoopLocalMode) { tempConf.set(hadoopSysDir, hconf.get(hadoopSysDir) + "/" + Utilities.randGen.nextInt()); tempConf.set(hadoopWorkDir, hconf.get(hadoopWorkDir) + "/" + Utilities.randGen.nextInt()); } try { out = FileSystem.getLocal(hconf).create(hConfFilePath); tempConf.writeXml(out); } finally { if (out != null) { out.close(); } } return " -jobconffile " + hConfFilePath.toString(); }
public static HiveServer create( Map<String, String> properties, File baseDir, File confDir, File logDir, FileSystem fileSystem) throws Exception { if (!properties.containsKey(WAREHOUSE_DIR)) { LOGGER.info("fileSystem " + fileSystem.getClass().getSimpleName()); if (fileSystem instanceof DistributedFileSystem) { String dfsUri = FileSystem.getDefaultUri(fileSystem.getConf()).toString(); LOGGER.info("dfsUri " + dfsUri); properties.put(WAREHOUSE_DIR, dfsUri + "/data"); fileSystem.mkdirs(new Path("/data/"), new FsPermission((short) 0777)); } else { properties.put(WAREHOUSE_DIR, new File(baseDir, "warehouse").getPath()); fileSystem.mkdirs(new Path("/", "warehouse"), new FsPermission((short) 0777)); } } LOGGER.info("Setting an readable path to hive.exec.scratchdir"); properties.put("hive.exec.scratchdir", new File(baseDir, "scratchdir").getPath()); if (!properties.containsKey(METASTORE_CONNECTION_URL)) { properties.put( METASTORE_CONNECTION_URL, String.format( "jdbc:derby:;databaseName=%s;create=true", new File(baseDir, "metastore").getPath())); } if (!properties.containsKey(HS2_PORT)) { properties.put(HS2_PORT, String.valueOf(findPort())); } if (!properties.containsKey(SUPPORT_CONCURRENCY)) { properties.put(SUPPORT_CONCURRENCY, "false"); } if (!properties.containsKey(HADOOPBIN)) { properties.put(HADOOPBIN, "./target/test-classes/hadoop"); } // Modify the test resource to have executable permission java.nio.file.Path hadoopPath = FileSystems.getDefault().getPath("target/test-classes", "hadoop"); if (hadoopPath != null) { hadoopPath.toFile().setExecutable(true); } properties.put(METASTORE_SETUGI, "true"); properties.put(METASTORE_CLIENT_TIMEOUT, "100"); properties.put(ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS.varname, "true"); properties.put(ConfVars.HIVESTATSAUTOGATHER.varname, "false"); properties.put(ConfVars.HIVE_STATS_COLLECT_SCANCOLS.varname, "true"); String hadoopBinPath = properties.get(HADOOPBIN); Assert.assertNotNull(hadoopBinPath, "Hadoop Bin"); File hadoopBin = new File(hadoopBinPath); if (!hadoopBin.isFile()) { Assert.fail( "Path to hadoop bin " + hadoopBin.getPath() + " is invalid. " + "Perhaps you missed the download-hadoop profile."); } /* * This hack, setting the hiveSiteURL field removes a previous hack involving * setting of system properties for each property. Although both are hacks, * I prefer this hack because once the system properties are set they can * affect later tests unless those tests clear them. This hack allows for * a clean switch to a new set of defaults when a new HiveConf object is created. */ Reflection.staticField("hiveSiteURL").ofType(URL.class).in(HiveConf.class).set(null); HiveConf hiveConf = new HiveConf(); for (Map.Entry<String, String> entry : properties.entrySet()) { LOGGER.info(entry.getKey() + " => " + entry.getValue()); hiveConf.set(entry.getKey(), entry.getValue()); } File hiveSite = new File(confDir, "hive-site.xml"); hiveConf.set(HIVESERVER2_IMPERSONATION, "false"); OutputStream out = new FileOutputStream(hiveSite); hiveConf.writeXml(out); out.close(); Reflection.staticField("hiveSiteURL") .ofType(URL.class) .in(HiveConf.class) .set(hiveSite.toURI().toURL()); LOGGER.info("Creating InternalHiveServer"); return new InternalHiveServer(hiveConf); }