@Override
public String dump_config() throws TException {
HiveConf c = new HiveConf();
ByteArrayOutputStream b = new ByteArrayOutputStream();
try {
c.writeXml(b);
return new String(b.toByteArray(), "UTF-8");
} catch (IOException e) {
throw new TException(e);
}
}
/**
* Given a Hive Configuration object - generate a command line fragment for passing such
* configuration information to ExecDriver.
*/
public static String generateCmdLine(HiveConf hconf, Context ctx) throws IOException {
HiveConf tempConf = new HiveConf();
Path hConfFilePath = new Path(ctx.getLocalTmpPath(), JOBCONF_FILENAME);
OutputStream out = null;
Properties deltaP = hconf.getChangedProperties();
boolean hadoopLocalMode = ShimLoader.getHadoopShims().isLocalMode(hconf);
String hadoopSysDir = "mapred.system.dir";
String hadoopWorkDir = "mapred.local.dir";
for (Object one : deltaP.keySet()) {
String oneProp = (String) one;
if (hadoopLocalMode && (oneProp.equals(hadoopSysDir) || oneProp.equals(hadoopWorkDir))) {
continue;
}
tempConf.set(oneProp, hconf.get(oneProp));
}
// Multiple concurrent local mode job submissions can cause collisions in
// working dirs and system dirs
// Workaround is to rename map red working dir to a temp dir in such cases
if (hadoopLocalMode) {
tempConf.set(hadoopSysDir, hconf.get(hadoopSysDir) + "/" + Utilities.randGen.nextInt());
tempConf.set(hadoopWorkDir, hconf.get(hadoopWorkDir) + "/" + Utilities.randGen.nextInt());
}
try {
out = FileSystem.getLocal(hconf).create(hConfFilePath);
tempConf.writeXml(out);
} finally {
if (out != null) {
out.close();
}
}
return " -jobconffile " + hConfFilePath.toString();
}
public static HiveServer create(
Map<String, String> properties,
File baseDir,
File confDir,
File logDir,
FileSystem fileSystem)
throws Exception {
if (!properties.containsKey(WAREHOUSE_DIR)) {
LOGGER.info("fileSystem " + fileSystem.getClass().getSimpleName());
if (fileSystem instanceof DistributedFileSystem) {
String dfsUri = FileSystem.getDefaultUri(fileSystem.getConf()).toString();
LOGGER.info("dfsUri " + dfsUri);
properties.put(WAREHOUSE_DIR, dfsUri + "/data");
fileSystem.mkdirs(new Path("/data/"), new FsPermission((short) 0777));
} else {
properties.put(WAREHOUSE_DIR, new File(baseDir, "warehouse").getPath());
fileSystem.mkdirs(new Path("/", "warehouse"), new FsPermission((short) 0777));
}
}
LOGGER.info("Setting an readable path to hive.exec.scratchdir");
properties.put("hive.exec.scratchdir", new File(baseDir, "scratchdir").getPath());
if (!properties.containsKey(METASTORE_CONNECTION_URL)) {
properties.put(
METASTORE_CONNECTION_URL,
String.format(
"jdbc:derby:;databaseName=%s;create=true", new File(baseDir, "metastore").getPath()));
}
if (!properties.containsKey(HS2_PORT)) {
properties.put(HS2_PORT, String.valueOf(findPort()));
}
if (!properties.containsKey(SUPPORT_CONCURRENCY)) {
properties.put(SUPPORT_CONCURRENCY, "false");
}
if (!properties.containsKey(HADOOPBIN)) {
properties.put(HADOOPBIN, "./target/test-classes/hadoop");
}
// Modify the test resource to have executable permission
java.nio.file.Path hadoopPath =
FileSystems.getDefault().getPath("target/test-classes", "hadoop");
if (hadoopPath != null) {
hadoopPath.toFile().setExecutable(true);
}
properties.put(METASTORE_SETUGI, "true");
properties.put(METASTORE_CLIENT_TIMEOUT, "100");
properties.put(ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS.varname, "true");
properties.put(ConfVars.HIVESTATSAUTOGATHER.varname, "false");
properties.put(ConfVars.HIVE_STATS_COLLECT_SCANCOLS.varname, "true");
String hadoopBinPath = properties.get(HADOOPBIN);
Assert.assertNotNull(hadoopBinPath, "Hadoop Bin");
File hadoopBin = new File(hadoopBinPath);
if (!hadoopBin.isFile()) {
Assert.fail(
"Path to hadoop bin "
+ hadoopBin.getPath()
+ " is invalid. "
+ "Perhaps you missed the download-hadoop profile.");
}
/*
* This hack, setting the hiveSiteURL field removes a previous hack involving
* setting of system properties for each property. Although both are hacks,
* I prefer this hack because once the system properties are set they can
* affect later tests unless those tests clear them. This hack allows for
* a clean switch to a new set of defaults when a new HiveConf object is created.
*/
Reflection.staticField("hiveSiteURL").ofType(URL.class).in(HiveConf.class).set(null);
HiveConf hiveConf = new HiveConf();
for (Map.Entry<String, String> entry : properties.entrySet()) {
LOGGER.info(entry.getKey() + " => " + entry.getValue());
hiveConf.set(entry.getKey(), entry.getValue());
}
File hiveSite = new File(confDir, "hive-site.xml");
hiveConf.set(HIVESERVER2_IMPERSONATION, "false");
OutputStream out = new FileOutputStream(hiveSite);
hiveConf.writeXml(out);
out.close();
Reflection.staticField("hiveSiteURL")
.ofType(URL.class)
.in(HiveConf.class)
.set(hiveSite.toURI().toURL());
LOGGER.info("Creating InternalHiveServer");
return new InternalHiveServer(hiveConf);
}
