public void createSources() throws Exception {
if (!isSessionStateStarted) {
startSessionState();
}
conf.setBoolean("hive.test.init.phase", true);
if (cliDriver == null) {
cliDriver = new CliDriver();
}
cliDriver.processLine("set test.data.dir=" + testFiles + ";");
conf.setBoolean("hive.test.init.phase", false);
}
public static HiveConf getHiveConf(Configuration conf) throws IOException {
HiveConf hiveConf = new HiveConf(conf, HCatUtil.class);
// copy the hive conf into the job conf and restore it
// in the backend context
if (conf.get(HCatConstants.HCAT_KEY_HIVE_CONF) == null) {
conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(hiveConf.getAllProperties()));
} else {
// Copy configuration properties into the hive conf
Properties properties =
(Properties) HCatUtil.deserialize(conf.get(HCatConstants.HCAT_KEY_HIVE_CONF));
for (Map.Entry<Object, Object> prop : properties.entrySet()) {
if (prop.getValue() instanceof String) {
hiveConf.set((String) prop.getKey(), (String) prop.getValue());
} else if (prop.getValue() instanceof Integer) {
hiveConf.setInt((String) prop.getKey(), (Integer) prop.getValue());
} else if (prop.getValue() instanceof Boolean) {
hiveConf.setBoolean((String) prop.getKey(), (Boolean) prop.getValue());
} else if (prop.getValue() instanceof Long) {
hiveConf.setLong((String) prop.getKey(), (Long) prop.getValue());
} else if (prop.getValue() instanceof Float) {
hiveConf.setFloat((String) prop.getKey(), (Float) prop.getValue());
}
}
}
if (conf.get(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE) != null) {
hiveConf.set(
"hive.metastore.token.signature", conf.get(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE));
}
return hiveConf;
}
public TestHadoop20SAuthBridge(String name) {
super(name);
System.setProperty(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.varname, "true");
System.setProperty(HiveConf.ConfVars.METASTOREURIS.varname, "thrift://localhost:" + port);
System.setProperty(
HiveConf.ConfVars.METASTOREWAREHOUSE.varname,
new Path(System.getProperty("test.build.data", "/tmp")).toString());
conf = new HiveConf(TestHadoop20SAuthBridge.class);
conf.setBoolean("hive.metastore.local", false);
}
/**
* Creates and initializes a JobConf object that can be used to execute the DAG. The configuration
* object will contain configurations from mapred-site overlaid with key/value pairs from the
* hiveConf object. Finally it will also contain some hive specific configurations that do not
* change from DAG to DAG.
*
* @param hiveConf Current hiveConf for the execution
* @return JobConf base configuration for job execution
* @throws IOException
*/
public JobConf createConfiguration(HiveConf hiveConf) throws IOException {
hiveConf.setBoolean("mapred.mapper.new-api", false);
JobConf conf = (JobConf) MRHelpers.getBaseMRConfiguration(hiveConf);
conf.set("mapred.output.committer.class", NullOutputCommitter.class.getName());
conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);
conf.setBoolean("mapred.committer.job.task.cleanup.needed", false);
conf.setClass("mapred.output.format.class", HiveOutputFormatImpl.class, OutputFormat.class);
conf.set(MRJobConfig.OUTPUT_KEY_CLASS, HiveKey.class.getName());
conf.set(MRJobConfig.OUTPUT_VALUE_CLASS, BytesWritable.class.getName());
conf.set("mapred.partitioner.class", HiveConf.getVar(conf, HiveConf.ConfVars.HIVEPARTITIONER));
conf.set("tez.runtime.partitioner.class", MRPartitioner.class.getName());
return conf;
}