Esempio n. 1
0
  public void createSources() throws Exception {
    if (!isSessionStateStarted) {
      startSessionState();
    }
    conf.setBoolean("hive.test.init.phase", true);

    if (cliDriver == null) {
      cliDriver = new CliDriver();
    }
    cliDriver.processLine("set test.data.dir=" + testFiles + ";");

    conf.setBoolean("hive.test.init.phase", false);
  }
Esempio n. 2
0
  public static HiveConf getHiveConf(Configuration conf) throws IOException {

    HiveConf hiveConf = new HiveConf(conf, HCatUtil.class);

    // copy the hive conf into the job conf and restore it
    // in the backend context
    if (conf.get(HCatConstants.HCAT_KEY_HIVE_CONF) == null) {
      conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(hiveConf.getAllProperties()));
    } else {
      // Copy configuration properties into the hive conf
      Properties properties =
          (Properties) HCatUtil.deserialize(conf.get(HCatConstants.HCAT_KEY_HIVE_CONF));

      for (Map.Entry<Object, Object> prop : properties.entrySet()) {
        if (prop.getValue() instanceof String) {
          hiveConf.set((String) prop.getKey(), (String) prop.getValue());
        } else if (prop.getValue() instanceof Integer) {
          hiveConf.setInt((String) prop.getKey(), (Integer) prop.getValue());
        } else if (prop.getValue() instanceof Boolean) {
          hiveConf.setBoolean((String) prop.getKey(), (Boolean) prop.getValue());
        } else if (prop.getValue() instanceof Long) {
          hiveConf.setLong((String) prop.getKey(), (Long) prop.getValue());
        } else if (prop.getValue() instanceof Float) {
          hiveConf.setFloat((String) prop.getKey(), (Float) prop.getValue());
        }
      }
    }

    if (conf.get(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE) != null) {
      hiveConf.set(
          "hive.metastore.token.signature", conf.get(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE));
    }

    return hiveConf;
  }
 public TestHadoop20SAuthBridge(String name) {
   super(name);
   System.setProperty(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL.varname, "true");
   System.setProperty(HiveConf.ConfVars.METASTOREURIS.varname, "thrift://localhost:" + port);
   System.setProperty(
       HiveConf.ConfVars.METASTOREWAREHOUSE.varname,
       new Path(System.getProperty("test.build.data", "/tmp")).toString());
   conf = new HiveConf(TestHadoop20SAuthBridge.class);
   conf.setBoolean("hive.metastore.local", false);
 }
Esempio n. 4
0
  /**
   * Creates and initializes a JobConf object that can be used to execute the DAG. The configuration
   * object will contain configurations from mapred-site overlaid with key/value pairs from the
   * hiveConf object. Finally it will also contain some hive specific configurations that do not
   * change from DAG to DAG.
   *
   * @param hiveConf Current hiveConf for the execution
   * @return JobConf base configuration for job execution
   * @throws IOException
   */
  public JobConf createConfiguration(HiveConf hiveConf) throws IOException {
    hiveConf.setBoolean("mapred.mapper.new-api", false);

    JobConf conf = (JobConf) MRHelpers.getBaseMRConfiguration(hiveConf);

    conf.set("mapred.output.committer.class", NullOutputCommitter.class.getName());

    conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);
    conf.setBoolean("mapred.committer.job.task.cleanup.needed", false);

    conf.setClass("mapred.output.format.class", HiveOutputFormatImpl.class, OutputFormat.class);

    conf.set(MRJobConfig.OUTPUT_KEY_CLASS, HiveKey.class.getName());
    conf.set(MRJobConfig.OUTPUT_VALUE_CLASS, BytesWritable.class.getName());

    conf.set("mapred.partitioner.class", HiveConf.getVar(conf, HiveConf.ConfVars.HIVEPARTITIONER));
    conf.set("tez.runtime.partitioner.class", MRPartitioner.class.getName());

    return conf;
  }