Пример #1
0
  /**
   * @param db
   * @param table
   * @param filter
   * @param path
   * @param jobConf
   * @return
   */
  public static boolean setDataStorageLocation(
      String db, String table, String filter, String path, JobConf jobConf) {
    Preconditions.checkNotNull(table, "Table name must not be null");

    HiveMetaStoreClient client = null;
    List<String> locations = new ArrayList<String>();

    try {
      client = getHiveMetaStoreClient(jobConf);

      Table hiveTable = HCatUtil.getTable(client, db, table);
      hiveTable.setDataLocation(new URI(path));

      client.alter_table(db, table, hiveTable.getTTable());
    } catch (IOException e) {
      logError("Error occured when getting hiveconf", e);
    } catch (URISyntaxException e) {
      logError("Error occured when convert path to URI", e);
    } catch (MetaException e) {
      logError("Error occured when getting HiveMetaStoreClient", e);
    } catch (NoSuchObjectException e) {
      logError("Table doesn't exist in HCatalog: " + table, e);
    } catch (TException e) {
      logError("Error occured when getting Table", e);
    } finally {
      HCatUtil.closeHiveClientQuietly(client);
    }

    return true;
  }
Пример #2
0
  public static HiveConf getHiveConf(Configuration conf) throws IOException {

    HiveConf hiveConf = new HiveConf(conf, HCatUtil.class);

    // copy the hive conf into the job conf and restore it
    // in the backend context
    if (conf.get(HCatConstants.HCAT_KEY_HIVE_CONF) == null) {
      conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(hiveConf.getAllProperties()));
    } else {
      // Copy configuration properties into the hive conf
      Properties properties =
          (Properties) HCatUtil.deserialize(conf.get(HCatConstants.HCAT_KEY_HIVE_CONF));

      for (Map.Entry<Object, Object> prop : properties.entrySet()) {
        if (prop.getValue() instanceof String) {
          hiveConf.set((String) prop.getKey(), (String) prop.getValue());
        } else if (prop.getValue() instanceof Integer) {
          hiveConf.setInt((String) prop.getKey(), (Integer) prop.getValue());
        } else if (prop.getValue() instanceof Boolean) {
          hiveConf.setBoolean((String) prop.getKey(), (Boolean) prop.getValue());
        } else if (prop.getValue() instanceof Long) {
          hiveConf.setLong((String) prop.getKey(), (Long) prop.getValue());
        } else if (prop.getValue() instanceof Float) {
          hiveConf.setFloat((String) prop.getKey(), (Float) prop.getValue());
        }
      }
    }

    if (conf.get(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE) != null) {
      hiveConf.set(
          "hive.metastore.token.signature", conf.get(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE));
    }

    return hiveConf;
  }
Пример #3
0
  /**
   * @param db
   * @param table
   * @param filter
   * @param jobConf
   * @return A list of locations
   */
  public static List<String> getDataStorageLocation(
      String db, String table, String filter, JobConf jobConf) {
    Preconditions.checkNotNull(table, "Table name must not be null");

    HiveMetaStoreClient client = null;
    List<String> locations = new ArrayList<String>();

    try {
      client = getHiveMetaStoreClient(jobConf);
      Table hiveTable = HCatUtil.getTable(client, db, table);

      if (hiveTable.isPartitioned()) {
        List<Partition> parts = null;
        if (null != StringUtils.stripToNull(filter)) {
          parts = client.listPartitionsByFilter(db, table, filter, (short) -1);
        } else {
          parts = client.listPartitions(db, table, (short) -1);
        }

        if (parts.size() > 0) {
          // Return more than one partitions when filter is
          // something
          // like ds >= 1234
          for (Partition part : parts) {
            locations.addAll(getFilesInHivePartition(part, jobConf));
          }
        } else {
          logError(
              "Table "
                  + hiveTable.getTableName()
                  + " doesn't have the specified partition:"
                  + filter,
              null);
        }

      } else {
        locations.add(hiveTable.getTTable().getSd().getLocation());
      }
    } catch (IOException e) {
      logError("Error occured when getting hiveconf", e);
    } catch (MetaException e) {
      logError("Error occured when getting HiveMetaStoreClient", e);
    } catch (NoSuchObjectException e) {
      logError("Table doesn't exist in HCatalog: " + table, e);
    } catch (TException e) {
      logError("Error occured when getting Table", e);
    } finally {
      HCatUtil.closeHiveClientQuietly(client);
    }

    return locations;
  }
Пример #4
0
  @InterfaceAudience.Private
  @InterfaceStability.Evolving
  public static void configureOutputStorageHandler(
      HCatStorageHandler storageHandler, Configuration conf, OutputJobInfo outputJobInfo) {
    // TODO replace IgnoreKeyTextOutputFormat with a
    // HiveOutputFormatWrapper in StorageHandler
    TableDesc tableDesc =
        new TableDesc(
            storageHandler.getSerDeClass(),
            storageHandler.getInputFormatClass(),
            IgnoreKeyTextOutputFormat.class,
            outputJobInfo.getTableInfo().getStorerInfo().getProperties());
    if (tableDesc.getJobProperties() == null)
      tableDesc.setJobProperties(new HashMap<String, String>());
    for (Map.Entry<String, String> el : conf) {
      tableDesc.getJobProperties().put(el.getKey(), el.getValue());
    }

    Map<String, String> jobProperties = new HashMap<String, String>();
    try {
      tableDesc
          .getJobProperties()
          .put(HCatConstants.HCAT_KEY_OUTPUT_INFO, HCatUtil.serialize(outputJobInfo));

      storageHandler.configureOutputJobProperties(tableDesc, jobProperties);

      for (Map.Entry<String, String> el : jobProperties.entrySet()) {
        conf.set(el.getKey(), el.getValue());
      }
    } catch (IOException e) {
      throw new IllegalStateException("Failed to configure StorageHandler", e);
    }
  }
Пример #5
0
  public static Map<String, String> getInputJobProperties(
      HCatStorageHandler storageHandler, InputJobInfo inputJobInfo) {
    TableDesc tableDesc =
        new TableDesc(
            storageHandler.getSerDeClass(),
            storageHandler.getInputFormatClass(),
            storageHandler.getOutputFormatClass(),
            inputJobInfo.getTableInfo().getStorerInfo().getProperties());
    if (tableDesc.getJobProperties() == null) {
      tableDesc.setJobProperties(new HashMap<String, String>());
    }

    Map<String, String> jobProperties = new HashMap<String, String>();
    try {
      tableDesc
          .getJobProperties()
          .put(HCatConstants.HCAT_KEY_JOB_INFO, HCatUtil.serialize(inputJobInfo));

      storageHandler.configureInputJobProperties(tableDesc, jobProperties);

    } catch (IOException e) {
      throw new IllegalStateException("Failed to configure StorageHandler", e);
    }

    return jobProperties;
  }
Пример #6
0
 public static HCatStorageHandler getStorageHandler(Configuration conf, PartInfo partitionInfo)
     throws IOException {
   return HCatUtil.getStorageHandler(
       conf,
       partitionInfo.getStorageHandlerClassName(),
       partitionInfo.getSerdeClassName(),
       partitionInfo.getInputFormatClassName(),
       partitionInfo.getOutputFormatClassName());
 }
Пример #7
0
  public static HCatSchema buildHCatSchema(List<FieldSchema> columns) {
    HCatSchema schema = null;

    try {
      schema = new HCatSchema(HCatUtil.getHCatFieldSchemaList(columns));
    } catch (HCatException e) {
      logError("Error occured when building table schema", e);
    }

    return schema;
  }
Пример #8
0
  public static Table getHiveTable(String db, String table, JobConf conf) {
    HiveMetaStoreClient client = null;
    Table hiveTable = null;

    try {
      client = getHiveMetaStoreClient(conf);
      hiveTable = HCatUtil.getTable(client, db, table);
    } catch (IOException e) {
      logError("Error occured when getting hiveconf", e);
    } catch (MetaException e) {
      logError("Error occured when getting HiveMetaStoreClient", e);
    } catch (NoSuchObjectException e) {
      logError("Table doesn't exist in HCatalog: " + table, e);
    } catch (TException e) {
      logError("Error occured when getting Table", e);
    } finally {
      HCatUtil.closeHiveClientQuietly(client);
    }

    return hiveTable;
  }
Пример #9
0
  public static HCatSchema getTableSchemaWithPtnCols(Table table) throws IOException {
    HCatSchema tableSchema = new HCatSchema(HCatUtil.getHCatFieldSchemaList(table.getCols()));

    if (table.getPartitionKeys().size() != 0) {

      // add partition keys to table schema
      // NOTE : this assumes that we do not ever have ptn keys as columns
      // inside the table schema as well!
      for (FieldSchema fs : table.getPartitionKeys()) {
        tableSchema.append(HCatSchemaUtils.getHCatFieldSchema(fs));
      }
    }
    return tableSchema;
  }
Пример #10
0
  private static HiveMetaStoreClient getHiveMetaStoreClient(JobConf jobConf)
      throws IOException, MetaException {
    HiveConf hiveConf = HCatUtil.getHiveConf(jobConf);

    return HCatUtil.getHiveClient(hiveConf);
  }
Пример #11
0
 public static HCatSchema extractSchema(Partition partition) throws HCatException {
   return new HCatSchema(HCatUtil.getHCatFieldSchemaList(partition.getCols()));
 }
Пример #12
0
 public static HCatSchema extractSchema(Table table) throws HCatException {
   return new HCatSchema(HCatUtil.getHCatFieldSchemaList(table.getCols()));
 }