/**
* @param db
* @param table
* @param filter
* @param path
* @param jobConf
* @return
*/
public static boolean setDataStorageLocation(
String db, String table, String filter, String path, JobConf jobConf) {
Preconditions.checkNotNull(table, "Table name must not be null");
HiveMetaStoreClient client = null;
List<String> locations = new ArrayList<String>();
try {
client = getHiveMetaStoreClient(jobConf);
Table hiveTable = HCatUtil.getTable(client, db, table);
hiveTable.setDataLocation(new URI(path));
client.alter_table(db, table, hiveTable.getTTable());
} catch (IOException e) {
logError("Error occured when getting hiveconf", e);
} catch (URISyntaxException e) {
logError("Error occured when convert path to URI", e);
} catch (MetaException e) {
logError("Error occured when getting HiveMetaStoreClient", e);
} catch (NoSuchObjectException e) {
logError("Table doesn't exist in HCatalog: " + table, e);
} catch (TException e) {
logError("Error occured when getting Table", e);
} finally {
HCatUtil.closeHiveClientQuietly(client);
}
return true;
}
public static HiveConf getHiveConf(Configuration conf) throws IOException {
HiveConf hiveConf = new HiveConf(conf, HCatUtil.class);
// copy the hive conf into the job conf and restore it
// in the backend context
if (conf.get(HCatConstants.HCAT_KEY_HIVE_CONF) == null) {
conf.set(HCatConstants.HCAT_KEY_HIVE_CONF, HCatUtil.serialize(hiveConf.getAllProperties()));
} else {
// Copy configuration properties into the hive conf
Properties properties =
(Properties) HCatUtil.deserialize(conf.get(HCatConstants.HCAT_KEY_HIVE_CONF));
for (Map.Entry<Object, Object> prop : properties.entrySet()) {
if (prop.getValue() instanceof String) {
hiveConf.set((String) prop.getKey(), (String) prop.getValue());
} else if (prop.getValue() instanceof Integer) {
hiveConf.setInt((String) prop.getKey(), (Integer) prop.getValue());
} else if (prop.getValue() instanceof Boolean) {
hiveConf.setBoolean((String) prop.getKey(), (Boolean) prop.getValue());
} else if (prop.getValue() instanceof Long) {
hiveConf.setLong((String) prop.getKey(), (Long) prop.getValue());
} else if (prop.getValue() instanceof Float) {
hiveConf.setFloat((String) prop.getKey(), (Float) prop.getValue());
}
}
}
if (conf.get(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE) != null) {
hiveConf.set(
"hive.metastore.token.signature", conf.get(HCatConstants.HCAT_KEY_TOKEN_SIGNATURE));
}
return hiveConf;
}
/**
* @param db
* @param table
* @param filter
* @param jobConf
* @return A list of locations
*/
public static List<String> getDataStorageLocation(
String db, String table, String filter, JobConf jobConf) {
Preconditions.checkNotNull(table, "Table name must not be null");
HiveMetaStoreClient client = null;
List<String> locations = new ArrayList<String>();
try {
client = getHiveMetaStoreClient(jobConf);
Table hiveTable = HCatUtil.getTable(client, db, table);
if (hiveTable.isPartitioned()) {
List<Partition> parts = null;
if (null != StringUtils.stripToNull(filter)) {
parts = client.listPartitionsByFilter(db, table, filter, (short) -1);
} else {
parts = client.listPartitions(db, table, (short) -1);
}
if (parts.size() > 0) {
// Return more than one partitions when filter is
// something
// like ds >= 1234
for (Partition part : parts) {
locations.addAll(getFilesInHivePartition(part, jobConf));
}
} else {
logError(
"Table "
+ hiveTable.getTableName()
+ " doesn't have the specified partition:"
+ filter,
null);
}
} else {
locations.add(hiveTable.getTTable().getSd().getLocation());
}
} catch (IOException e) {
logError("Error occured when getting hiveconf", e);
} catch (MetaException e) {
logError("Error occured when getting HiveMetaStoreClient", e);
} catch (NoSuchObjectException e) {
logError("Table doesn't exist in HCatalog: " + table, e);
} catch (TException e) {
logError("Error occured when getting Table", e);
} finally {
HCatUtil.closeHiveClientQuietly(client);
}
return locations;
}
@InterfaceAudience.Private
@InterfaceStability.Evolving
public static void configureOutputStorageHandler(
HCatStorageHandler storageHandler, Configuration conf, OutputJobInfo outputJobInfo) {
// TODO replace IgnoreKeyTextOutputFormat with a
// HiveOutputFormatWrapper in StorageHandler
TableDesc tableDesc =
new TableDesc(
storageHandler.getSerDeClass(),
storageHandler.getInputFormatClass(),
IgnoreKeyTextOutputFormat.class,
outputJobInfo.getTableInfo().getStorerInfo().getProperties());
if (tableDesc.getJobProperties() == null)
tableDesc.setJobProperties(new HashMap<String, String>());
for (Map.Entry<String, String> el : conf) {
tableDesc.getJobProperties().put(el.getKey(), el.getValue());
}
Map<String, String> jobProperties = new HashMap<String, String>();
try {
tableDesc
.getJobProperties()
.put(HCatConstants.HCAT_KEY_OUTPUT_INFO, HCatUtil.serialize(outputJobInfo));
storageHandler.configureOutputJobProperties(tableDesc, jobProperties);
for (Map.Entry<String, String> el : jobProperties.entrySet()) {
conf.set(el.getKey(), el.getValue());
}
} catch (IOException e) {
throw new IllegalStateException("Failed to configure StorageHandler", e);
}
}
public static Map<String, String> getInputJobProperties(
HCatStorageHandler storageHandler, InputJobInfo inputJobInfo) {
TableDesc tableDesc =
new TableDesc(
storageHandler.getSerDeClass(),
storageHandler.getInputFormatClass(),
storageHandler.getOutputFormatClass(),
inputJobInfo.getTableInfo().getStorerInfo().getProperties());
if (tableDesc.getJobProperties() == null) {
tableDesc.setJobProperties(new HashMap<String, String>());
}
Map<String, String> jobProperties = new HashMap<String, String>();
try {
tableDesc
.getJobProperties()
.put(HCatConstants.HCAT_KEY_JOB_INFO, HCatUtil.serialize(inputJobInfo));
storageHandler.configureInputJobProperties(tableDesc, jobProperties);
} catch (IOException e) {
throw new IllegalStateException("Failed to configure StorageHandler", e);
}
return jobProperties;
}
public static HCatStorageHandler getStorageHandler(Configuration conf, PartInfo partitionInfo)
throws IOException {
return HCatUtil.getStorageHandler(
conf,
partitionInfo.getStorageHandlerClassName(),
partitionInfo.getSerdeClassName(),
partitionInfo.getInputFormatClassName(),
partitionInfo.getOutputFormatClassName());
}
public static HCatSchema buildHCatSchema(List<FieldSchema> columns) {
HCatSchema schema = null;
try {
schema = new HCatSchema(HCatUtil.getHCatFieldSchemaList(columns));
} catch (HCatException e) {
logError("Error occured when building table schema", e);
}
return schema;
}
public static Table getHiveTable(String db, String table, JobConf conf) {
HiveMetaStoreClient client = null;
Table hiveTable = null;
try {
client = getHiveMetaStoreClient(conf);
hiveTable = HCatUtil.getTable(client, db, table);
} catch (IOException e) {
logError("Error occured when getting hiveconf", e);
} catch (MetaException e) {
logError("Error occured when getting HiveMetaStoreClient", e);
} catch (NoSuchObjectException e) {
logError("Table doesn't exist in HCatalog: " + table, e);
} catch (TException e) {
logError("Error occured when getting Table", e);
} finally {
HCatUtil.closeHiveClientQuietly(client);
}
return hiveTable;
}
public static HCatSchema getTableSchemaWithPtnCols(Table table) throws IOException {
HCatSchema tableSchema = new HCatSchema(HCatUtil.getHCatFieldSchemaList(table.getCols()));
if (table.getPartitionKeys().size() != 0) {
// add partition keys to table schema
// NOTE : this assumes that we do not ever have ptn keys as columns
// inside the table schema as well!
for (FieldSchema fs : table.getPartitionKeys()) {
tableSchema.append(HCatSchemaUtils.getHCatFieldSchema(fs));
}
}
return tableSchema;
}
private static HiveMetaStoreClient getHiveMetaStoreClient(JobConf jobConf)
throws IOException, MetaException {
HiveConf hiveConf = HCatUtil.getHiveConf(jobConf);
return HCatUtil.getHiveClient(hiveConf);
}
public static HCatSchema extractSchema(Partition partition) throws HCatException {
return new HCatSchema(HCatUtil.getHCatFieldSchemaList(partition.getCols()));
}
public static HCatSchema extractSchema(Table table) throws HCatException {
return new HCatSchema(HCatUtil.getHCatFieldSchemaList(table.getCols()));
}
