/**
* Create an instance of a storage handler. If storageHandler == null, then surrrogate
* StorageHandler is used to encapsulate the InputFormat, OutputFormat and SerDe. This
* StorageHandler assumes the other supplied storage artifacts are for a file-based storage
* system.
*
* @param conf job's configuration will be used to configure the Configurable StorageHandler
* @param storageHandler fully qualified class name of the desired StorageHandle instance
* @param serDe fully qualified class name of the desired SerDe instance
* @param inputFormat fully qualified class name of the desired InputFormat instance
* @param outputFormat fully qualified class name of the desired outputFormat instance
* @return storageHandler instance
* @throws IOException
*/
public static HCatStorageHandler getStorageHandler(
Configuration conf,
String storageHandler,
String serDe,
String inputFormat,
String outputFormat)
throws IOException {
if ((storageHandler == null) || (storageHandler.equals(FosterStorageHandler.class.getName()))) {
try {
FosterStorageHandler fosterStorageHandler =
new FosterStorageHandler(inputFormat, outputFormat, serDe);
fosterStorageHandler.setConf(conf);
return fosterStorageHandler;
} catch (ClassNotFoundException e) {
throw new IOException("Failed to load " + "foster storage handler", e);
}
}
try {
Class<? extends HCatStorageHandler> handlerClass =
(Class<? extends HCatStorageHandler>)
Class.forName(storageHandler, true, JavaUtils.getClassLoader());
return (HCatStorageHandler) ReflectionUtils.newInstance(handlerClass, conf);
} catch (ClassNotFoundException e) {
throw new IOException("Error in loading storage handler." + e.getMessage(), e);
}
}
public SessionState(HiveConf conf, String userName) {
this.conf = conf;
this.userName = userName;
isSilent = conf.getBoolVar(HiveConf.ConfVars.HIVESESSIONSILENT);
ls = new LineageState();
// Must be deterministic order map for consistent q-test output across Java versions
overriddenConfigurations = new LinkedHashMap<String, String>();
overriddenConfigurations.putAll(HiveConf.getConfSystemProperties());
// if there isn't already a session name, go ahead and create it.
if (StringUtils.isEmpty(conf.getVar(HiveConf.ConfVars.HIVESESSIONID))) {
conf.setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId());
}
parentLoader = JavaUtils.getClassLoader();
}
@SuppressWarnings("deprecation")
private static Class<? extends Deserializer> getDeserializerClass(String name) {
// CDH uses different names for Parquet
if ("parquet.hive.serde.ParquetHiveSerDe".equals(name)) {
return ParquetHiveSerDe.class;
}
try {
return Class.forName(name, true, JavaUtils.getClassLoader()).asSubclass(Deserializer.class);
} catch (ClassNotFoundException e) {
throw new PrestoException(HIVE_SERDE_NOT_FOUND, "deserializer does not exist: " + name);
} catch (ClassCastException e) {
throw new RuntimeException("invalid deserializer class: " + name);
}
}
/**
* Returns the hooks specified in a configuration variable. The hooks are returned in a list in
* the order they were specified in the configuration variable.
*
* @param hookConfVar The configuration variable specifying a comma separated list of the hook
* class names.
* @return A list of the hooks, in the order they are listed in the value of hookConfVar
* @throws Exception
*/
private static <T extends Hook> List<T> getHooks(String csHooks) throws Exception {
List<T> hooks = new ArrayList<T>();
if (csHooks.isEmpty()) {
return hooks;
}
for (String hookClass : Splitter.on(",").omitEmptyStrings().trimResults().split(csHooks)) {
try {
@SuppressWarnings("unchecked")
T hook = (T) Class.forName(hookClass, true, JavaUtils.getClassLoader()).newInstance();
hooks.add(hook);
} catch (ClassNotFoundException e) {
LOG.error(hookClass + " Class not found:" + e.getMessage());
throw e;
}
}
return hooks;
}
public void validate() throws SemanticException {
if ((this.getCols() == null) || (this.getCols().size() == 0)) {
// for now make sure that serde exists
if (StringUtils.isEmpty(this.getSerName())
|| !SerDeUtils.shouldGetColsFromSerDe(this.getSerName())) {
throw new SemanticException(ErrorMsg.INVALID_TBL_DDL_SERDE.getMsg());
}
return;
}
if (this.getStorageHandler() == null) {
try {
Class<?> origin = Class.forName(this.getOutputFormat(), true, JavaUtils.getClassLoader());
Class<? extends HiveOutputFormat> replaced =
HiveFileFormatUtils.getOutputFormatSubstitute(origin);
if (replaced == null) {
throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE.getMsg());
}
} catch (ClassNotFoundException e) {
throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE.getMsg());
}
}
List<String> colNames = ParseUtils.validateColumnNameUniqueness(this.getCols());
if (this.getBucketCols() != null) {
// all columns in cluster and sort are valid columns
Iterator<String> bucketCols = this.getBucketCols().iterator();
while (bucketCols.hasNext()) {
String bucketCol = bucketCols.next();
boolean found = false;
Iterator<String> colNamesIter = colNames.iterator();
while (colNamesIter.hasNext()) {
String colName = colNamesIter.next();
if (bucketCol.equalsIgnoreCase(colName)) {
found = true;
break;
}
}
if (!found) {
throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg());
}
}
}
if (this.getSortCols() != null) {
// all columns in cluster and sort are valid columns
Iterator<Order> sortCols = this.getSortCols().iterator();
while (sortCols.hasNext()) {
String sortCol = sortCols.next().getCol();
boolean found = false;
Iterator<String> colNamesIter = colNames.iterator();
while (colNamesIter.hasNext()) {
String colName = colNamesIter.next();
if (sortCol.equalsIgnoreCase(colName)) {
found = true;
break;
}
}
if (!found) {
throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg());
}
}
}
if (this.getPartCols() != null) {
// there is no overlap between columns and partitioning columns
Iterator<FieldSchema> partColsIter = this.getPartCols().iterator();
while (partColsIter.hasNext()) {
FieldSchema fs = partColsIter.next();
String partCol = fs.getName();
PrimitiveObjectInspectorUtils.PrimitiveTypeEntry pte =
PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(fs.getType());
if (null == pte) {
throw new SemanticException(
ErrorMsg.PARTITION_COLUMN_NON_PRIMITIVE.getMsg()
+ " Found "
+ partCol
+ " of type: "
+ fs.getType());
}
Iterator<String> colNamesIter = colNames.iterator();
while (colNamesIter.hasNext()) {
String colName = BaseSemanticAnalyzer.unescapeIdentifier(colNamesIter.next());
if (partCol.equalsIgnoreCase(colName)) {
throw new SemanticException(ErrorMsg.COLUMN_REPEATED_IN_PARTITIONING_COLS.getMsg());
}
}
}
}
/* Validate skewed information. */
ValidationUtility.validateSkewedInformation(
colNames, this.getSkewedColNames(), this.getSkewedColValues());
}