/** * Create an instance of a storage handler. If storageHandler == null, then surrrogate * StorageHandler is used to encapsulate the InputFormat, OutputFormat and SerDe. This * StorageHandler assumes the other supplied storage artifacts are for a file-based storage * system. * * @param conf job's configuration will be used to configure the Configurable StorageHandler * @param storageHandler fully qualified class name of the desired StorageHandle instance * @param serDe fully qualified class name of the desired SerDe instance * @param inputFormat fully qualified class name of the desired InputFormat instance * @param outputFormat fully qualified class name of the desired outputFormat instance * @return storageHandler instance * @throws IOException */ public static HCatStorageHandler getStorageHandler( Configuration conf, String storageHandler, String serDe, String inputFormat, String outputFormat) throws IOException { if ((storageHandler == null) || (storageHandler.equals(FosterStorageHandler.class.getName()))) { try { FosterStorageHandler fosterStorageHandler = new FosterStorageHandler(inputFormat, outputFormat, serDe); fosterStorageHandler.setConf(conf); return fosterStorageHandler; } catch (ClassNotFoundException e) { throw new IOException("Failed to load " + "foster storage handler", e); } } try { Class<? extends HCatStorageHandler> handlerClass = (Class<? extends HCatStorageHandler>) Class.forName(storageHandler, true, JavaUtils.getClassLoader()); return (HCatStorageHandler) ReflectionUtils.newInstance(handlerClass, conf); } catch (ClassNotFoundException e) { throw new IOException("Error in loading storage handler." + e.getMessage(), e); } }
public SessionState(HiveConf conf, String userName) { this.conf = conf; this.userName = userName; isSilent = conf.getBoolVar(HiveConf.ConfVars.HIVESESSIONSILENT); ls = new LineageState(); // Must be deterministic order map for consistent q-test output across Java versions overriddenConfigurations = new LinkedHashMap<String, String>(); overriddenConfigurations.putAll(HiveConf.getConfSystemProperties()); // if there isn't already a session name, go ahead and create it. if (StringUtils.isEmpty(conf.getVar(HiveConf.ConfVars.HIVESESSIONID))) { conf.setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId()); } parentLoader = JavaUtils.getClassLoader(); }
@SuppressWarnings("deprecation") private static Class<? extends Deserializer> getDeserializerClass(String name) { // CDH uses different names for Parquet if ("parquet.hive.serde.ParquetHiveSerDe".equals(name)) { return ParquetHiveSerDe.class; } try { return Class.forName(name, true, JavaUtils.getClassLoader()).asSubclass(Deserializer.class); } catch (ClassNotFoundException e) { throw new PrestoException(HIVE_SERDE_NOT_FOUND, "deserializer does not exist: " + name); } catch (ClassCastException e) { throw new RuntimeException("invalid deserializer class: " + name); } }
/** * Returns the hooks specified in a configuration variable. The hooks are returned in a list in * the order they were specified in the configuration variable. * * @param hookConfVar The configuration variable specifying a comma separated list of the hook * class names. * @return A list of the hooks, in the order they are listed in the value of hookConfVar * @throws Exception */ private static <T extends Hook> List<T> getHooks(String csHooks) throws Exception { List<T> hooks = new ArrayList<T>(); if (csHooks.isEmpty()) { return hooks; } for (String hookClass : Splitter.on(",").omitEmptyStrings().trimResults().split(csHooks)) { try { @SuppressWarnings("unchecked") T hook = (T) Class.forName(hookClass, true, JavaUtils.getClassLoader()).newInstance(); hooks.add(hook); } catch (ClassNotFoundException e) { LOG.error(hookClass + " Class not found:" + e.getMessage()); throw e; } } return hooks; }
public void validate() throws SemanticException { if ((this.getCols() == null) || (this.getCols().size() == 0)) { // for now make sure that serde exists if (StringUtils.isEmpty(this.getSerName()) || !SerDeUtils.shouldGetColsFromSerDe(this.getSerName())) { throw new SemanticException(ErrorMsg.INVALID_TBL_DDL_SERDE.getMsg()); } return; } if (this.getStorageHandler() == null) { try { Class<?> origin = Class.forName(this.getOutputFormat(), true, JavaUtils.getClassLoader()); Class<? extends HiveOutputFormat> replaced = HiveFileFormatUtils.getOutputFormatSubstitute(origin); if (replaced == null) { throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE.getMsg()); } } catch (ClassNotFoundException e) { throw new SemanticException(ErrorMsg.INVALID_OUTPUT_FORMAT_TYPE.getMsg()); } } List<String> colNames = ParseUtils.validateColumnNameUniqueness(this.getCols()); if (this.getBucketCols() != null) { // all columns in cluster and sort are valid columns Iterator<String> bucketCols = this.getBucketCols().iterator(); while (bucketCols.hasNext()) { String bucketCol = bucketCols.next(); boolean found = false; Iterator<String> colNamesIter = colNames.iterator(); while (colNamesIter.hasNext()) { String colName = colNamesIter.next(); if (bucketCol.equalsIgnoreCase(colName)) { found = true; break; } } if (!found) { throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg()); } } } if (this.getSortCols() != null) { // all columns in cluster and sort are valid columns Iterator<Order> sortCols = this.getSortCols().iterator(); while (sortCols.hasNext()) { String sortCol = sortCols.next().getCol(); boolean found = false; Iterator<String> colNamesIter = colNames.iterator(); while (colNamesIter.hasNext()) { String colName = colNamesIter.next(); if (sortCol.equalsIgnoreCase(colName)) { found = true; break; } } if (!found) { throw new SemanticException(ErrorMsg.INVALID_COLUMN.getMsg()); } } } if (this.getPartCols() != null) { // there is no overlap between columns and partitioning columns Iterator<FieldSchema> partColsIter = this.getPartCols().iterator(); while (partColsIter.hasNext()) { FieldSchema fs = partColsIter.next(); String partCol = fs.getName(); PrimitiveObjectInspectorUtils.PrimitiveTypeEntry pte = PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(fs.getType()); if (null == pte) { throw new SemanticException( ErrorMsg.PARTITION_COLUMN_NON_PRIMITIVE.getMsg() + " Found " + partCol + " of type: " + fs.getType()); } Iterator<String> colNamesIter = colNames.iterator(); while (colNamesIter.hasNext()) { String colName = BaseSemanticAnalyzer.unescapeIdentifier(colNamesIter.next()); if (partCol.equalsIgnoreCase(colName)) { throw new SemanticException(ErrorMsg.COLUMN_REPEATED_IN_PARTITIONING_COLS.getMsg()); } } } } /* Validate skewed information. */ ValidationUtility.validateSkewedInformation( colNames, this.getSkewedColNames(), this.getSkewedColValues()); }