@Override public void configureInputJobProperties(TableDesc tableDesc, Map<String, String> jobProperties) { try { Map<String, String> tableProperties = tableDesc.getJobProperties(); String jobInfoProperty = tableProperties.get(HCatConstants.HCAT_KEY_JOB_INFO); if (jobInfoProperty != null) { InputJobInfo inputJobInfo = (InputJobInfo) HCatUtil.deserialize(jobInfoProperty); HCatTableInfo tableInfo = inputJobInfo.getTableInfo(); HCatSchema dataColumns = tableInfo.getDataColumns(); List<HCatFieldSchema> dataFields = dataColumns.getFields(); StringBuilder columnNamesSb = new StringBuilder(); StringBuilder typeNamesSb = new StringBuilder(); for (HCatFieldSchema dataField : dataFields) { if (columnNamesSb.length() > 0) { columnNamesSb.append(","); typeNamesSb.append(":"); } columnNamesSb.append(dataField.getName()); typeNamesSb.append(dataField.getTypeString()); } jobProperties.put(IOConstants.SCHEMA_EVOLUTION_COLUMNS, columnNamesSb.toString()); jobProperties.put(IOConstants.SCHEMA_EVOLUTION_COLUMNS_TYPES, typeNamesSb.toString()); boolean isAcidTable = AcidUtils.isTablePropertyTransactional(tableProperties); AcidUtils.setTransactionalTableScan(jobProperties, isAcidTable); } } catch (IOException e) { throw new IllegalStateException("Failed to set output path", e); } }
private static Properties getSerdeProperties(HCatTableInfo info, HCatSchema s) throws SerDeException { Properties props = new Properties(); List<FieldSchema> fields = HCatUtil.getFieldSchemaList(s.getFields()); props.setProperty( org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMNS, MetaStoreUtils.getColumnNamesFromFieldSchema(fields)); props.setProperty( org.apache.hadoop.hive.serde.serdeConstants.LIST_COLUMN_TYPES, MetaStoreUtils.getColumnTypesFromFieldSchema(fields)); // setting these props to match LazySimpleSerde props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_NULL_FORMAT, "\\N"); props.setProperty(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1"); // add props from params set in table schema props.putAll(info.getStorerInfo().getProperties()); return props; }