protected void propagateCommonCatalogTableProperties(
CatalogStorage tableStorage, Properties props, String prefix) {
props.put(prefix + "_storage_type", tableStorage.getType().name());
props.put(prefix + "_catalog_url", tableStorage.getCatalogUrl());
props.put(prefix + "_database", tableStorage.getDatabase());
props.put(prefix + "_table", tableStorage.getTable());
}
protected void propagateCatalogTableProperties(
Output output, CatalogStorage tableStorage, Properties props) {
String prefix = "falcon_" + output.getName();
propagateCommonCatalogTableProperties(tableStorage, props, prefix);
// pig and java actions require partition expression as "key1=val1, key2=val2"
props.put(prefix + "_partitions_pig", "${coord:dataOutPartitions('" + output.getName() + "')}");
props.put(
prefix + "_partitions_java", "${coord:dataOutPartitions('" + output.getName() + "')}");
// hive requires partition expression as "key1='val1', key2='val2'" (with quotes around values)
// there is no direct EL expression in oozie
List<String> partitions = new ArrayList<String>();
for (String key : tableStorage.getDatedPartitionKeys()) {
StringBuilder expr = new StringBuilder();
expr.append("${coord:dataOutPartitionValue('")
.append(output.getName())
.append("', '")
.append(key)
.append("')}");
props.put(prefix + "_dated_partition_value_" + key, expr.toString());
partitions.add(key + "='" + expr + "'");
}
props.put(prefix + "_partitions_hive", StringUtils.join(partitions, ","));
}
private static String getTableFeedInstanceName(
Feed feed, String feedInstancePath, Storage.TYPE storageType) throws URISyntaxException {
CatalogStorage instanceStorage =
(CatalogStorage) FeedHelper.createStorage(storageType.name(), feedInstancePath);
return feed.getName() + "/" + instanceStorage.toPartitionAsPath();
}