private ConversionConfig(Config config, Table table, String destinationFormat) {
Preconditions.checkArgument(
config.hasPath(DESTINATION_TABLE_KEY),
String.format("Key %s.%s is not specified", destinationFormat, DESTINATION_TABLE_KEY));
Preconditions.checkArgument(
config.hasPath(DESTINATION_DB_KEY),
String.format("Key %s.%s is not specified", destinationFormat, DESTINATION_DB_KEY));
Preconditions.checkArgument(
config.hasPath(DESTINATION_DATA_PATH_KEY),
String.format(
"Key %s.%s is not specified", destinationFormat, DESTINATION_DATA_PATH_KEY));
// Required
this.destinationFormat = destinationFormat;
this.destinationTableName = resolveTemplate(config.getString(DESTINATION_TABLE_KEY), table);
this.destinationStagingTableName =
String.format(
"%s_%s", this.destinationTableName, "staging"); // Fixed and non-configurable
this.destinationDbName = resolveTemplate(config.getString(DESTINATION_DB_KEY), table);
this.destinationDataPath =
resolveTemplate(config.getString(DESTINATION_DATA_PATH_KEY), table);
// Optional
this.clusterBy = ConfigUtils.getStringList(config, CLUSTER_BY_KEY);
this.numBuckets = Optional.fromNullable(ConfigUtils.getInt(config, NUM_BUCKETS_KEY, null));
this.hiveRuntimeProperties =
ConfigUtils.configToProperties(
ConfigUtils.getConfig(
config, HIVE_RUNTIME_PROPERTIES_KEY_PREFIX, ConfigFactory.empty()));
this.evolutionEnabled = ConfigUtils.getBoolean(config, EVOLUTION_ENABLED, false);
this.rowLimit = Optional.fromNullable(ConfigUtils.getInt(config, ROW_LIMIT_KEY, null));
this.sourceDataPathIdentifier =
ConfigUtils.getStringList(config, SOURCE_DATA_PATH_IDENTIFIER_KEY);
}
private GobblinMetrics buildGobblinMetrics() {
// Create tags list
ImmutableList.Builder<Tag<?>> tags = new ImmutableList.Builder<>();
tags.add(new Tag<>(GobblinClusterMetricTagNames.APPLICATION_ID, this.applicationId));
tags.add(new Tag<>(GobblinClusterMetricTagNames.APPLICATION_NAME, this.applicationName));
// Intialize Gobblin metrics and start reporters
GobblinMetrics gobblinMetrics = GobblinMetrics.get(this.applicationId, null, tags.build());
gobblinMetrics.startMetricReporting(ConfigUtils.configToProperties(config));
return gobblinMetrics;
}
