private GobblinMetrics buildGobblinMetrics() {
// Create tags list
ImmutableList.Builder<Tag<?>> tags = new ImmutableList.Builder<>();
tags.add(new Tag<>(GobblinClusterMetricTagNames.APPLICATION_ID, this.applicationId));
tags.add(new Tag<>(GobblinClusterMetricTagNames.APPLICATION_NAME, this.applicationName));
// Intialize Gobblin metrics and start reporters
GobblinMetrics gobblinMetrics = GobblinMetrics.get(this.applicationId, null, tags.build());
gobblinMetrics.startMetricReporting(ConfigUtils.configToProperties(config));
return gobblinMetrics;
}
public DatasetCleaner(FileSystem fs, Properties props) throws IOException {
Preconditions.checkArgument(props.containsKey(DATASET_PROFILE_CLASS_KEY));
this.closer = Closer.create();
try {
FileSystem optionalRateControlledFs = fs;
if (props.contains(DATASET_CLEAN_HDFS_CALLS_PER_SECOND_LIMIT)) {
optionalRateControlledFs =
this.closer.register(
new RateControlledFileSystem(
fs,
Long.parseLong(props.getProperty(DATASET_CLEAN_HDFS_CALLS_PER_SECOND_LIMIT))));
((RateControlledFileSystem) optionalRateControlledFs).startRateControl();
}
Class<?> datasetFinderClass = Class.forName(props.getProperty(DATASET_PROFILE_CLASS_KEY));
this.datasetFinder =
(DatasetFinder)
datasetFinderClass
.getConstructor(FileSystem.class, Properties.class)
.newInstance(optionalRateControlledFs, props);
} catch (ClassNotFoundException exception) {
throw new IOException(exception);
} catch (NoSuchMethodException exception) {
throw new IOException(exception);
} catch (InstantiationException exception) {
throw new IOException(exception);
} catch (IllegalAccessException exception) {
throw new IOException(exception);
} catch (InvocationTargetException exception) {
throw new IOException(exception);
} catch (NumberFormatException exception) {
throw new IOException(exception);
} catch (ExecutionException exception) {
throw new IOException(exception);
}
ExecutorService executor =
ScalingThreadPoolExecutor.newScalingThreadPool(
0,
Integer.parseInt(
props.getProperty(
MAX_CONCURRENT_DATASETS_CLEANED, DEFAULT_MAX_CONCURRENT_DATASETS_CLEANED)),
100,
ExecutorsUtils.newThreadFactory(
Optional.of(LOG), Optional.of("Dataset-cleaner-pool-%d")));
this.service = MoreExecutors.listeningDecorator(executor);
// TODO -- Remove the dependency on gobblin-core after new Gobblin Metrics does not depend on
// gobblin-core.
this.metricContext =
this.closer.register(Instrumented.getMetricContext(new State(props), DatasetCleaner.class));
this.isMetricEnabled = GobblinMetrics.isEnabled(props);
}