Пример #1
0
  private GobblinMetrics buildGobblinMetrics() {
    // Create tags list
    ImmutableList.Builder<Tag<?>> tags = new ImmutableList.Builder<>();
    tags.add(new Tag<>(GobblinClusterMetricTagNames.APPLICATION_ID, this.applicationId));
    tags.add(new Tag<>(GobblinClusterMetricTagNames.APPLICATION_NAME, this.applicationName));

    // Intialize Gobblin metrics and start reporters
    GobblinMetrics gobblinMetrics = GobblinMetrics.get(this.applicationId, null, tags.build());
    gobblinMetrics.startMetricReporting(ConfigUtils.configToProperties(config));

    return gobblinMetrics;
  }
Пример #2
0
  public DatasetCleaner(FileSystem fs, Properties props) throws IOException {

    Preconditions.checkArgument(props.containsKey(DATASET_PROFILE_CLASS_KEY));
    this.closer = Closer.create();
    try {
      FileSystem optionalRateControlledFs = fs;
      if (props.contains(DATASET_CLEAN_HDFS_CALLS_PER_SECOND_LIMIT)) {
        optionalRateControlledFs =
            this.closer.register(
                new RateControlledFileSystem(
                    fs,
                    Long.parseLong(props.getProperty(DATASET_CLEAN_HDFS_CALLS_PER_SECOND_LIMIT))));
        ((RateControlledFileSystem) optionalRateControlledFs).startRateControl();
      }
      Class<?> datasetFinderClass = Class.forName(props.getProperty(DATASET_PROFILE_CLASS_KEY));
      this.datasetFinder =
          (DatasetFinder)
              datasetFinderClass
                  .getConstructor(FileSystem.class, Properties.class)
                  .newInstance(optionalRateControlledFs, props);
    } catch (ClassNotFoundException exception) {
      throw new IOException(exception);
    } catch (NoSuchMethodException exception) {
      throw new IOException(exception);
    } catch (InstantiationException exception) {
      throw new IOException(exception);
    } catch (IllegalAccessException exception) {
      throw new IOException(exception);
    } catch (InvocationTargetException exception) {
      throw new IOException(exception);
    } catch (NumberFormatException exception) {
      throw new IOException(exception);
    } catch (ExecutionException exception) {
      throw new IOException(exception);
    }
    ExecutorService executor =
        ScalingThreadPoolExecutor.newScalingThreadPool(
            0,
            Integer.parseInt(
                props.getProperty(
                    MAX_CONCURRENT_DATASETS_CLEANED, DEFAULT_MAX_CONCURRENT_DATASETS_CLEANED)),
            100,
            ExecutorsUtils.newThreadFactory(
                Optional.of(LOG), Optional.of("Dataset-cleaner-pool-%d")));
    this.service = MoreExecutors.listeningDecorator(executor);

    // TODO -- Remove the dependency on gobblin-core after new Gobblin Metrics does not depend on
    // gobblin-core.
    this.metricContext =
        this.closer.register(Instrumented.getMetricContext(new State(props), DatasetCleaner.class));
    this.isMetricEnabled = GobblinMetrics.isEnabled(props);
  }