private void report(Map<TableId, LevelDBTableService.TableStats> datasetStat)
      throws DatasetManagementException {
    for (Map.Entry<TableId, LevelDBTableService.TableStats> statEntry : datasetStat.entrySet()) {
      String namespace = statEntry.getKey().getNamespace().getId();
      // emit metrics for only user datasets, tables in system namespace are ignored
      if (namespace.equals(Constants.SYSTEM_NAMESPACE)) {
        continue;
      }
      String tableName = statEntry.getKey().getTableName();

      Collection<DatasetSpecificationSummary> instances =
          dsFramework.getInstances(Id.Namespace.from(namespace));
      for (DatasetSpecificationSummary spec : instances) {
        DatasetSpecification specification =
            dsFramework.getDatasetSpec(Id.DatasetInstance.from(namespace, spec.getName()));
        if (specification.isParent(tableName)) {
          MetricsCollector collector =
              metricsService.getCollector(
                  ImmutableMap.of(
                      Constants.Metrics.Tag.NAMESPACE,
                      namespace,
                      Constants.Metrics.Tag.DATASET,
                      spec.getName()));
          int sizeInMb = (int) (statEntry.getValue().getDiskSizeBytes() / BYTES_IN_MB);
          collector.gauge("dataset.size.mb", sizeInMb);
          break;
        }
      }
    }
  }
示例#2
0
 @AfterClass
 public static void afterClass() throws Exception {
   deleteNamespaces();
   streamService.stopAndWait();
   appFabricServer.stopAndWait();
   metricsCollectionService.stopAndWait();
   metricsService.stopAndWait();
   datasetService.stopAndWait();
   dsOpService.stopAndWait();
   txManager.stopAndWait();
 }
示例#3
0
  private static MetricsContext getMetricCollector(
      MetricsCollectionService service,
      Program program,
      String flowletName,
      String runId,
      int instanceId) {
    if (service == null) {
      return null;
    }
    Map<String, String> tags = Maps.newHashMap(getMetricsContext(program, runId));
    tags.put(Constants.Metrics.Tag.FLOWLET, flowletName);
    tags.put(Constants.Metrics.Tag.INSTANCE_ID, String.valueOf(instanceId));

    return service.getContext(tags);
  }
示例#4
0
  @BeforeClass
  public static void beforeClass() throws Throwable {
    CConfiguration conf = CConfiguration.create();

    conf.set(Constants.AppFabric.SERVER_ADDRESS, hostname);
    conf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder("data").getAbsolutePath());
    conf.set(Constants.AppFabric.OUTPUT_DIR, System.getProperty("java.io.tmpdir"));
    conf.set(Constants.AppFabric.TEMP_DIR, System.getProperty("java.io.tmpdir"));
    conf.setBoolean(Constants.Scheduler.SCHEDULERS_LAZY_START, true);
    conf.set(
        Constants.AppFabric.APP_TEMPLATE_DIR, tmpFolder.newFolder("templates").getAbsolutePath());
    conf.setBoolean(Constants.Dangerous.UNRECOVERABLE_RESET, true);

    DirUtils.mkdirs(new File(conf.get(Constants.AppFabric.APP_TEMPLATE_DIR)));
    DirUtils.mkdirs(new File(conf.get(Constants.AppFabric.APP_TEMPLATE_PLUGIN_DIR)));

    injector = Guice.createInjector(new AppFabricTestModule(conf));

    txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    dsOpService = injector.getInstance(DatasetOpExecutor.class);
    dsOpService.startAndWait();
    datasetService = injector.getInstance(DatasetService.class);
    datasetService.startAndWait();
    appFabricServer = injector.getInstance(AppFabricServer.class);
    appFabricServer.startAndWait();
    DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
    ServiceDiscovered appFabricHttpDiscovered =
        discoveryClient.discover(Constants.Service.APP_FABRIC_HTTP);
    EndpointStrategy endpointStrategy = new RandomEndpointStrategy(appFabricHttpDiscovered);
    port = endpointStrategy.pick(1, TimeUnit.SECONDS).getSocketAddress().getPort();
    txClient = injector.getInstance(TransactionSystemClient.class);
    metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
    metricsCollectionService.startAndWait();
    metricsService = injector.getInstance(MetricsQueryService.class);
    metricsService.startAndWait();
    streamService = injector.getInstance(StreamService.class);
    streamService.startAndWait();
    serviceStore = injector.getInstance(ServiceStore.class);
    serviceStore.startAndWait();
    streamAdmin = injector.getInstance(StreamAdmin.class);
    locationFactory = getInjector().getInstance(LocationFactory.class);
    adapterDir = new File(conf.get(Constants.AppFabric.APP_TEMPLATE_DIR));
    createNamespaces();
  }