private void report(Map<TableId, LevelDBTableService.TableStats> datasetStat)
throws DatasetManagementException {
for (Map.Entry<TableId, LevelDBTableService.TableStats> statEntry : datasetStat.entrySet()) {
String namespace = statEntry.getKey().getNamespace().getId();
// emit metrics for only user datasets, tables in system namespace are ignored
if (namespace.equals(Constants.SYSTEM_NAMESPACE)) {
continue;
}
String tableName = statEntry.getKey().getTableName();
Collection<DatasetSpecificationSummary> instances =
dsFramework.getInstances(Id.Namespace.from(namespace));
for (DatasetSpecificationSummary spec : instances) {
DatasetSpecification specification =
dsFramework.getDatasetSpec(Id.DatasetInstance.from(namespace, spec.getName()));
if (specification.isParent(tableName)) {
MetricsCollector collector =
metricsService.getCollector(
ImmutableMap.of(
Constants.Metrics.Tag.NAMESPACE,
namespace,
Constants.Metrics.Tag.DATASET,
spec.getName()));
int sizeInMb = (int) (statEntry.getValue().getDiskSizeBytes() / BYTES_IN_MB);
collector.gauge("dataset.size.mb", sizeInMb);
break;
}
}
}
}
@AfterClass
public static void afterClass() throws Exception {
deleteNamespaces();
streamService.stopAndWait();
appFabricServer.stopAndWait();
metricsCollectionService.stopAndWait();
metricsService.stopAndWait();
datasetService.stopAndWait();
dsOpService.stopAndWait();
txManager.stopAndWait();
}
private static MetricsContext getMetricCollector(
MetricsCollectionService service,
Program program,
String flowletName,
String runId,
int instanceId) {
if (service == null) {
return null;
}
Map<String, String> tags = Maps.newHashMap(getMetricsContext(program, runId));
tags.put(Constants.Metrics.Tag.FLOWLET, flowletName);
tags.put(Constants.Metrics.Tag.INSTANCE_ID, String.valueOf(instanceId));
return service.getContext(tags);
}
@BeforeClass
public static void beforeClass() throws Throwable {
CConfiguration conf = CConfiguration.create();
conf.set(Constants.AppFabric.SERVER_ADDRESS, hostname);
conf.set(Constants.CFG_LOCAL_DATA_DIR, tmpFolder.newFolder("data").getAbsolutePath());
conf.set(Constants.AppFabric.OUTPUT_DIR, System.getProperty("java.io.tmpdir"));
conf.set(Constants.AppFabric.TEMP_DIR, System.getProperty("java.io.tmpdir"));
conf.setBoolean(Constants.Scheduler.SCHEDULERS_LAZY_START, true);
conf.set(
Constants.AppFabric.APP_TEMPLATE_DIR, tmpFolder.newFolder("templates").getAbsolutePath());
conf.setBoolean(Constants.Dangerous.UNRECOVERABLE_RESET, true);
DirUtils.mkdirs(new File(conf.get(Constants.AppFabric.APP_TEMPLATE_DIR)));
DirUtils.mkdirs(new File(conf.get(Constants.AppFabric.APP_TEMPLATE_PLUGIN_DIR)));
injector = Guice.createInjector(new AppFabricTestModule(conf));
txManager = injector.getInstance(TransactionManager.class);
txManager.startAndWait();
dsOpService = injector.getInstance(DatasetOpExecutor.class);
dsOpService.startAndWait();
datasetService = injector.getInstance(DatasetService.class);
datasetService.startAndWait();
appFabricServer = injector.getInstance(AppFabricServer.class);
appFabricServer.startAndWait();
DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
ServiceDiscovered appFabricHttpDiscovered =
discoveryClient.discover(Constants.Service.APP_FABRIC_HTTP);
EndpointStrategy endpointStrategy = new RandomEndpointStrategy(appFabricHttpDiscovered);
port = endpointStrategy.pick(1, TimeUnit.SECONDS).getSocketAddress().getPort();
txClient = injector.getInstance(TransactionSystemClient.class);
metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
metricsCollectionService.startAndWait();
metricsService = injector.getInstance(MetricsQueryService.class);
metricsService.startAndWait();
streamService = injector.getInstance(StreamService.class);
streamService.startAndWait();
serviceStore = injector.getInstance(ServiceStore.class);
serviceStore.startAndWait();
streamAdmin = injector.getInstance(StreamAdmin.class);
locationFactory = getInjector().getInstance(LocationFactory.class);
adapterDir = new File(conf.get(Constants.AppFabric.APP_TEMPLATE_DIR));
createNamespaces();
}