private void report(Map<TableId, LevelDBTableService.TableStats> datasetStat)
      throws DatasetManagementException {
    for (Map.Entry<TableId, LevelDBTableService.TableStats> statEntry : datasetStat.entrySet()) {
      String namespace = statEntry.getKey().getNamespace().getId();
      // emit metrics for only user datasets, tables in system namespace are ignored
      if (namespace.equals(Constants.SYSTEM_NAMESPACE)) {
        continue;
      }
      String tableName = statEntry.getKey().getTableName();

      Collection<DatasetSpecificationSummary> instances =
          dsFramework.getInstances(Id.Namespace.from(namespace));
      for (DatasetSpecificationSummary spec : instances) {
        DatasetSpecification specification =
            dsFramework.getDatasetSpec(Id.DatasetInstance.from(namespace, spec.getName()));
        if (specification.isParent(tableName)) {
          MetricsCollector collector =
              metricsService.getCollector(
                  ImmutableMap.of(
                      Constants.Metrics.Tag.NAMESPACE,
                      namespace,
                      Constants.Metrics.Tag.DATASET,
                      spec.getName()));
          int sizeInMb = (int) (statEntry.getValue().getDiskSizeBytes() / BYTES_IN_MB);
          collector.gauge("dataset.size.mb", sizeInMb);
          break;
        }
      }
    }
  }
Exemplo n.º 2
0
 public void upgrade() throws IOException, DatasetManagementException {
   DatasetAdmin admin =
       dsFramework.getAdmin(Id.DatasetInstance.from(SYSTEM_NAMESPACE, getMetaTableName()), null);
   if (admin != null) {
     admin.upgrade();
   }
 }
Exemplo n.º 3
0
 private static Id.DatasetInstance getDatasetInstanceId(@Nullable Configuration conf) {
   return conf == null
       ? null
       : Id.DatasetInstance.from(
           conf.get(Constants.Explore.DATASET_NAMESPACE),
           conf.get(Constants.Explore.DATASET_NAME));
 }
Exemplo n.º 4
0
  @Test
  public void useTransactionTest() throws Exception {
    // Performing admin operations to create dataset instance
    // keyValueTable is a system dataset module
    Id.DatasetInstance myTableInstance = Id.DatasetInstance.from(namespace, "myTable");
    dsFramework.addInstance("keyValueTable", myTableInstance, DatasetProperties.EMPTY);

    Assert.assertTrue(feedManager.createFeed(FEED1));
    try {
      Cancellable cancellable =
          notificationService.subscribe(
              FEED1,
              new NotificationHandler<String>() {
                private int received = 0;

                @Override
                public Type getNotificationType() {
                  return String.class;
                }

                @Override
                public void received(
                    final String notification, NotificationContext notificationContext) {
                  notificationContext.execute(
                      new TxRunnable() {
                        @Override
                        public void run(DatasetContext context) throws Exception {
                          KeyValueTable table = context.getDataset("myTable");
                          table.write("foo", String.format("%s-%d", notification, received++));
                        }
                      },
                      TxRetryPolicy.maxRetries(5));
                }
              });
      TimeUnit.SECONDS.sleep(2);

      try {
        notificationService.publish(FEED1, "foobar");
        // Waiting for the subscriber to receive that notification
        TimeUnit.SECONDS.sleep(2);

        KeyValueTable table =
            dsFramework.getDataset(myTableInstance, DatasetDefinition.NO_ARGUMENTS, null);
        Assert.assertNotNull(table);
        Transaction tx1 = txManager.startShort(100);
        table.startTx(tx1);
        Assert.assertEquals("foobar-0", Bytes.toString(table.read("foo")));
        Assert.assertTrue(table.commitTx());
        txManager.canCommit(tx1, table.getTxChanges());
        txManager.commit(tx1);
        table.postTxCommit();
      } finally {
        cancellable.cancel();
      }
    } finally {
      dsFramework.deleteInstance(myTableInstance);
      feedManager.deleteFeed(FEED1);
    }
  }
Exemplo n.º 5
0
 public Table getMetaTable() throws IOException, DatasetManagementException {
   Id.DatasetInstance metaTableInstanceId =
       Id.DatasetInstance.from(SYSTEM_NAMESPACE, getMetaTableName());
   return DatasetsUtil.getOrCreateDataset(
       dsFramework,
       metaTableInstanceId,
       Table.class.getName(),
       DatasetProperties.EMPTY,
       DatasetDefinition.NO_ARGUMENTS,
       null);
 }
Exemplo n.º 6
0
public class UsageDatasetTest {
  @ClassRule
  public static DatasetFrameworkTestUtil dsFrameworkUtil = new DatasetFrameworkTestUtil();

  private final Id.Program flow11 = Id.Program.from("ns1", "app1", ProgramType.FLOW, "flow11");
  private final Id.Program flow12 = Id.Program.from("ns1", "app1", ProgramType.FLOW, "flow12");
  private final Id.Program service11 =
      Id.Program.from("ns1", "app1", ProgramType.SERVICE, "service11");

  private final Id.Program flow21 = Id.Program.from("ns1", "app2", ProgramType.FLOW, "flow21");
  private final Id.Program flow22 = Id.Program.from("ns1", "app2", ProgramType.FLOW, "flow22");
  private final Id.Program service21 =
      Id.Program.from("ns1", "app2", ProgramType.SERVICE, "service21");

  private final Id.DatasetInstance datasetInstance1 = Id.DatasetInstance.from("ns1", "ds1");
  private final Id.DatasetInstance datasetInstance2 = Id.DatasetInstance.from("ns1", "ds2");
  private final Id.DatasetInstance datasetInstance3 = Id.DatasetInstance.from("ns1", "ds3");

  private final Id.Stream stream1 = Id.Stream.from("ns1", "s1");
  private final Id.Stream stream2 = Id.Stream.from("ns2", "s1");

  private final Id.Adapter adapter1 = Id.Adapter.from("ns1", "adapter1");
  private final Id.Adapter adapter2 = Id.Adapter.from("ns2", "adapter1");

  @Test
  public void testOneMapping() throws Exception {
    UsageDataset usageDataset = getUsageDataset("testOneMapping");

    // Add mapping
    usageDataset.register(flow11, datasetInstance1);

    // Verify mapping
    Assert.assertEquals(ImmutableSet.of(datasetInstance1), usageDataset.getDatasets(flow11));
  }

  @Test
  public void testProgramDatasetMapping() throws Exception {
    UsageDataset usageDataset = getUsageDataset("testProgramDatasetMapping");

    // Add mappings
    usageDataset.register(flow11, datasetInstance1);
    usageDataset.register(flow11, datasetInstance3);
    usageDataset.register(flow12, datasetInstance2);
    usageDataset.register(service11, datasetInstance1);

    usageDataset.register(flow21, datasetInstance2);
    usageDataset.register(service21, datasetInstance1);

    // Verify program mappings
    Assert.assertEquals(
        ImmutableSet.of(datasetInstance1, datasetInstance3), usageDataset.getDatasets(flow11));
    Assert.assertEquals(ImmutableSet.of(datasetInstance2), usageDataset.getDatasets(flow12));
    Assert.assertEquals(ImmutableSet.of(datasetInstance1), usageDataset.getDatasets(service11));

    Assert.assertEquals(ImmutableSet.of(datasetInstance2), usageDataset.getDatasets(flow21));
    Assert.assertEquals(ImmutableSet.of(datasetInstance1), usageDataset.getDatasets(service21));

    // Verify app mappings
    Assert.assertEquals(
        ImmutableSet.of(datasetInstance1, datasetInstance2, datasetInstance3),
        usageDataset.getDatasets(flow11.getApplication()));
    Assert.assertEquals(
        ImmutableSet.of(datasetInstance1, datasetInstance2),
        usageDataset.getDatasets(flow21.getApplication()));

    // Verify dataset mappings
    Assert.assertEquals(
        ImmutableSet.of(flow11, service11, service21), usageDataset.getPrograms(datasetInstance1));
    Assert.assertEquals(
        ImmutableSet.of(flow12, flow21), usageDataset.getPrograms(datasetInstance2));
    Assert.assertEquals(ImmutableSet.of(flow11), usageDataset.getPrograms(datasetInstance3));

    // --------- Delete app1 -----------
    usageDataset.unregister(flow11.getApplication());

    // There should be no mappings for programs of app1 now
    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(flow11));
    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(flow12));
    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(service11));

    Assert.assertEquals(ImmutableSet.of(datasetInstance2), usageDataset.getDatasets(flow21));
    Assert.assertEquals(ImmutableSet.of(datasetInstance1), usageDataset.getDatasets(service21));

    // Verify app mappings
    Assert.assertEquals(
        ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(flow11.getApplication()));
    Assert.assertEquals(
        ImmutableSet.of(datasetInstance1, datasetInstance2),
        usageDataset.getDatasets(flow21.getApplication()));

    // Verify dataset mappings
    Assert.assertEquals(ImmutableSet.of(service21), usageDataset.getPrograms(datasetInstance1));
    Assert.assertEquals(ImmutableSet.of(flow21), usageDataset.getPrograms(datasetInstance2));
    Assert.assertEquals(ImmutableSet.<Id.Program>of(), usageDataset.getPrograms(datasetInstance3));
  }

  @Test
  public void testAllMappings() throws Exception {
    UsageDataset usageDataset = getUsageDataset("testAllMappings");

    // Add mappings
    usageDataset.register(flow11, datasetInstance1);
    usageDataset.register(service21, datasetInstance3);

    usageDataset.register(adapter1, datasetInstance1);
    usageDataset.register(adapter1, datasetInstance2);

    usageDataset.register(flow12, stream1);
    usageDataset.register(adapter2, stream2);

    // Verify app/adapter mappings
    Assert.assertEquals(ImmutableSet.of(datasetInstance1), usageDataset.getDatasets(flow11));
    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(flow12));
    Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));

    Assert.assertEquals(
        ImmutableSet.of(datasetInstance1, datasetInstance2), usageDataset.getDatasets(adapter1));

    Assert.assertEquals(ImmutableSet.of(stream1), usageDataset.getStreams(flow12));
    Assert.assertEquals(ImmutableSet.<Id.Stream>of(), usageDataset.getStreams(flow22));
    Assert.assertEquals(ImmutableSet.of(stream2), usageDataset.getStreams(adapter2));

    // Verify dataset/stream mappings
    Assert.assertEquals(ImmutableSet.of(flow11), usageDataset.getPrograms(datasetInstance1));
    Assert.assertEquals(ImmutableSet.of(adapter1), usageDataset.getAdapters(datasetInstance1));
    Assert.assertEquals(ImmutableSet.of(flow12), usageDataset.getPrograms(stream1));
    Assert.assertEquals(ImmutableSet.<Id.Program>of(), usageDataset.getPrograms(stream2));
    Assert.assertEquals(ImmutableSet.of(adapter2), usageDataset.getAdapters(stream2));

    // --------- Delete app1 -----------
    usageDataset.unregister(flow11.getApplication());

    // Verify app/adapter mappings
    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(flow11));
    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(flow12));
    Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));

    Assert.assertEquals(
        ImmutableSet.of(datasetInstance1, datasetInstance2), usageDataset.getDatasets(adapter1));

    Assert.assertEquals(ImmutableSet.<Id.Stream>of(), usageDataset.getStreams(flow12));
    Assert.assertEquals(ImmutableSet.<Id.Stream>of(), usageDataset.getStreams(flow22));
    Assert.assertEquals(ImmutableSet.of(stream2), usageDataset.getStreams(adapter2));

    // Verify dataset/stream mappings
    Assert.assertEquals(ImmutableSet.<Id.Program>of(), usageDataset.getPrograms(datasetInstance1));
    Assert.assertEquals(ImmutableSet.of(adapter1), usageDataset.getAdapters(datasetInstance1));
    Assert.assertEquals(ImmutableSet.<Id.Program>of(), usageDataset.getPrograms(stream1));
    Assert.assertEquals(ImmutableSet.<Id.Program>of(), usageDataset.getPrograms(stream2));
    Assert.assertEquals(ImmutableSet.of(adapter2), usageDataset.getAdapters(stream2));

    // --------- Delete adapter1 -----------
    usageDataset.unregister(adapter1);

    // Verify app/adapter mappings
    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(flow11));
    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(flow12));
    Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));

    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(adapter1));

    Assert.assertEquals(ImmutableSet.<Id.Stream>of(), usageDataset.getStreams(flow12));
    Assert.assertEquals(ImmutableSet.<Id.Stream>of(), usageDataset.getStreams(flow22));
    Assert.assertEquals(ImmutableSet.of(stream2), usageDataset.getStreams(adapter2));

    // Verify dataset/stream mappings
    Assert.assertEquals(ImmutableSet.<Id.Program>of(), usageDataset.getPrograms(datasetInstance1));
    Assert.assertEquals(ImmutableSet.<Id.Adapter>of(), usageDataset.getAdapters(datasetInstance1));
    Assert.assertEquals(ImmutableSet.<Id.Program>of(), usageDataset.getPrograms(stream1));
    Assert.assertEquals(ImmutableSet.<Id.Program>of(), usageDataset.getPrograms(stream2));
    Assert.assertEquals(ImmutableSet.of(adapter2), usageDataset.getAdapters(stream2));

    // --------- Delete adapter2 -----------
    usageDataset.unregister(adapter2);

    // Verify app/adapter mappings
    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(flow11));
    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(flow12));
    Assert.assertEquals(ImmutableSet.of(datasetInstance3), usageDataset.getDatasets(service21));

    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(adapter1));

    Assert.assertEquals(ImmutableSet.<Id.Stream>of(), usageDataset.getStreams(flow12));
    Assert.assertEquals(ImmutableSet.<Id.Stream>of(), usageDataset.getStreams(flow22));
    Assert.assertEquals(ImmutableSet.<Id.Stream>of(), usageDataset.getStreams(adapter2));

    // Verify dataset/stream mappings
    Assert.assertEquals(ImmutableSet.<Id.Program>of(), usageDataset.getPrograms(datasetInstance1));
    Assert.assertEquals(ImmutableSet.<Id.Adapter>of(), usageDataset.getAdapters(datasetInstance1));
    Assert.assertEquals(ImmutableSet.<Id.Program>of(), usageDataset.getPrograms(stream1));
    Assert.assertEquals(ImmutableSet.<Id.Program>of(), usageDataset.getPrograms(stream2));
    Assert.assertEquals(ImmutableSet.<Id.Adapter>of(), usageDataset.getAdapters(stream2));

    // --------- Delete app2 -----------
    usageDataset.unregister(flow21.getApplication());
    // Verify app/adapter mappings
    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(flow11));
    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(flow12));
    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(service21));

    Assert.assertEquals(ImmutableSet.<Id.DatasetInstance>of(), usageDataset.getDatasets(adapter1));

    Assert.assertEquals(ImmutableSet.<Id.Stream>of(), usageDataset.getStreams(flow12));
    Assert.assertEquals(ImmutableSet.<Id.Stream>of(), usageDataset.getStreams(flow22));
    Assert.assertEquals(ImmutableSet.<Id.Stream>of(), usageDataset.getStreams(adapter2));

    // Verify dataset/stream mappings
    Assert.assertEquals(ImmutableSet.<Id.Program>of(), usageDataset.getPrograms(datasetInstance1));
    Assert.assertEquals(ImmutableSet.<Id.Adapter>of(), usageDataset.getAdapters(datasetInstance1));
    Assert.assertEquals(ImmutableSet.<Id.Program>of(), usageDataset.getPrograms(stream1));
    Assert.assertEquals(ImmutableSet.<Id.Program>of(), usageDataset.getPrograms(stream2));
    Assert.assertEquals(ImmutableSet.<Id.Adapter>of(), usageDataset.getAdapters(stream2));
  }

  private static UsageDataset getUsageDataset(String instanceId) throws Exception {
    Id.DatasetInstance id =
        Id.DatasetInstance.from(DatasetFrameworkTestUtil.NAMESPACE_ID, instanceId);
    return UsageDatasets.get(dsFrameworkUtil.getFramework(), id);
  }
}
Exemplo n.º 7
0
 private static UsageDataset getUsageDataset(String instanceId) throws Exception {
   Id.DatasetInstance id =
       Id.DatasetInstance.from(DatasetFrameworkTestUtil.NAMESPACE_ID, instanceId);
   return UsageDatasets.get(dsFrameworkUtil.getFramework(), id);
 }