Esempio n. 1
0
  @Test
  public void testCreateTable() throws Exception {
    String tableName = tableName();
    String dbName = createDatabase();
    String colName = "col" + random();
    runCommand("create table " + dbName + "." + tableName + "(" + colName + " int, name string)");
    assertTableIsRegistered(dbName, tableName);

    // there is only one instance of column registered
    String colId = assertColumnIsRegistered(colName);
    Referenceable colEntity = dgiCLient.getEntity(colId);
    Assert.assertEquals(
        colEntity.get("qualifiedName"),
        String.format(
            "%s.%s.%s@%s",
            dbName.toLowerCase(), tableName.toLowerCase(), colName.toLowerCase(), CLUSTER_NAME));

    tableName = createTable();
    String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
    Referenceable tableRef = dgiCLient.getEntity(tableId);
    Assert.assertEquals(tableRef.get("tableType"), TableType.MANAGED_TABLE.name());
    Assert.assertEquals(tableRef.get(HiveDataModelGenerator.COMMENT), "table comment");
    String entityName =
        HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName);
    Assert.assertEquals(tableRef.get(HiveDataModelGenerator.NAME), entityName);
    Assert.assertEquals(
        tableRef.get("name"), "default." + tableName.toLowerCase() + "@" + CLUSTER_NAME);

    final Referenceable sdRef = (Referenceable) tableRef.get("sd");
    Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS), false);

    // Create table where database doesn't exist, will create database instance as well
    assertDatabaseIsRegistered(DEFAULT_DB);
  }
 @Test
 public void testUnsecuredClient() throws Exception {
   try {
     dgiClient.listTypes();
     Assert.fail("Should have failed with GSSException");
   } catch (Exception e) {
     e.printStackTrace();
     Assert.assertTrue(
         e.getMessage().contains("Mechanism level: Failed to find any Kerberos tgt"));
   }
 }
Esempio n. 3
0
  @Test
  public void testCreateDatabase() throws Exception {
    String dbName = "db" + random();
    runCommand("create database " + dbName + " WITH DBPROPERTIES ('p1'='v1', 'p2'='v2')");
    String dbId = assertDatabaseIsRegistered(dbName);

    Referenceable definition = dgiCLient.getEntity(dbId);
    Map params = (Map) definition.get("parameters");
    Assert.assertNotNull(params);
    Assert.assertEquals(params.size(), 2);
    Assert.assertEquals(params.get("p1"), "v1");

    // There should be just one entity per dbname
    runCommand("drop database " + dbName);
    runCommand("create database " + dbName);
    String dbid = assertDatabaseIsRegistered(dbName);

    // assert on qualified name
    Referenceable dbEntity = dgiCLient.getEntity(dbid);
    Assert.assertEquals(dbEntity.get("qualifiedName"), dbName.toLowerCase() + "@" + CLUSTER_NAME);
  }
  /**
   * Submits the entity definitions (instances). The body contains the JSONArray of entity json. The
   * service takes care of de-duping the entities based on any unique attribute for the give type.
   */
  @POST
  @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON})
  @Produces(Servlets.JSON_MEDIA_TYPE)
  public Response submit(@Context HttpServletRequest request) {

    String entityJson = null;
    try {
      String entities = Servlets.getRequestPayload(request);

      // Handle backward compatibility - if entities is not JSONArray, convert to JSONArray
      try {
        new JSONArray(entities);
      } catch (JSONException e) {
        final String finalEntities = entities;
        entities =
            new JSONArray() {
              {
                put(finalEntities);
              }
            }.toString();
      }

      entityJson = AtlasClient.toString(new JSONArray(entities));
      LOG.info("submitting entities {} ", entityJson);

      final List<String> guids = metadataService.createEntities(entities);
      LOG.info("Created entities {}", guids);
      JSONObject response = getResponse(new AtlasClient.EntityResult(guids, null, null));

      URI locationURI = getLocationURI(guids);

      return Response.created(locationURI).entity(response).build();

    } catch (EntityExistsException e) {
      LOG.error("Unique constraint violation for entity entityDef={}", entityJson, e);
      throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
    } catch (ValueConversionException ve) {
      LOG.error(
          "Unable to persist entity instance due to a deserialization error entityDef={}",
          entityJson,
          ve);
      throw new WebApplicationException(
          Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST));
    } catch (AtlasException | IllegalArgumentException e) {
      LOG.error("Unable to persist entity instance entityDef={}", entityJson, e);
      throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
    } catch (Throwable e) {
      LOG.error("Unable to persist entity instance entityDef={}", entityJson, e);
      throw new WebApplicationException(
          Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
    }
  }
Esempio n. 5
0
  @Test
  public void testSelect() throws Exception {
    String tableName = createTable();
    String query = "select * from " + tableName;
    runCommand(query);
    String pid = assertProcessIsRegistered(query);
    Referenceable processEntity = dgiCLient.getEntity(pid);
    Assert.assertEquals(processEntity.get("name"), query.toLowerCase());

    // single entity per query
    query = "SELECT * from " + tableName.toUpperCase();
    runCommand(query);
    assertProcessIsRegistered(query);
  }
Esempio n. 6
0
  @Test
  public void testLineage() throws Exception {
    String table1 = createTable(false);

    String db2 = createDatabase();
    String table2 = tableName();

    String query = String.format("create table %s.%s as select * from %s", db2, table2, table1);
    runCommand(query);
    String table1Id = assertTableIsRegistered(DEFAULT_DB, table1);
    String table2Id = assertTableIsRegistered(db2, table2);

    String datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, db2, table2);
    JSONObject response = dgiCLient.getInputGraph(datasetName);
    JSONObject vertices = response.getJSONObject("values").getJSONObject("vertices");
    Assert.assertTrue(vertices.has(table1Id));
    Assert.assertTrue(vertices.has(table2Id));

    datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, table1);
    response = dgiCLient.getOutputGraph(datasetName);
    vertices = response.getJSONObject("values").getJSONObject("vertices");
    Assert.assertTrue(vertices.has(table1Id));
    Assert.assertTrue(vertices.has(table2Id));
  }
Esempio n. 7
0
  @Test
  public void testInsert() throws Exception {
    String tableName = createTable();
    String insertTableName = createTable();
    String query =
        "insert into "
            + insertTableName
            + " partition(dt = '2015-01-01') select id, name from "
            + tableName
            + " where dt = '2015-01-01'";

    runCommand(query);
    assertProcessIsRegistered(query);
    String partId = assertPartitionIsRegistered(DEFAULT_DB, insertTableName, "2015-01-01");
    Referenceable partitionEntity = dgiCLient.getEntity(partId);
    Assert.assertEquals(
        partitionEntity.get("qualifiedName"),
        String.format(
            "%s.%s.%s@%s", "default", insertTableName.toLowerCase(), "2015-01-01", CLUSTER_NAME));
  }
Esempio n. 8
0
  private String assertEntityIsRegistered(final String query, String... arg) throws Exception {
    waitFor(
        2000,
        new Predicate() {
          @Override
          public boolean evaluate() throws Exception {
            JSONArray results = dgiCLient.search(query);
            return results.length() == 1;
          }
        });

    String column = (arg.length > 0) ? arg[0] : "_col_0";

    JSONArray results = dgiCLient.search(query);
    JSONObject row = results.getJSONObject(0);
    if (row.has("__guid")) {
      return row.getString("__guid");
    } else if (row.has("$id$")) {
      return row.getJSONObject("$id$").getString("id");
    } else {
      return row.getJSONObject(column).getString("id");
    }
  }
  /**
   * Complete update of a set of entities - the values not specified will be replaced with
   * null/removed Adds/Updates given entities identified by its GUID or unique attribute
   *
   * @return response payload as json
   */
  @PUT
  @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON})
  @Produces(Servlets.JSON_MEDIA_TYPE)
  public Response updateEntities(@Context HttpServletRequest request) {

    String entityJson = null;
    try {
      final String entities = Servlets.getRequestPayload(request);

      entityJson = AtlasClient.toString(new JSONArray(entities));
      LOG.info("updating entities {} ", entityJson);

      AtlasClient.EntityResult entityResult = metadataService.updateEntities(entities);
      LOG.info("Updated entities: {}", entityResult);

      JSONObject response = getResponse(entityResult);
      return Response.ok(response).build();
    } catch (EntityExistsException e) {
      LOG.error("Unique constraint violation for entityDef={}", entityJson, e);
      throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
    } catch (ValueConversionException ve) {
      LOG.error(
          "Unable to persist entity instance due to a deserialization error entityDef={}",
          entityJson,
          ve);
      throw new WebApplicationException(
          Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST));
    } catch (AtlasException | IllegalArgumentException e) {
      LOG.error("Unable to persist entity instance entityDef={}", entityJson, e);
      throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
    } catch (Throwable e) {
      LOG.error("Unable to persist entity instance entityDef={}", entityJson, e);
      throw new WebApplicationException(
          Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
    }
  }
Esempio n. 10
0
 private void assertEntityIsNotRegistered(String dslQuery) throws Exception {
   JSONArray results = dgiCLient.searchByDSL(dslQuery);
   Assert.assertEquals(results.length(), 0);
 }