@Test public void testCreateTable() throws Exception { String tableName = tableName(); String dbName = createDatabase(); String colName = "col" + random(); runCommand("create table " + dbName + "." + tableName + "(" + colName + " int, name string)"); assertTableIsRegistered(dbName, tableName); // there is only one instance of column registered String colId = assertColumnIsRegistered(colName); Referenceable colEntity = dgiCLient.getEntity(colId); Assert.assertEquals( colEntity.get("qualifiedName"), String.format( "%s.%s.%s@%s", dbName.toLowerCase(), tableName.toLowerCase(), colName.toLowerCase(), CLUSTER_NAME)); tableName = createTable(); String tableId = assertTableIsRegistered(DEFAULT_DB, tableName); Referenceable tableRef = dgiCLient.getEntity(tableId); Assert.assertEquals(tableRef.get("tableType"), TableType.MANAGED_TABLE.name()); Assert.assertEquals(tableRef.get(HiveDataModelGenerator.COMMENT), "table comment"); String entityName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName); Assert.assertEquals(tableRef.get(HiveDataModelGenerator.NAME), entityName); Assert.assertEquals( tableRef.get("name"), "default." + tableName.toLowerCase() + "@" + CLUSTER_NAME); final Referenceable sdRef = (Referenceable) tableRef.get("sd"); Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS), false); // Create table where database doesn't exist, will create database instance as well assertDatabaseIsRegistered(DEFAULT_DB); }
@Test public void testUnsecuredClient() throws Exception { try { dgiClient.listTypes(); Assert.fail("Should have failed with GSSException"); } catch (Exception e) { e.printStackTrace(); Assert.assertTrue( e.getMessage().contains("Mechanism level: Failed to find any Kerberos tgt")); } }
@Test public void testCreateDatabase() throws Exception { String dbName = "db" + random(); runCommand("create database " + dbName + " WITH DBPROPERTIES ('p1'='v1', 'p2'='v2')"); String dbId = assertDatabaseIsRegistered(dbName); Referenceable definition = dgiCLient.getEntity(dbId); Map params = (Map) definition.get("parameters"); Assert.assertNotNull(params); Assert.assertEquals(params.size(), 2); Assert.assertEquals(params.get("p1"), "v1"); // There should be just one entity per dbname runCommand("drop database " + dbName); runCommand("create database " + dbName); String dbid = assertDatabaseIsRegistered(dbName); // assert on qualified name Referenceable dbEntity = dgiCLient.getEntity(dbid); Assert.assertEquals(dbEntity.get("qualifiedName"), dbName.toLowerCase() + "@" + CLUSTER_NAME); }
/** * Submits the entity definitions (instances). The body contains the JSONArray of entity json. The * service takes care of de-duping the entities based on any unique attribute for the give type. */ @POST @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces(Servlets.JSON_MEDIA_TYPE) public Response submit(@Context HttpServletRequest request) { String entityJson = null; try { String entities = Servlets.getRequestPayload(request); // Handle backward compatibility - if entities is not JSONArray, convert to JSONArray try { new JSONArray(entities); } catch (JSONException e) { final String finalEntities = entities; entities = new JSONArray() { { put(finalEntities); } }.toString(); } entityJson = AtlasClient.toString(new JSONArray(entities)); LOG.info("submitting entities {} ", entityJson); final List<String> guids = metadataService.createEntities(entities); LOG.info("Created entities {}", guids); JSONObject response = getResponse(new AtlasClient.EntityResult(guids, null, null)); URI locationURI = getLocationURI(guids); return Response.created(locationURI).entity(response).build(); } catch (EntityExistsException e) { LOG.error("Unique constraint violation for entity entityDef={}", entityJson, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT)); } catch (ValueConversionException ve) { LOG.error( "Unable to persist entity instance due to a deserialization error entityDef={}", entityJson, ve); throw new WebApplicationException( Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST)); } catch (AtlasException | IllegalArgumentException e) { LOG.error("Unable to persist entity instance entityDef={}", entityJson, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); } catch (Throwable e) { LOG.error("Unable to persist entity instance entityDef={}", entityJson, e); throw new WebApplicationException( Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); } }
@Test public void testSelect() throws Exception { String tableName = createTable(); String query = "select * from " + tableName; runCommand(query); String pid = assertProcessIsRegistered(query); Referenceable processEntity = dgiCLient.getEntity(pid); Assert.assertEquals(processEntity.get("name"), query.toLowerCase()); // single entity per query query = "SELECT * from " + tableName.toUpperCase(); runCommand(query); assertProcessIsRegistered(query); }
@Test public void testLineage() throws Exception { String table1 = createTable(false); String db2 = createDatabase(); String table2 = tableName(); String query = String.format("create table %s.%s as select * from %s", db2, table2, table1); runCommand(query); String table1Id = assertTableIsRegistered(DEFAULT_DB, table1); String table2Id = assertTableIsRegistered(db2, table2); String datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, db2, table2); JSONObject response = dgiCLient.getInputGraph(datasetName); JSONObject vertices = response.getJSONObject("values").getJSONObject("vertices"); Assert.assertTrue(vertices.has(table1Id)); Assert.assertTrue(vertices.has(table2Id)); datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, table1); response = dgiCLient.getOutputGraph(datasetName); vertices = response.getJSONObject("values").getJSONObject("vertices"); Assert.assertTrue(vertices.has(table1Id)); Assert.assertTrue(vertices.has(table2Id)); }
@Test public void testInsert() throws Exception { String tableName = createTable(); String insertTableName = createTable(); String query = "insert into " + insertTableName + " partition(dt = '2015-01-01') select id, name from " + tableName + " where dt = '2015-01-01'"; runCommand(query); assertProcessIsRegistered(query); String partId = assertPartitionIsRegistered(DEFAULT_DB, insertTableName, "2015-01-01"); Referenceable partitionEntity = dgiCLient.getEntity(partId); Assert.assertEquals( partitionEntity.get("qualifiedName"), String.format( "%s.%s.%s@%s", "default", insertTableName.toLowerCase(), "2015-01-01", CLUSTER_NAME)); }
private String assertEntityIsRegistered(final String query, String... arg) throws Exception { waitFor( 2000, new Predicate() { @Override public boolean evaluate() throws Exception { JSONArray results = dgiCLient.search(query); return results.length() == 1; } }); String column = (arg.length > 0) ? arg[0] : "_col_0"; JSONArray results = dgiCLient.search(query); JSONObject row = results.getJSONObject(0); if (row.has("__guid")) { return row.getString("__guid"); } else if (row.has("$id$")) { return row.getJSONObject("$id$").getString("id"); } else { return row.getJSONObject(column).getString("id"); } }
/** * Complete update of a set of entities - the values not specified will be replaced with * null/removed Adds/Updates given entities identified by its GUID or unique attribute * * @return response payload as json */ @PUT @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces(Servlets.JSON_MEDIA_TYPE) public Response updateEntities(@Context HttpServletRequest request) { String entityJson = null; try { final String entities = Servlets.getRequestPayload(request); entityJson = AtlasClient.toString(new JSONArray(entities)); LOG.info("updating entities {} ", entityJson); AtlasClient.EntityResult entityResult = metadataService.updateEntities(entities); LOG.info("Updated entities: {}", entityResult); JSONObject response = getResponse(entityResult); return Response.ok(response).build(); } catch (EntityExistsException e) { LOG.error("Unique constraint violation for entityDef={}", entityJson, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT)); } catch (ValueConversionException ve) { LOG.error( "Unable to persist entity instance due to a deserialization error entityDef={}", entityJson, ve); throw new WebApplicationException( Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST)); } catch (AtlasException | IllegalArgumentException e) { LOG.error("Unable to persist entity instance entityDef={}", entityJson, e); throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST)); } catch (Throwable e) { LOG.error("Unable to persist entity instance entityDef={}", entityJson, e); throw new WebApplicationException( Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR)); } }
private void assertEntityIsNotRegistered(String dslQuery) throws Exception { JSONArray results = dgiCLient.searchByDSL(dslQuery); Assert.assertEquals(results.length(), 0); }