@Test
public void testCreateTable() throws Exception {
String tableName = tableName();
String dbName = createDatabase();
String colName = "col" + random();
runCommand("create table " + dbName + "." + tableName + "(" + colName + " int, name string)");
assertTableIsRegistered(dbName, tableName);
// there is only one instance of column registered
String colId = assertColumnIsRegistered(colName);
Referenceable colEntity = dgiCLient.getEntity(colId);
Assert.assertEquals(
colEntity.get("qualifiedName"),
String.format(
"%s.%s.%s@%s",
dbName.toLowerCase(), tableName.toLowerCase(), colName.toLowerCase(), CLUSTER_NAME));
tableName = createTable();
String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
Referenceable tableRef = dgiCLient.getEntity(tableId);
Assert.assertEquals(tableRef.get("tableType"), TableType.MANAGED_TABLE.name());
Assert.assertEquals(tableRef.get(HiveDataModelGenerator.COMMENT), "table comment");
String entityName =
HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName);
Assert.assertEquals(tableRef.get(HiveDataModelGenerator.NAME), entityName);
Assert.assertEquals(
tableRef.get("name"), "default." + tableName.toLowerCase() + "@" + CLUSTER_NAME);
final Referenceable sdRef = (Referenceable) tableRef.get("sd");
Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS), false);
// Create table where database doesn't exist, will create database instance as well
assertDatabaseIsRegistered(DEFAULT_DB);
}
@Test
public void testUnsecuredClient() throws Exception {
try {
dgiClient.listTypes();
Assert.fail("Should have failed with GSSException");
} catch (Exception e) {
e.printStackTrace();
Assert.assertTrue(
e.getMessage().contains("Mechanism level: Failed to find any Kerberos tgt"));
}
}
@Test
public void testCreateDatabase() throws Exception {
String dbName = "db" + random();
runCommand("create database " + dbName + " WITH DBPROPERTIES ('p1'='v1', 'p2'='v2')");
String dbId = assertDatabaseIsRegistered(dbName);
Referenceable definition = dgiCLient.getEntity(dbId);
Map params = (Map) definition.get("parameters");
Assert.assertNotNull(params);
Assert.assertEquals(params.size(), 2);
Assert.assertEquals(params.get("p1"), "v1");
// There should be just one entity per dbname
runCommand("drop database " + dbName);
runCommand("create database " + dbName);
String dbid = assertDatabaseIsRegistered(dbName);
// assert on qualified name
Referenceable dbEntity = dgiCLient.getEntity(dbid);
Assert.assertEquals(dbEntity.get("qualifiedName"), dbName.toLowerCase() + "@" + CLUSTER_NAME);
}
/**
* Submits the entity definitions (instances). The body contains the JSONArray of entity json. The
* service takes care of de-duping the entities based on any unique attribute for the give type.
*/
@POST
@Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON})
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response submit(@Context HttpServletRequest request) {
String entityJson = null;
try {
String entities = Servlets.getRequestPayload(request);
// Handle backward compatibility - if entities is not JSONArray, convert to JSONArray
try {
new JSONArray(entities);
} catch (JSONException e) {
final String finalEntities = entities;
entities =
new JSONArray() {
{
put(finalEntities);
}
}.toString();
}
entityJson = AtlasClient.toString(new JSONArray(entities));
LOG.info("submitting entities {} ", entityJson);
final List<String> guids = metadataService.createEntities(entities);
LOG.info("Created entities {}", guids);
JSONObject response = getResponse(new AtlasClient.EntityResult(guids, null, null));
URI locationURI = getLocationURI(guids);
return Response.created(locationURI).entity(response).build();
} catch (EntityExistsException e) {
LOG.error("Unique constraint violation for entity entityDef={}", entityJson, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
} catch (ValueConversionException ve) {
LOG.error(
"Unable to persist entity instance due to a deserialization error entityDef={}",
entityJson,
ve);
throw new WebApplicationException(
Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to persist entity instance entityDef={}", entityJson, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to persist entity instance entityDef={}", entityJson, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
@Test
public void testSelect() throws Exception {
String tableName = createTable();
String query = "select * from " + tableName;
runCommand(query);
String pid = assertProcessIsRegistered(query);
Referenceable processEntity = dgiCLient.getEntity(pid);
Assert.assertEquals(processEntity.get("name"), query.toLowerCase());
// single entity per query
query = "SELECT * from " + tableName.toUpperCase();
runCommand(query);
assertProcessIsRegistered(query);
}
@Test
public void testLineage() throws Exception {
String table1 = createTable(false);
String db2 = createDatabase();
String table2 = tableName();
String query = String.format("create table %s.%s as select * from %s", db2, table2, table1);
runCommand(query);
String table1Id = assertTableIsRegistered(DEFAULT_DB, table1);
String table2Id = assertTableIsRegistered(db2, table2);
String datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, db2, table2);
JSONObject response = dgiCLient.getInputGraph(datasetName);
JSONObject vertices = response.getJSONObject("values").getJSONObject("vertices");
Assert.assertTrue(vertices.has(table1Id));
Assert.assertTrue(vertices.has(table2Id));
datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, table1);
response = dgiCLient.getOutputGraph(datasetName);
vertices = response.getJSONObject("values").getJSONObject("vertices");
Assert.assertTrue(vertices.has(table1Id));
Assert.assertTrue(vertices.has(table2Id));
}
@Test
public void testInsert() throws Exception {
String tableName = createTable();
String insertTableName = createTable();
String query =
"insert into "
+ insertTableName
+ " partition(dt = '2015-01-01') select id, name from "
+ tableName
+ " where dt = '2015-01-01'";
runCommand(query);
assertProcessIsRegistered(query);
String partId = assertPartitionIsRegistered(DEFAULT_DB, insertTableName, "2015-01-01");
Referenceable partitionEntity = dgiCLient.getEntity(partId);
Assert.assertEquals(
partitionEntity.get("qualifiedName"),
String.format(
"%s.%s.%s@%s", "default", insertTableName.toLowerCase(), "2015-01-01", CLUSTER_NAME));
}
private String assertEntityIsRegistered(final String query, String... arg) throws Exception {
waitFor(
2000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
JSONArray results = dgiCLient.search(query);
return results.length() == 1;
}
});
String column = (arg.length > 0) ? arg[0] : "_col_0";
JSONArray results = dgiCLient.search(query);
JSONObject row = results.getJSONObject(0);
if (row.has("__guid")) {
return row.getString("__guid");
} else if (row.has("$id$")) {
return row.getJSONObject("$id$").getString("id");
} else {
return row.getJSONObject(column).getString("id");
}
}
/**
* Complete update of a set of entities - the values not specified will be replaced with
* null/removed Adds/Updates given entities identified by its GUID or unique attribute
*
* @return response payload as json
*/
@PUT
@Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON})
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response updateEntities(@Context HttpServletRequest request) {
String entityJson = null;
try {
final String entities = Servlets.getRequestPayload(request);
entityJson = AtlasClient.toString(new JSONArray(entities));
LOG.info("updating entities {} ", entityJson);
AtlasClient.EntityResult entityResult = metadataService.updateEntities(entities);
LOG.info("Updated entities: {}", entityResult);
JSONObject response = getResponse(entityResult);
return Response.ok(response).build();
} catch (EntityExistsException e) {
LOG.error("Unique constraint violation for entityDef={}", entityJson, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
} catch (ValueConversionException ve) {
LOG.error(
"Unable to persist entity instance due to a deserialization error entityDef={}",
entityJson,
ve);
throw new WebApplicationException(
Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST));
} catch (AtlasException | IllegalArgumentException e) {
LOG.error("Unable to persist entity instance entityDef={}", entityJson, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to persist entity instance entityDef={}", entityJson, e);
throw new WebApplicationException(
Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
}
}
private void assertEntityIsNotRegistered(String dslQuery) throws Exception {
JSONArray results = dgiCLient.searchByDSL(dslQuery);
Assert.assertEquals(results.length(), 0);
}