@Test
public void testUpdate() throws Exception {
HierarchicalTypeDefinition<ClassType> typeDefinition =
TypesUtil.createClassTypeDef(
randomString(),
ImmutableSet.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE));
List<String> typesCreated =
serviceClient.createType(TypesSerialization.toJson(typeDefinition, false));
assertEquals(typesCreated.size(), 1);
assertEquals(typesCreated.get(0), typeDefinition.typeName);
// Add attribute description
typeDefinition =
TypesUtil.createClassTypeDef(
typeDefinition.typeName,
ImmutableSet.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
createOptionalAttrDef("description", DataTypes.STRING_TYPE));
TypesDef typeDef =
TypesUtil.getTypesDef(
ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(typeDefinition));
List<String> typesUpdated = serviceClient.updateType(typeDef);
assertEquals(typesUpdated.size(), 1);
Assert.assertTrue(typesUpdated.contains(typeDefinition.typeName));
HierarchicalTypeDefinition<ClassType> updatedType =
serviceClient.getType(typeDefinition.typeName).classTypesAsJavaList().get(0);
assertEquals(updatedType.attributeDefinitions.length, 2);
}
@Test
public void testListTypesByFilter() throws Exception {
AttributeDefinition attr = TypesUtil.createOptionalAttrDef("attr", DataTypes.STRING_TYPE);
String a =
createType(
TypesSerialization.toJson(
TypesUtil.createClassTypeDef(
"A" + randomString(), ImmutableSet.<String>of(), attr),
false))
.get(0);
String a1 =
createType(
TypesSerialization.toJson(
TypesUtil.createClassTypeDef("A1" + randomString(), ImmutableSet.of(a), attr),
false))
.get(0);
String b =
createType(
TypesSerialization.toJson(
TypesUtil.createClassTypeDef(
"B" + randomString(), ImmutableSet.<String>of(), attr),
false))
.get(0);
String c =
createType(
TypesSerialization.toJson(
TypesUtil.createClassTypeDef("C" + randomString(), ImmutableSet.of(a, b), attr),
false))
.get(0);
List<String> results = serviceClient.listTypes(DataTypes.TypeCategory.CLASS, a, b);
assertEquals(results, Arrays.asList(a1), "Results: " + results);
}
@Test
public void testUTF8() throws Exception {
String classType = random();
String attrName = random();
String attrValue = random();
HierarchicalTypeDefinition<ClassType> classTypeDefinition =
TypesUtil.createClassTypeDef(
classType,
ImmutableList.<String>of(),
TypesUtil.createUniqueRequiredAttrDef(attrName, DataTypes.STRING_TYPE));
TypesDef typesDef =
TypeUtils.getTypesDef(
ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(classTypeDefinition));
createType(typesDef);
Referenceable instance = new Referenceable(classType);
instance.set(attrName, attrValue);
Id guid = createInstance(instance);
ClientResponse response = getEntityDefinition(guid._getId());
String definition = getEntityDefinition(response);
Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
Assert.assertEquals(getReferenceable.get(attrName), attrValue);
}
private List<HierarchicalTypeDefinition> createHiveTypes() throws Exception {
ArrayList<HierarchicalTypeDefinition> typeDefinitions = new ArrayList<>();
HierarchicalTypeDefinition<ClassType> databaseTypeDefinition =
TypesUtil.createClassTypeDef(
"database",
ImmutableSet.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
typeDefinitions.add(databaseTypeDefinition);
HierarchicalTypeDefinition<ClassType> tableTypeDefinition =
TypesUtil.createClassTypeDef(
"table",
ImmutableSet.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE),
createOptionalAttrDef("columnNames", DataTypes.arrayTypeName(DataTypes.STRING_TYPE)),
createOptionalAttrDef("created", DataTypes.DATE_TYPE),
createOptionalAttrDef(
"parameters", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE)),
TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE),
new AttributeDefinition(
"database", "database", Multiplicity.REQUIRED, false, "database"));
typeDefinitions.add(tableTypeDefinition);
HierarchicalTypeDefinition<TraitType> fetlTypeDefinition =
TypesUtil.createTraitTypeDef(
"fetl",
ImmutableSet.<String>of(),
TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE));
typeDefinitions.add(fetlTypeDefinition);
return typeDefinitions;
}
private String addNewType() throws Exception {
String typeName = "test" + randomString();
HierarchicalTypeDefinition<ClassType> testTypeDefinition =
TypesUtil.createClassTypeDef(
typeName,
ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
TypesUtil.createRequiredAttrDef("description", DataTypes.STRING_TYPE));
String typesAsJSON = TypesSerialization.toJson(testTypeDefinition, false);
createType(typesAsJSON);
return typeName;
}
@Test
public void testSerDeWithDescription() throws Exception {
HierarchicalTypeDefinition<ClassType> clsType =
TypesUtil.createClassTypeDef(
"Random",
"Random-description",
ImmutableSet.<String>of(),
TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE));
TypesDef typesDef = getTypesDef(clsType);
String json = TypesSerialization.toJson(typesDef);
System.out.println("json " + json);
TypesSerialization.fromJson(json);
}
@Test(dependsOnMethods = "testGetTraitNames")
public void testAddTraitWithAttribute() throws Exception {
final String traitName = "PII_Trait" + randomString();
HierarchicalTypeDefinition<TraitType> piiTrait =
TypesUtil.createTraitTypeDef(
traitName,
ImmutableList.<String>of(),
TypesUtil.createRequiredAttrDef("type", DataTypes.STRING_TYPE));
String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true);
LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON);
createType(traitDefinitionAsJSON);
Struct traitInstance = new Struct(traitName);
traitInstance.set("type", "SSN");
String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true);
LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
final String guid = tableId._getId();
ClientResponse clientResponse =
service
.path(ENTITIES)
.path(guid)
.path(TRAITS)
.accept(Servlets.JSON_MEDIA_TYPE)
.type(Servlets.JSON_MEDIA_TYPE)
.method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode());
String responseAsString = clientResponse.getEntity(String.class);
Assert.assertNotNull(responseAsString);
JSONObject response = new JSONObject(responseAsString);
Assert.assertNotNull(response.get(AtlasClient.REQUEST_ID));
Assert.assertNotNull(response.get(AtlasClient.GUID));
// verify the response
clientResponse = getEntityDefinition(guid);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
responseAsString = clientResponse.getEntity(String.class);
Assert.assertNotNull(responseAsString);
response = new JSONObject(responseAsString);
Assert.assertNotNull(response.get(AtlasClient.REQUEST_ID));
final String definition = response.getString(AtlasClient.DEFINITION);
Assert.assertNotNull(definition);
Referenceable entityRef = InstanceSerialization.fromJsonReferenceable(definition, true);
IStruct traitRef = entityRef.getTrait(traitName);
String type = (String) traitRef.get("type");
Assert.assertEquals(type, "SSN");
}
@Test(dependsOnMethods = "testGetTraitNames")
public void testAddTrait() throws Exception {
traitName = "PII_Trait" + randomString();
HierarchicalTypeDefinition<TraitType> piiTrait =
TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of());
String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true);
LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON);
createType(traitDefinitionAsJSON);
Struct traitInstance = new Struct(traitName);
String traitInstanceAsJSON = InstanceSerialization.toJson(traitInstance, true);
LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
final String guid = tableId._getId();
ClientResponse clientResponse =
service
.path(ENTITIES)
.path(guid)
.path(TRAITS)
.accept(Servlets.JSON_MEDIA_TYPE)
.type(Servlets.JSON_MEDIA_TYPE)
.method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.CREATED.getStatusCode());
String responseAsString = clientResponse.getEntity(String.class);
Assert.assertNotNull(responseAsString);
JSONObject response = new JSONObject(responseAsString);
Assert.assertNotNull(response.get(AtlasClient.REQUEST_ID));
Assert.assertNotNull(response.get(AtlasClient.GUID));
}
@Override
protected TypesDef getTypesDef(HierarchicalTypeDefinition<ClassType>... typeDefinitions) {
return TypesUtil.getTypesDef(
ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.copyOf(typeDefinitions));
}
@Override
protected TypesDef getTypesDef(StructTypeDefinition typeDefinition) {
return TypesUtil.getTypesDef(
ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of((HierarchicalTypeDefinition<ClassType>) typeDefinition));
}
@Test
public void testDuplicateSubmit() throws Exception {
HierarchicalTypeDefinition<ClassType> type =
TypesUtil.createClassTypeDef(
randomString(),
ImmutableSet.<String>of(),
TypesUtil.createUniqueRequiredAttrDef("name", DataTypes.STRING_TYPE));
TypesDef typesDef =
TypesUtil.getTypesDef(
ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(type));
serviceClient.createType(typesDef);
try {
serviceClient.createType(typesDef);
fail("Expected 409");
} catch (AtlasServiceException e) {
assertEquals(e.getStatus().getStatusCode(), Response.Status.CONFLICT.getStatusCode());
}
}
private String[] addTraits() throws Exception {
String[] traitNames = {
"class_trait", "secure_trait", "pii_trait", "ssn_trait", "salary_trait", "sox_trait",
};
for (String traitName : traitNames) {
HierarchicalTypeDefinition<TraitType> traitTypeDef =
TypesUtil.createTraitTypeDef(traitName, ImmutableSet.<String>of());
String json = TypesSerialization$.MODULE$.toJson(traitTypeDef, true);
createType(json);
}
return traitNames;
}
@Test
public void testAddTraitWithNoRegistration() throws Exception {
final String traitName = "PII_Trait" + randomString();
HierarchicalTypeDefinition<TraitType> piiTrait =
TypesUtil.createTraitTypeDef(traitName, ImmutableList.<String>of());
String traitDefinitionAsJSON = TypesSerialization$.MODULE$.toJson(piiTrait, true);
LOG.debug("traitDefinitionAsJSON = " + traitDefinitionAsJSON);
Struct traitInstance = new Struct(traitName);
String traitInstanceAsJSON = InstanceSerialization$.MODULE$.toJson(traitInstance, true);
LOG.debug("traitInstanceAsJSON = " + traitInstanceAsJSON);
ClientResponse clientResponse =
service
.path(ENTITIES)
.path("random")
.path(TRAITS)
.accept(Servlets.JSON_MEDIA_TYPE)
.type(Servlets.JSON_MEDIA_TYPE)
.method(HttpMethod.POST, ClientResponse.class, traitInstanceAsJSON);
Assert.assertEquals(clientResponse.getStatus(), Response.Status.NOT_FOUND.getStatusCode());
}
@Test(dependsOnMethods = "testStore")
public void testTypeUpdate() throws Exception {
// Add enum value
String _description = "_description_updated";
EnumTypeDefinition orgLevelEnum =
new EnumTypeDefinition(
"OrgLevel",
"OrgLevel" + _description,
new EnumValue("L1", 1),
new EnumValue("L2", 2),
new EnumValue("L3", 3));
// Add attribute
StructTypeDefinition addressDetails =
createStructTypeDef(
"Address",
createRequiredAttrDef("street", DataTypes.STRING_TYPE),
createRequiredAttrDef("city", DataTypes.STRING_TYPE),
createOptionalAttrDef("state", DataTypes.STRING_TYPE));
HierarchicalTypeDefinition<ClassType> deptTypeDef =
createClassTypeDef(
"Department",
"Department" + _description,
ImmutableSet.<String>of(),
createRequiredAttrDef("name", DataTypes.STRING_TYPE),
new AttributeDefinition(
"employees",
String.format("array<%s>", "Person"),
Multiplicity.OPTIONAL,
true,
"department"));
TypesDef typesDef =
TypesUtil.getTypesDef(
ImmutableList.of(orgLevelEnum),
ImmutableList.of(addressDetails),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(deptTypeDef));
Map<String, IDataType> typesAdded = ts.updateTypes(typesDef);
typeStore.store(ts, ImmutableList.copyOf(typesAdded.keySet()));
verifyEdges();
// Validate the updated types
TypesDef types = typeStore.restore();
ts.reset();
ts.defineTypes(types);
// Assert new enum value
EnumType orgLevel = ts.getDataType(EnumType.class, orgLevelEnum.name);
Assert.assertEquals(orgLevel.name, orgLevelEnum.name);
Assert.assertEquals(orgLevel.description, orgLevelEnum.description);
Assert.assertEquals(orgLevel.values().size(), orgLevelEnum.enumValues.length);
Assert.assertEquals(orgLevel.fromValue("L3").ordinal, 3);
// Assert new attribute
StructType addressType = ts.getDataType(StructType.class, addressDetails.typeName);
Assert.assertEquals(addressType.numFields, 3);
Assert.assertEquals(
addressType.fieldMapping.fields.get("state").dataType(), DataTypes.STRING_TYPE);
// Updating the definition again shouldn't add another edge
typesDef =
TypesUtil.getTypesDef(
ImmutableList.<EnumTypeDefinition>of(),
ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(deptTypeDef));
typesAdded = ts.updateTypes(typesDef);
typeStore.store(ts, ImmutableList.copyOf(typesAdded.keySet()));
verifyEdges();
}