private void dumpGraph() {
AtlasGraph<?, ?> graph = TestUtils.getGraph();
for (AtlasVertex<?, ?> v : graph.getVertices()) {
System.out.println("****v = " + GraphHelper.vertexString(v));
for (AtlasEdge<?, ?> e : v.getEdges(AtlasEdgeDirection.OUT)) {
System.out.println("****e = " + GraphHelper.edgeString(e));
}
}
}
public static void updateVertexPreUpdate(
AtlasStructDef structDef,
AtlasStructType structType,
AtlasVertex vertex,
AtlasTypeDefGraphStoreV1 typeDefStore)
throws AtlasBaseException {
List<String> attrNames = new ArrayList<>();
if (CollectionUtils.isNotEmpty(structDef.getAttributeDefs())) {
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
attrNames.add(attributeDef.getName());
}
}
List<String> currAttrNames =
vertex.getProperty(AtlasGraphUtilsV1.getPropertyKey(structDef), List.class);
// delete attributes that are not present in updated structDef
if (CollectionUtils.isNotEmpty(currAttrNames)) {
for (String currAttrName : currAttrNames) {
if (!attrNames.contains(currAttrName)) {
throw new AtlasBaseException(
AtlasErrorCode.ATTRIBUTE_DELETION_NOT_SUPPORTED, structDef.getName(), currAttrName);
}
}
}
typeDefStore.updateTypeVertex(structDef, vertex);
// add/update attributes that are present in updated structDef
if (CollectionUtils.isNotEmpty(structDef.getAttributeDefs())) {
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
if (CollectionUtils.isEmpty(currAttrNames)
|| !currAttrNames.contains(attributeDef.getName())) {
// new attribute - only allow if optional
if (!attributeDef.isOptional()) {
throw new AtlasBaseException(
AtlasErrorCode.CANNOT_ADD_MANDATORY_ATTRIBUTE,
structDef.getName(),
attributeDef.getName());
}
}
String propertyKey = AtlasGraphUtilsV1.getPropertyKey(structDef, attributeDef.getName());
AtlasGraphUtilsV1.setProperty(
vertex, propertyKey, toJsonFromAttributeDef(attributeDef, structType));
}
}
AtlasGraphUtilsV1.setProperty(vertex, AtlasGraphUtilsV1.getPropertyKey(structDef), attrNames);
}
@Override
public AtlasStructDef getByName(String name) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasStructDefStoreV1.getByName({})", name);
}
AtlasVertex vertex = typeDefStore.findTypeVertexByNameAndCategory(name, TypeCategory.STRUCT);
if (vertex == null) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_NOT_FOUND, name);
}
vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY, String.class);
AtlasStructDef ret = toStructDef(vertex);
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasStructDefStoreV1.getByName({}): {}", name, ret);
}
return ret;
}
public static AtlasStructDef toStructDef(
AtlasVertex vertex, AtlasStructDef structDef, AtlasTypeDefGraphStoreV1 typeDefStore)
throws AtlasBaseException {
AtlasStructDef ret = (structDef != null) ? structDef : new AtlasStructDef();
typeDefStore.vertexToTypeDef(vertex, ret);
List<AtlasAttributeDef> attributeDefs = new ArrayList<>();
List<String> attrNames = vertex.getProperty(AtlasGraphUtilsV1.getPropertyKey(ret), List.class);
if (CollectionUtils.isNotEmpty(attrNames)) {
for (String attrName : attrNames) {
String propertyKey = AtlasGraphUtilsV1.getPropertyKey(ret, attrName);
String attribJson = vertex.getProperty(propertyKey, String.class);
attributeDefs.add(
toAttributeDefFromJson(
structDef, AtlasType.fromJson(attribJson, Map.class), typeDefStore));
}
}
ret.setAttributeDefs(attributeDefs);
return ret;
}
protected void deleteVertex(AtlasVertex instanceVertex, boolean force) throws AtlasException {
// Update external references(incoming edges) to this vertex
LOG.debug(
"Setting the external references to {} to null(removing edges)", string(instanceVertex));
Iterator<AtlasEdge> edges = instanceVertex.getEdges(AtlasEdgeDirection.IN).iterator();
while (edges.hasNext()) {
AtlasEdge edge = edges.next();
Id.EntityState edgeState = GraphHelper.getState(edge);
if (edgeState == Id.EntityState.ACTIVE) {
// Delete only the active edge references
AttributeInfo attribute = getAttributeForEdge(edge.getLabel());
// TODO use delete edge instead??
deleteEdgeBetweenVertices(edge.getOutVertex(), edge.getInVertex(), attribute.name);
}
}
_deleteVertex(instanceVertex, force);
}
private static void addReferencesForAttribute(
AtlasVertex vertex, AtlasAttributeDef attributeDef, AtlasTypeDefGraphStoreV1 typeDefStore)
throws AtlasBaseException {
Set<String> referencedTypeNames =
AtlasTypeUtil.getReferencedTypeNames(attributeDef.getTypeName());
String typeName = vertex.getProperty(Constants.TYPENAME_PROPERTY_KEY, String.class);
for (String referencedTypeName : referencedTypeNames) {
if (!AtlasTypeUtil.isBuiltInType(referencedTypeName)) {
AtlasVertex referencedTypeVertex = typeDefStore.findTypeVertexByName(referencedTypeName);
if (referencedTypeVertex == null) {
throw new AtlasBaseException(
AtlasErrorCode.UNKNOWN_TYPE, referencedTypeName, typeName, attributeDef.getName());
}
String label = AtlasGraphUtilsV1.getEdgeLabel(typeName, attributeDef.getName());
typeDefStore.getOrCreateEdge(vertex, referencedTypeVertex, label);
}
}
}
/**
* Deletes the edge between outvertex and inVertex. The edge is for attribute attributeName of
* outVertex
*
* @param outVertex
* @param inVertex
* @param attributeName
* @throws AtlasException
*/
protected void deleteEdgeBetweenVertices(
AtlasVertex outVertex, AtlasVertex inVertex, String attributeName) throws AtlasException {
LOG.debug(
"Removing edge from {} to {} with attribute name {}",
string(outVertex),
string(inVertex),
attributeName);
String typeName = GraphHelper.getTypeName(outVertex);
String outId = GraphHelper.getIdFromVertex(outVertex);
Id.EntityState state = GraphHelper.getState(outVertex);
if ((outId != null && RequestContext.get().isDeletedEntity(outId))
|| state == Id.EntityState.DELETED) {
// If the reference vertex is marked for deletion, skip updating the reference
return;
}
IDataType type = typeSystem.getDataType(IDataType.class, typeName);
AttributeInfo attributeInfo = getFieldMapping(type).fields.get(attributeName);
String propertyName = GraphHelper.getQualifiedFieldName(type, attributeName);
String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
AtlasEdge edge = null;
switch (attributeInfo.dataType().getTypeCategory()) {
case CLASS:
// If its class attribute, its the only edge between two vertices
if (attributeInfo.multiplicity.nullAllowed()) {
edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel);
if (shouldUpdateReverseAttribute) {
GraphHelper.setProperty(outVertex, propertyName, null);
}
} else {
// Cannot unset a required attribute.
throw new NullRequiredAttributeException(
"Cannot unset required attribute "
+ GraphHelper.getQualifiedFieldName(type, attributeName)
+ " on "
+ GraphHelper.getVertexDetails(outVertex)
+ " edge = "
+ edgeLabel);
}
break;
case ARRAY:
// If its array attribute, find the right edge between the two vertices and update array
// property
List<String> elements = GraphHelper.getListProperty(outVertex, propertyName);
if (elements != null) {
elements =
new ArrayList<>(
elements); // Make a copy, else list.remove reflects on titan.getProperty()
for (String elementEdgeId : elements) {
AtlasEdge elementEdge =
graphHelper.getEdgeByEdgeId(outVertex, edgeLabel, elementEdgeId);
if (elementEdge == null) {
continue;
}
AtlasVertex elementVertex = elementEdge.getInVertex();
if (elementVertex.equals(inVertex)) {
edge = elementEdge;
// TODO element.size includes deleted items as well. should exclude
if (!attributeInfo.multiplicity.nullAllowed()
&& elements.size() <= attributeInfo.multiplicity.lower) {
// Deleting this edge would violate the attribute's lower bound.
throw new NullRequiredAttributeException(
"Cannot remove array element from required attribute "
+ GraphHelper.getQualifiedFieldName(type, attributeName)
+ " on "
+ GraphHelper.getVertexDetails(outVertex)
+ " "
+ GraphHelper.getEdgeDetails(elementEdge));
}
if (shouldUpdateReverseAttribute) {
// if composite attribute, remove the reference as well. else, just remove the edge
// for example, when table is deleted, process still references the table
// but when column is deleted, table will not reference the deleted column
LOG.debug(
"Removing edge {} from the array attribute {}",
string(elementEdge),
attributeName);
elements.remove(elementEdge.getId().toString());
GraphHelper.setProperty(outVertex, propertyName, elements);
break;
}
}
}
}
break;
case MAP:
// If its map attribute, find the right edge between two vertices and update map property
List<String> keys = GraphHelper.getListProperty(outVertex, propertyName);
if (keys != null) {
keys =
new ArrayList<>(
keys); // Make a copy, else list.remove reflects on titan.getProperty()
for (String key : keys) {
String keyPropertyName = GraphHelper.getQualifiedNameForMapKey(propertyName, key);
String mapEdgeId =
GraphHelper.getSingleValuedProperty(outVertex, keyPropertyName, String.class);
AtlasEdge mapEdge = graphHelper.getEdgeByEdgeId(outVertex, keyPropertyName, mapEdgeId);
if (mapEdge != null) {
AtlasVertex mapVertex = mapEdge.getInVertex();
if (mapVertex.getId().toString().equals(inVertex.getId().toString())) {
// TODO keys.size includes deleted items as well. should exclude
if (attributeInfo.multiplicity.nullAllowed()
|| keys.size() > attributeInfo.multiplicity.lower) {
edge = mapEdge;
} else {
// Deleting this entry would violate the attribute's lower bound.
throw new NullRequiredAttributeException(
"Cannot remove map entry "
+ keyPropertyName
+ " from required attribute "
+ GraphHelper.getQualifiedFieldName(type, attributeName)
+ " on "
+ GraphHelper.getVertexDetails(outVertex)
+ " "
+ GraphHelper.getEdgeDetails(mapEdge));
}
if (shouldUpdateReverseAttribute) {
// remove this key
LOG.debug(
"Removing edge {}, key {} from the map attribute {}",
string(mapEdge),
key,
attributeName);
keys.remove(key);
GraphHelper.setProperty(outVertex, propertyName, keys);
GraphHelper.setProperty(outVertex, keyPropertyName, null);
}
break;
}
}
}
}
break;
case STRUCT:
case TRAIT:
break;
default:
throw new IllegalStateException(
"There can't be an edge from "
+ GraphHelper.getVertexDetails(outVertex)
+ " to "
+ GraphHelper.getVertexDetails(inVertex)
+ " with attribute name "
+ attributeName
+ " which is not class/array/map attribute");
}
if (edge != null) {
deleteEdge(edge, false);
RequestContext requestContext = RequestContext.get();
GraphHelper.setProperty(
outVertex,
Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
requestContext.getRequestTime());
requestContext.recordEntityUpdate(outId);
}
}
private static AtlasAttributeDef toAttributeDefFromJson(
AtlasStructDef structDef, Map attribInfo, AtlasTypeDefGraphStoreV1 typeDefStore)
throws AtlasBaseException {
AtlasAttributeDef ret = new AtlasAttributeDef();
ret.setName((String) attribInfo.get("name"));
ret.setTypeName((String) attribInfo.get("dataType"));
ret.setUnique((Boolean) attribInfo.get("isUnique"));
ret.setIndexable((Boolean) attribInfo.get("isIndexable"));
String attrTypeName = ret.getTypeName();
if (AtlasTypeUtil.isArrayType(attrTypeName)) {
Set<String> typeNames = AtlasTypeUtil.getReferencedTypeNames(ret.getTypeName());
if (typeNames.size() > 0) {
attrTypeName = typeNames.iterator().next();
}
}
if (!AtlasTypeUtil.isBuiltInType(attrTypeName)) {
AtlasVertex attributeType = typeDefStore.findTypeVertexByName(attrTypeName);
// check for isComposite/reverseAttributeName for entity types
if (attributeType != null && typeDefStore.isTypeVertex(attributeType, TypeCategory.CLASS)) {
String reverseAttribName = (String) attribInfo.get("reverseAttributeName");
Boolean isComposite = (Boolean) attribInfo.get("isComposite");
if (StringUtils.isNotBlank(reverseAttribName) || isComposite) {
if (AtlasTypeUtil.isMapType(attrTypeName)) {
throw new AtlasBaseException(
AtlasErrorCode.CONSTRAINT_NOT_SUPPORTED_ON_MAP_TYPE,
structDef.getName(),
ret.getName(),
attrTypeName);
}
String refAttributeName = null;
List<String> attrNames =
attributeType.getProperty(AtlasGraphUtilsV1.getPropertyKey(attrTypeName), List.class);
if (CollectionUtils.isNotEmpty(attrNames)) {
for (String attrName : attrNames) {
String attribJson =
attributeType.getProperty(
AtlasGraphUtilsV1.getPropertyKey(attrTypeName, attrName), String.class);
Map refAttrInfo = AtlasType.fromJson(attribJson, Map.class);
String refAttribType = (String) refAttrInfo.get("dataType");
String refAttribRevAttribName = (String) refAttrInfo.get("reverseAttributeName");
if (StringUtils.equals(refAttribType, structDef.getName())
&& StringUtils.equals(refAttribRevAttribName, ret.getName())) {
refAttributeName = (String) refAttrInfo.get("name");
break;
}
}
}
if (isComposite) {
if (StringUtils.isNotBlank(
refAttributeName)) { // ex: hive_table.columns, hive_column.table
Map<String, Object> params = new HashMap<>();
params.put(AtlasConstraintDef.CONSTRAINT_PARAM_REF_ATTRIBUTE, refAttributeName);
ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_MAPPED_FROM_REF, params));
} else { // ex: hive_table.partitionKeys, with no reverseAttribute-reference
ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY));
}
}
if (StringUtils.isNotBlank(reverseAttribName)) { // ex: hive_column.table
Map<String, Object> params = new HashMap<>();
params.put(CONSTRAINT_PARAM_ON_DELETE, CONSTRAINT_PARAM_VAL_CASCADE);
ret.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY, params));
}
}
}
}
Map multiplicity = AtlasType.fromJson((String) attribInfo.get("multiplicity"), Map.class);
Number minCount = (Number) multiplicity.get("lower");
Number maxCount = (Number) multiplicity.get("upper");
Boolean isUnique = (Boolean) multiplicity.get("isUnique");
if (minCount == null || minCount.intValue() == 0) {
ret.setOptional(true);
ret.setValuesMinCount(0);
} else {
ret.setOptional(false);
ret.setValuesMinCount(minCount.intValue());
}
if (maxCount == null || maxCount.intValue() < 2) {
ret.setCardinality(AtlasAttributeDef.Cardinality.SINGLE);
ret.setValuesMaxCount(1);
} else {
if (isUnique == null || isUnique == Boolean.FALSE) {
ret.setCardinality(AtlasAttributeDef.Cardinality.LIST);
} else {
ret.setCardinality(AtlasAttributeDef.Cardinality.SET);
}
ret.setValuesMaxCount(maxCount.intValue());
}
return ret;
}