private void writeEdges(Writer writer, List<Edge> edges, Map<Vertex, Integer> ids)
throws IOException {
for (Edge e : edges) {
writeEdgeProperties(
writer, e, ids.get(e.getVertex(Direction.OUT)), ids.get(e.getVertex(Direction.IN)));
}
}
private void verifyUpdatedEdges(Process newProcess) {
Vertex processVertex = getEntityVertex(newProcess.getName(), RelationshipType.PROCESS_ENTITY);
// cluster
Edge edge =
processVertex
.getEdges(Direction.OUT, RelationshipLabel.PROCESS_CLUSTER_EDGE.getName())
.iterator()
.next();
Assert.assertEquals(edge.getVertex(Direction.IN).getProperty("name"), anotherCluster.getName());
// inputs
edge =
processVertex
.getEdges(Direction.IN, RelationshipLabel.FEED_PROCESS_EDGE.getName())
.iterator()
.next();
Assert.assertEquals(
edge.getVertex(Direction.OUT).getProperty("name"),
newProcess.getInputs().getInputs().get(0).getFeed());
// outputs
for (Edge e :
processVertex.getEdges(Direction.OUT, RelationshipLabel.PROCESS_FEED_EDGE.getName())) {
Assert.fail("there should not be any edges to output feeds" + e);
}
}
private static void debugEdge(final Edge e) {
System.out.println("edge " + e + ":");
for (String key : e.getPropertyKeys()) {
System.out.println("\t" + key + ":\t'" + e.getProperty(key) + "'");
}
System.out.println("\t[in vertex]: " + e.getVertex(Direction.IN));
System.out.println("\t[out vertex]: " + e.getVertex(Direction.OUT));
}
public static String edgeString(final Edge edge) {
return "e["
+ edge.getLabel()
+ "], ["
+ edge.getVertex(Direction.OUT)
+ " -> "
+ edge.getLabel()
+ " -> "
+ edge.getVertex(Direction.IN)
+ "]";
}
static Edge addEdge(Graph graph, Edge edge) {
Edge newEdge = graph.getEdge(edge.getId());
if (null == newEdge) {
Vertex outVertex = addNode(graph, edge.getVertex(Direction.OUT));
Vertex inVertex = addNode(graph, edge.getVertex(Direction.IN));
String label = edge.getLabel();
newEdge = graph.addEdge(edge.getId(), outVertex, inVertex, label);
copyProperties(edge, edge);
}
return newEdge;
}
private void verifyProcessEntityEdges() {
Vertex processVertex = getEntityVertex(PROCESS_ENTITY_NAME, RelationshipType.PROCESS_ENTITY);
// verify edge to cluster vertex
verifyVertexForEdge(
processVertex,
Direction.OUT,
RelationshipLabel.FEED_CLUSTER_EDGE.getName(),
CLUSTER_ENTITY_NAME,
RelationshipType.CLUSTER_ENTITY.getName());
// verify edge to user vertex
verifyVertexForEdge(
processVertex,
Direction.OUT,
RelationshipLabel.USER.getName(),
FALCON_USER,
RelationshipType.USER.getName());
// verify edge to tags vertex
verifyVertexForEdge(
processVertex, Direction.OUT, "classified-as", "Critical", RelationshipType.TAGS.getName());
// verify edges to inputs
List<String> actual = new ArrayList<String>();
for (Edge edge :
processVertex.getEdges(Direction.IN, RelationshipLabel.FEED_PROCESS_EDGE.getName())) {
Vertex outVertex = edge.getVertex(Direction.OUT);
Assert.assertEquals(
RelationshipType.FEED_ENTITY.getName(),
outVertex.getProperty(RelationshipProperty.TYPE.getName()));
actual.add(outVertex.<String>getProperty(RelationshipProperty.NAME.getName()));
}
Assert.assertTrue(
actual.containsAll(Arrays.asList("impression-feed", "clicks-feed")),
"Actual does not contain expected: " + actual);
actual.clear();
// verify edges to outputs
for (Edge edge :
processVertex.getEdges(Direction.OUT, RelationshipLabel.PROCESS_FEED_EDGE.getName())) {
Vertex outVertex = edge.getVertex(Direction.IN);
Assert.assertEquals(
RelationshipType.FEED_ENTITY.getName(),
outVertex.getProperty(RelationshipProperty.TYPE.getName()));
actual.add(outVertex.<String>getProperty(RelationshipProperty.NAME.getName()));
}
Assert.assertTrue(
actual.containsAll(Arrays.asList("imp-click-join1", "imp-click-join2")),
"Actual does not contain expected: " + actual);
}
protected void deleteEdge(Edge edge, boolean updateReverseAttribute, boolean force)
throws AtlasException {
// update reverse attribute
if (updateReverseAttribute) {
AttributeInfo attributeInfo = getAttributeForEdge(edge.getLabel());
if (attributeInfo.reverseAttributeName != null) {
deleteEdgeBetweenVertices(
edge.getVertex(Direction.IN),
edge.getVertex(Direction.OUT),
attributeInfo.reverseAttributeName);
}
}
deleteEdge(edge, force);
}
public Object processEdge(
final Range annotation,
final Method method,
final Object[] arguments,
final FramedGraph framedGraph,
final Edge edge,
final Direction direction) {
return framedGraph.frame(edge.getVertex(direction.opposite()), method.getReturnType());
}
protected void deleteVertex(Vertex instanceVertex, boolean force) throws AtlasException {
// Update external references(incoming edges) to this vertex
LOG.debug(
"Setting the external references to {} to null(removing edges)", string(instanceVertex));
Iterator<Edge> edges = instanceVertex.getEdges(Direction.IN).iterator();
while (edges.hasNext()) {
Edge edge = edges.next();
Id.EntityState edgeState = GraphHelper.getState(edge);
if (edgeState == Id.EntityState.ACTIVE) {
// Delete only the active edge references
AttributeInfo attribute = getAttributeForEdge(edge.getLabel());
// TODO use delete edge instead??
deleteEdgeBetweenVertices(
edge.getVertex(Direction.OUT), edge.getVertex(Direction.IN), attribute.name);
}
}
_deleteVertex(instanceVertex, force);
}
/**
* Creates a Jackson ObjectNode from a graph element.
*
* @param element the graph element to convert to JSON.
* @param propertyKeys The property keys at the root of the element to serialize. If null, then
* all keys are serialized.
* @param showTypes Data types are written to the JSON explicitly if true.
*/
public static ObjectNode createJSONElementAsObjectNode(
final Element element, final List<String> propertyKeys, final boolean showTypes) {
ObjectNode jsonElement =
createJSONMap(createPropertyMap(element, propertyKeys), propertyKeys, showTypes);
putObject(jsonElement, GraphSONTokens._ID, element.getId());
if (element instanceof Vertex) {
jsonElement.put(GraphSONTokens._TYPE, GraphSONTokens.VERTEX);
} else if (element instanceof Edge) {
final Edge edge = (Edge) element;
jsonElement.put(GraphSONTokens._TYPE, GraphSONTokens.EDGE);
putObject(jsonElement, GraphSONTokens._OUT_V, edge.getVertex(Direction.OUT).getId());
putObject(jsonElement, GraphSONTokens._IN_V, edge.getVertex(Direction.IN).getId());
jsonElement.put(GraphSONTokens._LABEL, edge.getLabel());
}
return jsonElement;
}
private void verifyVertexForEdge(
Vertex fromVertex,
Direction direction,
String label,
String expectedName,
String expectedType) {
for (Edge edge : fromVertex.getEdges(direction, label)) {
Vertex outVertex = edge.getVertex(Direction.IN);
Assert.assertEquals(outVertex.getProperty(RelationshipProperty.NAME.getName()), expectedName);
Assert.assertEquals(outVertex.getProperty(RelationshipProperty.TYPE.getName()), expectedType);
}
}
public Edge removeRelation(String edgeId, boolean cascade) {
LOG.debug("Removing edge with id {}", edgeId);
final Edge edge = titanGraph.getEdge(edgeId);
titanGraph.removeEdge(edge);
LOG.info("Removed edge {}", edge);
if (cascade) {
Vertex referredVertex = edge.getVertex(Direction.IN);
titanGraph.removeVertex(referredVertex);
LOG.info("Removed vertex {}", referredVertex);
}
return edge;
}
@Override
public void outputGraph(Graph graph, OutputStream out) throws IOException {
Iterable<Edge> iterable = graph.getEdges();
Iterator<Edge> it = iterable.iterator();
writeCSVField("SourceId", out, true, false);
writeCSVField("SourceVirtual", out, false, false);
writeCSVField("SourceFileType", out, false, false);
writeCSVField("SourceName", out, false, false);
writeCSVField("SourceAuthor", out, false, false);
writeCSVField("SourceModified", out, false, false);
writeCSVField("LinkType", out, false, false);
writeCSVField("DestinationId", out, false, false);
writeCSVField("DestinationVirtual", out, false, false);
writeCSVField("DestinationFileType", out, false, false);
writeCSVField("DestinationName", out, false, false);
writeCSVField("DestinationAuthor", out, false, false);
writeCSVField("DestinationModified", out, false, true);
while (it.hasNext()) {
Edge edge = it.next();
Vertex fromV = edge.getVertex(Direction.OUT);
Vertex toV = edge.getVertex(Direction.IN);
writeCSVField(fromV.getId(), out, true, false);
writeCSVField(fromV.getProperty(DictionaryConst.NODE_VIRTUAL), out, false, false);
writeCSVField(fromV.getProperty(DictionaryConst.PROPERTY_TYPE), out, false, false);
writeCSVField(fromV.getProperty(DictionaryConst.PROPERTY_NAME), out, false, false);
writeCSVField(fromV.getProperty(DictionaryConst.PROPERTY_AUTHOR), out, false, false);
writeCSVField(fromV.getProperty(DictionaryConst.PROPERTY_LAST_MODIFIED), out, false, false);
writeCSVField(edge.getLabel(), out, false, false);
writeCSVField(toV.getId(), out, false, false);
writeCSVField(toV.getProperty(DictionaryConst.NODE_VIRTUAL), out, false, false);
writeCSVField(toV.getProperty(DictionaryConst.PROPERTY_TYPE), out, false, false);
writeCSVField(toV.getProperty(DictionaryConst.PROPERTY_NAME), out, false, false);
writeCSVField(toV.getProperty(DictionaryConst.PROPERTY_AUTHOR), out, false, false);
writeCSVField(toV.getProperty(DictionaryConst.PROPERTY_LAST_MODIFIED), out, false, true);
}
}
private void verifyUpdatedEdges(Feed newFeed) {
Vertex feedVertex = getEntityVertex(newFeed.getName(), RelationshipType.FEED_ENTITY);
// groups
Edge edge =
feedVertex.getEdges(Direction.OUT, RelationshipLabel.GROUPS.getName()).iterator().next();
Assert.assertEquals(edge.getVertex(Direction.IN).getProperty("name"), "reporting");
// tags
edge = feedVertex.getEdges(Direction.OUT, "classified-as").iterator().next();
Assert.assertEquals(edge.getVertex(Direction.IN).getProperty("name"), "Secured");
edge = feedVertex.getEdges(Direction.OUT, "source").iterator().next();
Assert.assertEquals(edge.getVertex(Direction.IN).getProperty("name"), "data-warehouse");
// new cluster
List<String> actual = new ArrayList<String>();
for (Edge clusterEdge :
feedVertex.getEdges(Direction.OUT, RelationshipLabel.FEED_CLUSTER_EDGE.getName())) {
actual.add(clusterEdge.getVertex(Direction.IN).<String>getProperty("name"));
}
Assert.assertTrue(
actual.containsAll(Arrays.asList("primary-cluster", "another-cluster")),
"Actual does not contain expected: " + actual);
}
/**
* Force delete is used to remove struct/trait in case of entity updates
*
* @param edge
* @param typeCategory
* @param isComposite
* @param forceDeleteStructTrait
* @return returns true if the edge reference is hard deleted
* @throws AtlasException
*/
public boolean deleteEdgeReference(
Edge edge,
DataTypes.TypeCategory typeCategory,
boolean isComposite,
boolean forceDeleteStructTrait)
throws AtlasException {
LOG.debug("Deleting {}", string(edge));
boolean forceDelete =
(typeCategory == DataTypes.TypeCategory.STRUCT
|| typeCategory == DataTypes.TypeCategory.TRAIT)
? forceDeleteStructTrait
: false;
if (typeCategory == DataTypes.TypeCategory.STRUCT
|| typeCategory == DataTypes.TypeCategory.TRAIT
|| (typeCategory == DataTypes.TypeCategory.CLASS && isComposite)) {
// If the vertex is of type struct/trait, delete the edge and then the reference vertex as the
// vertex is not shared by any other entities.
// If the vertex is of type class, and its composite attribute, this reference vertex'
// lifecycle is controlled
// through this delete, hence delete the edge and the reference vertex.
Vertex vertexForDelete = edge.getVertex(Direction.IN);
// If deleting the edge and then the in vertex, reverse attribute shouldn't be updated
deleteEdge(edge, false, forceDelete);
deleteTypeVertex(vertexForDelete, typeCategory, forceDelete);
} else {
// If the vertex is of type class, and its not a composite attributes, the reference vertex'
// lifecycle is not controlled
// through this delete. Hence just remove the reference edge. Leave the reference vertex as is
// If deleting just the edge, reverse attribute should be updated for any references
// For example, for the department type system, if the person's manager edge is deleted,
// subordinates of manager should be updated
deleteEdge(edge, true, false);
}
return !softDelete || forceDelete;
}
private void verifyLineageGraphForReplicationOrEviction(
String feedName,
String feedInstanceDataPath,
WorkflowExecutionContext context,
RelationshipLabel edgeLabel)
throws Exception {
String feedInstanceName =
InstanceRelationshipGraphBuilder.getFeedInstanceName(
feedName,
context.getClusterName(),
feedInstanceDataPath,
context.getNominalTimeAsISO8601());
Vertex feedVertex = getEntityVertex(feedInstanceName, RelationshipType.FEED_INSTANCE);
Edge edge = feedVertex.getEdges(Direction.OUT, edgeLabel.getName()).iterator().next();
Assert.assertNotNull(edge);
Assert.assertEquals(
edge.getProperty(RelationshipProperty.TIMESTAMP.getName()),
context.getTimeStampAsISO8601());
Vertex clusterVertex = edge.getVertex(Direction.IN);
Assert.assertEquals(
clusterVertex.getProperty(RelationshipProperty.NAME.getName()), context.getClusterName());
}
public boolean execute(Object... objects) throws Exception {
Map<String, Object> inputMap = (Map<String, Object>) objects[0];
Map<String, Object> data = (Map<String, Object>) inputMap.get("data");
String host = (String) data.get("host");
String source = (String) data.get("source");
String dest = (String) data.get("desc");
String error = null;
Map<String, Object> payload = (Map<String, Object>) inputMap.get("payload");
Map<String, Object> user = (Map<String, Object>) payload.get("user");
String userHost = (String) user.get("host");
OrientGraph graph = ServiceLocator.getInstance().getGraph();
try {
Vertex sourceRule = DbService.getVertexByRid(graph, source);
Vertex destRule = DbService.getVertexByRid(graph, dest);
if (sourceRule == null || destRule == null) {
error = "source rule or destination rule doesn't exist";
inputMap.put("responseCode", 400);
} else {
String sourceRuleClass = sourceRule.getProperty("ruleClass");
String destRuleClass = destRule.getProperty("ruleClass");
if (userHost != null) {
if (!userHost.equals(host)) {
error = "You can only add dependency from host: " + host;
inputMap.put("responseCode", 403);
} else {
// make sure dest ruleClass contains host.
if (!destRuleClass.contains(host)) {
error = "Destination rule doesn't belong to the host " + host;
inputMap.put("responseCode", 403);
} else {
// check if there is an depend edge from source to dest
boolean hasEdge = false;
for (Edge edge : (Iterable<Edge>) sourceRule.getEdges(Direction.OUT, "Own")) {
if (edge.getVertex(Direction.IN) == destRule) hasEdge = true;
}
if (hasEdge) {
error = "There is depend edge between source rule and dest rule";
inputMap.put("responseCode", 400);
} else {
Map eventMap = getEventMap(inputMap);
Map<String, Object> eventData = (Map<String, Object>) eventMap.get("data");
inputMap.put("eventMap", eventMap);
eventData.put("sourceRuleClass", sourceRuleClass);
eventData.put("destRuleClass", destRuleClass);
eventData.put("content", data.get("content"));
eventData.put("createDate", new java.util.Date());
eventData.put("createUserId", user.get("userId"));
}
}
}
}
}
} catch (Exception e) {
logger.error("Exception:", e);
throw e;
} finally {
graph.shutdown();
}
if (error != null) {
inputMap.put("result", error);
return false;
} else {
return true;
}
}
public MicroEdge(final Edge edge) {
super(edge);
this.outVertex = new MicroVertex(edge.getVertex(Direction.OUT));
this.inVertex = new MicroVertex(edge.getVertex(Direction.IN));
}
@Override
public Object load(ObjectCache objectsBeingAccessed) {
Vertex value = toLoad.getVertex(Direction.IN);
return loadValue(objectsBeingAccessed, value);
}
/**
* Deletes the edge between outvertex and inVertex. The edge is for attribute attributeName of
* outVertex
*
* @param outVertex
* @param inVertex
* @param attributeName
* @throws AtlasException
*/
protected void deleteEdgeBetweenVertices(Vertex outVertex, Vertex inVertex, String attributeName)
throws AtlasException {
LOG.debug(
"Removing edge from {} to {} with attribute name {}",
string(outVertex),
string(inVertex),
attributeName);
String typeName = GraphHelper.getTypeName(outVertex);
String outId = GraphHelper.getIdFromVertex(outVertex);
Id.EntityState state = GraphHelper.getState(outVertex);
if ((outId != null && RequestContext.get().isDeletedEntity(outId))
|| state == Id.EntityState.DELETED) {
// If the reference vertex is marked for deletion, skip updating the reference
return;
}
IDataType type = typeSystem.getDataType(IDataType.class, typeName);
AttributeInfo attributeInfo = getFieldMapping(type).fields.get(attributeName);
String propertyName = GraphHelper.getQualifiedFieldName(type, attributeName);
String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
Edge edge = null;
switch (attributeInfo.dataType().getTypeCategory()) {
case CLASS:
// If its class attribute, its the only edge between two vertices
if (attributeInfo.multiplicity.nullAllowed()) {
edge = GraphHelper.getEdgeForLabel(outVertex, edgeLabel);
if (shouldUpdateReverseAttribute) {
GraphHelper.setProperty(outVertex, propertyName, null);
}
} else {
// Cannot unset a required attribute.
throw new NullRequiredAttributeException(
"Cannot unset required attribute "
+ GraphHelper.getQualifiedFieldName(type, attributeName)
+ " on "
+ string(outVertex)
+ " edge = "
+ edgeLabel);
}
break;
case ARRAY:
// If its array attribute, find the right edge between the two vertices and update array
// property
List<String> elements = outVertex.getProperty(propertyName);
if (elements != null) {
elements =
new ArrayList<>(
elements); // Make a copy, else list.remove reflects on titan.getProperty()
for (String elementEdgeId : elements) {
Edge elementEdge = graphHelper.getEdgeByEdgeId(outVertex, edgeLabel, elementEdgeId);
if (elementEdge == null) {
continue;
}
Vertex elementVertex = elementEdge.getVertex(Direction.IN);
if (elementVertex.getId().toString().equals(inVertex.getId().toString())) {
edge = elementEdge;
// TODO element.size includes deleted items as well. should exclude
if (!attributeInfo.multiplicity.nullAllowed()
&& elements.size() <= attributeInfo.multiplicity.lower) {
// Deleting this edge would violate the attribute's lower bound.
throw new NullRequiredAttributeException(
"Cannot remove array element from required attribute "
+ GraphHelper.getQualifiedFieldName(type, attributeName)
+ " on "
+ string(outVertex)
+ " "
+ string(elementEdge));
}
if (shouldUpdateReverseAttribute) {
// if composite attribute, remove the reference as well. else, just remove the edge
// for example, when table is deleted, process still references the table
// but when column is deleted, table will not reference the deleted column
LOG.debug(
"Removing edge {} from the array attribute {}",
string(elementEdge),
attributeName);
elements.remove(elementEdge.getId().toString());
GraphHelper.setProperty(outVertex, propertyName, elements);
break;
}
}
}
}
break;
case MAP:
// If its map attribute, find the right edge between two vertices and update map property
List<String> keys = outVertex.getProperty(propertyName);
if (keys != null) {
keys =
new ArrayList<>(
keys); // Make a copy, else list.remove reflects on titan.getProperty()
for (String key : keys) {
String keyPropertyName = GraphHelper.getQualifiedNameForMapKey(propertyName, key);
String mapEdgeId = outVertex.getProperty(keyPropertyName);
Edge mapEdge = graphHelper.getEdgeByEdgeId(outVertex, keyPropertyName, mapEdgeId);
Vertex mapVertex = mapEdge.getVertex(Direction.IN);
if (mapVertex.getId().toString().equals(inVertex.getId().toString())) {
// TODO keys.size includes deleted items as well. should exclude
if (attributeInfo.multiplicity.nullAllowed()
|| keys.size() > attributeInfo.multiplicity.lower) {
edge = mapEdge;
} else {
// Deleting this entry would violate the attribute's lower bound.
throw new NullRequiredAttributeException(
"Cannot remove map entry "
+ keyPropertyName
+ " from required attribute "
+ GraphHelper.getQualifiedFieldName(type, attributeName)
+ " on "
+ string(outVertex)
+ " "
+ string(mapEdge));
}
if (shouldUpdateReverseAttribute) {
// remove this key
LOG.debug(
"Removing edge {}, key {} from the map attribute {}",
string(mapEdge),
key,
attributeName);
keys.remove(key);
GraphHelper.setProperty(outVertex, propertyName, keys);
GraphHelper.setProperty(outVertex, keyPropertyName, null);
}
break;
}
}
}
break;
case STRUCT:
case TRAIT:
break;
default:
throw new IllegalStateException(
"There can't be an edge from "
+ string(outVertex)
+ " to "
+ string(inVertex)
+ " with attribute name "
+ attributeName
+ " which is not class/array/map attribute");
}
if (edge != null) {
deleteEdge(edge, false);
RequestContext requestContext = RequestContext.get();
GraphHelper.setProperty(
outVertex,
Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
requestContext.getRequestTime());
requestContext.recordEntityUpdate(outId);
}
}
