private void verifyLineageGraph(
String feedType,
List<String> expectedFeeds,
List<String> secureFeeds,
List<String> ownedAndSecureFeeds) {
// feeds owned by a user
List<String> feedNamesOwnedByUser = getFeedsOwnedByAUser(feedType);
Assert.assertTrue(feedNamesOwnedByUser.containsAll(expectedFeeds));
Graph graph = service.getGraph();
Iterator<Vertex> vertices =
graph.getVertices("name", "impression-feed/2014-01-01T00:00Z").iterator();
Assert.assertTrue(vertices.hasNext());
Vertex feedInstanceVertex = vertices.next();
Assert.assertEquals(
feedInstanceVertex.getProperty(RelationshipProperty.TYPE.getName()),
RelationshipType.FEED_INSTANCE.getName());
Object vertexId = feedInstanceVertex.getId();
Vertex vertexById = graph.getVertex(vertexId);
Assert.assertEquals(vertexById, feedInstanceVertex);
// feeds classified as secure
verifyFeedsClassifiedAsSecure(feedType, secureFeeds);
// feeds owned by a user and classified as secure
verifyFeedsOwnedByUserAndClassification(feedType, "Financial", ownedAndSecureFeeds);
}
@Test
public void testLineageForNoDateInFeedPath() throws Exception {
setupForNoDateInFeedPath();
WorkflowExecutionContext context =
WorkflowExecutionContext.create(
getTestMessageArgs(
EntityOperations.GENERATE,
GENERATE_WORKFLOW_NAME,
null,
OUTPUT_INSTANCE_PATHS_NO_DATE,
INPUT_INSTANCE_PATHS_NO_DATE,
null),
WorkflowExecutionContext.Type.POST_PROCESSING);
service.onSuccess(context);
debug(service.getGraph());
GraphUtils.dump(service.getGraph());
// Verify if instance name has nominal time
List<String> feedNamesOwnedByUser =
getFeedsOwnedByAUser(RelationshipType.FEED_INSTANCE.getName());
List<String> expected =
Arrays.asList(
"impression-feed/2014-01-01T01:00Z",
"clicks-feed/2014-01-01T01:00Z",
"imp-click-join1/2014-01-01T01:00Z",
"imp-click-join2/2014-01-01T01:00Z");
Assert.assertTrue(feedNamesOwnedByUser.containsAll(expected));
// +5 = 1 process, 2 inputs, 2 outputs
Assert.assertEquals(getVerticesCount(service.getGraph()), 22);
// +34 = +26 for feed instances + 8 for process instance
Assert.assertEquals(getEdgesCount(service.getGraph()), 65);
}
@Test
public void testLineageForRetention() throws Exception {
setupForLineageEviction();
WorkflowExecutionContext context =
WorkflowExecutionContext.create(
getTestMessageArgs(
EntityOperations.DELETE,
EVICTION_WORKFLOW_NAME,
EVICTED_FEED,
EVICTED_INSTANCE_PATHS,
"IGNORE",
EVICTED_FEED),
WorkflowExecutionContext.Type.POST_PROCESSING);
service.onSuccess(context);
debug(service.getGraph());
GraphUtils.dump(service.getGraph());
List<String> expectedFeeds =
Arrays.asList(
"impression-feed/2014-01-01T00:00Z",
"clicks-feed/2014-01-01T00:00Z",
"imp-click-join1/2014-01-01T00:00Z",
"imp-click-join1/2014-01-02T00:00Z");
List<String> secureFeeds =
Arrays.asList("impression-feed/2014-01-01T00:00Z", "clicks-feed/2014-01-01T00:00Z");
List<String> ownedAndSecureFeeds =
Arrays.asList(
"clicks-feed/2014-01-01T00:00Z",
"imp-click-join1/2014-01-01T00:00Z",
"imp-click-join1/2014-01-02T00:00Z");
verifyLineageGraph(
RelationshipType.FEED_INSTANCE.getName(), expectedFeeds, secureFeeds, ownedAndSecureFeeds);
String[] paths = EVICTED_INSTANCE_PATHS.split(EvictedInstanceSerDe.INSTANCEPATH_SEPARATOR);
for (String feedInstanceDataPath : paths) {
verifyLineageGraphForReplicationOrEviction(
EVICTED_FEED, feedInstanceDataPath, context, RelationshipLabel.FEED_CLUSTER_EVICTED_EDGE);
}
// No new vertices added
Assert.assertEquals(getVerticesCount(service.getGraph()), 23);
// +1 = +2 for evicted-from edge from Feed Instance vertex to cluster.
// -1 imp-click-join1 is added twice instead of imp-click-join2 so there is one less edge as
// there is no
// classified-as -> Secure edge.
Assert.assertEquals(getEdgesCount(service.getGraph()), 72);
}
@Test
public void testMapLineage() throws Exception {
setup();
WorkflowExecutionContext context =
WorkflowExecutionContext.create(
getTestMessageArgs(
EntityOperations.GENERATE, GENERATE_WORKFLOW_NAME, null, null, null, null),
WorkflowExecutionContext.Type.POST_PROCESSING);
service.onSuccess(context);
debug(service.getGraph());
GraphUtils.dump(service.getGraph());
verifyLineageGraph(RelationshipType.FEED_INSTANCE.getName());
// +6 = 1 process, 2 inputs = 3 instances,2 outputs
Assert.assertEquals(getVerticesCount(service.getGraph()), 23);
// +40 = +26 for feed instances + 8 for process instance + 6 for second feed instance
Assert.assertEquals(getEdgesCount(service.getGraph()), 71);
}