@Test(dependsOnMethods = "testOnAddClusterEntity")
public void testOnAddFeedEntity() throws Exception {
Feed impressionsFeed =
addFeedEntity(
"impression-feed",
clusterEntity,
"classified-as=Secure",
"analytics",
Storage.TYPE.FILESYSTEM,
"/falcon/impression-feed/${YEAR}/${MONTH}/${DAY}");
inputFeeds.add(impressionsFeed);
verifyEntityWasAddedToGraph(impressionsFeed.getName(), RelationshipType.FEED_ENTITY);
verifyFeedEntityEdges(impressionsFeed.getName(), "Secure", "analytics");
Assert.assertEquals(getVerticesCount(service.getGraph()), 7); // +4 = feed, tag, group, user
Assert.assertEquals(getEdgesCount(service.getGraph()), 6); // +4 = cluster, tag, group, user
Feed clicksFeed =
addFeedEntity(
"clicks-feed",
clusterEntity,
"classified-as=Secure,classified-as=Financial",
"analytics",
Storage.TYPE.FILESYSTEM,
"/falcon/clicks-feed/${YEAR}-${MONTH}-${DAY}");
inputFeeds.add(clicksFeed);
verifyEntityWasAddedToGraph(clicksFeed.getName(), RelationshipType.FEED_ENTITY);
Assert.assertEquals(getVerticesCount(service.getGraph()), 9); // feed and financial vertex
Assert.assertEquals(
getEdgesCount(service.getGraph()), 11); // +5 = cluster + user + 2Group + Tag
Feed join1Feed =
addFeedEntity(
"imp-click-join1",
clusterEntity,
"classified-as=Financial",
"reporting,bi",
Storage.TYPE.FILESYSTEM,
"/falcon/imp-click-join1/${YEAR}${MONTH}${DAY}");
outputFeeds.add(join1Feed);
verifyEntityWasAddedToGraph(join1Feed.getName(), RelationshipType.FEED_ENTITY);
Assert.assertEquals(getVerticesCount(service.getGraph()), 12); // + 3 = 1 feed and 2 groups
Assert.assertEquals(getEdgesCount(service.getGraph()), 16); // +5 = cluster + user +
// Group + 2Tags
Feed join2Feed =
addFeedEntity(
"imp-click-join2",
clusterEntity,
"classified-as=Secure,classified-as=Financial",
"reporting,bi",
Storage.TYPE.FILESYSTEM,
"/falcon/imp-click-join2/${YEAR}${MONTH}${DAY}");
outputFeeds.add(join2Feed);
verifyEntityWasAddedToGraph(join2Feed.getName(), RelationshipType.FEED_ENTITY);
Assert.assertEquals(getVerticesCount(service.getGraph()), 13); // +1 feed
// +6 = user + 2tags + 2Groups + Cluster
Assert.assertEquals(getEdgesCount(service.getGraph()), 22);
}
public void updateFeedEntity(Feed oldFeed, Feed newFeed) {
LOG.info("Updating feed entity: {}", newFeed.getName());
Vertex feedEntityVertex = findVertex(oldFeed.getName(), RelationshipType.FEED_ENTITY);
if (feedEntityVertex == null) {
LOG.error("Illegal State: Feed entity vertex must exist for {}", oldFeed.getName());
throw new IllegalStateException(oldFeed.getName() + " entity vertex must exist.");
}
updateDataClassification(oldFeed.getTags(), newFeed.getTags(), feedEntityVertex);
updateGroups(oldFeed.getGroups(), newFeed.getGroups(), feedEntityVertex);
updateFeedClusters(
oldFeed.getClusters().getClusters(), newFeed.getClusters().getClusters(), feedEntityVertex);
}
@Test(dependsOnMethods = "testOnChange")
public void testOnFeedEntityChange() throws Exception {
Feed oldFeed = inputFeeds.get(0);
Feed newFeed =
EntityBuilderTestUtil.buildFeed(
oldFeed.getName(),
clusterEntity,
"classified-as=Secured,source=data-warehouse",
"reporting");
addStorage(
newFeed, Storage.TYPE.FILESYSTEM, "jail://global:00/falcon/impression-feed/20140101");
try {
configStore.initiateUpdate(newFeed);
// add cluster
org.apache.falcon.entity.v0.feed.Cluster feedCluster =
new org.apache.falcon.entity.v0.feed.Cluster();
feedCluster.setName(anotherCluster.getName());
newFeed.getClusters().getClusters().add(feedCluster);
configStore.update(EntityType.FEED, newFeed);
} finally {
configStore.cleanupUpdateInit();
}
verifyUpdatedEdges(newFeed);
Assert.assertEquals(getVerticesCount(service.getGraph()), 22); // +2 = 2 new tags
Assert.assertEquals(getEdgesCount(service.getGraph()), 35); // +2 = 1 new cluster, 1 new tag
}
@Test
public void testGetEntityDefinition() throws Exception {
ClientResponse response;
Map<String, String> overlay = getUniqueOverlay();
response = submitToFalcon(CLUSTER_FILE_TEMPLATE, overlay, EntityType.CLUSTER);
assertSuccessful(response);
response = submitToFalcon(FEED_TEMPLATE1, overlay, EntityType.FEED);
assertSuccessful(response);
response =
this.service
.path("api/entities/definition/feed/" + overlay.get("inputFeedName"))
.header("Remote-User", REMOTE_USER)
.accept(MediaType.TEXT_XML)
.get(ClientResponse.class);
String feedXML = response.getEntity(String.class);
try {
Feed result = (Feed) unmarshaller.unmarshal(new StringReader(feedXML));
Assert.assertEquals(result.getName(), overlay.get("inputFeedName"));
} catch (JAXBException e) {
Assert.fail("Reponse " + feedXML + " is not valid", e);
}
}
@Override
public void onAdd(Entity entity) throws FalconException {
if (entity.getEntityType().equals(EntityType.FEED)) {
Feed feed = (Feed) entity;
if (feed.getGroups() == null || feed.getGroups().equals("")) {
return;
}
Set<FeedGroup> groupSet = getGroups(feed);
addGroups(feed.getName(), groupSet);
}
}
public void addFeedEntity(Feed feed) {
LOG.info("Adding feed entity: {}", feed.getName());
Vertex feedVertex = addVertex(feed.getName(), RelationshipType.FEED_ENTITY);
addUserRelation(feedVertex);
addDataClassification(feed.getTags(), feedVertex);
addGroups(feed.getGroups(), feedVertex);
for (org.apache.falcon.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) {
if (ClusterType.TARGET != feedCluster.getType()) {
addRelationToCluster(
feedVertex, feedCluster.getName(), RelationshipLabel.FEED_CLUSTER_EDGE);
}
}
for (org.apache.falcon.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) {
if (FeedHelper.isImportEnabled(feedCluster)) {
addRelationToDatasource(
feedVertex,
FeedHelper.getImportDatasourceName(feedCluster),
RelationshipLabel.DATASOURCE_IMPORT_EDGE);
}
}
}
private static String getFileSystemFeedInstanceName(
String feedInstancePath, Feed feed, Cluster cluster, String nominalTime)
throws FalconException {
Storage rawStorage = FeedHelper.createStorage(cluster, feed);
String feedPathTemplate = rawStorage.getUriTemplate(LocationType.DATA);
String instance = feedInstancePath;
String[] elements = FeedDataPath.PATTERN.split(feedPathTemplate);
for (String element : elements) {
instance = instance.replaceFirst(element, "");
}
return StringUtils.isEmpty(instance)
? feed.getName() + "/" + nominalTime
: feed.getName()
+ "/"
+ SchemaHelper.formatDateUTCToISO8601(instance, FEED_INSTANCE_FORMAT);
}
private void verifyUpdatedEdges(Feed newFeed) {
Vertex feedVertex = getEntityVertex(newFeed.getName(), RelationshipType.FEED_ENTITY);
// groups
Edge edge =
feedVertex.getEdges(Direction.OUT, RelationshipLabel.GROUPS.getName()).iterator().next();
Assert.assertEquals(edge.getVertex(Direction.IN).getProperty("name"), "reporting");
// tags
edge = feedVertex.getEdges(Direction.OUT, "classified-as").iterator().next();
Assert.assertEquals(edge.getVertex(Direction.IN).getProperty("name"), "Secured");
edge = feedVertex.getEdges(Direction.OUT, "source").iterator().next();
Assert.assertEquals(edge.getVertex(Direction.IN).getProperty("name"), "data-warehouse");
// new cluster
List<String> actual = new ArrayList<String>();
for (Edge clusterEdge :
feedVertex.getEdges(Direction.OUT, RelationshipLabel.FEED_CLUSTER_EDGE.getName())) {
actual.add(clusterEdge.getVertex(Direction.IN).<String>getProperty("name"));
}
Assert.assertTrue(
actual.containsAll(Arrays.asList("primary-cluster", "another-cluster")),
"Actual does not contain expected: " + actual);
}
private static String getTableFeedInstanceName(
Feed feed, String feedInstancePath, Storage.TYPE storageType) throws URISyntaxException {
CatalogStorage instanceStorage =
(CatalogStorage) FeedHelper.createStorage(storageType.name(), feedInstancePath);
return feed.getName() + "/" + instanceStorage.toPartitionAsPath();
}