@Test(dependsOnMethods = "testOnChange")
public void testOnFeedEntityChange() throws Exception {
Feed oldFeed = inputFeeds.get(0);
Feed newFeed =
EntityBuilderTestUtil.buildFeed(
oldFeed.getName(),
clusterEntity,
"classified-as=Secured,source=data-warehouse",
"reporting");
addStorage(
newFeed, Storage.TYPE.FILESYSTEM, "jail://global:00/falcon/impression-feed/20140101");
try {
configStore.initiateUpdate(newFeed);
// add cluster
org.apache.falcon.entity.v0.feed.Cluster feedCluster =
new org.apache.falcon.entity.v0.feed.Cluster();
feedCluster.setName(anotherCluster.getName());
newFeed.getClusters().getClusters().add(feedCluster);
configStore.update(EntityType.FEED, newFeed);
} finally {
configStore.cleanupUpdateInit();
}
verifyUpdatedEdges(newFeed);
Assert.assertEquals(getVerticesCount(service.getGraph()), 22); // +2 = 2 new tags
Assert.assertEquals(getEdgesCount(service.getGraph()), 35); // +2 = 1 new cluster, 1 new tag
}
public List<String> getFeedClusterNames(List<org.apache.falcon.entity.v0.feed.Cluster> clusters) {
List<String> clusterNames = new ArrayList<String>(clusters.size());
for (org.apache.falcon.entity.v0.feed.Cluster cluster : clusters) {
clusterNames.add(cluster.getName());
}
return clusterNames;
}
private org.apache.falcon.entity.v0.feed.Clusters createBlankClusters() {
org.apache.falcon.entity.v0.feed.Clusters clusters =
new org.apache.falcon.entity.v0.feed.Clusters();
Cluster cluster = new Cluster();
cluster.setName("blankCluster1");
clusters.getClusters().add(cluster);
Cluster cluster2 = new Cluster();
cluster2.setName("blankCluster2");
clusters.getClusters().add(cluster2);
return clusters;
}
@Override
public List<Properties> buildCoords(Cluster cluster, Path buildPath) throws FalconException {
org.apache.falcon.entity.v0.feed.Cluster feedCluster =
FeedHelper.getCluster(entity, cluster.getName());
if (feedCluster == null) {
return null;
}
COORDINATORAPP coord = new COORDINATORAPP();
String coordName = getEntityName();
coord.setName(coordName);
Date endDate = feedCluster.getValidity().getEnd();
coord.setEnd(SchemaHelper.formatDateUTC(endDate));
if (feedCluster.getValidity().getEnd().before(new Date())) {
Date startDate = DateUtils.addMinutes(endDate, -1);
coord.setStart(SchemaHelper.formatDateUTC(startDate));
} else {
coord.setStart(SchemaHelper.formatDateUTC(new Date()));
}
coord.setTimezone(entity.getTimezone().getID());
TimeUnit timeUnit = entity.getFrequency().getTimeUnit();
if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) {
coord.setFrequency("${coord:hours(6)}");
} else {
coord.setFrequency("${coord:days(1)}");
}
Path coordPath = getBuildPath(buildPath);
Properties props = createCoordDefaultConfiguration(coordName);
WORKFLOW workflow = new WORKFLOW();
Properties wfProps =
OozieOrchestrationWorkflowBuilder.get(entity, cluster, Tag.RETENTION)
.build(cluster, coordPath);
workflow.setAppPath(getStoragePath(wfProps.getProperty(OozieEntityBuilder.ENTITY_PATH)));
props.putAll(getProperties(coordPath, coordName));
// Add the custom properties set in feed. Else, dryrun won't catch any missing props.
props.putAll(EntityUtil.getEntityProperties(entity));
workflow.setConfiguration(getConfig(props));
ACTION action = new ACTION();
action.setWorkflow(workflow);
coord.setAction(action);
Path marshalPath = marshal(cluster, coord, coordPath);
return Arrays.asList(getProperties(marshalPath, coordName));
}
private void addClusterAndFeedForReplication() throws Exception {
// Add cluster
clusterEntity = addClusterEntity(CLUSTER_ENTITY_NAME, COLO_NAME, "classification=production");
// Add backup cluster
Cluster bcpCluster =
addClusterEntity(BCP_CLUSTER_ENTITY_NAME, "east-coast", "classification=bcp");
Cluster[] clusters = {clusterEntity, bcpCluster};
// Add feed
Feed rawFeed =
addFeedEntity(
REPLICATED_FEED,
clusters,
"classified-as=Secure",
"analytics",
Storage.TYPE.FILESYSTEM,
"/falcon/raw-click/${YEAR}/${MONTH}/${DAY}");
// Add uri template for each cluster
for (org.apache.falcon.entity.v0.feed.Cluster feedCluster :
rawFeed.getClusters().getClusters()) {
if (feedCluster.getName().equals(CLUSTER_ENTITY_NAME)) {
addStorage(
feedCluster,
rawFeed,
Storage.TYPE.FILESYSTEM,
"/falcon/raw-click/primary/${YEAR}/${MONTH}/${DAY}");
} else {
addStorage(
feedCluster,
rawFeed,
Storage.TYPE.FILESYSTEM,
"/falcon/raw-click/bcp/${YEAR}/${MONTH}/${DAY}");
}
}
// update config store
try {
configStore.initiateUpdate(rawFeed);
configStore.update(EntityType.FEED, rawFeed);
} finally {
configStore.cleanupUpdateInit();
}
inputFeeds.add(rawFeed);
}
private Feed addFeedEntity(
String feedName,
Cluster[] clusters,
String tags,
String groups,
Storage.TYPE storageType,
String uriTemplate)
throws Exception {
Feed feed = EntityBuilderTestUtil.buildFeed(feedName, clusters, tags, groups);
addStorage(feed, storageType, uriTemplate);
for (org.apache.falcon.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) {
if (feedCluster.getName().equals(BCP_CLUSTER_ENTITY_NAME)) {
feedCluster.setType(ClusterType.TARGET);
}
}
configStore.publish(EntityType.FEED, feed);
return feed;
}
private static void addStorage(
org.apache.falcon.entity.v0.feed.Cluster cluster,
Feed feed,
Storage.TYPE storageType,
String uriTemplate) {
if (storageType == Storage.TYPE.FILESYSTEM) {
Locations locations = new Locations();
feed.setLocations(locations);
Location location = new Location();
location.setType(LocationType.DATA);
location.setPath(uriTemplate);
cluster.setLocations(new Locations());
cluster.getLocations().getLocations().add(location);
} else {
CatalogTable table = new CatalogTable();
table.setUri(uriTemplate);
cluster.setTable(table);
}
}
public void addFeedEntity(Feed feed) {
LOG.info("Adding feed entity: {}", feed.getName());
Vertex feedVertex = addVertex(feed.getName(), RelationshipType.FEED_ENTITY);
addUserRelation(feedVertex);
addDataClassification(feed.getTags(), feedVertex);
addGroups(feed.getGroups(), feedVertex);
for (org.apache.falcon.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) {
if (ClusterType.TARGET != feedCluster.getType()) {
addRelationToCluster(
feedVertex, feedCluster.getName(), RelationshipLabel.FEED_CLUSTER_EDGE);
}
}
for (org.apache.falcon.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) {
if (FeedHelper.isImportEnabled(feedCluster)) {
addRelationToDatasource(
feedVertex,
FeedHelper.getImportDatasourceName(feedCluster),
RelationshipLabel.DATASOURCE_IMPORT_EDGE);
}
}
}
public void updateFeedClusters(
List<org.apache.falcon.entity.v0.feed.Cluster> oldClusters,
List<org.apache.falcon.entity.v0.feed.Cluster> newClusters,
Vertex feedEntityVertex) {
if (areFeedClustersSame(oldClusters, newClusters)) {
return;
}
// remove edges to old clusters
for (org.apache.falcon.entity.v0.feed.Cluster oldCuster : oldClusters) {
if (ClusterType.TARGET != oldCuster.getType()) {
removeEdge(
feedEntityVertex, oldCuster.getName(), RelationshipLabel.FEED_CLUSTER_EDGE.getName());
}
}
// add edges to new clusters
for (org.apache.falcon.entity.v0.feed.Cluster newCluster : newClusters) {
if (ClusterType.TARGET != newCluster.getType()) {
addRelationToCluster(
feedEntityVertex, newCluster.getName(), RelationshipLabel.FEED_CLUSTER_EDGE);
}
}
}