@Test(dependsOnMethods = "testOnChange") public void testOnFeedEntityChange() throws Exception { Feed oldFeed = inputFeeds.get(0); Feed newFeed = EntityBuilderTestUtil.buildFeed( oldFeed.getName(), clusterEntity, "classified-as=Secured,source=data-warehouse", "reporting"); addStorage( newFeed, Storage.TYPE.FILESYSTEM, "jail://global:00/falcon/impression-feed/20140101"); try { configStore.initiateUpdate(newFeed); // add cluster org.apache.falcon.entity.v0.feed.Cluster feedCluster = new org.apache.falcon.entity.v0.feed.Cluster(); feedCluster.setName(anotherCluster.getName()); newFeed.getClusters().getClusters().add(feedCluster); configStore.update(EntityType.FEED, newFeed); } finally { configStore.cleanupUpdateInit(); } verifyUpdatedEdges(newFeed); Assert.assertEquals(getVerticesCount(service.getGraph()), 22); // +2 = 2 new tags Assert.assertEquals(getEdgesCount(service.getGraph()), 35); // +2 = 1 new cluster, 1 new tag }
public List<String> getFeedClusterNames(List<org.apache.falcon.entity.v0.feed.Cluster> clusters) { List<String> clusterNames = new ArrayList<String>(clusters.size()); for (org.apache.falcon.entity.v0.feed.Cluster cluster : clusters) { clusterNames.add(cluster.getName()); } return clusterNames; }
private org.apache.falcon.entity.v0.feed.Clusters createBlankClusters() { org.apache.falcon.entity.v0.feed.Clusters clusters = new org.apache.falcon.entity.v0.feed.Clusters(); Cluster cluster = new Cluster(); cluster.setName("blankCluster1"); clusters.getClusters().add(cluster); Cluster cluster2 = new Cluster(); cluster2.setName("blankCluster2"); clusters.getClusters().add(cluster2); return clusters; }
@Override public List<Properties> buildCoords(Cluster cluster, Path buildPath) throws FalconException { org.apache.falcon.entity.v0.feed.Cluster feedCluster = FeedHelper.getCluster(entity, cluster.getName()); if (feedCluster == null) { return null; } COORDINATORAPP coord = new COORDINATORAPP(); String coordName = getEntityName(); coord.setName(coordName); Date endDate = feedCluster.getValidity().getEnd(); coord.setEnd(SchemaHelper.formatDateUTC(endDate)); if (feedCluster.getValidity().getEnd().before(new Date())) { Date startDate = DateUtils.addMinutes(endDate, -1); coord.setStart(SchemaHelper.formatDateUTC(startDate)); } else { coord.setStart(SchemaHelper.formatDateUTC(new Date())); } coord.setTimezone(entity.getTimezone().getID()); TimeUnit timeUnit = entity.getFrequency().getTimeUnit(); if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) { coord.setFrequency("${coord:hours(6)}"); } else { coord.setFrequency("${coord:days(1)}"); } Path coordPath = getBuildPath(buildPath); Properties props = createCoordDefaultConfiguration(coordName); WORKFLOW workflow = new WORKFLOW(); Properties wfProps = OozieOrchestrationWorkflowBuilder.get(entity, cluster, Tag.RETENTION) .build(cluster, coordPath); workflow.setAppPath(getStoragePath(wfProps.getProperty(OozieEntityBuilder.ENTITY_PATH))); props.putAll(getProperties(coordPath, coordName)); // Add the custom properties set in feed. Else, dryrun won't catch any missing props. props.putAll(EntityUtil.getEntityProperties(entity)); workflow.setConfiguration(getConfig(props)); ACTION action = new ACTION(); action.setWorkflow(workflow); coord.setAction(action); Path marshalPath = marshal(cluster, coord, coordPath); return Arrays.asList(getProperties(marshalPath, coordName)); }
private void addClusterAndFeedForReplication() throws Exception { // Add cluster clusterEntity = addClusterEntity(CLUSTER_ENTITY_NAME, COLO_NAME, "classification=production"); // Add backup cluster Cluster bcpCluster = addClusterEntity(BCP_CLUSTER_ENTITY_NAME, "east-coast", "classification=bcp"); Cluster[] clusters = {clusterEntity, bcpCluster}; // Add feed Feed rawFeed = addFeedEntity( REPLICATED_FEED, clusters, "classified-as=Secure", "analytics", Storage.TYPE.FILESYSTEM, "/falcon/raw-click/${YEAR}/${MONTH}/${DAY}"); // Add uri template for each cluster for (org.apache.falcon.entity.v0.feed.Cluster feedCluster : rawFeed.getClusters().getClusters()) { if (feedCluster.getName().equals(CLUSTER_ENTITY_NAME)) { addStorage( feedCluster, rawFeed, Storage.TYPE.FILESYSTEM, "/falcon/raw-click/primary/${YEAR}/${MONTH}/${DAY}"); } else { addStorage( feedCluster, rawFeed, Storage.TYPE.FILESYSTEM, "/falcon/raw-click/bcp/${YEAR}/${MONTH}/${DAY}"); } } // update config store try { configStore.initiateUpdate(rawFeed); configStore.update(EntityType.FEED, rawFeed); } finally { configStore.cleanupUpdateInit(); } inputFeeds.add(rawFeed); }
private Feed addFeedEntity( String feedName, Cluster[] clusters, String tags, String groups, Storage.TYPE storageType, String uriTemplate) throws Exception { Feed feed = EntityBuilderTestUtil.buildFeed(feedName, clusters, tags, groups); addStorage(feed, storageType, uriTemplate); for (org.apache.falcon.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) { if (feedCluster.getName().equals(BCP_CLUSTER_ENTITY_NAME)) { feedCluster.setType(ClusterType.TARGET); } } configStore.publish(EntityType.FEED, feed); return feed; }
private static void addStorage( org.apache.falcon.entity.v0.feed.Cluster cluster, Feed feed, Storage.TYPE storageType, String uriTemplate) { if (storageType == Storage.TYPE.FILESYSTEM) { Locations locations = new Locations(); feed.setLocations(locations); Location location = new Location(); location.setType(LocationType.DATA); location.setPath(uriTemplate); cluster.setLocations(new Locations()); cluster.getLocations().getLocations().add(location); } else { CatalogTable table = new CatalogTable(); table.setUri(uriTemplate); cluster.setTable(table); } }
public void addFeedEntity(Feed feed) { LOG.info("Adding feed entity: {}", feed.getName()); Vertex feedVertex = addVertex(feed.getName(), RelationshipType.FEED_ENTITY); addUserRelation(feedVertex); addDataClassification(feed.getTags(), feedVertex); addGroups(feed.getGroups(), feedVertex); for (org.apache.falcon.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) { if (ClusterType.TARGET != feedCluster.getType()) { addRelationToCluster( feedVertex, feedCluster.getName(), RelationshipLabel.FEED_CLUSTER_EDGE); } } for (org.apache.falcon.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) { if (FeedHelper.isImportEnabled(feedCluster)) { addRelationToDatasource( feedVertex, FeedHelper.getImportDatasourceName(feedCluster), RelationshipLabel.DATASOURCE_IMPORT_EDGE); } } }
public void updateFeedClusters( List<org.apache.falcon.entity.v0.feed.Cluster> oldClusters, List<org.apache.falcon.entity.v0.feed.Cluster> newClusters, Vertex feedEntityVertex) { if (areFeedClustersSame(oldClusters, newClusters)) { return; } // remove edges to old clusters for (org.apache.falcon.entity.v0.feed.Cluster oldCuster : oldClusters) { if (ClusterType.TARGET != oldCuster.getType()) { removeEdge( feedEntityVertex, oldCuster.getName(), RelationshipLabel.FEED_CLUSTER_EDGE.getName()); } } // add edges to new clusters for (org.apache.falcon.entity.v0.feed.Cluster newCluster : newClusters) { if (ClusterType.TARGET != newCluster.getType()) { addRelationToCluster( feedEntityVertex, newCluster.getName(), RelationshipLabel.FEED_CLUSTER_EDGE); } } }