@Test(dependsOnMethods = "testOnChange")
  public void testOnFeedEntityChange() throws Exception {
    Feed oldFeed = inputFeeds.get(0);
    Feed newFeed =
        EntityBuilderTestUtil.buildFeed(
            oldFeed.getName(),
            clusterEntity,
            "classified-as=Secured,source=data-warehouse",
            "reporting");
    addStorage(
        newFeed, Storage.TYPE.FILESYSTEM, "jail://global:00/falcon/impression-feed/20140101");

    try {
      configStore.initiateUpdate(newFeed);

      // add cluster
      org.apache.falcon.entity.v0.feed.Cluster feedCluster =
          new org.apache.falcon.entity.v0.feed.Cluster();
      feedCluster.setName(anotherCluster.getName());
      newFeed.getClusters().getClusters().add(feedCluster);

      configStore.update(EntityType.FEED, newFeed);
    } finally {
      configStore.cleanupUpdateInit();
    }

    verifyUpdatedEdges(newFeed);
    Assert.assertEquals(getVerticesCount(service.getGraph()), 22); // +2 = 2 new tags
    Assert.assertEquals(getEdgesCount(service.getGraph()), 35); // +2 = 1 new cluster, 1 new tag
  }
  public List<String> getFeedClusterNames(List<org.apache.falcon.entity.v0.feed.Cluster> clusters) {
    List<String> clusterNames = new ArrayList<String>(clusters.size());
    for (org.apache.falcon.entity.v0.feed.Cluster cluster : clusters) {
      clusterNames.add(cluster.getName());
    }

    return clusterNames;
  }
Example #3
0
  private org.apache.falcon.entity.v0.feed.Clusters createBlankClusters() {
    org.apache.falcon.entity.v0.feed.Clusters clusters =
        new org.apache.falcon.entity.v0.feed.Clusters();

    Cluster cluster = new Cluster();
    cluster.setName("blankCluster1");
    clusters.getClusters().add(cluster);

    Cluster cluster2 = new Cluster();
    cluster2.setName("blankCluster2");
    clusters.getClusters().add(cluster2);

    return clusters;
  }
  @Override
  public List<Properties> buildCoords(Cluster cluster, Path buildPath) throws FalconException {
    org.apache.falcon.entity.v0.feed.Cluster feedCluster =
        FeedHelper.getCluster(entity, cluster.getName());
    if (feedCluster == null) {
      return null;
    }

    COORDINATORAPP coord = new COORDINATORAPP();
    String coordName = getEntityName();
    coord.setName(coordName);
    Date endDate = feedCluster.getValidity().getEnd();
    coord.setEnd(SchemaHelper.formatDateUTC(endDate));
    if (feedCluster.getValidity().getEnd().before(new Date())) {
      Date startDate = DateUtils.addMinutes(endDate, -1);
      coord.setStart(SchemaHelper.formatDateUTC(startDate));
    } else {
      coord.setStart(SchemaHelper.formatDateUTC(new Date()));
    }
    coord.setTimezone(entity.getTimezone().getID());
    TimeUnit timeUnit = entity.getFrequency().getTimeUnit();
    if (timeUnit == TimeUnit.hours || timeUnit == TimeUnit.minutes) {
      coord.setFrequency("${coord:hours(6)}");
    } else {
      coord.setFrequency("${coord:days(1)}");
    }

    Path coordPath = getBuildPath(buildPath);
    Properties props = createCoordDefaultConfiguration(coordName);

    WORKFLOW workflow = new WORKFLOW();
    Properties wfProps =
        OozieOrchestrationWorkflowBuilder.get(entity, cluster, Tag.RETENTION)
            .build(cluster, coordPath);
    workflow.setAppPath(getStoragePath(wfProps.getProperty(OozieEntityBuilder.ENTITY_PATH)));
    props.putAll(getProperties(coordPath, coordName));
    // Add the custom properties set in feed. Else, dryrun won't catch any missing props.
    props.putAll(EntityUtil.getEntityProperties(entity));
    workflow.setConfiguration(getConfig(props));
    ACTION action = new ACTION();
    action.setWorkflow(workflow);

    coord.setAction(action);

    Path marshalPath = marshal(cluster, coord, coordPath);
    return Arrays.asList(getProperties(marshalPath, coordName));
  }
  private void addClusterAndFeedForReplication() throws Exception {
    // Add cluster
    clusterEntity = addClusterEntity(CLUSTER_ENTITY_NAME, COLO_NAME, "classification=production");
    // Add backup cluster
    Cluster bcpCluster =
        addClusterEntity(BCP_CLUSTER_ENTITY_NAME, "east-coast", "classification=bcp");

    Cluster[] clusters = {clusterEntity, bcpCluster};

    // Add feed
    Feed rawFeed =
        addFeedEntity(
            REPLICATED_FEED,
            clusters,
            "classified-as=Secure",
            "analytics",
            Storage.TYPE.FILESYSTEM,
            "/falcon/raw-click/${YEAR}/${MONTH}/${DAY}");
    // Add uri template for each cluster
    for (org.apache.falcon.entity.v0.feed.Cluster feedCluster :
        rawFeed.getClusters().getClusters()) {
      if (feedCluster.getName().equals(CLUSTER_ENTITY_NAME)) {
        addStorage(
            feedCluster,
            rawFeed,
            Storage.TYPE.FILESYSTEM,
            "/falcon/raw-click/primary/${YEAR}/${MONTH}/${DAY}");
      } else {
        addStorage(
            feedCluster,
            rawFeed,
            Storage.TYPE.FILESYSTEM,
            "/falcon/raw-click/bcp/${YEAR}/${MONTH}/${DAY}");
      }
    }

    // update config store
    try {
      configStore.initiateUpdate(rawFeed);
      configStore.update(EntityType.FEED, rawFeed);
    } finally {
      configStore.cleanupUpdateInit();
    }
    inputFeeds.add(rawFeed);
  }
 private Feed addFeedEntity(
     String feedName,
     Cluster[] clusters,
     String tags,
     String groups,
     Storage.TYPE storageType,
     String uriTemplate)
     throws Exception {
   Feed feed = EntityBuilderTestUtil.buildFeed(feedName, clusters, tags, groups);
   addStorage(feed, storageType, uriTemplate);
   for (org.apache.falcon.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) {
     if (feedCluster.getName().equals(BCP_CLUSTER_ENTITY_NAME)) {
       feedCluster.setType(ClusterType.TARGET);
     }
   }
   configStore.publish(EntityType.FEED, feed);
   return feed;
 }
  private static void addStorage(
      org.apache.falcon.entity.v0.feed.Cluster cluster,
      Feed feed,
      Storage.TYPE storageType,
      String uriTemplate) {
    if (storageType == Storage.TYPE.FILESYSTEM) {
      Locations locations = new Locations();
      feed.setLocations(locations);

      Location location = new Location();
      location.setType(LocationType.DATA);
      location.setPath(uriTemplate);
      cluster.setLocations(new Locations());
      cluster.getLocations().getLocations().add(location);
    } else {
      CatalogTable table = new CatalogTable();
      table.setUri(uriTemplate);
      cluster.setTable(table);
    }
  }
  public void addFeedEntity(Feed feed) {
    LOG.info("Adding feed entity: {}", feed.getName());
    Vertex feedVertex = addVertex(feed.getName(), RelationshipType.FEED_ENTITY);

    addUserRelation(feedVertex);
    addDataClassification(feed.getTags(), feedVertex);
    addGroups(feed.getGroups(), feedVertex);

    for (org.apache.falcon.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) {
      if (ClusterType.TARGET != feedCluster.getType()) {
        addRelationToCluster(
            feedVertex, feedCluster.getName(), RelationshipLabel.FEED_CLUSTER_EDGE);
      }
    }

    for (org.apache.falcon.entity.v0.feed.Cluster feedCluster : feed.getClusters().getClusters()) {
      if (FeedHelper.isImportEnabled(feedCluster)) {
        addRelationToDatasource(
            feedVertex,
            FeedHelper.getImportDatasourceName(feedCluster),
            RelationshipLabel.DATASOURCE_IMPORT_EDGE);
      }
    }
  }
  public void updateFeedClusters(
      List<org.apache.falcon.entity.v0.feed.Cluster> oldClusters,
      List<org.apache.falcon.entity.v0.feed.Cluster> newClusters,
      Vertex feedEntityVertex) {
    if (areFeedClustersSame(oldClusters, newClusters)) {
      return;
    }

    // remove edges to old clusters
    for (org.apache.falcon.entity.v0.feed.Cluster oldCuster : oldClusters) {
      if (ClusterType.TARGET != oldCuster.getType()) {
        removeEdge(
            feedEntityVertex, oldCuster.getName(), RelationshipLabel.FEED_CLUSTER_EDGE.getName());
      }
    }

    // add edges to new clusters
    for (org.apache.falcon.entity.v0.feed.Cluster newCluster : newClusters) {
      if (ClusterType.TARGET != newCluster.getType()) {
        addRelationToCluster(
            feedEntityVertex, newCluster.getName(), RelationshipLabel.FEED_CLUSTER_EDGE);
      }
    }
  }