/**
   * KLUDGE - Until ACL is mandated entity passed should be decorated for equals check to pass.
   * existingEntity in config store will have teh decoration and equals check fails if entity passed
   * is not decorated for checking if entity already exists.
   *
   * @param entity entity
   */
  private void decorateEntityWithACL(Entity entity) {
    if (SecurityUtil.isAuthorizationEnabled() || entity.getACL() != null) {
      return; // not necessary to decorate
    }

    final String proxyUser = CurrentUser.getUser();
    final String defaultGroupName = CurrentUser.getPrimaryGroupName();
    switch (entity.getEntityType()) {
      case CLUSTER:
        org.apache.falcon.entity.v0.cluster.ACL clusterACL =
            new org.apache.falcon.entity.v0.cluster.ACL();
        clusterACL.setOwner(proxyUser);
        clusterACL.setGroup(defaultGroupName);
        ((org.apache.falcon.entity.v0.cluster.Cluster) entity).setACL(clusterACL);
        break;

      case FEED:
        org.apache.falcon.entity.v0.feed.ACL feedACL = new org.apache.falcon.entity.v0.feed.ACL();
        feedACL.setOwner(proxyUser);
        feedACL.setGroup(defaultGroupName);
        ((org.apache.falcon.entity.v0.feed.Feed) entity).setACL(feedACL);
        break;

      case PROCESS:
        org.apache.falcon.entity.v0.process.ACL processACL =
            new org.apache.falcon.entity.v0.process.ACL();
        processACL.setOwner(proxyUser);
        processACL.setGroup(defaultGroupName);
        ((org.apache.falcon.entity.v0.process.Process) entity).setACL(processACL);
        break;

      default:
        break;
    }
  }
  private boolean isEntityFiltered(
      Entity entity, EntityList.EntityFilterByFields filter, Map.Entry<String, List<String>> pair) {
    switch (filter) {
      case TYPE:
        return !containsIgnoreCase(pair.getValue(), entity.getEntityType().toString());

      case NAME:
        return !containsIgnoreCase(pair.getValue(), entity.getName());

      case STATUS:
        return !containsIgnoreCase(pair.getValue(), getStatusString(entity));

      case PIPELINES:
        if (!entity.getEntityType().equals(EntityType.PROCESS)) {
          throw FalconWebException.newException(
              "Invalid filterBy key for non process entities " + pair.getKey(),
              Response.Status.BAD_REQUEST);
        }
        return !EntityUtil.getPipelines(entity).contains(pair.getValue().get(0));

      case CLUSTER:
        return !EntityUtil.getClustersDefined(entity).contains(pair.getValue().get(0));

      case TAGS:
        return isFilteredByTags(getFilterByTags(pair.getValue()), EntityUtil.getTags(entity));

      default:
        return false;
    }
  }
  private List<Entity> obtainUpdateEntityLocks(Entity entity) throws FalconException {
    List<Entity> tokenList = new ArrayList<Entity>();

    // first obtain lock for the entity for which update is issued.
    if (memoryLocks.acquireLock(entity)) {
      tokenList.add(entity);
    } else {
      throw new FalconException(
          "Looks like an update command is already issued for " + entity.toShortString());
    }

    // now obtain locks for all dependent entities.
    Set<Entity> affectedEntities = EntityGraph.get().getDependents(entity);
    for (Entity e : affectedEntities) {
      if (memoryLocks.acquireLock(e)) {
        tokenList.add(e);
      } else {
        LOG.error(
            "Error while trying to acquire lock for {}. Releasing already obtained locks",
            e.toShortString());
        throw new FalconException(
            "There are multiple update commands running for dependent entity " + e.toShortString());
      }
    }
    return tokenList;
  }
  protected synchronized Entity submitInternal(HttpServletRequest request, String type)
      throws IOException, FalconException {

    EntityType entityType = EntityType.getEnum(type);
    Entity entity = deserializeEntity(request, entityType);
    // KLUDGE - Until ACL is mandated entity passed should be decorated for equals check to pass
    decorateEntityWithACL(entity);

    Entity existingEntity = configStore.get(entityType, entity.getName());
    if (existingEntity != null) {
      if (EntityUtil.equals(existingEntity, entity)) {
        return existingEntity;
      }

      throw new EntityAlreadyExistsException(
          entity.toShortString()
              + " already registered with configuration store. "
              + "Can't be submitted again. Try removing before submitting.");
    }

    SecurityUtil.tryProxy(entity); // proxy before validating since FS/Oozie needs to be proxied
    validate(entity);
    configStore.publish(entityType, entity);
    LOG.info("Submit successful: ({}): {}", type, entity.getName());
    return entity;
  }
  /**
   * Post an entity XML with entity type. Validates the XML which can be Process, Feed or
   * Dataendpoint
   *
   * @param type entity type
   * @return APIResule -Succeeded or Failed
   */
  public APIResult validate(HttpServletRequest request, String type) {
    try {
      EntityType entityType = EntityType.getEnum(type);
      Entity entity = deserializeEntity(request, entityType);
      validate(entity);

      // Validate that the entity can be scheduled in the cluster
      if (entity.getEntityType().isSchedulable()) {
        Set<String> clusters = EntityUtil.getClustersDefinedInColos(entity);
        for (String cluster : clusters) {
          try {
            getWorkflowEngine().dryRun(entity, cluster);
          } catch (FalconException e) {
            throw new FalconException("dryRun failed on cluster " + cluster, e);
          }
        }
      }
      return new APIResult(
          APIResult.Status.SUCCEEDED,
          "Validated successfully (" + entityType + ") " + entity.getName());
    } catch (Throwable e) {
      LOG.error("Validation failed for entity ({})", type, e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
 @Override
 public String delete(Entity entity) throws FalconException {
   if (isActive(entity)) {
     EXECUTION_SERVICE.delete(entity);
   }
   // This should remove it from state store too as state store listens to config store changes.
   CONFIG_STORE.remove(entity.getEntityType(), entity.getName());
   return "SUCCESS";
 }
  /**
   * Submit a new entity. Entities can be of type feed, process or data end points. Entity
   * definitions are validated structurally against schema and subsequently for other rules before
   * they are admitted into the system
   *
   * <p>Entity name acts as the key and an entity once added, can't be added again unless deleted.
   *
   * @param request - Servlet Request
   * @param type - entity type - feed, process or data end point
   * @param colo - applicable colo
   * @return result of the operation
   */
  public APIResult submit(HttpServletRequest request, String type, String colo) {

    checkColo(colo);
    try {
      Entity entity = submitInternal(request, type);
      return new APIResult(
          APIResult.Status.SUCCEEDED, "Submit successful (" + type + ") " + entity.getName());
    } catch (Throwable e) {
      LOG.error("Unable to persist entity object", e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
  public static OozieEntityBuilder get(Entity entity) {
    switch (entity.getEntityType()) {
      case FEED:
        return new FeedBundleBuilder((Feed) entity);

      case PROCESS:
        return new ProcessBundleBuilder((Process) entity);

      default:
    }
    throw new IllegalArgumentException("Unhandled type: " + entity.getEntityType());
  }
  private void validateUpdate(Entity oldEntity, Entity newEntity) throws FalconException {
    if (oldEntity.getEntityType() != newEntity.getEntityType() || !oldEntity.equals(newEntity)) {
      throw new FalconException(
          oldEntity.toShortString() + " can't be updated with " + newEntity.toShortString());
    }

    if (oldEntity.getEntityType() == EntityType.CLUSTER) {
      throw new FalconException("Update not supported for clusters");
    }

    String[] props = oldEntity.getEntityType().getImmutableProperties();
    for (String prop : props) {
      Object oldProp, newProp;
      try {
        oldProp = PropertyUtils.getProperty(oldEntity, prop);
        newProp = PropertyUtils.getProperty(newEntity, prop);
      } catch (Exception e) {
        throw new FalconException(e);
      }
      if (!ObjectUtils.equals(oldProp, newProp)) {
        throw new ValidationException(
            oldEntity.toShortString() + ": " + prop + " can't be changed");
      }
    }
  }
 /**
  * Returns the entity definition as an XML based on name.
  *
  * @param type entity type
  * @param entityName entity name
  * @return String
  */
 public String getEntityDefinition(String type, String entityName) {
   try {
     EntityType entityType = EntityType.getEnum(type);
     Entity entity = configStore.get(entityType, entityName);
     if (entity == null) {
       throw new NoSuchElementException(entityName + " (" + type + ") not found");
     }
     return entity.toString();
   } catch (Throwable e) {
     LOG.error(
         "Unable to get entity definition from config store for ({}): {}", type, entityName, e);
     throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
   }
 }
 @Override
 public String touch(Entity entity, String cluster, Boolean skipDryRun) throws FalconException {
   EntityID id = new EntityID(entity);
   // Ideally state store should have all entities, but, check anyway.
   if (STATE_STORE.entityExists(id)) {
     Date endTime = EntityUtil.getEndTime(entity, cluster);
     if (endTime.before(DateUtil.now())) {
       throw new FalconException(
           "Entity's end time "
               + SchemaHelper.formatDateUTC(endTime)
               + " is before current time. Entity can't be touch-ed as it has completed.");
     }
     Collection<InstanceState> instances =
         STATE_STORE.getExecutionInstances(entity, cluster, InstanceState.getRunningStates());
     // touch should happen irrespective of the state the entity is in.
     DAGEngineFactory.getDAGEngine(cluster)
         .touch(entity, (skipDryRun == null) ? Boolean.FALSE : skipDryRun);
     StringBuilder builder = new StringBuilder();
     builder
         .append(entity.toShortString())
         .append("/Effective Time: ")
         .append(getEffectiveTime(entity, cluster, instances));
     return builder.toString();
   }
   throw new FalconException("Could not find entity " + id + " in state store.");
 }
 private void canRemove(Entity entity) throws FalconException {
   Pair<String, EntityType>[] referencedBy = EntityIntegrityChecker.referencedBy(entity);
   if (referencedBy != null && referencedBy.length > 0) {
     StringBuilder messages = new StringBuilder();
     for (Pair<String, EntityType> ref : referencedBy) {
       messages.append(ref).append("\n");
     }
     throw new FalconException(
         entity.getName()
             + "("
             + entity.getEntityType()
             + ") cant "
             + "be removed as it is referred by "
             + messages);
   }
 }
 @Override
 public void onRemove(Entity entity) throws FalconException {
   if (entity.getEntityType().equals(EntityType.FEED)) {
     Feed feed = (Feed) entity;
     if (StringUtils.isEmpty(feed.getGroups())) {
       return;
     }
     String[] groups = feed.getGroups().split(",");
     for (String group : groups) {
       groupsMapping.get(group).getFeeds().remove(entity.getName());
       if (groupsMapping.get(group).getFeeds().size() == 0) {
         groupsMapping.remove(group);
       }
     }
   }
 }
  @Override
  public String update(Entity oldEntity, Entity newEntity, String cluster, Boolean skipDryRun)
      throws FalconException {
    org.apache.falcon.entity.v0.cluster.Cluster clusterEntity =
        ConfigurationStore.get().get(EntityType.CLUSTER, cluster);
    boolean entityUpdated =
        UpdateHelper.isEntityUpdated(
            oldEntity,
            newEntity,
            cluster,
            EntityUtil.getLatestStagingPath(clusterEntity, oldEntity));
    StringBuilder result = new StringBuilder();
    if (!entityUpdated) {
      // Ideally should throw an exception, but, keeping it backward-compatible.
      LOG.warn(
          "No relevant updates detected in the new entity definition for entity {}!",
          newEntity.getName());
      return result.toString();
    }

    Date oldEndTime = EntityUtil.getEndTime(oldEntity, cluster);
    Date newEndTime = EntityUtil.getEndTime(newEntity, cluster);
    if (newEndTime.before(DateUtil.now()) || newEndTime.before(oldEndTime)) {
      throw new FalconException(
          "New Entity's end time "
              + SchemaHelper.formatDateUTC(newEndTime)
              + " is before current time or before old end time. Entity can't be updated.");
    }

    // The steps required are the same as touch.
    DAGEngineFactory.getDAGEngine(cluster)
        .touch(newEntity, (skipDryRun == null) ? Boolean.FALSE : skipDryRun);
    // Additionally, update the executor.
    // The update will kick in for new instances created and not for READY/WAITING instances, as
    // with Oozie.
    Collection<InstanceState> instances = new ArrayList<>();
    instances.add(STATE_STORE.getLastExecutionInstance(oldEntity, cluster));
    EXECUTION_SERVICE.getEntityExecutor(oldEntity, cluster).update(newEntity);

    result
        .append(newEntity.toShortString())
        .append("/Effective Time: ")
        .append(getEffectiveTime(newEntity, cluster, instances));
    return result.toString();
  }
 private EntityElement getEntityElement(Entity entity, HashSet<String> fields) {
   EntityElement elem = new EntityElement();
   elem.type = entity.getEntityType().toString();
   elem.name = entity.getName();
   if (fields.contains(EntityList.EntityFieldList.STATUS.name())) {
     elem.status = getStatusString(entity);
   }
   if (fields.contains(EntityList.EntityFieldList.PIPELINES.name())) {
     elem.pipeline = EntityUtil.getPipelines(entity);
   }
   if (fields.contains(EntityList.EntityFieldList.TAGS.name())) {
     elem.tag = EntityUtil.getTags(entity);
   }
   if (fields.contains(EntityList.EntityFieldList.CLUSTERS.name())) {
     elem.cluster = new ArrayList<String>(EntityUtil.getClustersDefined(entity));
   }
   return elem;
 }
 protected String getStatusString(Entity entity) {
   String statusString;
   try {
     statusString = getStatus(entity, entity.getEntityType()).name();
   } catch (Throwable throwable) {
     // Unable to fetch statusString, setting it to unknown for backwards compatibility
     statusString = "UNKNOWN";
   }
   return statusString;
 }
  protected boolean isEntityAuthorized(Entity entity) {
    try {
      SecurityUtil.getAuthorizationProvider()
          .authorizeEntity(
              entity.getName(),
              entity.getEntityType().toString(),
              entity.getACL(),
              "list",
              CurrentUser.getAuthenticatedUGI());
    } catch (Exception e) {
      LOG.info(
          "Authorization failed for entity="
              + entity.getName()
              + " for user="
              + CurrentUser.getUser(),
          e);
      return false;
    }

    return true;
  }
  @Override
  public void onAdd(Entity entity) throws FalconException {

    if (entity.getEntityType().equals(EntityType.FEED)) {
      Feed feed = (Feed) entity;
      if (feed.getGroups() == null || feed.getGroups().equals("")) {
        return;
      }
      Set<FeedGroup> groupSet = getGroups(feed);
      addGroups(feed.getName(), groupSet);
    }
  }
  @Override
  public void onChange(final Entity oldEntity, final Entity newEntity) throws FalconException {
    EntityType entityType = newEntity.getEntityType();
    LOG.info("Updating lineage for entity: {}, type: {}", newEntity.getName(), entityType);
    try {
      new TransactionRetryHelper.Builder<Void>(getTransactionalGraph())
          .perform(
              new TransactionWork<Void>() {
                @Override
                public Void execute(TransactionalGraph transactionalGraph) throws Exception {
                  entityGraphBuilder.updateEntity(oldEntity, newEntity);
                  transactionalGraph.commit();
                  return null;
                }
              })
          .build()
          .exponentialBackoff(transactionRetries, transactionRetryDelayInMillis);

    } catch (Exception e) {
      getTransactionalGraph().rollback();
      throw new FalconException(e);
    }
  }
示例#20
0
  protected Properties getEntityProperties(Entity myEntity) {
    Properties properties = new Properties();
    switch (myEntity.getEntityType()) {
      case CLUSTER:
        org.apache.falcon.entity.v0.cluster.Properties clusterProps =
            ((Cluster) myEntity).getProperties();
        if (clusterProps != null) {
          for (Property prop : clusterProps.getProperties()) {
            properties.put(prop.getName(), prop.getValue());
          }
        }
        break;

      case FEED:
        org.apache.falcon.entity.v0.feed.Properties feedProps = ((Feed) myEntity).getProperties();
        if (feedProps != null) {
          for (org.apache.falcon.entity.v0.feed.Property prop : feedProps.getProperties()) {
            properties.put(prop.getName(), prop.getValue());
          }
        }
        break;

      case PROCESS:
        org.apache.falcon.entity.v0.process.Properties processProps =
            ((Process) myEntity).getProperties();
        if (processProps != null) {
          for (org.apache.falcon.entity.v0.process.Property prop : processProps.getProperties()) {
            properties.put(prop.getName(), prop.getValue());
          }
        }
        break;

      default:
        throw new IllegalArgumentException("Unhandled entity type " + myEntity.getEntityType());
    }
    return properties;
  }
示例#21
0
 protected void verifyEntityProperties(
     Entity entity,
     Cluster cluster,
     Cluster srcCluster,
     WorkflowExecutionContext.EntityOperations operation,
     HashMap<String, String> props)
     throws Exception {
   Assert.assertEquals(props.get(WorkflowExecutionArgs.ENTITY_NAME.getName()), entity.getName());
   Assert.assertEquals(
       props.get(WorkflowExecutionArgs.ENTITY_TYPE.getName()), entity.getEntityType().name());
   if (WorkflowExecutionContext.EntityOperations.REPLICATE == operation) {
     Assert.assertEquals(
         props.get(WorkflowExecutionArgs.CLUSTER_NAME.getName()),
         cluster.getName()
             + WorkflowExecutionContext.CLUSTER_NAME_SEPARATOR
             + srcCluster.getName());
   } else {
     Assert.assertEquals(
         props.get(WorkflowExecutionArgs.CLUSTER_NAME.getName()), cluster.getName());
   }
   Assert.assertEquals(
       props.get(WorkflowExecutionArgs.LOG_DIR.getName()), getLogPath(cluster, entity));
   Assert.assertEquals(props.get("falconDataOperation"), operation.name());
 }
 public void updateEntity(Entity oldEntity, Entity newEntity) {
   EntityType entityType = oldEntity.getEntityType();
   switch (entityType) {
     case CLUSTER:
       updateClusterEntity((Cluster) oldEntity, (Cluster) newEntity);
       break;
     case PROCESS:
       updateProcessEntity((Process) oldEntity, (Process) newEntity);
       break;
     case FEED:
       updateFeedEntity((Feed) oldEntity, (Feed) newEntity);
       break;
     case DATASOURCE:
       updateDatasourceEntity((Datasource) oldEntity, (Datasource) newEntity);
       break;
     default:
       throw new IllegalArgumentException("Invalid EntityType " + entityType);
   }
 }
  public void addEntity(Entity entity) {
    EntityType entityType = entity.getEntityType();
    switch (entityType) {
      case CLUSTER:
        addClusterEntity((Cluster) entity);
        break;
      case PROCESS:
        addProcessEntity((Process) entity);
        break;
      case FEED:
        addFeedEntity((Feed) entity);
        break;
      case DATASOURCE:
        addDatasourceEntity((Datasource) entity);
        break;

      default:
        throw new IllegalArgumentException("Invalid EntityType " + entityType);
    }
  }
 @SuppressWarnings({"unchecked", "rawtypes"})
 private void validate(Entity entity) throws FalconException {
   EntityParser entityParser = EntityParserFactory.getParser(entity.getEntityType());
   entityParser.validate(entity);
 }
  protected List<Entity> getFilteredEntities(
      EntityType entityType,
      String nameSubsequence,
      String tagKeywords,
      Map<String, List<String>> filterByFieldsValues,
      String startDate,
      String endDate,
      String cluster)
      throws FalconException, IOException {
    Collection<String> entityNames = configStore.getEntities(entityType);
    if (entityNames.isEmpty()) {
      return Collections.emptyList();
    }

    List<Entity> entities = new ArrayList<Entity>();
    char[] subsequence = nameSubsequence.toLowerCase().toCharArray();
    List<String> tagKeywordsList;
    if (StringUtils.isEmpty(tagKeywords)) {
      tagKeywordsList = new ArrayList<>();
    } else {
      tagKeywordsList = getFilterByTags(Arrays.asList(tagKeywords.toLowerCase()));
    }
    for (String entityName : entityNames) {
      Entity entity;
      try {
        entity = configStore.get(entityType, entityName);
        if (entity == null) {
          continue;
        }
      } catch (FalconException e1) {
        LOG.error(
            "Unable to get list for entities for ({})",
            entityType.getEntityClass().getSimpleName(),
            e1);
        throw FalconWebException.newException(e1, Response.Status.BAD_REQUEST);
      }

      if (SecurityUtil.isAuthorizationEnabled() && !isEntityAuthorized(entity)) {
        // the user who requested list query has no permission to access this entity. Skip this
        // entity
        continue;
      }
      if (isFilteredByDatesAndCluster(entity, startDate, endDate, cluster)) {
        // this is for entity summary
        continue;
      }
      SecurityUtil.tryProxy(entity);

      // filter by fields
      if (isFilteredByFields(entity, filterByFieldsValues)) {
        continue;
      }

      // filter by subsequence of name
      if (subsequence.length > 0
          && !matchesNameSubsequence(subsequence, entityName.toLowerCase())) {
        continue;
      }

      // filter by tag keywords
      if (!matchTagKeywords(tagKeywordsList, entity.getTags())) {
        continue;
      }

      entities.add(entity);
    }

    return entities;
  }
示例#26
0
  @Override
  // SUSPEND CHECKSTYLE CHECK ParameterNumberCheck
  public void handleRerun(
      String clusterName,
      String entityType,
      String entityName,
      String nominalTime,
      String runId,
      String wfId,
      String workflowUser,
      long msgReceivedTime) {
    try {
      Entity entity = EntityUtil.getEntity(entityType, entityName);
      Retry retry = getRetry(entity);

      if (retry == null) {
        LOG.warn(
            "Retry not configured for entity: {} ({}), ignoring failed retried",
            entityType,
            entity.getName());
        return;
      }

      int attempts = retry.getAttempts();
      Frequency delay = retry.getDelay();
      PolicyType policy = retry.getPolicy();
      int intRunId = Integer.parseInt(runId);

      if (attempts > intRunId) {
        AbstractRerunPolicy rerunPolicy = RerunPolicyFactory.getRetryPolicy(policy);
        long delayTime = rerunPolicy.getDelay(delay, Integer.parseInt(runId));
        RetryEvent event =
            new RetryEvent(
                clusterName,
                wfId,
                msgReceivedTime,
                delayTime,
                entityType,
                entityName,
                nominalTime,
                intRunId,
                attempts,
                0,
                workflowUser);
        offerToQueue(event);
      } else {
        LOG.warn(
            "All retry attempt failed out of configured: {} attempt for entity instance: {}:{} "
                + "And WorkflowId: {}",
            attempts,
            entityName,
            nominalTime,
            wfId);

        GenericAlert.alertRetryFailed(
            entityType,
            entityName,
            nominalTime,
            wfId,
            workflowUser,
            runId,
            "All retry attempt failed out of configured: "
                + attempts
                + " attempt for entity instance::");
      }
    } catch (FalconException e) {
      LOG.error("Error during retry of entity instance {}:{}", entityName, nominalTime, e);
      GenericAlert.alertRetryFailed(
          entityType, entityName, nominalTime, wfId, workflowUser, runId, e.getMessage());
    }
  }