Exemplo n.º 1
0
  protected synchronized Entity submitInternal(HttpServletRequest request, String type)
      throws IOException, FalconException {

    EntityType entityType = EntityType.getEnum(type);
    Entity entity = deserializeEntity(request, entityType);
    // KLUDGE - Until ACL is mandated entity passed should be decorated for equals check to pass
    decorateEntityWithACL(entity);

    Entity existingEntity = configStore.get(entityType, entity.getName());
    if (existingEntity != null) {
      if (EntityUtil.equals(existingEntity, entity)) {
        return existingEntity;
      }

      throw new EntityAlreadyExistsException(
          entity.toShortString()
              + " already registered with configuration store. "
              + "Can't be submitted again. Try removing before submitting.");
    }

    SecurityUtil.tryProxy(entity); // proxy before validating since FS/Oozie needs to be proxied
    validate(entity);
    configStore.publish(entityType, entity);
    LOG.info("Submit successful: ({}): {}", type, entity.getName());
    return entity;
  }
Exemplo n.º 2
0
  private boolean isEntityFiltered(
      Entity entity, EntityList.EntityFilterByFields filter, Map.Entry<String, List<String>> pair) {
    switch (filter) {
      case TYPE:
        return !containsIgnoreCase(pair.getValue(), entity.getEntityType().toString());

      case NAME:
        return !containsIgnoreCase(pair.getValue(), entity.getName());

      case STATUS:
        return !containsIgnoreCase(pair.getValue(), getStatusString(entity));

      case PIPELINES:
        if (!entity.getEntityType().equals(EntityType.PROCESS)) {
          throw FalconWebException.newException(
              "Invalid filterBy key for non process entities " + pair.getKey(),
              Response.Status.BAD_REQUEST);
        }
        return !EntityUtil.getPipelines(entity).contains(pair.getValue().get(0));

      case CLUSTER:
        return !EntityUtil.getClustersDefined(entity).contains(pair.getValue().get(0));

      case TAGS:
        return isFilteredByTags(getFilterByTags(pair.getValue()), EntityUtil.getTags(entity));

      default:
        return false;
    }
  }
Exemplo n.º 3
0
  /**
   * Post an entity XML with entity type. Validates the XML which can be Process, Feed or
   * Dataendpoint
   *
   * @param type entity type
   * @return APIResule -Succeeded or Failed
   */
  public APIResult validate(HttpServletRequest request, String type) {
    try {
      EntityType entityType = EntityType.getEnum(type);
      Entity entity = deserializeEntity(request, entityType);
      validate(entity);

      // Validate that the entity can be scheduled in the cluster
      if (entity.getEntityType().isSchedulable()) {
        Set<String> clusters = EntityUtil.getClustersDefinedInColos(entity);
        for (String cluster : clusters) {
          try {
            getWorkflowEngine().dryRun(entity, cluster);
          } catch (FalconException e) {
            throw new FalconException("dryRun failed on cluster " + cluster, e);
          }
        }
      }
      return new APIResult(
          APIResult.Status.SUCCEEDED,
          "Validated successfully (" + entityType + ") " + entity.getName());
    } catch (Throwable e) {
      LOG.error("Validation failed for entity ({})", type, e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
Exemplo n.º 4
0
 @Override
 public String delete(Entity entity) throws FalconException {
   if (isActive(entity)) {
     EXECUTION_SERVICE.delete(entity);
   }
   // This should remove it from state store too as state store listens to config store changes.
   CONFIG_STORE.remove(entity.getEntityType(), entity.getName());
   return "SUCCESS";
 }
Exemplo n.º 5
0
  protected boolean isEntityAuthorized(Entity entity) {
    try {
      SecurityUtil.getAuthorizationProvider()
          .authorizeEntity(
              entity.getName(),
              entity.getEntityType().toString(),
              entity.getACL(),
              "list",
              CurrentUser.getAuthenticatedUGI());
    } catch (Exception e) {
      LOG.info(
          "Authorization failed for entity="
              + entity.getName()
              + " for user="
              + CurrentUser.getUser(),
          e);
      return false;
    }

    return true;
  }
Exemplo n.º 6
0
  /**
   * Submit a new entity. Entities can be of type feed, process or data end points. Entity
   * definitions are validated structurally against schema and subsequently for other rules before
   * they are admitted into the system
   *
   * <p>Entity name acts as the key and an entity once added, can't be added again unless deleted.
   *
   * @param request - Servlet Request
   * @param type - entity type - feed, process or data end point
   * @param colo - applicable colo
   * @return result of the operation
   */
  public APIResult submit(HttpServletRequest request, String type, String colo) {

    checkColo(colo);
    try {
      Entity entity = submitInternal(request, type);
      return new APIResult(
          APIResult.Status.SUCCEEDED, "Submit successful (" + type + ") " + entity.getName());
    } catch (Throwable e) {
      LOG.error("Unable to persist entity object", e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
 @Override
 public void onRemove(Entity entity) throws FalconException {
   if (entity.getEntityType().equals(EntityType.FEED)) {
     Feed feed = (Feed) entity;
     if (StringUtils.isEmpty(feed.getGroups())) {
       return;
     }
     String[] groups = feed.getGroups().split(",");
     for (String group : groups) {
       groupsMapping.get(group).getFeeds().remove(entity.getName());
       if (groupsMapping.get(group).getFeeds().size() == 0) {
         groupsMapping.remove(group);
       }
     }
   }
 }
Exemplo n.º 8
0
 private void canRemove(Entity entity) throws FalconException {
   Pair<String, EntityType>[] referencedBy = EntityIntegrityChecker.referencedBy(entity);
   if (referencedBy != null && referencedBy.length > 0) {
     StringBuilder messages = new StringBuilder();
     for (Pair<String, EntityType> ref : referencedBy) {
       messages.append(ref).append("\n");
     }
     throw new FalconException(
         entity.getName()
             + "("
             + entity.getEntityType()
             + ") cant "
             + "be removed as it is referred by "
             + messages);
   }
 }
Exemplo n.º 9
0
  @Override
  public String update(Entity oldEntity, Entity newEntity, String cluster, Boolean skipDryRun)
      throws FalconException {
    org.apache.falcon.entity.v0.cluster.Cluster clusterEntity =
        ConfigurationStore.get().get(EntityType.CLUSTER, cluster);
    boolean entityUpdated =
        UpdateHelper.isEntityUpdated(
            oldEntity,
            newEntity,
            cluster,
            EntityUtil.getLatestStagingPath(clusterEntity, oldEntity));
    StringBuilder result = new StringBuilder();
    if (!entityUpdated) {
      // Ideally should throw an exception, but, keeping it backward-compatible.
      LOG.warn(
          "No relevant updates detected in the new entity definition for entity {}!",
          newEntity.getName());
      return result.toString();
    }

    Date oldEndTime = EntityUtil.getEndTime(oldEntity, cluster);
    Date newEndTime = EntityUtil.getEndTime(newEntity, cluster);
    if (newEndTime.before(DateUtil.now()) || newEndTime.before(oldEndTime)) {
      throw new FalconException(
          "New Entity's end time "
              + SchemaHelper.formatDateUTC(newEndTime)
              + " is before current time or before old end time. Entity can't be updated.");
    }

    // The steps required are the same as touch.
    DAGEngineFactory.getDAGEngine(cluster)
        .touch(newEntity, (skipDryRun == null) ? Boolean.FALSE : skipDryRun);
    // Additionally, update the executor.
    // The update will kick in for new instances created and not for READY/WAITING instances, as
    // with Oozie.
    Collection<InstanceState> instances = new ArrayList<>();
    instances.add(STATE_STORE.getLastExecutionInstance(oldEntity, cluster));
    EXECUTION_SERVICE.getEntityExecutor(oldEntity, cluster).update(newEntity);

    result
        .append(newEntity.toShortString())
        .append("/Effective Time: ")
        .append(getEffectiveTime(newEntity, cluster, instances));
    return result.toString();
  }
Exemplo n.º 10
0
 private EntityElement getEntityElement(Entity entity, HashSet<String> fields) {
   EntityElement elem = new EntityElement();
   elem.type = entity.getEntityType().toString();
   elem.name = entity.getName();
   if (fields.contains(EntityList.EntityFieldList.STATUS.name())) {
     elem.status = getStatusString(entity);
   }
   if (fields.contains(EntityList.EntityFieldList.PIPELINES.name())) {
     elem.pipeline = EntityUtil.getPipelines(entity);
   }
   if (fields.contains(EntityList.EntityFieldList.TAGS.name())) {
     elem.tag = EntityUtil.getTags(entity);
   }
   if (fields.contains(EntityList.EntityFieldList.CLUSTERS.name())) {
     elem.cluster = new ArrayList<String>(EntityUtil.getClustersDefined(entity));
   }
   return elem;
 }
  @Override
  public void onChange(final Entity oldEntity, final Entity newEntity) throws FalconException {
    EntityType entityType = newEntity.getEntityType();
    LOG.info("Updating lineage for entity: {}, type: {}", newEntity.getName(), entityType);
    try {
      new TransactionRetryHelper.Builder<Void>(getTransactionalGraph())
          .perform(
              new TransactionWork<Void>() {
                @Override
                public Void execute(TransactionalGraph transactionalGraph) throws Exception {
                  entityGraphBuilder.updateEntity(oldEntity, newEntity);
                  transactionalGraph.commit();
                  return null;
                }
              })
          .build()
          .exponentialBackoff(transactionRetries, transactionRetryDelayInMillis);

    } catch (Exception e) {
      getTransactionalGraph().rollback();
      throw new FalconException(e);
    }
  }
Exemplo n.º 12
0
 protected void verifyEntityProperties(
     Entity entity,
     Cluster cluster,
     Cluster srcCluster,
     WorkflowExecutionContext.EntityOperations operation,
     HashMap<String, String> props)
     throws Exception {
   Assert.assertEquals(props.get(WorkflowExecutionArgs.ENTITY_NAME.getName()), entity.getName());
   Assert.assertEquals(
       props.get(WorkflowExecutionArgs.ENTITY_TYPE.getName()), entity.getEntityType().name());
   if (WorkflowExecutionContext.EntityOperations.REPLICATE == operation) {
     Assert.assertEquals(
         props.get(WorkflowExecutionArgs.CLUSTER_NAME.getName()),
         cluster.getName()
             + WorkflowExecutionContext.CLUSTER_NAME_SEPARATOR
             + srcCluster.getName());
   } else {
     Assert.assertEquals(
         props.get(WorkflowExecutionArgs.CLUSTER_NAME.getName()), cluster.getName());
   }
   Assert.assertEquals(
       props.get(WorkflowExecutionArgs.LOG_DIR.getName()), getLogPath(cluster, entity));
   Assert.assertEquals(props.get("falconDataOperation"), operation.name());
 }
Exemplo n.º 13
0
  @Override
  // SUSPEND CHECKSTYLE CHECK ParameterNumberCheck
  public void handleRerun(
      String clusterName,
      String entityType,
      String entityName,
      String nominalTime,
      String runId,
      String wfId,
      String workflowUser,
      long msgReceivedTime) {
    try {
      Entity entity = EntityUtil.getEntity(entityType, entityName);
      Retry retry = getRetry(entity);

      if (retry == null) {
        LOG.warn(
            "Retry not configured for entity: {} ({}), ignoring failed retried",
            entityType,
            entity.getName());
        return;
      }

      int attempts = retry.getAttempts();
      Frequency delay = retry.getDelay();
      PolicyType policy = retry.getPolicy();
      int intRunId = Integer.parseInt(runId);

      if (attempts > intRunId) {
        AbstractRerunPolicy rerunPolicy = RerunPolicyFactory.getRetryPolicy(policy);
        long delayTime = rerunPolicy.getDelay(delay, Integer.parseInt(runId));
        RetryEvent event =
            new RetryEvent(
                clusterName,
                wfId,
                msgReceivedTime,
                delayTime,
                entityType,
                entityName,
                nominalTime,
                intRunId,
                attempts,
                0,
                workflowUser);
        offerToQueue(event);
      } else {
        LOG.warn(
            "All retry attempt failed out of configured: {} attempt for entity instance: {}:{} "
                + "And WorkflowId: {}",
            attempts,
            entityName,
            nominalTime,
            wfId);

        GenericAlert.alertRetryFailed(
            entityType,
            entityName,
            nominalTime,
            wfId,
            workflowUser,
            runId,
            "All retry attempt failed out of configured: "
                + attempts
                + " attempt for entity instance::");
      }
    } catch (FalconException e) {
      LOG.error("Error during retry of entity instance {}:{}", entityName, nominalTime, e);
      GenericAlert.alertRetryFailed(
          entityType, entityName, nominalTime, wfId, workflowUser, runId, e.getMessage());
    }
  }