Exemplo n.º 1
0
  /**
   * Deletes a scheduled entity, a deleted entity is removed completely from execution pool.
   *
   * @param type entity type
   * @param entity entity name
   * @return APIResult
   */
  public APIResult delete(HttpServletRequest request, String type, String entity, String colo) {
    checkColo(colo);
    try {
      EntityType entityType = EntityType.getEnum(type);
      String removedFromEngine = "";
      try {
        Entity entityObj = EntityUtil.getEntity(type, entity);

        canRemove(entityObj);
        if (entityType.isSchedulable() && !DeploymentUtil.isPrism()) {
          getWorkflowEngine().delete(entityObj);
          removedFromEngine = "(KILLED in ENGINE)";
        }

        configStore.remove(entityType, entity);
      } catch (EntityNotRegisteredException e) { // already deleted
        return new APIResult(
            APIResult.Status.SUCCEEDED, entity + "(" + type + ") doesn't exist. Nothing to do");
      }

      return new APIResult(
          APIResult.Status.SUCCEEDED,
          entity + "(" + type + ") removed successfully " + removedFromEngine);
    } catch (Throwable e) {
      LOG.error("Unable to reach workflow engine for deletion or deletion failed", e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
Exemplo n.º 2
0
  /**
   * Returns the list of filtered entities as well as the total number of results.
   *
   * @param fieldStr Fields that the query is interested in, separated by comma
   * @param nameSubsequence Name subsequence to match
   * @param tagKeywords Tag keywords to match, separated by commma
   * @param filterType Only return entities of this type
   * @param filterTags Full tag matching, separated by comma
   * @param filterBy Specific fields to match (i.e. TYPE, NAME, STATUS, PIPELINES, CLUSTER)
   * @param orderBy Order result by these fields.
   * @param sortOrder Valid options are "asc" and “desc”
   * @param offset Pagination offset.
   * @param resultsPerPage Number of results that should be returned starting at the offset.
   * @return EntityList
   */
  public EntityList getEntityList(
      String fieldStr,
      String nameSubsequence,
      String tagKeywords,
      String filterType,
      String filterTags,
      String filterBy,
      String orderBy,
      String sortOrder,
      Integer offset,
      Integer resultsPerPage) {

    HashSet<String> fields = new HashSet<String>(Arrays.asList(fieldStr.toUpperCase().split(",")));
    Map<String, List<String>> filterByFieldsValues = getFilterByFieldsValues(filterBy);
    validateEntityFilterByClause(filterByFieldsValues);
    if (StringUtils.isNotEmpty(filterTags)) {
      filterByFieldsValues.put(
          EntityList.EntityFilterByFields.TAGS.name(), Arrays.asList(filterTags));
    }

    // get filtered entities
    List<Entity> entities = new ArrayList<Entity>();
    try {
      if (StringUtils.isEmpty(filterType)) {
        // return entities of all types if no entity type specified
        for (EntityType entityType : EntityType.values()) {
          entities.addAll(
              getFilteredEntities(
                  entityType, nameSubsequence, tagKeywords, filterByFieldsValues, "", "", ""));
        }
      } else {
        String[] types = filterType.split(",");
        for (String type : types) {
          EntityType entityType = EntityType.getEnum(type);
          entities.addAll(
              getFilteredEntities(
                  entityType, nameSubsequence, tagKeywords, filterByFieldsValues, "", "", ""));
        }
      }
    } catch (Exception e) {
      LOG.error("Failed to get entity list", e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }

    // sort entities and pagination
    List<Entity> entitiesReturn =
        sortEntitiesPagination(entities, orderBy, sortOrder, offset, resultsPerPage);

    // add total number of results
    EntityList entityList =
        entitiesReturn.size() == 0
            ? new EntityList(new Entity[] {}, 0)
            : new EntityList(
                buildEntityElements(new HashSet<String>(fields), entitiesReturn), entities.size());
    return entityList;
  }
Exemplo n.º 3
0
  @CliCommand(
      value = {METADATA_COMMAND_PREFIX + LIST_OPT},
      help = LIST_OPT_DESCRIPTION)
  public String list(
      @CliOption(
              key = {TYPE_OPT},
              mandatory = true,
              help = TYPE_OPT_DESCRIPTION)
          final String dimensionType,
      @CliOption(
              key = {CLUSTER_OPT},
              mandatory = false,
              help = CLUSTER_OPT_DESCRIPTION)
          final String cluster,
      @CliOption(
              key = {FEED_OPT},
              mandatory = false,
              help = FEED_OPT_DESCRIPTION)
          final String feed,
      @CliOption(
              key = {PROCESS_OPT},
              mandatory = false,
              help = PROCESS_OPT_DESCRIPTION)
          final String process,
      @CliOption(
              key = {NUM_RESULTS_OPT},
              mandatory = false,
              help = NUM_RESULTS_OPT_DESCRIPTION)
          final Integer numResults) {
    validateDimensionType(dimensionType.toUpperCase());
    if (!(dimensionType.toUpperCase()).equals(RelationshipType.REPLICATION_METRICS.name())) {
      return getFalconClient().getDimensionList(dimensionType, cluster, getDoAs());
    } else {
      String schedEntityType = null;
      String schedEntityName = null;
      if (StringUtils.isNotEmpty(feed)) {
        schedEntityType = EntityType.getEnum(FEED_OPT).name();
        schedEntityName = feed;
      } else if (StringUtils.isNotEmpty(process)) {
        schedEntityType = EntityType.getEnum(PROCESS_OPT).name();
        schedEntityName = process;
      }
      validateScheduleEntity(schedEntityType, schedEntityName);

      return getFalconClient()
          .getReplicationMetricsDimensionList(
              schedEntityType, schedEntityName, numResults, getDoAs());
    }
  }
  protected void storeEntity(EntityType type, String name, String resource) throws Exception {
    Unmarshaller unmarshaller = type.getUnmarshaller();
    ConfigurationStore store = ConfigurationStore.get();
    store.remove(type, name);
    switch (type) {
      case CLUSTER:
        Cluster cluster = (Cluster) unmarshaller.unmarshal(this.getClass().getResource(resource));
        cluster.setName(name);
        store.publish(type, cluster);
        break;

      case FEED:
        Feed feed = (Feed) unmarshaller.unmarshal(this.getClass().getResource(resource));
        feed.setName(name);
        store.publish(type, feed);
        break;

      case PROCESS:
        Process process = (Process) unmarshaller.unmarshal(this.getClass().getResource(resource));
        process.setName(name);
        store.publish(type, process);
        break;

      default:
    }
  }
Exemplo n.º 5
0
  protected synchronized Entity submitInternal(HttpServletRequest request, String type)
      throws IOException, FalconException {

    EntityType entityType = EntityType.getEnum(type);
    Entity entity = deserializeEntity(request, entityType);
    // KLUDGE - Until ACL is mandated entity passed should be decorated for equals check to pass
    decorateEntityWithACL(entity);

    Entity existingEntity = configStore.get(entityType, entity.getName());
    if (existingEntity != null) {
      if (EntityUtil.equals(existingEntity, entity)) {
        return existingEntity;
      }

      throw new EntityAlreadyExistsException(
          entity.toShortString()
              + " already registered with configuration store. "
              + "Can't be submitted again. Try removing before submitting.");
    }

    SecurityUtil.tryProxy(entity); // proxy before validating since FS/Oozie needs to be proxied
    validate(entity);
    configStore.publish(entityType, entity);
    LOG.info("Submit successful: ({}): {}", type, entity.getName());
    return entity;
  }
Exemplo n.º 6
0
  /**
   * Post an entity XML with entity type. Validates the XML which can be Process, Feed or
   * Dataendpoint
   *
   * @param type entity type
   * @return APIResule -Succeeded or Failed
   */
  public APIResult validate(HttpServletRequest request, String type) {
    try {
      EntityType entityType = EntityType.getEnum(type);
      Entity entity = deserializeEntity(request, entityType);
      validate(entity);

      // Validate that the entity can be scheduled in the cluster
      if (entity.getEntityType().isSchedulable()) {
        Set<String> clusters = EntityUtil.getClustersDefinedInColos(entity);
        for (String cluster : clusters) {
          try {
            getWorkflowEngine().dryRun(entity, cluster);
          } catch (FalconException e) {
            throw new FalconException("dryRun failed on cluster " + cluster, e);
          }
        }
      }
      return new APIResult(
          APIResult.Status.SUCCEEDED,
          "Validated successfully (" + entityType + ") " + entity.getName());
    } catch (Throwable e) {
      LOG.error("Validation failed for entity ({})", type, e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
Exemplo n.º 7
0
 protected void assertLibExtensions(
     FileSystem fs, COORDINATORAPP coord, EntityType type, String lifecycle) throws Exception {
   WORKFLOWAPP wf = getWorkflowapp(fs, coord);
   List<Object> actions = wf.getDecisionOrForkOrJoin();
   String lifeCyclePath = lifecycle == null ? "" : "/" + lifecycle;
   for (Object obj : actions) {
     if (!(obj instanceof ACTION)) {
       continue;
     }
     ACTION action = (ACTION) obj;
     List<String> files = null;
     if (action.getJava() != null) {
       files = action.getJava().getFile();
     } else if (action.getPig() != null) {
       files = action.getPig().getFile();
     } else if (action.getMapReduce() != null) {
       files = action.getMapReduce().getFile();
     }
     if (files != null) {
       Assert.assertTrue(
           files
               .get(files.size() - 1)
               .endsWith(
                   "/projects/falcon/working/libext/" + type.name() + lifeCyclePath + "/ext.jar"));
     }
   }
 }
 private static void cleanupConfigurationStore(ConfigurationStore store) throws Exception {
   for (EntityType type : EntityType.values()) {
     Collection<String> entities = store.getEntities(type);
     for (String entity : entities) {
       store.remove(type, entity);
     }
   }
 }
Exemplo n.º 9
0
 protected void cleanupStore() throws FalconException {
   ConfigurationStore store = ConfigurationStore.get();
   for (EntityType type : EntityType.values()) {
     Collection<String> entities = store.getEntities(type);
     for (String entity : entities) {
       store.remove(type, entity);
     }
   }
 }
Exemplo n.º 10
0
  protected Entity storeEntity(EntityType type, String name, String resource, String writeEndpoint)
      throws Exception {
    Unmarshaller unmarshaller = type.getUnmarshaller();
    ConfigurationStore store = ConfigurationStore.get();
    switch (type) {
      case CLUSTER:
        Cluster cluster = (Cluster) unmarshaller.unmarshal(this.getClass().getResource(resource));
        if (name != null) {
          store.remove(type, name);
          cluster.setName(name);
        }
        store.publish(type, cluster);

        if (writeEndpoint != null) {
          ClusterHelper.getInterface(cluster, Interfacetype.WRITE).setEndpoint(writeEndpoint);
          FileSystem fs = new Path(writeEndpoint).getFileSystem(EmbeddedCluster.newConfiguration());
          fs.create(
                  new Path(
                      ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath(),
                      "libext/FEED/retention/ext.jar"))
              .close();
          fs.create(
                  new Path(
                      ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath(),
                      "libext/FEED/replication/ext.jar"))
              .close();
        }

        return cluster;

      case FEED:
        Feed feed = (Feed) unmarshaller.unmarshal(this.getClass().getResource(resource));
        if (name != null) {
          store.remove(type, name);
          feed.setName(name);
        }
        store.publish(type, feed);
        return feed;

      case PROCESS:
        Process process = (Process) unmarshaller.unmarshal(this.getClass().getResource(resource));
        if (name != null) {
          store.remove(type, name);
          process.setName(name);
        }
        store.publish(type, process);
        return process;

      default:
    }

    throw new IllegalArgumentException("Unhandled type: " + type);
  }
Exemplo n.º 11
0
 /**
  * Returns the entity definition as an XML based on name.
  *
  * @param type entity type
  * @param entityName entity name
  * @return String
  */
 public String getEntityDefinition(String type, String entityName) {
   try {
     EntityType entityType = EntityType.getEnum(type);
     Entity entity = configStore.get(entityType, entityName);
     if (entity == null) {
       throw new NoSuchElementException(entityName + " (" + type + ") not found");
     }
     return entity.toString();
   } catch (Throwable e) {
     LOG.error(
         "Unable to get entity definition from config store for ({}): {}", type, entityName, e);
     throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
   }
 }
Exemplo n.º 12
0
  protected Set<String> getApplicableColos(String type, String name) {
    try {
      if (DeploymentUtil.isEmbeddedMode()) {
        return DeploymentUtil.getDefaultColos();
      }

      if (EntityType.getEnum(type) == EntityType.CLUSTER) {
        return getAllColos();
      }

      return getApplicableColos(type, EntityUtil.getEntity(type, name));
    } catch (FalconException e) {
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
  // The method retrieves the conf from the cache if it is in cache.
  // Else, queries WF Engine to retrieve the conf of the workflow
  private void updateContextFromWFConf(WorkflowExecutionContext context) {
    try {
      Properties wfProps = contextMap.get(context.getWorkflowId());
      if (wfProps == null) {
        Entity entity =
            CONFIG_STORE.get(EntityType.valueOf(context.getEntityType()), context.getEntityName());
        // Entity can be null in case of delete. Engine will generate notifications for instance
        // kills.
        // But, the entity would no longer be in the config store.
        if (entity == null) {
          return;
        }
        for (String cluster : EntityUtil.getClustersDefinedInColos(entity)) {
          try {
            InstancesResult.Instance[] instances =
                WorkflowEngineFactory.getWorkflowEngine()
                    .getJobDetails(cluster, context.getWorkflowId())
                    .getInstances();
            if (instances != null && instances.length > 0) {
              wfProps = getWFProps(instances[0].getWfParams());
              // Required by RetryService. But, is not part of conf.
              wfProps.setProperty(
                  WorkflowExecutionArgs.RUN_ID.getName(),
                  Integer.toString(instances[0].getRunId()));
            }
          } catch (FalconException e) {
            // Do Nothing. The workflow may not have been deployed on this cluster.
            continue;
          }
          contextMap.put(context.getWorkflowId(), wfProps);
        }
      }

      // No extra props to enhance the context with.
      if (wfProps == null || wfProps.isEmpty()) {
        return;
      }

      for (WorkflowExecutionArgs arg : WorkflowExecutionArgs.values()) {
        if (wfProps.containsKey(arg.getName())) {
          context.setValue(arg, wfProps.getProperty(arg.getName()));
        }
      }

    } catch (FalconException e) {
      LOG.error("Unable to retrieve entity {} of type {} from config store.", e);
    }
  }
Exemplo n.º 14
0
  /**
   * Returns the status of requested entity.
   *
   * @param type entity type
   * @param entity entity name
   * @return String
   */
  public APIResult getStatus(String type, String entity, String colo) {

    checkColo(colo);
    Entity entityObj;
    try {
      entityObj = EntityUtil.getEntity(type, entity);
      EntityType entityType = EntityType.getEnum(type);
      EntityStatus status = getStatus(entityObj, entityType);
      return new APIResult(Status.SUCCEEDED, status.name());
    } catch (FalconWebException e) {
      throw e;
    } catch (Exception e) {

      LOG.error("Unable to get status for entity {} ({})", entity, type, e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
Exemplo n.º 15
0
  protected EntityStatus getStatus(Entity entity, EntityType type) throws FalconException {
    EntityStatus status;

    if (type.isSchedulable()) {
      if (workflowEngine.isActive(entity)) {
        if (workflowEngine.isSuspended(entity)) {
          status = EntityStatus.SUSPENDED;
        } else {
          status = EntityStatus.RUNNING;
        }
      } else {
        status = EntityStatus.SUBMITTED;
      }
    } else {
      status = EntityStatus.SUBMITTED;
    }
    return status;
  }
Exemplo n.º 16
0
  public APIResult update(HttpServletRequest request, String type, String entityName, String colo) {
    checkColo(colo);
    List<Entity> tokenList = null;
    try {
      EntityType entityType = EntityType.getEnum(type);
      Entity oldEntity = EntityUtil.getEntity(type, entityName);
      Entity newEntity = deserializeEntity(request, entityType);
      // KLUDGE - Until ACL is mandated entity passed should be decorated for equals check to pass
      decorateEntityWithACL(newEntity);
      validate(newEntity);

      validateUpdate(oldEntity, newEntity);
      configStore.initiateUpdate(newEntity);

      tokenList = obtainUpdateEntityLocks(oldEntity);

      StringBuilder result = new StringBuilder("Updated successfully");
      // Update in workflow engine
      if (!DeploymentUtil.isPrism()) {
        Set<String> oldClusters = EntityUtil.getClustersDefinedInColos(oldEntity);
        Set<String> newClusters = EntityUtil.getClustersDefinedInColos(newEntity);
        newClusters.retainAll(oldClusters); // common clusters for update
        oldClusters.removeAll(newClusters); // deleted clusters

        for (String cluster : newClusters) {
          result.append(getWorkflowEngine().update(oldEntity, newEntity, cluster));
        }
        for (String cluster : oldClusters) {
          getWorkflowEngine().delete(oldEntity, cluster);
        }
      }

      configStore.update(entityType, newEntity);

      return new APIResult(APIResult.Status.SUCCEEDED, result.toString());
    } catch (Throwable e) {
      LOG.error("Update failed", e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    } finally {
      ConfigurationStore.get().cleanupUpdateInit();
      releaseUpdateEntityLocks(entityName, tokenList);
    }
  }
Exemplo n.º 17
0
  protected Set<String> getApplicableColos(String type, Entity entity) {
    try {
      if (DeploymentUtil.isEmbeddedMode()) {
        return DeploymentUtil.getDefaultColos();
      }

      if (EntityType.getEnum(type) == EntityType.CLUSTER) {
        return getAllColos();
      }

      Set<String> clusters = EntityUtil.getClustersDefined(entity);
      Set<String> colos = new HashSet<String>();
      for (String cluster : clusters) {
        Cluster clusterEntity = EntityUtil.getEntity(EntityType.CLUSTER, cluster);
        colos.add(clusterEntity.getColo());
      }
      return colos;
    } catch (FalconException e) {
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
Exemplo n.º 18
0
  /**
   * Given the location of data, returns the feed.
   *
   * @param type type of the entity, is valid only for feeds.
   * @param instancePath location of the data
   * @return Feed Name, type of the data and cluster name.
   */
  public FeedLookupResult reverseLookup(String type, String instancePath) {
    try {
      EntityType entityType = EntityType.getEnum(type);
      if (entityType != EntityType.FEED) {
        LOG.error("Reverse Lookup is not supported for entitytype: {}", type);
        throw new IllegalArgumentException("Reverse lookup is not supported for " + type);
      }

      instancePath = StringUtils.trim(instancePath);
      String instancePathWithoutSlash =
          instancePath.endsWith("/") ? StringUtils.removeEnd(instancePath, "/") : instancePath;
      // treat strings with and without trailing slash as same for purpose of searching e.g.
      // /data/cas and /data/cas/ should be treated as same.
      String instancePathWithSlash = instancePathWithoutSlash + "/";
      FeedLocationStore store = FeedLocationStore.get();
      Collection<FeedLookupResult.FeedProperties> feeds = new ArrayList<>();
      Collection<FeedLookupResult.FeedProperties> res =
          store.reverseLookup(instancePathWithoutSlash);
      if (res != null) {
        feeds.addAll(res);
      }
      res = store.reverseLookup(instancePathWithSlash);
      if (res != null) {
        feeds.addAll(res);
      }
      FeedLookupResult result = new FeedLookupResult(APIResult.Status.SUCCEEDED, "SUCCESS");
      FeedLookupResult.FeedProperties[] props =
          feeds.toArray(new FeedLookupResult.FeedProperties[0]);
      result.setElements(props);
      return result;

    } catch (IllegalArgumentException e) {
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    } catch (Throwable throwable) {
      LOG.error("reverse look up failed", throwable);
      throw FalconWebException.newException(throwable, Response.Status.INTERNAL_SERVER_ERROR);
    }
  }
Exemplo n.º 19
0
  protected List<Entity> getFilteredEntities(
      EntityType entityType,
      String nameSubsequence,
      String tagKeywords,
      Map<String, List<String>> filterByFieldsValues,
      String startDate,
      String endDate,
      String cluster)
      throws FalconException, IOException {
    Collection<String> entityNames = configStore.getEntities(entityType);
    if (entityNames.isEmpty()) {
      return Collections.emptyList();
    }

    List<Entity> entities = new ArrayList<Entity>();
    char[] subsequence = nameSubsequence.toLowerCase().toCharArray();
    List<String> tagKeywordsList;
    if (StringUtils.isEmpty(tagKeywords)) {
      tagKeywordsList = new ArrayList<>();
    } else {
      tagKeywordsList = getFilterByTags(Arrays.asList(tagKeywords.toLowerCase()));
    }
    for (String entityName : entityNames) {
      Entity entity;
      try {
        entity = configStore.get(entityType, entityName);
        if (entity == null) {
          continue;
        }
      } catch (FalconException e1) {
        LOG.error(
            "Unable to get list for entities for ({})",
            entityType.getEntityClass().getSimpleName(),
            e1);
        throw FalconWebException.newException(e1, Response.Status.BAD_REQUEST);
      }

      if (SecurityUtil.isAuthorizationEnabled() && !isEntityAuthorized(entity)) {
        // the user who requested list query has no permission to access this entity. Skip this
        // entity
        continue;
      }
      if (isFilteredByDatesAndCluster(entity, startDate, endDate, cluster)) {
        // this is for entity summary
        continue;
      }
      SecurityUtil.tryProxy(entity);

      // filter by fields
      if (isFilteredByFields(entity, filterByFieldsValues)) {
        continue;
      }

      // filter by subsequence of name
      if (subsequence.length > 0
          && !matchesNameSubsequence(subsequence, entityName.toLowerCase())) {
        continue;
      }

      // filter by tag keywords
      if (!matchTagKeywords(tagKeywordsList, entity.getTags())) {
        continue;
      }

      entities.add(entity);
    }

    return entities;
  }