/**
   * Deletes a scheduled entity, a deleted entity is removed completely from execution pool.
   *
   * @param type entity type
   * @param entity entity name
   * @return APIResult
   */
  public APIResult delete(HttpServletRequest request, String type, String entity, String colo) {
    checkColo(colo);
    try {
      EntityType entityType = EntityType.getEnum(type);
      String removedFromEngine = "";
      try {
        Entity entityObj = EntityUtil.getEntity(type, entity);

        canRemove(entityObj);
        if (entityType.isSchedulable() && !DeploymentUtil.isPrism()) {
          getWorkflowEngine().delete(entityObj);
          removedFromEngine = "(KILLED in ENGINE)";
        }

        configStore.remove(entityType, entity);
      } catch (EntityNotRegisteredException e) { // already deleted
        return new APIResult(
            APIResult.Status.SUCCEEDED, entity + "(" + type + ") doesn't exist. Nothing to do");
      }

      return new APIResult(
          APIResult.Status.SUCCEEDED,
          entity + "(" + type + ") removed successfully " + removedFromEngine);
    } catch (Throwable e) {
      LOG.error("Unable to reach workflow engine for deletion or deletion failed", e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
  protected int getRequiredNumberOfResults(int arraySize, int offset, int numresults) {
    /* Get a subset of elements based on offset and count. When returning subset of elements,
    elements[offset] is included. Size 10, offset 10, return empty list.
    Size 10, offset 5, count 3, return elements[5,6,7].
    Size 10, offset 5, count >= 5, return elements[5,6,7,8,9]
    return elements starting from elements[offset] until the end OR offset+numResults*/

    if (numresults < 1) {
      LOG.error("Value for param numResults should be > than 0  : {}", numresults);
      throw FalconWebException.newException(
          "Value for param numResults should be > than 0  : " + numresults,
          Response.Status.BAD_REQUEST);
    }

    if (offset < 0) {
      offset = 0;
    }

    if (offset >= arraySize || arraySize == 0) {
      // No elements to return
      return 0;
    }

    numresults = numresults <= MAX_RESULTS ? numresults : MAX_RESULTS;
    int retLen = arraySize - offset;
    if (retLen > numresults) {
      retLen = numresults;
    }
    return retLen;
  }
 protected void checkColo(String colo) {
   if (!DeploymentUtil.getCurrentColo().equals(colo)) {
     throw FalconWebException.newException(
         "Current colo (" + DeploymentUtil.getCurrentColo() + ") is not " + colo,
         Response.Status.BAD_REQUEST);
   }
 }
  /**
   * Post an entity XML with entity type. Validates the XML which can be Process, Feed or
   * Dataendpoint
   *
   * @param type entity type
   * @return APIResule -Succeeded or Failed
   */
  public APIResult validate(HttpServletRequest request, String type) {
    try {
      EntityType entityType = EntityType.getEnum(type);
      Entity entity = deserializeEntity(request, entityType);
      validate(entity);

      // Validate that the entity can be scheduled in the cluster
      if (entity.getEntityType().isSchedulable()) {
        Set<String> clusters = EntityUtil.getClustersDefinedInColos(entity);
        for (String cluster : clusters) {
          try {
            getWorkflowEngine().dryRun(entity, cluster);
          } catch (FalconException e) {
            throw new FalconException("dryRun failed on cluster " + cluster, e);
          }
        }
      }
      return new APIResult(
          APIResult.Status.SUCCEEDED,
          "Validated successfully (" + entityType + ") " + entity.getName());
    } catch (Throwable e) {
      LOG.error("Validation failed for entity ({})", type, e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
  private boolean isEntityFiltered(
      Entity entity, EntityList.EntityFilterByFields filter, Map.Entry<String, List<String>> pair) {
    switch (filter) {
      case TYPE:
        return !containsIgnoreCase(pair.getValue(), entity.getEntityType().toString());

      case NAME:
        return !containsIgnoreCase(pair.getValue(), entity.getName());

      case STATUS:
        return !containsIgnoreCase(pair.getValue(), getStatusString(entity));

      case PIPELINES:
        if (!entity.getEntityType().equals(EntityType.PROCESS)) {
          throw FalconWebException.newException(
              "Invalid filterBy key for non process entities " + pair.getKey(),
              Response.Status.BAD_REQUEST);
        }
        return !EntityUtil.getPipelines(entity).contains(pair.getValue().get(0));

      case CLUSTER:
        return !EntityUtil.getClustersDefined(entity).contains(pair.getValue().get(0));

      case TAGS:
        return isFilteredByTags(getFilterByTags(pair.getValue()), EntityUtil.getTags(entity));

      default:
        return false;
    }
  }
  /**
   * Returns the list of filtered entities as well as the total number of results.
   *
   * @param fieldStr Fields that the query is interested in, separated by comma
   * @param nameSubsequence Name subsequence to match
   * @param tagKeywords Tag keywords to match, separated by commma
   * @param filterType Only return entities of this type
   * @param filterTags Full tag matching, separated by comma
   * @param filterBy Specific fields to match (i.e. TYPE, NAME, STATUS, PIPELINES, CLUSTER)
   * @param orderBy Order result by these fields.
   * @param sortOrder Valid options are "asc" and “desc”
   * @param offset Pagination offset.
   * @param resultsPerPage Number of results that should be returned starting at the offset.
   * @return EntityList
   */
  public EntityList getEntityList(
      String fieldStr,
      String nameSubsequence,
      String tagKeywords,
      String filterType,
      String filterTags,
      String filterBy,
      String orderBy,
      String sortOrder,
      Integer offset,
      Integer resultsPerPage) {

    HashSet<String> fields = new HashSet<String>(Arrays.asList(fieldStr.toUpperCase().split(",")));
    Map<String, List<String>> filterByFieldsValues = getFilterByFieldsValues(filterBy);
    validateEntityFilterByClause(filterByFieldsValues);
    if (StringUtils.isNotEmpty(filterTags)) {
      filterByFieldsValues.put(
          EntityList.EntityFilterByFields.TAGS.name(), Arrays.asList(filterTags));
    }

    // get filtered entities
    List<Entity> entities = new ArrayList<Entity>();
    try {
      if (StringUtils.isEmpty(filterType)) {
        // return entities of all types if no entity type specified
        for (EntityType entityType : EntityType.values()) {
          entities.addAll(
              getFilteredEntities(
                  entityType, nameSubsequence, tagKeywords, filterByFieldsValues, "", "", ""));
        }
      } else {
        String[] types = filterType.split(",");
        for (String type : types) {
          EntityType entityType = EntityType.getEnum(type);
          entities.addAll(
              getFilteredEntities(
                  entityType, nameSubsequence, tagKeywords, filterByFieldsValues, "", "", ""));
        }
      }
    } catch (Exception e) {
      LOG.error("Failed to get entity list", e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }

    // sort entities and pagination
    List<Entity> entitiesReturn =
        sortEntitiesPagination(entities, orderBy, sortOrder, offset, resultsPerPage);

    // add total number of results
    EntityList entityList =
        entitiesReturn.size() == 0
            ? new EntityList(new Entity[] {}, 0)
            : new EntityList(
                buildEntityElements(new HashSet<String>(fields), entitiesReturn), entities.size());
    return entityList;
  }
  /**
   * Returns dependencies.
   *
   * @param type entity type
   * @param entityName entity name
   * @return EntityList
   */
  public EntityList getDependencies(String type, String entityName) {

    try {
      Entity entityObj = EntityUtil.getEntity(type, entityName);
      return EntityUtil.getEntityDependencies(entityObj);
    } catch (Exception e) {
      LOG.error("Unable to get dependencies for entityName {} ({})", entityName, type, e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
  /**
   * Submit a new entity. Entities can be of type feed, process or data end points. Entity
   * definitions are validated structurally against schema and subsequently for other rules before
   * they are admitted into the system
   *
   * <p>Entity name acts as the key and an entity once added, can't be added again unless deleted.
   *
   * @param request - Servlet Request
   * @param type - entity type - feed, process or data end point
   * @param colo - applicable colo
   * @return result of the operation
   */
  public APIResult submit(HttpServletRequest request, String type, String colo) {

    checkColo(colo);
    try {
      Entity entity = submitInternal(request, type);
      return new APIResult(
          APIResult.Status.SUCCEEDED, "Submit successful (" + type + ") " + entity.getName());
    } catch (Throwable e) {
      LOG.error("Unable to persist entity object", e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
  /**
   * Given the location of data, returns the feed.
   *
   * @param type type of the entity, is valid only for feeds.
   * @param instancePath location of the data
   * @return Feed Name, type of the data and cluster name.
   */
  public FeedLookupResult reverseLookup(String type, String instancePath) {
    try {
      EntityType entityType = EntityType.getEnum(type);
      if (entityType != EntityType.FEED) {
        LOG.error("Reverse Lookup is not supported for entitytype: {}", type);
        throw new IllegalArgumentException("Reverse lookup is not supported for " + type);
      }

      instancePath = StringUtils.trim(instancePath);
      String instancePathWithoutSlash =
          instancePath.endsWith("/") ? StringUtils.removeEnd(instancePath, "/") : instancePath;
      // treat strings with and without trailing slash as same for purpose of searching e.g.
      // /data/cas and /data/cas/ should be treated as same.
      String instancePathWithSlash = instancePathWithoutSlash + "/";
      FeedLocationStore store = FeedLocationStore.get();
      Collection<FeedLookupResult.FeedProperties> feeds = new ArrayList<>();
      Collection<FeedLookupResult.FeedProperties> res =
          store.reverseLookup(instancePathWithoutSlash);
      if (res != null) {
        feeds.addAll(res);
      }
      res = store.reverseLookup(instancePathWithSlash);
      if (res != null) {
        feeds.addAll(res);
      }
      FeedLookupResult result = new FeedLookupResult(APIResult.Status.SUCCEEDED, "SUCCESS");
      FeedLookupResult.FeedProperties[] props =
          feeds.toArray(new FeedLookupResult.FeedProperties[0]);
      result.setElements(props);
      return result;

    } catch (IllegalArgumentException e) {
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    } catch (Throwable throwable) {
      LOG.error("reverse look up failed", throwable);
      throw FalconWebException.newException(throwable, Response.Status.INTERNAL_SERVER_ERROR);
    }
  }
 protected Set<String> getColosFromExpression(String coloExpr, String type, String entity) {
   Set<String> colos;
   final Set<String> applicableColos = getApplicableColos(type, entity);
   if (coloExpr == null || coloExpr.equals("*") || coloExpr.isEmpty()) {
     colos = applicableColos;
   } else {
     colos = new HashSet<String>(Arrays.asList(coloExpr.split(",")));
     if (!applicableColos.containsAll(colos)) {
       throw FalconWebException.newException(
           "Given colos not applicable for entity operation", Response.Status.BAD_REQUEST);
     }
   }
   return colos;
 }
 /**
  * Returns the entity definition as an XML based on name.
  *
  * @param type entity type
  * @param entityName entity name
  * @return String
  */
 public String getEntityDefinition(String type, String entityName) {
   try {
     EntityType entityType = EntityType.getEnum(type);
     Entity entity = configStore.get(entityType, entityName);
     if (entity == null) {
       throw new NoSuchElementException(entityName + " (" + type + ") not found");
     }
     return entity.toString();
   } catch (Throwable e) {
     LOG.error(
         "Unable to get entity definition from config store for ({}): {}", type, entityName, e);
     throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
   }
 }
  protected String getValidSortOrder(String sortOrder, String orderBy) {
    if (StringUtils.isEmpty(sortOrder)) {
      return (orderBy.equalsIgnoreCase("starttime") || orderBy.equalsIgnoreCase("endtime"))
          ? "desc"
          : "asc";
    }

    if (sortOrder.equalsIgnoreCase("asc") || sortOrder.equalsIgnoreCase("desc")) {
      return sortOrder;
    }

    String err = "Value for param sortOrder should be \"asc\" or \"desc\". It is  : " + sortOrder;
    LOG.error(err);
    throw FalconWebException.newException(err, Response.Status.BAD_REQUEST);
  }
  protected Set<String> getApplicableColos(String type, String name) {
    try {
      if (DeploymentUtil.isEmbeddedMode()) {
        return DeploymentUtil.getDefaultColos();
      }

      if (EntityType.getEnum(type) == EntityType.CLUSTER) {
        return getAllColos();
      }

      return getApplicableColos(type, EntityUtil.getEntity(type, name));
    } catch (FalconException e) {
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
  /**
   * Returns the status of requested entity.
   *
   * @param type entity type
   * @param entity entity name
   * @return String
   */
  public APIResult getStatus(String type, String entity, String colo) {

    checkColo(colo);
    Entity entityObj;
    try {
      entityObj = EntityUtil.getEntity(type, entity);
      EntityType entityType = EntityType.getEnum(type);
      EntityStatus status = getStatus(entityObj, entityType);
      return new APIResult(Status.SUCCEEDED, status.name());
    } catch (FalconWebException e) {
      throw e;
    } catch (Exception e) {

      LOG.error("Unable to get status for entity {} ({})", entity, type, e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
  public APIResult update(HttpServletRequest request, String type, String entityName, String colo) {
    checkColo(colo);
    List<Entity> tokenList = null;
    try {
      EntityType entityType = EntityType.getEnum(type);
      Entity oldEntity = EntityUtil.getEntity(type, entityName);
      Entity newEntity = deserializeEntity(request, entityType);
      // KLUDGE - Until ACL is mandated entity passed should be decorated for equals check to pass
      decorateEntityWithACL(newEntity);
      validate(newEntity);

      validateUpdate(oldEntity, newEntity);
      configStore.initiateUpdate(newEntity);

      tokenList = obtainUpdateEntityLocks(oldEntity);

      StringBuilder result = new StringBuilder("Updated successfully");
      // Update in workflow engine
      if (!DeploymentUtil.isPrism()) {
        Set<String> oldClusters = EntityUtil.getClustersDefinedInColos(oldEntity);
        Set<String> newClusters = EntityUtil.getClustersDefinedInColos(newEntity);
        newClusters.retainAll(oldClusters); // common clusters for update
        oldClusters.removeAll(newClusters); // deleted clusters

        for (String cluster : newClusters) {
          result.append(getWorkflowEngine().update(oldEntity, newEntity, cluster));
        }
        for (String cluster : oldClusters) {
          getWorkflowEngine().delete(oldEntity, cluster);
        }
      }

      configStore.update(entityType, newEntity);

      return new APIResult(APIResult.Status.SUCCEEDED, result.toString());
    } catch (Throwable e) {
      LOG.error("Update failed", e);
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    } finally {
      ConfigurationStore.get().cleanupUpdateInit();
      releaseUpdateEntityLocks(entityName, tokenList);
    }
  }
  protected Set<String> getApplicableColos(String type, Entity entity) {
    try {
      if (DeploymentUtil.isEmbeddedMode()) {
        return DeploymentUtil.getDefaultColos();
      }

      if (EntityType.getEnum(type) == EntityType.CLUSTER) {
        return getAllColos();
      }

      Set<String> clusters = EntityUtil.getClustersDefined(entity);
      Set<String> colos = new HashSet<String>();
      for (String cluster : clusters) {
        Cluster clusterEntity = EntityUtil.getEntity(EntityType.CLUSTER, cluster);
        colos.add(clusterEntity.getColo());
      }
      return colos;
    } catch (FalconException e) {
      throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
    }
  }
    public T execute(String coloExpr, String type, String name) {
      Set<String> colos = getColosFromExpression(coloExpr, type, name);

      Map<String, T> results = new HashMap<String, T>();
      for (String colo : colos) {
        try {
          T resultHolder = doExecute(colo);
          results.put(colo, resultHolder);
        } catch (FalconException e) {
          results.put(
              colo,
              getResultInstance(
                  APIResult.Status.FAILED, e.getClass().getName() + "::" + e.getMessage()));
        }
      }
      T finalResult = consolidateResult(results, clazz);
      if (finalResult.getStatus() != APIResult.Status.SUCCEEDED) {
        throw FalconWebException.newException(finalResult, Response.Status.BAD_REQUEST);
      } else {
        return finalResult;
      }
    }
  protected List<Entity> getFilteredEntities(
      EntityType entityType,
      String nameSubsequence,
      String tagKeywords,
      Map<String, List<String>> filterByFieldsValues,
      String startDate,
      String endDate,
      String cluster)
      throws FalconException, IOException {
    Collection<String> entityNames = configStore.getEntities(entityType);
    if (entityNames.isEmpty()) {
      return Collections.emptyList();
    }

    List<Entity> entities = new ArrayList<Entity>();
    char[] subsequence = nameSubsequence.toLowerCase().toCharArray();
    List<String> tagKeywordsList;
    if (StringUtils.isEmpty(tagKeywords)) {
      tagKeywordsList = new ArrayList<>();
    } else {
      tagKeywordsList = getFilterByTags(Arrays.asList(tagKeywords.toLowerCase()));
    }
    for (String entityName : entityNames) {
      Entity entity;
      try {
        entity = configStore.get(entityType, entityName);
        if (entity == null) {
          continue;
        }
      } catch (FalconException e1) {
        LOG.error(
            "Unable to get list for entities for ({})",
            entityType.getEntityClass().getSimpleName(),
            e1);
        throw FalconWebException.newException(e1, Response.Status.BAD_REQUEST);
      }

      if (SecurityUtil.isAuthorizationEnabled() && !isEntityAuthorized(entity)) {
        // the user who requested list query has no permission to access this entity. Skip this
        // entity
        continue;
      }
      if (isFilteredByDatesAndCluster(entity, startDate, endDate, cluster)) {
        // this is for entity summary
        continue;
      }
      SecurityUtil.tryProxy(entity);

      // filter by fields
      if (isFilteredByFields(entity, filterByFieldsValues)) {
        continue;
      }

      // filter by subsequence of name
      if (subsequence.length > 0
          && !matchesNameSubsequence(subsequence, entityName.toLowerCase())) {
        continue;
      }

      // filter by tag keywords
      if (!matchTagKeywords(tagKeywordsList, entity.getTags())) {
        continue;
      }

      entities.add(entity);
    }

    return entities;
  }