/**
* Deletes a scheduled entity, a deleted entity is removed completely from execution pool.
*
* @param type entity type
* @param entity entity name
* @return APIResult
*/
public APIResult delete(HttpServletRequest request, String type, String entity, String colo) {
checkColo(colo);
try {
EntityType entityType = EntityType.getEnum(type);
String removedFromEngine = "";
try {
Entity entityObj = EntityUtil.getEntity(type, entity);
canRemove(entityObj);
if (entityType.isSchedulable() && !DeploymentUtil.isPrism()) {
getWorkflowEngine().delete(entityObj);
removedFromEngine = "(KILLED in ENGINE)";
}
configStore.remove(entityType, entity);
} catch (EntityNotRegisteredException e) { // already deleted
return new APIResult(
APIResult.Status.SUCCEEDED, entity + "(" + type + ") doesn't exist. Nothing to do");
}
return new APIResult(
APIResult.Status.SUCCEEDED,
entity + "(" + type + ") removed successfully " + removedFromEngine);
} catch (Throwable e) {
LOG.error("Unable to reach workflow engine for deletion or deletion failed", e);
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
}
}
protected int getRequiredNumberOfResults(int arraySize, int offset, int numresults) {
/* Get a subset of elements based on offset and count. When returning subset of elements,
elements[offset] is included. Size 10, offset 10, return empty list.
Size 10, offset 5, count 3, return elements[5,6,7].
Size 10, offset 5, count >= 5, return elements[5,6,7,8,9]
return elements starting from elements[offset] until the end OR offset+numResults*/
if (numresults < 1) {
LOG.error("Value for param numResults should be > than 0 : {}", numresults);
throw FalconWebException.newException(
"Value for param numResults should be > than 0 : " + numresults,
Response.Status.BAD_REQUEST);
}
if (offset < 0) {
offset = 0;
}
if (offset >= arraySize || arraySize == 0) {
// No elements to return
return 0;
}
numresults = numresults <= MAX_RESULTS ? numresults : MAX_RESULTS;
int retLen = arraySize - offset;
if (retLen > numresults) {
retLen = numresults;
}
return retLen;
}
protected void checkColo(String colo) {
if (!DeploymentUtil.getCurrentColo().equals(colo)) {
throw FalconWebException.newException(
"Current colo (" + DeploymentUtil.getCurrentColo() + ") is not " + colo,
Response.Status.BAD_REQUEST);
}
}
/**
* Post an entity XML with entity type. Validates the XML which can be Process, Feed or
* Dataendpoint
*
* @param type entity type
* @return APIResule -Succeeded or Failed
*/
public APIResult validate(HttpServletRequest request, String type) {
try {
EntityType entityType = EntityType.getEnum(type);
Entity entity = deserializeEntity(request, entityType);
validate(entity);
// Validate that the entity can be scheduled in the cluster
if (entity.getEntityType().isSchedulable()) {
Set<String> clusters = EntityUtil.getClustersDefinedInColos(entity);
for (String cluster : clusters) {
try {
getWorkflowEngine().dryRun(entity, cluster);
} catch (FalconException e) {
throw new FalconException("dryRun failed on cluster " + cluster, e);
}
}
}
return new APIResult(
APIResult.Status.SUCCEEDED,
"Validated successfully (" + entityType + ") " + entity.getName());
} catch (Throwable e) {
LOG.error("Validation failed for entity ({})", type, e);
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
}
}
private boolean isEntityFiltered(
Entity entity, EntityList.EntityFilterByFields filter, Map.Entry<String, List<String>> pair) {
switch (filter) {
case TYPE:
return !containsIgnoreCase(pair.getValue(), entity.getEntityType().toString());
case NAME:
return !containsIgnoreCase(pair.getValue(), entity.getName());
case STATUS:
return !containsIgnoreCase(pair.getValue(), getStatusString(entity));
case PIPELINES:
if (!entity.getEntityType().equals(EntityType.PROCESS)) {
throw FalconWebException.newException(
"Invalid filterBy key for non process entities " + pair.getKey(),
Response.Status.BAD_REQUEST);
}
return !EntityUtil.getPipelines(entity).contains(pair.getValue().get(0));
case CLUSTER:
return !EntityUtil.getClustersDefined(entity).contains(pair.getValue().get(0));
case TAGS:
return isFilteredByTags(getFilterByTags(pair.getValue()), EntityUtil.getTags(entity));
default:
return false;
}
}
/**
* Returns the list of filtered entities as well as the total number of results.
*
* @param fieldStr Fields that the query is interested in, separated by comma
* @param nameSubsequence Name subsequence to match
* @param tagKeywords Tag keywords to match, separated by commma
* @param filterType Only return entities of this type
* @param filterTags Full tag matching, separated by comma
* @param filterBy Specific fields to match (i.e. TYPE, NAME, STATUS, PIPELINES, CLUSTER)
* @param orderBy Order result by these fields.
* @param sortOrder Valid options are "asc" and “desc”
* @param offset Pagination offset.
* @param resultsPerPage Number of results that should be returned starting at the offset.
* @return EntityList
*/
public EntityList getEntityList(
String fieldStr,
String nameSubsequence,
String tagKeywords,
String filterType,
String filterTags,
String filterBy,
String orderBy,
String sortOrder,
Integer offset,
Integer resultsPerPage) {
HashSet<String> fields = new HashSet<String>(Arrays.asList(fieldStr.toUpperCase().split(",")));
Map<String, List<String>> filterByFieldsValues = getFilterByFieldsValues(filterBy);
validateEntityFilterByClause(filterByFieldsValues);
if (StringUtils.isNotEmpty(filterTags)) {
filterByFieldsValues.put(
EntityList.EntityFilterByFields.TAGS.name(), Arrays.asList(filterTags));
}
// get filtered entities
List<Entity> entities = new ArrayList<Entity>();
try {
if (StringUtils.isEmpty(filterType)) {
// return entities of all types if no entity type specified
for (EntityType entityType : EntityType.values()) {
entities.addAll(
getFilteredEntities(
entityType, nameSubsequence, tagKeywords, filterByFieldsValues, "", "", ""));
}
} else {
String[] types = filterType.split(",");
for (String type : types) {
EntityType entityType = EntityType.getEnum(type);
entities.addAll(
getFilteredEntities(
entityType, nameSubsequence, tagKeywords, filterByFieldsValues, "", "", ""));
}
}
} catch (Exception e) {
LOG.error("Failed to get entity list", e);
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
}
// sort entities and pagination
List<Entity> entitiesReturn =
sortEntitiesPagination(entities, orderBy, sortOrder, offset, resultsPerPage);
// add total number of results
EntityList entityList =
entitiesReturn.size() == 0
? new EntityList(new Entity[] {}, 0)
: new EntityList(
buildEntityElements(new HashSet<String>(fields), entitiesReturn), entities.size());
return entityList;
}
/**
* Returns dependencies.
*
* @param type entity type
* @param entityName entity name
* @return EntityList
*/
public EntityList getDependencies(String type, String entityName) {
try {
Entity entityObj = EntityUtil.getEntity(type, entityName);
return EntityUtil.getEntityDependencies(entityObj);
} catch (Exception e) {
LOG.error("Unable to get dependencies for entityName {} ({})", entityName, type, e);
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
}
}
protected Map<String, List<String>> validateEntityFilterByClause(
Map<String, List<String>> filterByFieldsValues) {
for (Map.Entry<String, List<String>> entry : filterByFieldsValues.entrySet()) {
try {
EntityList.EntityFilterByFields.valueOf(entry.getKey().toUpperCase());
} catch (IllegalArgumentException e) {
throw FalconWebException.newInstanceException(
"Invalid filter key: " + entry.getKey(), Response.Status.BAD_REQUEST);
}
}
return filterByFieldsValues;
}
/**
* Submit a new entity. Entities can be of type feed, process or data end points. Entity
* definitions are validated structurally against schema and subsequently for other rules before
* they are admitted into the system
*
* <p>Entity name acts as the key and an entity once added, can't be added again unless deleted.
*
* @param request - Servlet Request
* @param type - entity type - feed, process or data end point
* @param colo - applicable colo
* @return result of the operation
*/
public APIResult submit(HttpServletRequest request, String type, String colo) {
checkColo(colo);
try {
Entity entity = submitInternal(request, type);
return new APIResult(
APIResult.Status.SUCCEEDED, "Submit successful (" + type + ") " + entity.getName());
} catch (Throwable e) {
LOG.error("Unable to persist entity object", e);
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
}
}
/**
* Given the location of data, returns the feed.
*
* @param type type of the entity, is valid only for feeds.
* @param instancePath location of the data
* @return Feed Name, type of the data and cluster name.
*/
public FeedLookupResult reverseLookup(String type, String instancePath) {
try {
EntityType entityType = EntityType.getEnum(type);
if (entityType != EntityType.FEED) {
LOG.error("Reverse Lookup is not supported for entitytype: {}", type);
throw new IllegalArgumentException("Reverse lookup is not supported for " + type);
}
instancePath = StringUtils.trim(instancePath);
String instancePathWithoutSlash =
instancePath.endsWith("/") ? StringUtils.removeEnd(instancePath, "/") : instancePath;
// treat strings with and without trailing slash as same for purpose of searching e.g.
// /data/cas and /data/cas/ should be treated as same.
String instancePathWithSlash = instancePathWithoutSlash + "/";
FeedLocationStore store = FeedLocationStore.get();
Collection<FeedLookupResult.FeedProperties> feeds = new ArrayList<>();
Collection<FeedLookupResult.FeedProperties> res =
store.reverseLookup(instancePathWithoutSlash);
if (res != null) {
feeds.addAll(res);
}
res = store.reverseLookup(instancePathWithSlash);
if (res != null) {
feeds.addAll(res);
}
FeedLookupResult result = new FeedLookupResult(APIResult.Status.SUCCEEDED, "SUCCESS");
FeedLookupResult.FeedProperties[] props =
feeds.toArray(new FeedLookupResult.FeedProperties[0]);
result.setElements(props);
return result;
} catch (IllegalArgumentException e) {
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
} catch (Throwable throwable) {
LOG.error("reverse look up failed", throwable);
throw FalconWebException.newException(throwable, Response.Status.INTERNAL_SERVER_ERROR);
}
}
protected Set<String> getColosFromExpression(String coloExpr, String type, String entity) {
Set<String> colos;
final Set<String> applicableColos = getApplicableColos(type, entity);
if (coloExpr == null || coloExpr.equals("*") || coloExpr.isEmpty()) {
colos = applicableColos;
} else {
colos = new HashSet<String>(Arrays.asList(coloExpr.split(",")));
if (!applicableColos.containsAll(colos)) {
throw FalconWebException.newException(
"Given colos not applicable for entity operation", Response.Status.BAD_REQUEST);
}
}
return colos;
}
/**
* Returns the entity definition as an XML based on name.
*
* @param type entity type
* @param entityName entity name
* @return String
*/
public String getEntityDefinition(String type, String entityName) {
try {
EntityType entityType = EntityType.getEnum(type);
Entity entity = configStore.get(entityType, entityName);
if (entity == null) {
throw new NoSuchElementException(entityName + " (" + type + ") not found");
}
return entity.toString();
} catch (Throwable e) {
LOG.error(
"Unable to get entity definition from config store for ({}): {}", type, entityName, e);
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
}
}
protected String getValidSortOrder(String sortOrder, String orderBy) {
if (StringUtils.isEmpty(sortOrder)) {
return (orderBy.equalsIgnoreCase("starttime") || orderBy.equalsIgnoreCase("endtime"))
? "desc"
: "asc";
}
if (sortOrder.equalsIgnoreCase("asc") || sortOrder.equalsIgnoreCase("desc")) {
return sortOrder;
}
String err = "Value for param sortOrder should be \"asc\" or \"desc\". It is : " + sortOrder;
LOG.error(err);
throw FalconWebException.newException(err, Response.Status.BAD_REQUEST);
}
protected Set<String> getApplicableColos(String type, String name) {
try {
if (DeploymentUtil.isEmbeddedMode()) {
return DeploymentUtil.getDefaultColos();
}
if (EntityType.getEnum(type) == EntityType.CLUSTER) {
return getAllColos();
}
return getApplicableColos(type, EntityUtil.getEntity(type, name));
} catch (FalconException e) {
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
}
}
/**
* Returns the status of requested entity.
*
* @param type entity type
* @param entity entity name
* @return String
*/
public APIResult getStatus(String type, String entity, String colo) {
checkColo(colo);
Entity entityObj;
try {
entityObj = EntityUtil.getEntity(type, entity);
EntityType entityType = EntityType.getEnum(type);
EntityStatus status = getStatus(entityObj, entityType);
return new APIResult(Status.SUCCEEDED, status.name());
} catch (FalconWebException e) {
throw e;
} catch (Exception e) {
LOG.error("Unable to get status for entity {} ({})", entity, type, e);
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
}
}
public APIResult update(HttpServletRequest request, String type, String entityName, String colo) {
checkColo(colo);
List<Entity> tokenList = null;
try {
EntityType entityType = EntityType.getEnum(type);
Entity oldEntity = EntityUtil.getEntity(type, entityName);
Entity newEntity = deserializeEntity(request, entityType);
// KLUDGE - Until ACL is mandated entity passed should be decorated for equals check to pass
decorateEntityWithACL(newEntity);
validate(newEntity);
validateUpdate(oldEntity, newEntity);
configStore.initiateUpdate(newEntity);
tokenList = obtainUpdateEntityLocks(oldEntity);
StringBuilder result = new StringBuilder("Updated successfully");
// Update in workflow engine
if (!DeploymentUtil.isPrism()) {
Set<String> oldClusters = EntityUtil.getClustersDefinedInColos(oldEntity);
Set<String> newClusters = EntityUtil.getClustersDefinedInColos(newEntity);
newClusters.retainAll(oldClusters); // common clusters for update
oldClusters.removeAll(newClusters); // deleted clusters
for (String cluster : newClusters) {
result.append(getWorkflowEngine().update(oldEntity, newEntity, cluster));
}
for (String cluster : oldClusters) {
getWorkflowEngine().delete(oldEntity, cluster);
}
}
configStore.update(entityType, newEntity);
return new APIResult(APIResult.Status.SUCCEEDED, result.toString());
} catch (Throwable e) {
LOG.error("Update failed", e);
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
} finally {
ConfigurationStore.get().cleanupUpdateInit();
releaseUpdateEntityLocks(entityName, tokenList);
}
}
protected Set<String> getApplicableColos(String type, Entity entity) {
try {
if (DeploymentUtil.isEmbeddedMode()) {
return DeploymentUtil.getDefaultColos();
}
if (EntityType.getEnum(type) == EntityType.CLUSTER) {
return getAllColos();
}
Set<String> clusters = EntityUtil.getClustersDefined(entity);
Set<String> colos = new HashSet<String>();
for (String cluster : clusters) {
Cluster clusterEntity = EntityUtil.getEntity(EntityType.CLUSTER, cluster);
colos.add(clusterEntity.getColo());
}
return colos;
} catch (FalconException e) {
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
}
}
public T execute(String coloExpr, String type, String name) {
Set<String> colos = getColosFromExpression(coloExpr, type, name);
Map<String, T> results = new HashMap<String, T>();
for (String colo : colos) {
try {
T resultHolder = doExecute(colo);
results.put(colo, resultHolder);
} catch (FalconException e) {
results.put(
colo,
getResultInstance(
APIResult.Status.FAILED, e.getClass().getName() + "::" + e.getMessage()));
}
}
T finalResult = consolidateResult(results, clazz);
if (finalResult.getStatus() != APIResult.Status.SUCCEEDED) {
throw FalconWebException.newException(finalResult, Response.Status.BAD_REQUEST);
} else {
return finalResult;
}
}
protected List<Entity> getFilteredEntities(
EntityType entityType,
String nameSubsequence,
String tagKeywords,
Map<String, List<String>> filterByFieldsValues,
String startDate,
String endDate,
String cluster)
throws FalconException, IOException {
Collection<String> entityNames = configStore.getEntities(entityType);
if (entityNames.isEmpty()) {
return Collections.emptyList();
}
List<Entity> entities = new ArrayList<Entity>();
char[] subsequence = nameSubsequence.toLowerCase().toCharArray();
List<String> tagKeywordsList;
if (StringUtils.isEmpty(tagKeywords)) {
tagKeywordsList = new ArrayList<>();
} else {
tagKeywordsList = getFilterByTags(Arrays.asList(tagKeywords.toLowerCase()));
}
for (String entityName : entityNames) {
Entity entity;
try {
entity = configStore.get(entityType, entityName);
if (entity == null) {
continue;
}
} catch (FalconException e1) {
LOG.error(
"Unable to get list for entities for ({})",
entityType.getEntityClass().getSimpleName(),
e1);
throw FalconWebException.newException(e1, Response.Status.BAD_REQUEST);
}
if (SecurityUtil.isAuthorizationEnabled() && !isEntityAuthorized(entity)) {
// the user who requested list query has no permission to access this entity. Skip this
// entity
continue;
}
if (isFilteredByDatesAndCluster(entity, startDate, endDate, cluster)) {
// this is for entity summary
continue;
}
SecurityUtil.tryProxy(entity);
// filter by fields
if (isFilteredByFields(entity, filterByFieldsValues)) {
continue;
}
// filter by subsequence of name
if (subsequence.length > 0
&& !matchesNameSubsequence(subsequence, entityName.toLowerCase())) {
continue;
}
// filter by tag keywords
if (!matchTagKeywords(tagKeywordsList, entity.getTags())) {
continue;
}
entities.add(entity);
}
return entities;
}