@CliCommand( value = {METADATA_COMMAND_PREFIX + LIST_OPT}, help = LIST_OPT_DESCRIPTION) public String list( @CliOption( key = {TYPE_OPT}, mandatory = true, help = TYPE_OPT_DESCRIPTION) final String dimensionType, @CliOption( key = {CLUSTER_OPT}, mandatory = false, help = CLUSTER_OPT_DESCRIPTION) final String cluster, @CliOption( key = {FEED_OPT}, mandatory = false, help = FEED_OPT_DESCRIPTION) final String feed, @CliOption( key = {PROCESS_OPT}, mandatory = false, help = PROCESS_OPT_DESCRIPTION) final String process, @CliOption( key = {NUM_RESULTS_OPT}, mandatory = false, help = NUM_RESULTS_OPT_DESCRIPTION) final Integer numResults) { validateDimensionType(dimensionType.toUpperCase()); if (!(dimensionType.toUpperCase()).equals(RelationshipType.REPLICATION_METRICS.name())) { return getFalconClient().getDimensionList(dimensionType, cluster, getDoAs()); } else { String schedEntityType = null; String schedEntityName = null; if (StringUtils.isNotEmpty(feed)) { schedEntityType = EntityType.getEnum(FEED_OPT).name(); schedEntityName = feed; } else if (StringUtils.isNotEmpty(process)) { schedEntityType = EntityType.getEnum(PROCESS_OPT).name(); schedEntityName = process; } validateScheduleEntity(schedEntityType, schedEntityName); return getFalconClient() .getReplicationMetricsDimensionList( schedEntityType, schedEntityName, numResults, getDoAs()); } }
protected synchronized Entity submitInternal(HttpServletRequest request, String type) throws IOException, FalconException { EntityType entityType = EntityType.getEnum(type); Entity entity = deserializeEntity(request, entityType); // KLUDGE - Until ACL is mandated entity passed should be decorated for equals check to pass decorateEntityWithACL(entity); Entity existingEntity = configStore.get(entityType, entity.getName()); if (existingEntity != null) { if (EntityUtil.equals(existingEntity, entity)) { return existingEntity; } throw new EntityAlreadyExistsException( entity.toShortString() + " already registered with configuration store. " + "Can't be submitted again. Try removing before submitting."); } SecurityUtil.tryProxy(entity); // proxy before validating since FS/Oozie needs to be proxied validate(entity); configStore.publish(entityType, entity); LOG.info("Submit successful: ({}): {}", type, entity.getName()); return entity; }
/** * Deletes a scheduled entity, a deleted entity is removed completely from execution pool. * * @param type entity type * @param entity entity name * @return APIResult */ public APIResult delete(HttpServletRequest request, String type, String entity, String colo) { checkColo(colo); try { EntityType entityType = EntityType.getEnum(type); String removedFromEngine = ""; try { Entity entityObj = EntityUtil.getEntity(type, entity); canRemove(entityObj); if (entityType.isSchedulable() && !DeploymentUtil.isPrism()) { getWorkflowEngine().delete(entityObj); removedFromEngine = "(KILLED in ENGINE)"; } configStore.remove(entityType, entity); } catch (EntityNotRegisteredException e) { // already deleted return new APIResult( APIResult.Status.SUCCEEDED, entity + "(" + type + ") doesn't exist. Nothing to do"); } return new APIResult( APIResult.Status.SUCCEEDED, entity + "(" + type + ") removed successfully " + removedFromEngine); } catch (Throwable e) { LOG.error("Unable to reach workflow engine for deletion or deletion failed", e); throw FalconWebException.newException(e, Response.Status.BAD_REQUEST); } }
/** * Post an entity XML with entity type. Validates the XML which can be Process, Feed or * Dataendpoint * * @param type entity type * @return APIResule -Succeeded or Failed */ public APIResult validate(HttpServletRequest request, String type) { try { EntityType entityType = EntityType.getEnum(type); Entity entity = deserializeEntity(request, entityType); validate(entity); // Validate that the entity can be scheduled in the cluster if (entity.getEntityType().isSchedulable()) { Set<String> clusters = EntityUtil.getClustersDefinedInColos(entity); for (String cluster : clusters) { try { getWorkflowEngine().dryRun(entity, cluster); } catch (FalconException e) { throw new FalconException("dryRun failed on cluster " + cluster, e); } } } return new APIResult( APIResult.Status.SUCCEEDED, "Validated successfully (" + entityType + ") " + entity.getName()); } catch (Throwable e) { LOG.error("Validation failed for entity ({})", type, e); throw FalconWebException.newException(e, Response.Status.BAD_REQUEST); } }
/** * Returns the list of filtered entities as well as the total number of results. * * @param fieldStr Fields that the query is interested in, separated by comma * @param nameSubsequence Name subsequence to match * @param tagKeywords Tag keywords to match, separated by commma * @param filterType Only return entities of this type * @param filterTags Full tag matching, separated by comma * @param filterBy Specific fields to match (i.e. TYPE, NAME, STATUS, PIPELINES, CLUSTER) * @param orderBy Order result by these fields. * @param sortOrder Valid options are "asc" and “desc” * @param offset Pagination offset. * @param resultsPerPage Number of results that should be returned starting at the offset. * @return EntityList */ public EntityList getEntityList( String fieldStr, String nameSubsequence, String tagKeywords, String filterType, String filterTags, String filterBy, String orderBy, String sortOrder, Integer offset, Integer resultsPerPage) { HashSet<String> fields = new HashSet<String>(Arrays.asList(fieldStr.toUpperCase().split(","))); Map<String, List<String>> filterByFieldsValues = getFilterByFieldsValues(filterBy); validateEntityFilterByClause(filterByFieldsValues); if (StringUtils.isNotEmpty(filterTags)) { filterByFieldsValues.put( EntityList.EntityFilterByFields.TAGS.name(), Arrays.asList(filterTags)); } // get filtered entities List<Entity> entities = new ArrayList<Entity>(); try { if (StringUtils.isEmpty(filterType)) { // return entities of all types if no entity type specified for (EntityType entityType : EntityType.values()) { entities.addAll( getFilteredEntities( entityType, nameSubsequence, tagKeywords, filterByFieldsValues, "", "", "")); } } else { String[] types = filterType.split(","); for (String type : types) { EntityType entityType = EntityType.getEnum(type); entities.addAll( getFilteredEntities( entityType, nameSubsequence, tagKeywords, filterByFieldsValues, "", "", "")); } } } catch (Exception e) { LOG.error("Failed to get entity list", e); throw FalconWebException.newException(e, Response.Status.BAD_REQUEST); } // sort entities and pagination List<Entity> entitiesReturn = sortEntitiesPagination(entities, orderBy, sortOrder, offset, resultsPerPage); // add total number of results EntityList entityList = entitiesReturn.size() == 0 ? new EntityList(new Entity[] {}, 0) : new EntityList( buildEntityElements(new HashSet<String>(fields), entitiesReturn), entities.size()); return entityList; }
/** * Returns the entity definition as an XML based on name. * * @param type entity type * @param entityName entity name * @return String */ public String getEntityDefinition(String type, String entityName) { try { EntityType entityType = EntityType.getEnum(type); Entity entity = configStore.get(entityType, entityName); if (entity == null) { throw new NoSuchElementException(entityName + " (" + type + ") not found"); } return entity.toString(); } catch (Throwable e) { LOG.error( "Unable to get entity definition from config store for ({}): {}", type, entityName, e); throw FalconWebException.newException(e, Response.Status.BAD_REQUEST); } }
protected Set<String> getApplicableColos(String type, String name) { try { if (DeploymentUtil.isEmbeddedMode()) { return DeploymentUtil.getDefaultColos(); } if (EntityType.getEnum(type) == EntityType.CLUSTER) { return getAllColos(); } return getApplicableColos(type, EntityUtil.getEntity(type, name)); } catch (FalconException e) { throw FalconWebException.newException(e, Response.Status.BAD_REQUEST); } }
/** * Returns the status of requested entity. * * @param type entity type * @param entity entity name * @return String */ public APIResult getStatus(String type, String entity, String colo) { checkColo(colo); Entity entityObj; try { entityObj = EntityUtil.getEntity(type, entity); EntityType entityType = EntityType.getEnum(type); EntityStatus status = getStatus(entityObj, entityType); return new APIResult(Status.SUCCEEDED, status.name()); } catch (FalconWebException e) { throw e; } catch (Exception e) { LOG.error("Unable to get status for entity {} ({})", entity, type, e); throw FalconWebException.newException(e, Response.Status.BAD_REQUEST); } }
public APIResult update(HttpServletRequest request, String type, String entityName, String colo) { checkColo(colo); List<Entity> tokenList = null; try { EntityType entityType = EntityType.getEnum(type); Entity oldEntity = EntityUtil.getEntity(type, entityName); Entity newEntity = deserializeEntity(request, entityType); // KLUDGE - Until ACL is mandated entity passed should be decorated for equals check to pass decorateEntityWithACL(newEntity); validate(newEntity); validateUpdate(oldEntity, newEntity); configStore.initiateUpdate(newEntity); tokenList = obtainUpdateEntityLocks(oldEntity); StringBuilder result = new StringBuilder("Updated successfully"); // Update in workflow engine if (!DeploymentUtil.isPrism()) { Set<String> oldClusters = EntityUtil.getClustersDefinedInColos(oldEntity); Set<String> newClusters = EntityUtil.getClustersDefinedInColos(newEntity); newClusters.retainAll(oldClusters); // common clusters for update oldClusters.removeAll(newClusters); // deleted clusters for (String cluster : newClusters) { result.append(getWorkflowEngine().update(oldEntity, newEntity, cluster)); } for (String cluster : oldClusters) { getWorkflowEngine().delete(oldEntity, cluster); } } configStore.update(entityType, newEntity); return new APIResult(APIResult.Status.SUCCEEDED, result.toString()); } catch (Throwable e) { LOG.error("Update failed", e); throw FalconWebException.newException(e, Response.Status.BAD_REQUEST); } finally { ConfigurationStore.get().cleanupUpdateInit(); releaseUpdateEntityLocks(entityName, tokenList); } }
protected Set<String> getApplicableColos(String type, Entity entity) { try { if (DeploymentUtil.isEmbeddedMode()) { return DeploymentUtil.getDefaultColos(); } if (EntityType.getEnum(type) == EntityType.CLUSTER) { return getAllColos(); } Set<String> clusters = EntityUtil.getClustersDefined(entity); Set<String> colos = new HashSet<String>(); for (String cluster : clusters) { Cluster clusterEntity = EntityUtil.getEntity(EntityType.CLUSTER, cluster); colos.add(clusterEntity.getColo()); } return colos; } catch (FalconException e) { throw FalconWebException.newException(e, Response.Status.BAD_REQUEST); } }
/** * Given the location of data, returns the feed. * * @param type type of the entity, is valid only for feeds. * @param instancePath location of the data * @return Feed Name, type of the data and cluster name. */ public FeedLookupResult reverseLookup(String type, String instancePath) { try { EntityType entityType = EntityType.getEnum(type); if (entityType != EntityType.FEED) { LOG.error("Reverse Lookup is not supported for entitytype: {}", type); throw new IllegalArgumentException("Reverse lookup is not supported for " + type); } instancePath = StringUtils.trim(instancePath); String instancePathWithoutSlash = instancePath.endsWith("/") ? StringUtils.removeEnd(instancePath, "/") : instancePath; // treat strings with and without trailing slash as same for purpose of searching e.g. // /data/cas and /data/cas/ should be treated as same. String instancePathWithSlash = instancePathWithoutSlash + "/"; FeedLocationStore store = FeedLocationStore.get(); Collection<FeedLookupResult.FeedProperties> feeds = new ArrayList<>(); Collection<FeedLookupResult.FeedProperties> res = store.reverseLookup(instancePathWithoutSlash); if (res != null) { feeds.addAll(res); } res = store.reverseLookup(instancePathWithSlash); if (res != null) { feeds.addAll(res); } FeedLookupResult result = new FeedLookupResult(APIResult.Status.SUCCEEDED, "SUCCESS"); FeedLookupResult.FeedProperties[] props = feeds.toArray(new FeedLookupResult.FeedProperties[0]); result.setElements(props); return result; } catch (IllegalArgumentException e) { throw FalconWebException.newException(e, Response.Status.BAD_REQUEST); } catch (Throwable throwable) { LOG.error("reverse look up failed", throwable); throw FalconWebException.newException(throwable, Response.Status.INTERNAL_SERVER_ERROR); } }