/**
* KLUDGE - Until ACL is mandated entity passed should be decorated for equals check to pass.
* existingEntity in config store will have teh decoration and equals check fails if entity passed
* is not decorated for checking if entity already exists.
*
* @param entity entity
*/
private void decorateEntityWithACL(Entity entity) {
if (SecurityUtil.isAuthorizationEnabled() || entity.getACL() != null) {
return; // not necessary to decorate
}
final String proxyUser = CurrentUser.getUser();
final String defaultGroupName = CurrentUser.getPrimaryGroupName();
switch (entity.getEntityType()) {
case CLUSTER:
org.apache.falcon.entity.v0.cluster.ACL clusterACL =
new org.apache.falcon.entity.v0.cluster.ACL();
clusterACL.setOwner(proxyUser);
clusterACL.setGroup(defaultGroupName);
((org.apache.falcon.entity.v0.cluster.Cluster) entity).setACL(clusterACL);
break;
case FEED:
org.apache.falcon.entity.v0.feed.ACL feedACL = new org.apache.falcon.entity.v0.feed.ACL();
feedACL.setOwner(proxyUser);
feedACL.setGroup(defaultGroupName);
((org.apache.falcon.entity.v0.feed.Feed) entity).setACL(feedACL);
break;
case PROCESS:
org.apache.falcon.entity.v0.process.ACL processACL =
new org.apache.falcon.entity.v0.process.ACL();
processACL.setOwner(proxyUser);
processACL.setGroup(defaultGroupName);
((org.apache.falcon.entity.v0.process.Process) entity).setACL(processACL);
break;
default:
break;
}
}
private boolean isEntityFiltered(
Entity entity, EntityList.EntityFilterByFields filter, Map.Entry<String, List<String>> pair) {
switch (filter) {
case TYPE:
return !containsIgnoreCase(pair.getValue(), entity.getEntityType().toString());
case NAME:
return !containsIgnoreCase(pair.getValue(), entity.getName());
case STATUS:
return !containsIgnoreCase(pair.getValue(), getStatusString(entity));
case PIPELINES:
if (!entity.getEntityType().equals(EntityType.PROCESS)) {
throw FalconWebException.newException(
"Invalid filterBy key for non process entities " + pair.getKey(),
Response.Status.BAD_REQUEST);
}
return !EntityUtil.getPipelines(entity).contains(pair.getValue().get(0));
case CLUSTER:
return !EntityUtil.getClustersDefined(entity).contains(pair.getValue().get(0));
case TAGS:
return isFilteredByTags(getFilterByTags(pair.getValue()), EntityUtil.getTags(entity));
default:
return false;
}
}
private List<Entity> obtainUpdateEntityLocks(Entity entity) throws FalconException {
List<Entity> tokenList = new ArrayList<Entity>();
// first obtain lock for the entity for which update is issued.
if (memoryLocks.acquireLock(entity)) {
tokenList.add(entity);
} else {
throw new FalconException(
"Looks like an update command is already issued for " + entity.toShortString());
}
// now obtain locks for all dependent entities.
Set<Entity> affectedEntities = EntityGraph.get().getDependents(entity);
for (Entity e : affectedEntities) {
if (memoryLocks.acquireLock(e)) {
tokenList.add(e);
} else {
LOG.error(
"Error while trying to acquire lock for {}. Releasing already obtained locks",
e.toShortString());
throw new FalconException(
"There are multiple update commands running for dependent entity " + e.toShortString());
}
}
return tokenList;
}
protected synchronized Entity submitInternal(HttpServletRequest request, String type)
throws IOException, FalconException {
EntityType entityType = EntityType.getEnum(type);
Entity entity = deserializeEntity(request, entityType);
// KLUDGE - Until ACL is mandated entity passed should be decorated for equals check to pass
decorateEntityWithACL(entity);
Entity existingEntity = configStore.get(entityType, entity.getName());
if (existingEntity != null) {
if (EntityUtil.equals(existingEntity, entity)) {
return existingEntity;
}
throw new EntityAlreadyExistsException(
entity.toShortString()
+ " already registered with configuration store. "
+ "Can't be submitted again. Try removing before submitting.");
}
SecurityUtil.tryProxy(entity); // proxy before validating since FS/Oozie needs to be proxied
validate(entity);
configStore.publish(entityType, entity);
LOG.info("Submit successful: ({}): {}", type, entity.getName());
return entity;
}
/**
* Post an entity XML with entity type. Validates the XML which can be Process, Feed or
* Dataendpoint
*
* @param type entity type
* @return APIResule -Succeeded or Failed
*/
public APIResult validate(HttpServletRequest request, String type) {
try {
EntityType entityType = EntityType.getEnum(type);
Entity entity = deserializeEntity(request, entityType);
validate(entity);
// Validate that the entity can be scheduled in the cluster
if (entity.getEntityType().isSchedulable()) {
Set<String> clusters = EntityUtil.getClustersDefinedInColos(entity);
for (String cluster : clusters) {
try {
getWorkflowEngine().dryRun(entity, cluster);
} catch (FalconException e) {
throw new FalconException("dryRun failed on cluster " + cluster, e);
}
}
}
return new APIResult(
APIResult.Status.SUCCEEDED,
"Validated successfully (" + entityType + ") " + entity.getName());
} catch (Throwable e) {
LOG.error("Validation failed for entity ({})", type, e);
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
}
}
@Override
public String delete(Entity entity) throws FalconException {
if (isActive(entity)) {
EXECUTION_SERVICE.delete(entity);
}
// This should remove it from state store too as state store listens to config store changes.
CONFIG_STORE.remove(entity.getEntityType(), entity.getName());
return "SUCCESS";
}
/**
* Submit a new entity. Entities can be of type feed, process or data end points. Entity
* definitions are validated structurally against schema and subsequently for other rules before
* they are admitted into the system
*
* <p>Entity name acts as the key and an entity once added, can't be added again unless deleted.
*
* @param request - Servlet Request
* @param type - entity type - feed, process or data end point
* @param colo - applicable colo
* @return result of the operation
*/
public APIResult submit(HttpServletRequest request, String type, String colo) {
checkColo(colo);
try {
Entity entity = submitInternal(request, type);
return new APIResult(
APIResult.Status.SUCCEEDED, "Submit successful (" + type + ") " + entity.getName());
} catch (Throwable e) {
LOG.error("Unable to persist entity object", e);
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
}
}
public static OozieEntityBuilder get(Entity entity) {
switch (entity.getEntityType()) {
case FEED:
return new FeedBundleBuilder((Feed) entity);
case PROCESS:
return new ProcessBundleBuilder((Process) entity);
default:
}
throw new IllegalArgumentException("Unhandled type: " + entity.getEntityType());
}
private void validateUpdate(Entity oldEntity, Entity newEntity) throws FalconException {
if (oldEntity.getEntityType() != newEntity.getEntityType() || !oldEntity.equals(newEntity)) {
throw new FalconException(
oldEntity.toShortString() + " can't be updated with " + newEntity.toShortString());
}
if (oldEntity.getEntityType() == EntityType.CLUSTER) {
throw new FalconException("Update not supported for clusters");
}
String[] props = oldEntity.getEntityType().getImmutableProperties();
for (String prop : props) {
Object oldProp, newProp;
try {
oldProp = PropertyUtils.getProperty(oldEntity, prop);
newProp = PropertyUtils.getProperty(newEntity, prop);
} catch (Exception e) {
throw new FalconException(e);
}
if (!ObjectUtils.equals(oldProp, newProp)) {
throw new ValidationException(
oldEntity.toShortString() + ": " + prop + " can't be changed");
}
}
}
/**
* Returns the entity definition as an XML based on name.
*
* @param type entity type
* @param entityName entity name
* @return String
*/
public String getEntityDefinition(String type, String entityName) {
try {
EntityType entityType = EntityType.getEnum(type);
Entity entity = configStore.get(entityType, entityName);
if (entity == null) {
throw new NoSuchElementException(entityName + " (" + type + ") not found");
}
return entity.toString();
} catch (Throwable e) {
LOG.error(
"Unable to get entity definition from config store for ({}): {}", type, entityName, e);
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
}
}
@Override
public String touch(Entity entity, String cluster, Boolean skipDryRun) throws FalconException {
EntityID id = new EntityID(entity);
// Ideally state store should have all entities, but, check anyway.
if (STATE_STORE.entityExists(id)) {
Date endTime = EntityUtil.getEndTime(entity, cluster);
if (endTime.before(DateUtil.now())) {
throw new FalconException(
"Entity's end time "
+ SchemaHelper.formatDateUTC(endTime)
+ " is before current time. Entity can't be touch-ed as it has completed.");
}
Collection<InstanceState> instances =
STATE_STORE.getExecutionInstances(entity, cluster, InstanceState.getRunningStates());
// touch should happen irrespective of the state the entity is in.
DAGEngineFactory.getDAGEngine(cluster)
.touch(entity, (skipDryRun == null) ? Boolean.FALSE : skipDryRun);
StringBuilder builder = new StringBuilder();
builder
.append(entity.toShortString())
.append("/Effective Time: ")
.append(getEffectiveTime(entity, cluster, instances));
return builder.toString();
}
throw new FalconException("Could not find entity " + id + " in state store.");
}
private void canRemove(Entity entity) throws FalconException {
Pair<String, EntityType>[] referencedBy = EntityIntegrityChecker.referencedBy(entity);
if (referencedBy != null && referencedBy.length > 0) {
StringBuilder messages = new StringBuilder();
for (Pair<String, EntityType> ref : referencedBy) {
messages.append(ref).append("\n");
}
throw new FalconException(
entity.getName()
+ "("
+ entity.getEntityType()
+ ") cant "
+ "be removed as it is referred by "
+ messages);
}
}
@Override
public void onRemove(Entity entity) throws FalconException {
if (entity.getEntityType().equals(EntityType.FEED)) {
Feed feed = (Feed) entity;
if (StringUtils.isEmpty(feed.getGroups())) {
return;
}
String[] groups = feed.getGroups().split(",");
for (String group : groups) {
groupsMapping.get(group).getFeeds().remove(entity.getName());
if (groupsMapping.get(group).getFeeds().size() == 0) {
groupsMapping.remove(group);
}
}
}
}
@Override
public String update(Entity oldEntity, Entity newEntity, String cluster, Boolean skipDryRun)
throws FalconException {
org.apache.falcon.entity.v0.cluster.Cluster clusterEntity =
ConfigurationStore.get().get(EntityType.CLUSTER, cluster);
boolean entityUpdated =
UpdateHelper.isEntityUpdated(
oldEntity,
newEntity,
cluster,
EntityUtil.getLatestStagingPath(clusterEntity, oldEntity));
StringBuilder result = new StringBuilder();
if (!entityUpdated) {
// Ideally should throw an exception, but, keeping it backward-compatible.
LOG.warn(
"No relevant updates detected in the new entity definition for entity {}!",
newEntity.getName());
return result.toString();
}
Date oldEndTime = EntityUtil.getEndTime(oldEntity, cluster);
Date newEndTime = EntityUtil.getEndTime(newEntity, cluster);
if (newEndTime.before(DateUtil.now()) || newEndTime.before(oldEndTime)) {
throw new FalconException(
"New Entity's end time "
+ SchemaHelper.formatDateUTC(newEndTime)
+ " is before current time or before old end time. Entity can't be updated.");
}
// The steps required are the same as touch.
DAGEngineFactory.getDAGEngine(cluster)
.touch(newEntity, (skipDryRun == null) ? Boolean.FALSE : skipDryRun);
// Additionally, update the executor.
// The update will kick in for new instances created and not for READY/WAITING instances, as
// with Oozie.
Collection<InstanceState> instances = new ArrayList<>();
instances.add(STATE_STORE.getLastExecutionInstance(oldEntity, cluster));
EXECUTION_SERVICE.getEntityExecutor(oldEntity, cluster).update(newEntity);
result
.append(newEntity.toShortString())
.append("/Effective Time: ")
.append(getEffectiveTime(newEntity, cluster, instances));
return result.toString();
}
private EntityElement getEntityElement(Entity entity, HashSet<String> fields) {
EntityElement elem = new EntityElement();
elem.type = entity.getEntityType().toString();
elem.name = entity.getName();
if (fields.contains(EntityList.EntityFieldList.STATUS.name())) {
elem.status = getStatusString(entity);
}
if (fields.contains(EntityList.EntityFieldList.PIPELINES.name())) {
elem.pipeline = EntityUtil.getPipelines(entity);
}
if (fields.contains(EntityList.EntityFieldList.TAGS.name())) {
elem.tag = EntityUtil.getTags(entity);
}
if (fields.contains(EntityList.EntityFieldList.CLUSTERS.name())) {
elem.cluster = new ArrayList<String>(EntityUtil.getClustersDefined(entity));
}
return elem;
}
protected String getStatusString(Entity entity) {
String statusString;
try {
statusString = getStatus(entity, entity.getEntityType()).name();
} catch (Throwable throwable) {
// Unable to fetch statusString, setting it to unknown for backwards compatibility
statusString = "UNKNOWN";
}
return statusString;
}
protected boolean isEntityAuthorized(Entity entity) {
try {
SecurityUtil.getAuthorizationProvider()
.authorizeEntity(
entity.getName(),
entity.getEntityType().toString(),
entity.getACL(),
"list",
CurrentUser.getAuthenticatedUGI());
} catch (Exception e) {
LOG.info(
"Authorization failed for entity="
+ entity.getName()
+ " for user="
+ CurrentUser.getUser(),
e);
return false;
}
return true;
}
@Override
public void onAdd(Entity entity) throws FalconException {
if (entity.getEntityType().equals(EntityType.FEED)) {
Feed feed = (Feed) entity;
if (feed.getGroups() == null || feed.getGroups().equals("")) {
return;
}
Set<FeedGroup> groupSet = getGroups(feed);
addGroups(feed.getName(), groupSet);
}
}
@Override
public void onChange(final Entity oldEntity, final Entity newEntity) throws FalconException {
EntityType entityType = newEntity.getEntityType();
LOG.info("Updating lineage for entity: {}, type: {}", newEntity.getName(), entityType);
try {
new TransactionRetryHelper.Builder<Void>(getTransactionalGraph())
.perform(
new TransactionWork<Void>() {
@Override
public Void execute(TransactionalGraph transactionalGraph) throws Exception {
entityGraphBuilder.updateEntity(oldEntity, newEntity);
transactionalGraph.commit();
return null;
}
})
.build()
.exponentialBackoff(transactionRetries, transactionRetryDelayInMillis);
} catch (Exception e) {
getTransactionalGraph().rollback();
throw new FalconException(e);
}
}
protected Properties getEntityProperties(Entity myEntity) {
Properties properties = new Properties();
switch (myEntity.getEntityType()) {
case CLUSTER:
org.apache.falcon.entity.v0.cluster.Properties clusterProps =
((Cluster) myEntity).getProperties();
if (clusterProps != null) {
for (Property prop : clusterProps.getProperties()) {
properties.put(prop.getName(), prop.getValue());
}
}
break;
case FEED:
org.apache.falcon.entity.v0.feed.Properties feedProps = ((Feed) myEntity).getProperties();
if (feedProps != null) {
for (org.apache.falcon.entity.v0.feed.Property prop : feedProps.getProperties()) {
properties.put(prop.getName(), prop.getValue());
}
}
break;
case PROCESS:
org.apache.falcon.entity.v0.process.Properties processProps =
((Process) myEntity).getProperties();
if (processProps != null) {
for (org.apache.falcon.entity.v0.process.Property prop : processProps.getProperties()) {
properties.put(prop.getName(), prop.getValue());
}
}
break;
default:
throw new IllegalArgumentException("Unhandled entity type " + myEntity.getEntityType());
}
return properties;
}
protected void verifyEntityProperties(
Entity entity,
Cluster cluster,
Cluster srcCluster,
WorkflowExecutionContext.EntityOperations operation,
HashMap<String, String> props)
throws Exception {
Assert.assertEquals(props.get(WorkflowExecutionArgs.ENTITY_NAME.getName()), entity.getName());
Assert.assertEquals(
props.get(WorkflowExecutionArgs.ENTITY_TYPE.getName()), entity.getEntityType().name());
if (WorkflowExecutionContext.EntityOperations.REPLICATE == operation) {
Assert.assertEquals(
props.get(WorkflowExecutionArgs.CLUSTER_NAME.getName()),
cluster.getName()
+ WorkflowExecutionContext.CLUSTER_NAME_SEPARATOR
+ srcCluster.getName());
} else {
Assert.assertEquals(
props.get(WorkflowExecutionArgs.CLUSTER_NAME.getName()), cluster.getName());
}
Assert.assertEquals(
props.get(WorkflowExecutionArgs.LOG_DIR.getName()), getLogPath(cluster, entity));
Assert.assertEquals(props.get("falconDataOperation"), operation.name());
}
public void updateEntity(Entity oldEntity, Entity newEntity) {
EntityType entityType = oldEntity.getEntityType();
switch (entityType) {
case CLUSTER:
updateClusterEntity((Cluster) oldEntity, (Cluster) newEntity);
break;
case PROCESS:
updateProcessEntity((Process) oldEntity, (Process) newEntity);
break;
case FEED:
updateFeedEntity((Feed) oldEntity, (Feed) newEntity);
break;
case DATASOURCE:
updateDatasourceEntity((Datasource) oldEntity, (Datasource) newEntity);
break;
default:
throw new IllegalArgumentException("Invalid EntityType " + entityType);
}
}
public void addEntity(Entity entity) {
EntityType entityType = entity.getEntityType();
switch (entityType) {
case CLUSTER:
addClusterEntity((Cluster) entity);
break;
case PROCESS:
addProcessEntity((Process) entity);
break;
case FEED:
addFeedEntity((Feed) entity);
break;
case DATASOURCE:
addDatasourceEntity((Datasource) entity);
break;
default:
throw new IllegalArgumentException("Invalid EntityType " + entityType);
}
}
@SuppressWarnings({"unchecked", "rawtypes"})
private void validate(Entity entity) throws FalconException {
EntityParser entityParser = EntityParserFactory.getParser(entity.getEntityType());
entityParser.validate(entity);
}
protected List<Entity> getFilteredEntities(
EntityType entityType,
String nameSubsequence,
String tagKeywords,
Map<String, List<String>> filterByFieldsValues,
String startDate,
String endDate,
String cluster)
throws FalconException, IOException {
Collection<String> entityNames = configStore.getEntities(entityType);
if (entityNames.isEmpty()) {
return Collections.emptyList();
}
List<Entity> entities = new ArrayList<Entity>();
char[] subsequence = nameSubsequence.toLowerCase().toCharArray();
List<String> tagKeywordsList;
if (StringUtils.isEmpty(tagKeywords)) {
tagKeywordsList = new ArrayList<>();
} else {
tagKeywordsList = getFilterByTags(Arrays.asList(tagKeywords.toLowerCase()));
}
for (String entityName : entityNames) {
Entity entity;
try {
entity = configStore.get(entityType, entityName);
if (entity == null) {
continue;
}
} catch (FalconException e1) {
LOG.error(
"Unable to get list for entities for ({})",
entityType.getEntityClass().getSimpleName(),
e1);
throw FalconWebException.newException(e1, Response.Status.BAD_REQUEST);
}
if (SecurityUtil.isAuthorizationEnabled() && !isEntityAuthorized(entity)) {
// the user who requested list query has no permission to access this entity. Skip this
// entity
continue;
}
if (isFilteredByDatesAndCluster(entity, startDate, endDate, cluster)) {
// this is for entity summary
continue;
}
SecurityUtil.tryProxy(entity);
// filter by fields
if (isFilteredByFields(entity, filterByFieldsValues)) {
continue;
}
// filter by subsequence of name
if (subsequence.length > 0
&& !matchesNameSubsequence(subsequence, entityName.toLowerCase())) {
continue;
}
// filter by tag keywords
if (!matchTagKeywords(tagKeywordsList, entity.getTags())) {
continue;
}
entities.add(entity);
}
return entities;
}
@Override
// SUSPEND CHECKSTYLE CHECK ParameterNumberCheck
public void handleRerun(
String clusterName,
String entityType,
String entityName,
String nominalTime,
String runId,
String wfId,
String workflowUser,
long msgReceivedTime) {
try {
Entity entity = EntityUtil.getEntity(entityType, entityName);
Retry retry = getRetry(entity);
if (retry == null) {
LOG.warn(
"Retry not configured for entity: {} ({}), ignoring failed retried",
entityType,
entity.getName());
return;
}
int attempts = retry.getAttempts();
Frequency delay = retry.getDelay();
PolicyType policy = retry.getPolicy();
int intRunId = Integer.parseInt(runId);
if (attempts > intRunId) {
AbstractRerunPolicy rerunPolicy = RerunPolicyFactory.getRetryPolicy(policy);
long delayTime = rerunPolicy.getDelay(delay, Integer.parseInt(runId));
RetryEvent event =
new RetryEvent(
clusterName,
wfId,
msgReceivedTime,
delayTime,
entityType,
entityName,
nominalTime,
intRunId,
attempts,
0,
workflowUser);
offerToQueue(event);
} else {
LOG.warn(
"All retry attempt failed out of configured: {} attempt for entity instance: {}:{} "
+ "And WorkflowId: {}",
attempts,
entityName,
nominalTime,
wfId);
GenericAlert.alertRetryFailed(
entityType,
entityName,
nominalTime,
wfId,
workflowUser,
runId,
"All retry attempt failed out of configured: "
+ attempts
+ " attempt for entity instance::");
}
} catch (FalconException e) {
LOG.error("Error during retry of entity instance {}:{}", entityName, nominalTime, e);
GenericAlert.alertRetryFailed(
entityType, entityName, nominalTime, wfId, workflowUser, runId, e.getMessage());
}
}