protected synchronized Entity submitInternal(HttpServletRequest request, String type)
throws IOException, FalconException {
EntityType entityType = EntityType.getEnum(type);
Entity entity = deserializeEntity(request, entityType);
// KLUDGE - Until ACL is mandated entity passed should be decorated for equals check to pass
decorateEntityWithACL(entity);
Entity existingEntity = configStore.get(entityType, entity.getName());
if (existingEntity != null) {
if (EntityUtil.equals(existingEntity, entity)) {
return existingEntity;
}
throw new EntityAlreadyExistsException(
entity.toShortString()
+ " already registered with configuration store. "
+ "Can't be submitted again. Try removing before submitting.");
}
SecurityUtil.tryProxy(entity); // proxy before validating since FS/Oozie needs to be proxied
validate(entity);
configStore.publish(entityType, entity);
LOG.info("Submit successful: ({}): {}", type, entity.getName());
return entity;
}
private boolean isEntityFiltered(
Entity entity, EntityList.EntityFilterByFields filter, Map.Entry<String, List<String>> pair) {
switch (filter) {
case TYPE:
return !containsIgnoreCase(pair.getValue(), entity.getEntityType().toString());
case NAME:
return !containsIgnoreCase(pair.getValue(), entity.getName());
case STATUS:
return !containsIgnoreCase(pair.getValue(), getStatusString(entity));
case PIPELINES:
if (!entity.getEntityType().equals(EntityType.PROCESS)) {
throw FalconWebException.newException(
"Invalid filterBy key for non process entities " + pair.getKey(),
Response.Status.BAD_REQUEST);
}
return !EntityUtil.getPipelines(entity).contains(pair.getValue().get(0));
case CLUSTER:
return !EntityUtil.getClustersDefined(entity).contains(pair.getValue().get(0));
case TAGS:
return isFilteredByTags(getFilterByTags(pair.getValue()), EntityUtil.getTags(entity));
default:
return false;
}
}
/**
* Post an entity XML with entity type. Validates the XML which can be Process, Feed or
* Dataendpoint
*
* @param type entity type
* @return APIResule -Succeeded or Failed
*/
public APIResult validate(HttpServletRequest request, String type) {
try {
EntityType entityType = EntityType.getEnum(type);
Entity entity = deserializeEntity(request, entityType);
validate(entity);
// Validate that the entity can be scheduled in the cluster
if (entity.getEntityType().isSchedulable()) {
Set<String> clusters = EntityUtil.getClustersDefinedInColos(entity);
for (String cluster : clusters) {
try {
getWorkflowEngine().dryRun(entity, cluster);
} catch (FalconException e) {
throw new FalconException("dryRun failed on cluster " + cluster, e);
}
}
}
return new APIResult(
APIResult.Status.SUCCEEDED,
"Validated successfully (" + entityType + ") " + entity.getName());
} catch (Throwable e) {
LOG.error("Validation failed for entity ({})", type, e);
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
}
}
@Override
public String delete(Entity entity) throws FalconException {
if (isActive(entity)) {
EXECUTION_SERVICE.delete(entity);
}
// This should remove it from state store too as state store listens to config store changes.
CONFIG_STORE.remove(entity.getEntityType(), entity.getName());
return "SUCCESS";
}
protected boolean isEntityAuthorized(Entity entity) {
try {
SecurityUtil.getAuthorizationProvider()
.authorizeEntity(
entity.getName(),
entity.getEntityType().toString(),
entity.getACL(),
"list",
CurrentUser.getAuthenticatedUGI());
} catch (Exception e) {
LOG.info(
"Authorization failed for entity="
+ entity.getName()
+ " for user="
+ CurrentUser.getUser(),
e);
return false;
}
return true;
}
/**
* Submit a new entity. Entities can be of type feed, process or data end points. Entity
* definitions are validated structurally against schema and subsequently for other rules before
* they are admitted into the system
*
* <p>Entity name acts as the key and an entity once added, can't be added again unless deleted.
*
* @param request - Servlet Request
* @param type - entity type - feed, process or data end point
* @param colo - applicable colo
* @return result of the operation
*/
public APIResult submit(HttpServletRequest request, String type, String colo) {
checkColo(colo);
try {
Entity entity = submitInternal(request, type);
return new APIResult(
APIResult.Status.SUCCEEDED, "Submit successful (" + type + ") " + entity.getName());
} catch (Throwable e) {
LOG.error("Unable to persist entity object", e);
throw FalconWebException.newException(e, Response.Status.BAD_REQUEST);
}
}
@Override
public void onRemove(Entity entity) throws FalconException {
if (entity.getEntityType().equals(EntityType.FEED)) {
Feed feed = (Feed) entity;
if (StringUtils.isEmpty(feed.getGroups())) {
return;
}
String[] groups = feed.getGroups().split(",");
for (String group : groups) {
groupsMapping.get(group).getFeeds().remove(entity.getName());
if (groupsMapping.get(group).getFeeds().size() == 0) {
groupsMapping.remove(group);
}
}
}
}
private void canRemove(Entity entity) throws FalconException {
Pair<String, EntityType>[] referencedBy = EntityIntegrityChecker.referencedBy(entity);
if (referencedBy != null && referencedBy.length > 0) {
StringBuilder messages = new StringBuilder();
for (Pair<String, EntityType> ref : referencedBy) {
messages.append(ref).append("\n");
}
throw new FalconException(
entity.getName()
+ "("
+ entity.getEntityType()
+ ") cant "
+ "be removed as it is referred by "
+ messages);
}
}
@Override
public String update(Entity oldEntity, Entity newEntity, String cluster, Boolean skipDryRun)
throws FalconException {
org.apache.falcon.entity.v0.cluster.Cluster clusterEntity =
ConfigurationStore.get().get(EntityType.CLUSTER, cluster);
boolean entityUpdated =
UpdateHelper.isEntityUpdated(
oldEntity,
newEntity,
cluster,
EntityUtil.getLatestStagingPath(clusterEntity, oldEntity));
StringBuilder result = new StringBuilder();
if (!entityUpdated) {
// Ideally should throw an exception, but, keeping it backward-compatible.
LOG.warn(
"No relevant updates detected in the new entity definition for entity {}!",
newEntity.getName());
return result.toString();
}
Date oldEndTime = EntityUtil.getEndTime(oldEntity, cluster);
Date newEndTime = EntityUtil.getEndTime(newEntity, cluster);
if (newEndTime.before(DateUtil.now()) || newEndTime.before(oldEndTime)) {
throw new FalconException(
"New Entity's end time "
+ SchemaHelper.formatDateUTC(newEndTime)
+ " is before current time or before old end time. Entity can't be updated.");
}
// The steps required are the same as touch.
DAGEngineFactory.getDAGEngine(cluster)
.touch(newEntity, (skipDryRun == null) ? Boolean.FALSE : skipDryRun);
// Additionally, update the executor.
// The update will kick in for new instances created and not for READY/WAITING instances, as
// with Oozie.
Collection<InstanceState> instances = new ArrayList<>();
instances.add(STATE_STORE.getLastExecutionInstance(oldEntity, cluster));
EXECUTION_SERVICE.getEntityExecutor(oldEntity, cluster).update(newEntity);
result
.append(newEntity.toShortString())
.append("/Effective Time: ")
.append(getEffectiveTime(newEntity, cluster, instances));
return result.toString();
}
private EntityElement getEntityElement(Entity entity, HashSet<String> fields) {
EntityElement elem = new EntityElement();
elem.type = entity.getEntityType().toString();
elem.name = entity.getName();
if (fields.contains(EntityList.EntityFieldList.STATUS.name())) {
elem.status = getStatusString(entity);
}
if (fields.contains(EntityList.EntityFieldList.PIPELINES.name())) {
elem.pipeline = EntityUtil.getPipelines(entity);
}
if (fields.contains(EntityList.EntityFieldList.TAGS.name())) {
elem.tag = EntityUtil.getTags(entity);
}
if (fields.contains(EntityList.EntityFieldList.CLUSTERS.name())) {
elem.cluster = new ArrayList<String>(EntityUtil.getClustersDefined(entity));
}
return elem;
}
@Override
public void onChange(final Entity oldEntity, final Entity newEntity) throws FalconException {
EntityType entityType = newEntity.getEntityType();
LOG.info("Updating lineage for entity: {}, type: {}", newEntity.getName(), entityType);
try {
new TransactionRetryHelper.Builder<Void>(getTransactionalGraph())
.perform(
new TransactionWork<Void>() {
@Override
public Void execute(TransactionalGraph transactionalGraph) throws Exception {
entityGraphBuilder.updateEntity(oldEntity, newEntity);
transactionalGraph.commit();
return null;
}
})
.build()
.exponentialBackoff(transactionRetries, transactionRetryDelayInMillis);
} catch (Exception e) {
getTransactionalGraph().rollback();
throw new FalconException(e);
}
}
protected void verifyEntityProperties(
Entity entity,
Cluster cluster,
Cluster srcCluster,
WorkflowExecutionContext.EntityOperations operation,
HashMap<String, String> props)
throws Exception {
Assert.assertEquals(props.get(WorkflowExecutionArgs.ENTITY_NAME.getName()), entity.getName());
Assert.assertEquals(
props.get(WorkflowExecutionArgs.ENTITY_TYPE.getName()), entity.getEntityType().name());
if (WorkflowExecutionContext.EntityOperations.REPLICATE == operation) {
Assert.assertEquals(
props.get(WorkflowExecutionArgs.CLUSTER_NAME.getName()),
cluster.getName()
+ WorkflowExecutionContext.CLUSTER_NAME_SEPARATOR
+ srcCluster.getName());
} else {
Assert.assertEquals(
props.get(WorkflowExecutionArgs.CLUSTER_NAME.getName()), cluster.getName());
}
Assert.assertEquals(
props.get(WorkflowExecutionArgs.LOG_DIR.getName()), getLogPath(cluster, entity));
Assert.assertEquals(props.get("falconDataOperation"), operation.name());
}
@Override
// SUSPEND CHECKSTYLE CHECK ParameterNumberCheck
public void handleRerun(
String clusterName,
String entityType,
String entityName,
String nominalTime,
String runId,
String wfId,
String workflowUser,
long msgReceivedTime) {
try {
Entity entity = EntityUtil.getEntity(entityType, entityName);
Retry retry = getRetry(entity);
if (retry == null) {
LOG.warn(
"Retry not configured for entity: {} ({}), ignoring failed retried",
entityType,
entity.getName());
return;
}
int attempts = retry.getAttempts();
Frequency delay = retry.getDelay();
PolicyType policy = retry.getPolicy();
int intRunId = Integer.parseInt(runId);
if (attempts > intRunId) {
AbstractRerunPolicy rerunPolicy = RerunPolicyFactory.getRetryPolicy(policy);
long delayTime = rerunPolicy.getDelay(delay, Integer.parseInt(runId));
RetryEvent event =
new RetryEvent(
clusterName,
wfId,
msgReceivedTime,
delayTime,
entityType,
entityName,
nominalTime,
intRunId,
attempts,
0,
workflowUser);
offerToQueue(event);
} else {
LOG.warn(
"All retry attempt failed out of configured: {} attempt for entity instance: {}:{} "
+ "And WorkflowId: {}",
attempts,
entityName,
nominalTime,
wfId);
GenericAlert.alertRetryFailed(
entityType,
entityName,
nominalTime,
wfId,
workflowUser,
runId,
"All retry attempt failed out of configured: "
+ attempts
+ " attempt for entity instance::");
}
} catch (FalconException e) {
LOG.error("Error during retry of entity instance {}:{}", entityName, nominalTime, e);
GenericAlert.alertRetryFailed(
entityType, entityName, nominalTime, wfId, workflowUser, runId, e.getMessage());
}
}
