/** {@inheritDoc} */
@Override
protected DataExecutionResponse run(
DataBuilderContext dataBuilderContext,
DataFlowInstance dataFlowInstance,
DataDelta dataDelta,
DataFlow dataFlow,
DataBuilderFactory builderFactory)
throws DataBuilderFrameworkException, DataValidationException {
CompletionService<DataContainer> completionExecutor =
new ExecutorCompletionService<DataContainer>(executorService);
ExecutionGraph executionGraph = dataFlow.getExecutionGraph();
DataSet dataSet =
dataFlowInstance.getDataSet().accessor().copy(); // Create own copy to work with
DataSetAccessor dataSetAccessor = DataSet.accessor(dataSet);
dataSetAccessor.merge(dataDelta);
Map<String, Data> responseData = Maps.newTreeMap();
Set<String> activeDataSet = Sets.newHashSet();
for (Data data : dataDelta.getDelta()) {
activeDataSet.add(data.getData());
}
List<List<DataBuilderMeta>> dependencyHierarchy = executionGraph.getDependencyHierarchy();
Set<String> newlyGeneratedData = Sets.newHashSet();
Set<DataBuilderMeta> processedBuilders =
Collections.synchronizedSet(Sets.<DataBuilderMeta>newHashSet());
while (true) {
for (List<DataBuilderMeta> levelBuilders : dependencyHierarchy) {
List<Future<DataContainer>> dataFutures = Lists.newArrayList();
for (DataBuilderMeta builderMeta : levelBuilders) {
if (processedBuilders.contains(builderMeta)) {
continue;
}
// If there is an intersection, means some of it's inputs have changed. Reevaluate
if (Sets.intersection(builderMeta.getConsumes(), activeDataSet).isEmpty()) {
continue;
}
DataBuilder builder = builderFactory.create(builderMeta.getName());
if (!dataSetAccessor.checkForData(builder.getDataBuilderMeta().getConsumes())) {
break; // No need to run others, list is topo sorted
}
BuilderRunner builderRunner =
new BuilderRunner(
dataBuilderExecutionListener,
dataFlowInstance,
builderMeta,
dataDelta,
responseData,
builder,
dataBuilderContext,
processedBuilders,
dataSet);
Future<DataContainer> future = completionExecutor.submit(builderRunner);
dataFutures.add(future);
}
// Now wait for something to complete.
int listSize = dataFutures.size();
for (int i = 0; i < listSize; i++) {
try {
DataContainer responseContainer = completionExecutor.take().get();
Data response = responseContainer.getGeneratedData();
if (responseContainer.isHasError()) {
if (null != responseContainer.getValidationException()) {
throw responseContainer.getValidationException();
}
throw responseContainer.getException();
}
if (null != response) {
dataSetAccessor.merge(response);
responseData.put(response.getData(), response);
activeDataSet.add(response.getData());
if (null != dataFlow.getTransients()
&& !dataFlow.getTransients().contains(response.getData())) {
newlyGeneratedData.add(response.getData());
}
}
} catch (InterruptedException e) {
throw new DataBuilderFrameworkException(
DataBuilderFrameworkException.ErrorCode.BUILDER_EXECUTION_ERROR,
"Error while waiting for error ",
e);
} catch (ExecutionException e) {
throw new DataBuilderFrameworkException(
DataBuilderFrameworkException.ErrorCode.BUILDER_EXECUTION_ERROR,
"Error while waiting for error ",
e.getCause());
}
}
}
if (newlyGeneratedData.contains(dataFlow.getTargetData())) {
// logger.debug("Finished running this instance of the flow. Exiting.");
break;
}
if (newlyGeneratedData.isEmpty()) {
// logger.debug("Nothing happened in this loop, exiting..");
break;
}
// StringBuilder stringBuilder = new StringBuilder();
// for(String data : newlyGeneratedData) {
// stringBuilder.append(data + ", ");
// }
// logger.info("Newly generated: " + stringBuilder);
activeDataSet.clear();
activeDataSet.addAll(newlyGeneratedData);
newlyGeneratedData.clear();
if (!dataFlow.isLoopingEnabled()) {
break;
}
}
DataSet finalDataSet = dataSetAccessor.copy(dataFlow.getTransients());
dataFlowInstance.setDataSet(finalDataSet);
return new DataExecutionResponse(responseData);
}
@Override
public DataContainer call() throws Exception {
for (DataBuilderExecutionListener listener : dataBuilderExecutionListener) {
try {
listener.beforeExecute(dataFlowInstance, builderMeta, dataDelta, responseData);
} catch (Throwable t) {
logger.error("Error running pre-execution execution listener: ", t);
}
}
try {
Data response =
builder.process(
dataBuilderContext.immutableCopy(
dataSet.accessor().getAccesibleDataSetFor(builder)));
// logger.debug("Ran " + builderMeta.getName());
procesedBuilders.add(builderMeta);
for (DataBuilderExecutionListener listener : dataBuilderExecutionListener) {
try {
listener.afterExecute(dataFlowInstance, builderMeta, dataDelta, responseData, response);
} catch (Throwable t) {
logger.error("Error running post-execution listener: ", t);
}
}
if (null != response) {
Preconditions.checkArgument(
response.getData().equalsIgnoreCase(builderMeta.getProduces()),
String.format(
"Builder is supposed to produce %s but produces %s",
builderMeta.getProduces(), response.getData()));
response.setGeneratedBy(builderMeta.getName());
}
return new DataContainer(builderMeta, response);
} catch (DataBuilderException e) {
logger.error("Error running builder: " + builderMeta.getName());
for (DataBuilderExecutionListener listener : dataBuilderExecutionListener) {
try {
listener.afterException(dataFlowInstance, builderMeta, dataDelta, responseData, e);
} catch (Throwable error) {
logger.error("Error running post-execution listener: ", error);
}
}
return new DataContainer(
builderMeta,
new DataBuilderFrameworkException(
DataBuilderFrameworkException.ErrorCode.BUILDER_EXECUTION_ERROR,
"Error running builder: " + builderMeta.getName(),
e.getDetails(),
e));
} catch (DataValidationException e) {
logger.error("Validation error in data produced by builder" + builderMeta.getName());
for (DataBuilderExecutionListener listener : dataBuilderExecutionListener) {
try {
listener.afterException(dataFlowInstance, builderMeta, dataDelta, responseData, e);
} catch (Throwable error) {
logger.error("Error running post-execution listener: ", error);
}
}
return new DataContainer(
builderMeta,
new DataValidationException(
DataValidationException.ErrorCode.DATA_VALIDATION_EXCEPTION,
"Error running builder: " + builderMeta.getName(),
new DataExecutionResponse(responseData),
e.getDetails(),
e));
} catch (Throwable t) {
logger.error("Error running builder: " + builderMeta.getName());
for (DataBuilderExecutionListener listener : dataBuilderExecutionListener) {
try {
listener.afterException(dataFlowInstance, builderMeta, dataDelta, responseData, t);
} catch (Throwable error) {
logger.error("Error running post-execution listener: ", error);
}
}
Map<String, Object> objectMap = new HashMap<String, Object>();
objectMap.put("MESSAGE", t.getMessage());
return new DataContainer(
builderMeta,
new DataBuilderFrameworkException(
DataBuilderFrameworkException.ErrorCode.BUILDER_EXECUTION_ERROR,
"Error running builder: " + builderMeta.getName() + t.getMessage(),
objectMap,
t));
}
}