コード例 #1
0
  /** {@inheritDoc} */
  @Override
  protected DataExecutionResponse run(
      DataBuilderContext dataBuilderContext,
      DataFlowInstance dataFlowInstance,
      DataDelta dataDelta,
      DataFlow dataFlow,
      DataBuilderFactory builderFactory)
      throws DataBuilderFrameworkException, DataValidationException {
    CompletionService<DataContainer> completionExecutor =
        new ExecutorCompletionService<DataContainer>(executorService);
    ExecutionGraph executionGraph = dataFlow.getExecutionGraph();
    DataSet dataSet =
        dataFlowInstance.getDataSet().accessor().copy(); // Create own copy to work with
    DataSetAccessor dataSetAccessor = DataSet.accessor(dataSet);
    dataSetAccessor.merge(dataDelta);
    Map<String, Data> responseData = Maps.newTreeMap();
    Set<String> activeDataSet = Sets.newHashSet();

    for (Data data : dataDelta.getDelta()) {
      activeDataSet.add(data.getData());
    }
    List<List<DataBuilderMeta>> dependencyHierarchy = executionGraph.getDependencyHierarchy();
    Set<String> newlyGeneratedData = Sets.newHashSet();
    Set<DataBuilderMeta> processedBuilders =
        Collections.synchronizedSet(Sets.<DataBuilderMeta>newHashSet());
    while (true) {
      for (List<DataBuilderMeta> levelBuilders : dependencyHierarchy) {
        List<Future<DataContainer>> dataFutures = Lists.newArrayList();
        for (DataBuilderMeta builderMeta : levelBuilders) {
          if (processedBuilders.contains(builderMeta)) {
            continue;
          }
          // If there is an intersection, means some of it's inputs have changed. Reevaluate
          if (Sets.intersection(builderMeta.getConsumes(), activeDataSet).isEmpty()) {
            continue;
          }
          DataBuilder builder = builderFactory.create(builderMeta.getName());
          if (!dataSetAccessor.checkForData(builder.getDataBuilderMeta().getConsumes())) {
            break; // No need to run others, list is topo sorted
          }
          BuilderRunner builderRunner =
              new BuilderRunner(
                  dataBuilderExecutionListener,
                  dataFlowInstance,
                  builderMeta,
                  dataDelta,
                  responseData,
                  builder,
                  dataBuilderContext,
                  processedBuilders,
                  dataSet);
          Future<DataContainer> future = completionExecutor.submit(builderRunner);
          dataFutures.add(future);
        }

        // Now wait for something to complete.
        int listSize = dataFutures.size();
        for (int i = 0; i < listSize; i++) {
          try {
            DataContainer responseContainer = completionExecutor.take().get();
            Data response = responseContainer.getGeneratedData();
            if (responseContainer.isHasError()) {
              if (null != responseContainer.getValidationException()) {
                throw responseContainer.getValidationException();
              }

              throw responseContainer.getException();
            }
            if (null != response) {
              dataSetAccessor.merge(response);
              responseData.put(response.getData(), response);
              activeDataSet.add(response.getData());
              if (null != dataFlow.getTransients()
                  && !dataFlow.getTransients().contains(response.getData())) {
                newlyGeneratedData.add(response.getData());
              }
            }
          } catch (InterruptedException e) {
            throw new DataBuilderFrameworkException(
                DataBuilderFrameworkException.ErrorCode.BUILDER_EXECUTION_ERROR,
                "Error while waiting for error ",
                e);
          } catch (ExecutionException e) {
            throw new DataBuilderFrameworkException(
                DataBuilderFrameworkException.ErrorCode.BUILDER_EXECUTION_ERROR,
                "Error while waiting for error ",
                e.getCause());
          }
        }
      }
      if (newlyGeneratedData.contains(dataFlow.getTargetData())) {
        // logger.debug("Finished running this instance of the flow. Exiting.");
        break;
      }
      if (newlyGeneratedData.isEmpty()) {
        // logger.debug("Nothing happened in this loop, exiting..");
        break;
      }
      //            StringBuilder stringBuilder = new StringBuilder();
      //            for(String data : newlyGeneratedData) {
      //                stringBuilder.append(data + ", ");
      //            }
      // logger.info("Newly generated: " + stringBuilder);
      activeDataSet.clear();
      activeDataSet.addAll(newlyGeneratedData);
      newlyGeneratedData.clear();
      if (!dataFlow.isLoopingEnabled()) {
        break;
      }
    }
    DataSet finalDataSet = dataSetAccessor.copy(dataFlow.getTransients());
    dataFlowInstance.setDataSet(finalDataSet);
    return new DataExecutionResponse(responseData);
  }
コード例 #2
0
    @Override
    public DataContainer call() throws Exception {

      for (DataBuilderExecutionListener listener : dataBuilderExecutionListener) {
        try {
          listener.beforeExecute(dataFlowInstance, builderMeta, dataDelta, responseData);
        } catch (Throwable t) {
          logger.error("Error running pre-execution execution listener: ", t);
        }
      }
      try {
        Data response =
            builder.process(
                dataBuilderContext.immutableCopy(
                    dataSet.accessor().getAccesibleDataSetFor(builder)));
        // logger.debug("Ran " + builderMeta.getName());
        procesedBuilders.add(builderMeta);
        for (DataBuilderExecutionListener listener : dataBuilderExecutionListener) {
          try {
            listener.afterExecute(dataFlowInstance, builderMeta, dataDelta, responseData, response);
          } catch (Throwable t) {
            logger.error("Error running post-execution listener: ", t);
          }
        }
        if (null != response) {
          Preconditions.checkArgument(
              response.getData().equalsIgnoreCase(builderMeta.getProduces()),
              String.format(
                  "Builder is supposed to produce %s but produces %s",
                  builderMeta.getProduces(), response.getData()));
          response.setGeneratedBy(builderMeta.getName());
        }
        return new DataContainer(builderMeta, response);
      } catch (DataBuilderException e) {
        logger.error("Error running builder: " + builderMeta.getName());
        for (DataBuilderExecutionListener listener : dataBuilderExecutionListener) {
          try {
            listener.afterException(dataFlowInstance, builderMeta, dataDelta, responseData, e);

          } catch (Throwable error) {
            logger.error("Error running post-execution listener: ", error);
          }
        }
        return new DataContainer(
            builderMeta,
            new DataBuilderFrameworkException(
                DataBuilderFrameworkException.ErrorCode.BUILDER_EXECUTION_ERROR,
                "Error running builder: " + builderMeta.getName(),
                e.getDetails(),
                e));

      } catch (DataValidationException e) {
        logger.error("Validation error in data produced by builder" + builderMeta.getName());
        for (DataBuilderExecutionListener listener : dataBuilderExecutionListener) {
          try {
            listener.afterException(dataFlowInstance, builderMeta, dataDelta, responseData, e);

          } catch (Throwable error) {
            logger.error("Error running post-execution listener: ", error);
          }
        }
        return new DataContainer(
            builderMeta,
            new DataValidationException(
                DataValidationException.ErrorCode.DATA_VALIDATION_EXCEPTION,
                "Error running builder: " + builderMeta.getName(),
                new DataExecutionResponse(responseData),
                e.getDetails(),
                e));

      } catch (Throwable t) {
        logger.error("Error running builder: " + builderMeta.getName());
        for (DataBuilderExecutionListener listener : dataBuilderExecutionListener) {
          try {
            listener.afterException(dataFlowInstance, builderMeta, dataDelta, responseData, t);

          } catch (Throwable error) {
            logger.error("Error running post-execution listener: ", error);
          }
        }
        Map<String, Object> objectMap = new HashMap<String, Object>();
        objectMap.put("MESSAGE", t.getMessage());
        return new DataContainer(
            builderMeta,
            new DataBuilderFrameworkException(
                DataBuilderFrameworkException.ErrorCode.BUILDER_EXECUTION_ERROR,
                "Error running builder: " + builderMeta.getName() + t.getMessage(),
                objectMap,
                t));
      }
    }