示例#1
0
  // TODO - also use this scheme in the GPF GUIs (nf, 2012-03-02)
  // See also [BEAM-1375] Allow gpt to use template variables in parameter files
  private Map<String, String> getRawParameterMap() throws Exception {
    Map<String, String> parameterMap;
    String parameterFilePath = commandLineArgs.getParameterFilePath();
    if (parameterFilePath != null) {
      // put command line parameters in the Velocity context so that we can reference them in the
      // parameters file
      VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
      velocityContext.put("parameters", commandLineArgs.getParameterMap());

      Resource parameterFile =
          metadataResourceEngine.readResource("parameterFile", parameterFilePath);
      Map<String, String> configFilemap = parameterFile.getMap();
      if (!parameterFile.isXml()) {
        configFilemap.putAll(commandLineArgs.getParameterMap());
      }
      parameterMap = configFilemap;
    } else {
      parameterMap = new HashMap<String, String>();
    }

    // CLI parameters shall always overwrite file parameters
    parameterMap.putAll(commandLineArgs.getParameterMap());
    metadataResourceEngine.getVelocityContext().put("parameters", parameterMap);
    return parameterMap;
  }
  void processMetadata() throws Exception {
    VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
    velocityContext.put(KEY_DATE_FORMAT, new SimpleDateFormat("yyyy-MM-dd"));
    velocityContext.put(KEY_DATE, new Date());

    HashMap<String, String> metadataPaths = cliHandler.fetchGlobalMetadataFiles();
    for (String key : metadataPaths.keySet()) {
      metadataResourceEngine.readResource(key, metadataPaths.get(key));
    }

    Map<String, String> sourcePaths = cliHandler.fetchSourceItemFiles();
    for (String key : sourcePaths.keySet()) {
      metadataResourceEngine.readRelatedResource(key, sourcePaths.get(key));
    }
    velocityContext.put(KEY_XPATH, new XPathHandler());
    velocityContext.put(KEY_SOURCES, sourcePaths);

    velocityContext.put(KEY_SYSTEM, System.getProperties());
    velocityContext.put(KEY_ARGS, Arrays.asList(cliHandler.fetchArguments()));

    Map<String, String> templatePaths = cliHandler.fetchTemplateFiles();
    String outputItemPath = cliHandler.fetchTargetItemFile();
    velocityContext.put(KEY_TARGET, outputItemPath);
    for (String templateKey : templatePaths.keySet()) {
      metadataResourceEngine.writeRelatedResource(templatePaths.get(templateKey), outputItemPath);
    }
  }
示例#3
0
 private void readMetadata(String path, boolean fail) throws Exception {
   try {
     metadataResourceEngine.readResource("metadata", path);
   } catch (Exception e) {
     if (fail) {
       throw e;
     }
     final String message =
         String.format("Failed to read metadata file '%s': %s", path, e.getMessage());
     if (commandLineContext.fileExists(path)) {
       logSevereProblem(message, e);
     } else {
       commandLineContext.getLogger().warning(message);
     }
   }
 }
示例#4
0
  private void runGraph() throws Exception {
    final OperatorSpiRegistry operatorSpiRegistry =
        GPF.getDefaultInstance().getOperatorSpiRegistry();

    Map<String, String> templateVariables = getRawParameterMap();

    Map<String, String> sourceNodeIdMap = getSourceNodeIdMap();
    templateVariables.putAll(sourceNodeIdMap);
    // todo - use Velocity and the current Velocity context for reading the graph XML! (nf,
    // 20120610)
    Graph graph = readGraph(commandLineArgs.getGraphFilePath(), templateVariables);
    Node lastNode = graph.getNode(graph.getNodeCount() - 1);
    SortedMap<String, String> sourceFilePathsMap = commandLineArgs.getSourceFilePathMap();

    // For each source path add a ReadOp to the graph
    String readOperatorAlias = OperatorSpi.getOperatorAlias(ReadOp.class);
    for (Entry<String, String> entry : sourceFilePathsMap.entrySet()) {
      String sourceId = entry.getKey();
      String sourceFilePath = entry.getValue();
      String sourceNodeId = sourceNodeIdMap.get(sourceId);
      if (graph.getNode(sourceNodeId) == null) {

        DomElement configuration = new DefaultDomElement("parameters");
        configuration.createChild("file").setValue(sourceFilePath);

        Node sourceNode = new Node(sourceNodeId, readOperatorAlias);
        sourceNode.setConfiguration(configuration);

        graph.addNode(sourceNode);
      }
    }

    final String operatorName = lastNode.getOperatorName();
    final OperatorSpi lastOpSpi = operatorSpiRegistry.getOperatorSpi(operatorName);
    if (lastOpSpi == null) {
      throw new GraphException(
          String.format("Unknown operator name '%s'. No SPI found.", operatorName));
    }

    if (!Output.class.isAssignableFrom(lastOpSpi.getOperatorClass())) {

      // If the graph's last node does not implement Output, then add a WriteOp
      String writeOperatorAlias = OperatorSpi.getOperatorAlias(WriteOp.class);

      DomElement configuration = new DefaultDomElement("parameters");
      configuration.createChild("file").setValue(commandLineArgs.getTargetFilePath());
      configuration.createChild("formatName").setValue(commandLineArgs.getTargetFormatName());
      configuration
          .createChild("clearCacheAfterRowWrite")
          .setValue(Boolean.toString(commandLineArgs.isClearCacheAfterRowWrite()));

      Node targetNode = new Node(WRITE_OP_ID_PREFIX + lastNode.getId(), writeOperatorAlias);
      targetNode.addSource(new NodeSource("source", lastNode.getId()));
      targetNode.setConfiguration(configuration);

      graph.addNode(targetNode);
    }
    executeGraph(graph);
    VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
    File graphFile = new File(commandLineArgs.getGraphFilePath());
    velocityContext.put("graph", graph);

    metadataResourceEngine.readResource("graphXml", graphFile.getPath());
  }