void processMetadata() throws Exception {
    VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
    velocityContext.put(KEY_DATE_FORMAT, new SimpleDateFormat("yyyy-MM-dd"));
    velocityContext.put(KEY_DATE, new Date());

    HashMap<String, String> metadataPaths = cliHandler.fetchGlobalMetadataFiles();
    for (String key : metadataPaths.keySet()) {
      metadataResourceEngine.readResource(key, metadataPaths.get(key));
    }

    Map<String, String> sourcePaths = cliHandler.fetchSourceItemFiles();
    for (String key : sourcePaths.keySet()) {
      metadataResourceEngine.readRelatedResource(key, sourcePaths.get(key));
    }
    velocityContext.put(KEY_XPATH, new XPathHandler());
    velocityContext.put(KEY_SOURCES, sourcePaths);

    velocityContext.put(KEY_SYSTEM, System.getProperties());
    velocityContext.put(KEY_ARGS, Arrays.asList(cliHandler.fetchArguments()));

    Map<String, String> templatePaths = cliHandler.fetchTemplateFiles();
    String outputItemPath = cliHandler.fetchTargetItemFile();
    velocityContext.put(KEY_TARGET, outputItemPath);
    for (String templateKey : templatePaths.keySet()) {
      metadataResourceEngine.writeRelatedResource(templatePaths.get(templateKey), outputItemPath);
    }
  }
Exemplo n.º 2
0
  // TODO - also use this scheme in the GPF GUIs (nf, 2012-03-02)
  // See also [BEAM-1375] Allow gpt to use template variables in parameter files
  private Map<String, String> getRawParameterMap() throws Exception {
    Map<String, String> parameterMap;
    String parameterFilePath = commandLineArgs.getParameterFilePath();
    if (parameterFilePath != null) {
      // put command line parameters in the Velocity context so that we can reference them in the
      // parameters file
      VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
      velocityContext.put("parameters", commandLineArgs.getParameterMap());

      Resource parameterFile =
          metadataResourceEngine.readResource("parameterFile", parameterFilePath);
      Map<String, String> configFilemap = parameterFile.getMap();
      if (!parameterFile.isXml()) {
        configFilemap.putAll(commandLineArgs.getParameterMap());
      }
      parameterMap = configFilemap;
    } else {
      parameterMap = new HashMap<String, String>();
    }

    // CLI parameters shall always overwrite file parameters
    parameterMap.putAll(commandLineArgs.getParameterMap());
    metadataResourceEngine.getVelocityContext().put("parameters", parameterMap);
    return parameterMap;
  }
Exemplo n.º 3
0
  @Override
  public void graphProcessingStopped(GraphContext graphContext) {
    VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
    velocityContext.put("graph", graphContext.getGraph());
    Product[] outputProducts = graphContext.getOutputProducts();
    if (outputProducts.length >= 1) {
      velocityContext.put("targetProduct", outputProducts[0]);
    }
    velocityContext.put("targetProducts", outputProducts);

    Product sourceProduct = null;
    Map<String, Product> sourceProducts = new HashMap<String, Product>();
    for (Node node : graphContext.getGraph().getNodes()) {
      final NodeContext nodeContext = graphContext.getNodeContext(node);
      if (nodeContext.getOperator() instanceof ReadOp) {
        final Product product = nodeContext.getOperator().getTargetProduct();
        if (sourceProduct == null) {
          sourceProduct = product;
        }
        if (node.getId().startsWith(READ_OP_ID_PREFIX)) {
          final String sourceId = node.getId().substring(READ_OP_ID_PREFIX.length());
          sourceProducts.put(sourceId, product);
        }
      }
    }
    velocityContext.put("sourceProduct", sourceProduct);
    velocityContext.put("sourceProducts", sourceProducts);
  }
Exemplo n.º 4
0
 private void runGraphOrOperator() throws Exception {
   VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
   velocityContext.put("processingStartTime", DATETIME_FORMAT.format(new Date()));
   if (commandLineArgs.getOperatorName() != null) {
     // Operator name given: parameters and sources are parsed from command-line args
     runOperator();
   } else if (commandLineArgs.getGraphFilePath() != null) {
     // Path to Graph XML given: parameters and sources are parsed from command-line args
     runGraph();
   }
   velocityContext.put("processingStopTime", DATETIME_FORMAT.format(new Date()));
 }
Exemplo n.º 5
0
 void readSourceMetadataFiles() {
   final SortedMap<String, String> sourceFilePathMap = commandLineArgs.getSourceFilePathMap();
   for (String sourceId : sourceFilePathMap.keySet()) {
     final String sourcePath = sourceFilePathMap.get(sourceId);
     try {
       metadataResourceEngine.readRelatedResource(sourceId, sourcePath);
     } catch (IOException e) {
       String msgPattern = "Failed to load metadata file associated with '%s = %s': %s";
       logSevereProblem(String.format(msgPattern, sourceId, sourcePath, e.getMessage()), e);
     }
   }
 }
Exemplo n.º 6
0
  private void runVelocityTemplates() {
    String velocityDirPath = commandLineArgs.getVelocityTemplateDirPath();
    File velocityDir;
    if (velocityDirPath != null) {
      velocityDir = new File(velocityDirPath);
    } else {
      velocityDir = new File(CommandLineArgs.DEFAULT_VELOCITY_TEMPLATE_DIRPATH);
    }

    String[] templateNames =
        velocityDir.list(
            new FilenameFilter() {
              @Override
              public boolean accept(File dir, String name) {
                return name.toLowerCase().endsWith(CommandLineArgs.VELOCITY_TEMPLATE_EXTENSION);
              }
            });

    Logger logger = commandLineContext.getLogger();
    if (templateNames == null) {
      String msgPattern = "Velocity template directory '%s' does not exist or inaccessible";
      logger.severe(String.format(msgPattern, velocityDir));
      return;
    }
    if (templateNames.length == 0) {
      String msgPattern = "Velocity template directory '%s' does not contain any templates (*.vm)";
      logger.warning(String.format(msgPattern, velocityDir));
      return;
    }

    // It can happen that we have no target file when the operator implements the Output interface
    if (!commandLineContext.isFile(commandLineArgs.getTargetFilePath())) {
      String msgPattern =
          "Target file '%s' does not exist, but is required to process velocity templates";
      logger.warning(String.format(msgPattern, commandLineArgs.getTargetFilePath()));
      return;
    }

    for (String templateName : templateNames) {
      try {
        metadataResourceEngine.writeRelatedResource(
            velocityDir + "/" + templateName, commandLineArgs.getTargetFilePath());
      } catch (IOException e) {
        String msgPattern = "Can't write related resource using template file '%s': %s";
        logSevereProblem(String.format(msgPattern, templateName, e.getMessage()), e);
      }
    }
  }
Exemplo n.º 7
0
 private void readMetadata(String path, boolean fail) throws Exception {
   try {
     metadataResourceEngine.readResource("metadata", path);
   } catch (Exception e) {
     if (fail) {
       throw e;
     }
     final String message =
         String.format("Failed to read metadata file '%s': %s", path, e.getMessage());
     if (commandLineContext.fileExists(path)) {
       logSevereProblem(message, e);
     } else {
       commandLineContext.getLogger().warning(message);
     }
   }
 }
Exemplo n.º 8
0
  private Map<String, Object> convertParameterMap(
      String operatorName, Map<String, String> parameterMap) throws ValidationException {
    HashMap<String, Object> parameters = new HashMap<String, Object>();
    PropertyContainer container =
        ParameterDescriptorFactory.createMapBackedOperatorPropertyContainer(
            operatorName, parameters);
    // explicitly set default values for putting them into the backing map
    container.setDefaultValues();

    // handle xml parameters
    Object parametersObject = metadataResourceEngine.getVelocityContext().get("parameterFile");
    if (parametersObject instanceof Resource) {
      Resource paramatersResource = (Resource) parametersObject;
      if (paramatersResource.isXml()) {
        OperatorSpiRegistry operatorSpiRegistry = GPF.getDefaultInstance().getOperatorSpiRegistry();
        OperatorSpi operatorSpi = operatorSpiRegistry.getOperatorSpi(operatorName);
        Class<? extends Operator> operatorClass = operatorSpi.getOperatorClass();
        DefaultDomConverter domConverter =
            new DefaultDomConverter(operatorClass, new ParameterDescriptorFactory());

        DomElement parametersElement = createDomElement(paramatersResource.getContent());
        try {
          domConverter.convertDomToValue(parametersElement, container);
        } catch (ConversionException e) {
          String msgPattern = "Can not convert XML parameters for operator '%s'";
          throw new RuntimeException(String.format(msgPattern, operatorName));
        }
      }
    }

    for (Entry<String, String> entry : parameterMap.entrySet()) {
      String paramName = entry.getKey();
      String paramValue = entry.getValue();
      final Property property = container.getProperty(paramName);
      if (property != null) {
        property.setValueFromText(paramValue);
      } else {
        throw new RuntimeException(
            String.format("Parameter '%s' is not known by operator '%s'", paramName, operatorName));
      }
    }
    return parameters;
  }
Exemplo n.º 9
0
 private void runOperator() throws Exception {
   Map<String, String> parameterMap = getRawParameterMap();
   String operatorName = commandLineArgs.getOperatorName();
   Map<String, Object> parameters = convertParameterMap(operatorName, parameterMap);
   Map<String, Product> sourceProducts = getSourceProductMap();
   Product targetProduct = createOpProduct(operatorName, parameters, sourceProducts);
   // write product only if Operator does not implement the Output interface
   OperatorProductReader opProductReader = null;
   if (targetProduct.getProductReader() instanceof OperatorProductReader) {
     opProductReader = (OperatorProductReader) targetProduct.getProductReader();
   }
   Operator operator =
       opProductReader != null ? opProductReader.getOperatorContext().getOperator() : null;
   if (operator instanceof Output) {
     final OperatorExecutor executor = OperatorExecutor.create(operator);
     executor.execute(ProgressMonitor.NULL);
   } else {
     String filePath = commandLineArgs.getTargetFilePath();
     String formatName = commandLineArgs.getTargetFormatName();
     writeProduct(
         targetProduct, filePath, formatName, commandLineArgs.isClearCacheAfterRowWrite());
   }
   VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
   if (operator != null) {
     OperatorSpi spi = operator.getSpi();
     velocityContext.put("operator", operator);
     velocityContext.put("operatorSpi", spi);
     velocityContext.put(
         "operatorMetadata", spi.getOperatorClass().getAnnotation(OperatorMetadata.class));
   }
   velocityContext.put("operatorName", operatorName);
   velocityContext.put("parameters", parameters); // Check if we should use parameterMap here (nf)
   velocityContext.put("sourceProduct", sourceProducts.get("sourceProduct"));
   velocityContext.put(
       "sourceProducts", sourceProducts); // Check if we should use an array here (nf)
   velocityContext.put("targetProduct", targetProduct);
   velocityContext.put("targetProducts", new Product[] {targetProduct});
 }
Exemplo n.º 10
0
  private void initVelocityContext() throws Exception {
    VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
    velocityContext.put("system", System.getProperties());
    velocityContext.put("softwareName", "BEAM gpt");
    velocityContext.put("softwareVersion", System.getProperty("beam.version", ""));
    velocityContext.put("commandLineArgs", commandLineArgs);

    // Derived properties (shortcuts).
    // Check if we really want them, if so, we have to maintain them in the future (nf)
    File targetFile = new File(commandLineArgs.getTargetFilePath());
    File parentFile = targetFile.getParentFile();
    velocityContext.put("targetFile", targetFile);
    velocityContext.put("targetDir", parentFile != null ? parentFile : new File("."));
    velocityContext.put("targetBaseName", FileUtils.getFilenameWithoutExtension(targetFile));
    velocityContext.put("targetName", targetFile.getName());
    velocityContext.put("targetFormat", commandLineArgs.getTargetFormatName());

    // Check if we also put the following into the context?
    // Actually no, because this puts the ontext in an unknown state, because we don't know which
    // are the key's names (nf)
    // velocityContext.putAll(commandLineArgs.getParameterMap());
    // velocityContext.putAll(commandLineArgs.getTargetFilePathMap());
    // velocityContext.putAll(commandLineArgs.getSourceFilePathMap());
  }
Exemplo n.º 11
0
  private void runGraph() throws Exception {
    final OperatorSpiRegistry operatorSpiRegistry =
        GPF.getDefaultInstance().getOperatorSpiRegistry();

    Map<String, String> templateVariables = getRawParameterMap();

    Map<String, String> sourceNodeIdMap = getSourceNodeIdMap();
    templateVariables.putAll(sourceNodeIdMap);
    // todo - use Velocity and the current Velocity context for reading the graph XML! (nf,
    // 20120610)
    Graph graph = readGraph(commandLineArgs.getGraphFilePath(), templateVariables);
    Node lastNode = graph.getNode(graph.getNodeCount() - 1);
    SortedMap<String, String> sourceFilePathsMap = commandLineArgs.getSourceFilePathMap();

    // For each source path add a ReadOp to the graph
    String readOperatorAlias = OperatorSpi.getOperatorAlias(ReadOp.class);
    for (Entry<String, String> entry : sourceFilePathsMap.entrySet()) {
      String sourceId = entry.getKey();
      String sourceFilePath = entry.getValue();
      String sourceNodeId = sourceNodeIdMap.get(sourceId);
      if (graph.getNode(sourceNodeId) == null) {

        DomElement configuration = new DefaultDomElement("parameters");
        configuration.createChild("file").setValue(sourceFilePath);

        Node sourceNode = new Node(sourceNodeId, readOperatorAlias);
        sourceNode.setConfiguration(configuration);

        graph.addNode(sourceNode);
      }
    }

    final String operatorName = lastNode.getOperatorName();
    final OperatorSpi lastOpSpi = operatorSpiRegistry.getOperatorSpi(operatorName);
    if (lastOpSpi == null) {
      throw new GraphException(
          String.format("Unknown operator name '%s'. No SPI found.", operatorName));
    }

    if (!Output.class.isAssignableFrom(lastOpSpi.getOperatorClass())) {

      // If the graph's last node does not implement Output, then add a WriteOp
      String writeOperatorAlias = OperatorSpi.getOperatorAlias(WriteOp.class);

      DomElement configuration = new DefaultDomElement("parameters");
      configuration.createChild("file").setValue(commandLineArgs.getTargetFilePath());
      configuration.createChild("formatName").setValue(commandLineArgs.getTargetFormatName());
      configuration
          .createChild("clearCacheAfterRowWrite")
          .setValue(Boolean.toString(commandLineArgs.isClearCacheAfterRowWrite()));

      Node targetNode = new Node(WRITE_OP_ID_PREFIX + lastNode.getId(), writeOperatorAlias);
      targetNode.addSource(new NodeSource("source", lastNode.getId()));
      targetNode.setConfiguration(configuration);

      graph.addNode(targetNode);
    }
    executeGraph(graph);
    VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
    File graphFile = new File(commandLineArgs.getGraphFilePath());
    velocityContext.put("graph", graph);

    metadataResourceEngine.readResource("graphXml", graphFile.getPath());
  }