void processMetadata() throws Exception {
VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
velocityContext.put(KEY_DATE_FORMAT, new SimpleDateFormat("yyyy-MM-dd"));
velocityContext.put(KEY_DATE, new Date());
HashMap<String, String> metadataPaths = cliHandler.fetchGlobalMetadataFiles();
for (String key : metadataPaths.keySet()) {
metadataResourceEngine.readResource(key, metadataPaths.get(key));
}
Map<String, String> sourcePaths = cliHandler.fetchSourceItemFiles();
for (String key : sourcePaths.keySet()) {
metadataResourceEngine.readRelatedResource(key, sourcePaths.get(key));
}
velocityContext.put(KEY_XPATH, new XPathHandler());
velocityContext.put(KEY_SOURCES, sourcePaths);
velocityContext.put(KEY_SYSTEM, System.getProperties());
velocityContext.put(KEY_ARGS, Arrays.asList(cliHandler.fetchArguments()));
Map<String, String> templatePaths = cliHandler.fetchTemplateFiles();
String outputItemPath = cliHandler.fetchTargetItemFile();
velocityContext.put(KEY_TARGET, outputItemPath);
for (String templateKey : templatePaths.keySet()) {
metadataResourceEngine.writeRelatedResource(templatePaths.get(templateKey), outputItemPath);
}
}
// TODO - also use this scheme in the GPF GUIs (nf, 2012-03-02)
// See also [BEAM-1375] Allow gpt to use template variables in parameter files
private Map<String, String> getRawParameterMap() throws Exception {
Map<String, String> parameterMap;
String parameterFilePath = commandLineArgs.getParameterFilePath();
if (parameterFilePath != null) {
// put command line parameters in the Velocity context so that we can reference them in the
// parameters file
VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
velocityContext.put("parameters", commandLineArgs.getParameterMap());
Resource parameterFile =
metadataResourceEngine.readResource("parameterFile", parameterFilePath);
Map<String, String> configFilemap = parameterFile.getMap();
if (!parameterFile.isXml()) {
configFilemap.putAll(commandLineArgs.getParameterMap());
}
parameterMap = configFilemap;
} else {
parameterMap = new HashMap<String, String>();
}
// CLI parameters shall always overwrite file parameters
parameterMap.putAll(commandLineArgs.getParameterMap());
metadataResourceEngine.getVelocityContext().put("parameters", parameterMap);
return parameterMap;
}
@Override
public void graphProcessingStopped(GraphContext graphContext) {
VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
velocityContext.put("graph", graphContext.getGraph());
Product[] outputProducts = graphContext.getOutputProducts();
if (outputProducts.length >= 1) {
velocityContext.put("targetProduct", outputProducts[0]);
}
velocityContext.put("targetProducts", outputProducts);
Product sourceProduct = null;
Map<String, Product> sourceProducts = new HashMap<String, Product>();
for (Node node : graphContext.getGraph().getNodes()) {
final NodeContext nodeContext = graphContext.getNodeContext(node);
if (nodeContext.getOperator() instanceof ReadOp) {
final Product product = nodeContext.getOperator().getTargetProduct();
if (sourceProduct == null) {
sourceProduct = product;
}
if (node.getId().startsWith(READ_OP_ID_PREFIX)) {
final String sourceId = node.getId().substring(READ_OP_ID_PREFIX.length());
sourceProducts.put(sourceId, product);
}
}
}
velocityContext.put("sourceProduct", sourceProduct);
velocityContext.put("sourceProducts", sourceProducts);
}
private void runGraphOrOperator() throws Exception {
VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
velocityContext.put("processingStartTime", DATETIME_FORMAT.format(new Date()));
if (commandLineArgs.getOperatorName() != null) {
// Operator name given: parameters and sources are parsed from command-line args
runOperator();
} else if (commandLineArgs.getGraphFilePath() != null) {
// Path to Graph XML given: parameters and sources are parsed from command-line args
runGraph();
}
velocityContext.put("processingStopTime", DATETIME_FORMAT.format(new Date()));
}
void readSourceMetadataFiles() {
final SortedMap<String, String> sourceFilePathMap = commandLineArgs.getSourceFilePathMap();
for (String sourceId : sourceFilePathMap.keySet()) {
final String sourcePath = sourceFilePathMap.get(sourceId);
try {
metadataResourceEngine.readRelatedResource(sourceId, sourcePath);
} catch (IOException e) {
String msgPattern = "Failed to load metadata file associated with '%s = %s': %s";
logSevereProblem(String.format(msgPattern, sourceId, sourcePath, e.getMessage()), e);
}
}
}
private void runVelocityTemplates() {
String velocityDirPath = commandLineArgs.getVelocityTemplateDirPath();
File velocityDir;
if (velocityDirPath != null) {
velocityDir = new File(velocityDirPath);
} else {
velocityDir = new File(CommandLineArgs.DEFAULT_VELOCITY_TEMPLATE_DIRPATH);
}
String[] templateNames =
velocityDir.list(
new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.toLowerCase().endsWith(CommandLineArgs.VELOCITY_TEMPLATE_EXTENSION);
}
});
Logger logger = commandLineContext.getLogger();
if (templateNames == null) {
String msgPattern = "Velocity template directory '%s' does not exist or inaccessible";
logger.severe(String.format(msgPattern, velocityDir));
return;
}
if (templateNames.length == 0) {
String msgPattern = "Velocity template directory '%s' does not contain any templates (*.vm)";
logger.warning(String.format(msgPattern, velocityDir));
return;
}
// It can happen that we have no target file when the operator implements the Output interface
if (!commandLineContext.isFile(commandLineArgs.getTargetFilePath())) {
String msgPattern =
"Target file '%s' does not exist, but is required to process velocity templates";
logger.warning(String.format(msgPattern, commandLineArgs.getTargetFilePath()));
return;
}
for (String templateName : templateNames) {
try {
metadataResourceEngine.writeRelatedResource(
velocityDir + "/" + templateName, commandLineArgs.getTargetFilePath());
} catch (IOException e) {
String msgPattern = "Can't write related resource using template file '%s': %s";
logSevereProblem(String.format(msgPattern, templateName, e.getMessage()), e);
}
}
}
private void readMetadata(String path, boolean fail) throws Exception {
try {
metadataResourceEngine.readResource("metadata", path);
} catch (Exception e) {
if (fail) {
throw e;
}
final String message =
String.format("Failed to read metadata file '%s': %s", path, e.getMessage());
if (commandLineContext.fileExists(path)) {
logSevereProblem(message, e);
} else {
commandLineContext.getLogger().warning(message);
}
}
}
private Map<String, Object> convertParameterMap(
String operatorName, Map<String, String> parameterMap) throws ValidationException {
HashMap<String, Object> parameters = new HashMap<String, Object>();
PropertyContainer container =
ParameterDescriptorFactory.createMapBackedOperatorPropertyContainer(
operatorName, parameters);
// explicitly set default values for putting them into the backing map
container.setDefaultValues();
// handle xml parameters
Object parametersObject = metadataResourceEngine.getVelocityContext().get("parameterFile");
if (parametersObject instanceof Resource) {
Resource paramatersResource = (Resource) parametersObject;
if (paramatersResource.isXml()) {
OperatorSpiRegistry operatorSpiRegistry = GPF.getDefaultInstance().getOperatorSpiRegistry();
OperatorSpi operatorSpi = operatorSpiRegistry.getOperatorSpi(operatorName);
Class<? extends Operator> operatorClass = operatorSpi.getOperatorClass();
DefaultDomConverter domConverter =
new DefaultDomConverter(operatorClass, new ParameterDescriptorFactory());
DomElement parametersElement = createDomElement(paramatersResource.getContent());
try {
domConverter.convertDomToValue(parametersElement, container);
} catch (ConversionException e) {
String msgPattern = "Can not convert XML parameters for operator '%s'";
throw new RuntimeException(String.format(msgPattern, operatorName));
}
}
}
for (Entry<String, String> entry : parameterMap.entrySet()) {
String paramName = entry.getKey();
String paramValue = entry.getValue();
final Property property = container.getProperty(paramName);
if (property != null) {
property.setValueFromText(paramValue);
} else {
throw new RuntimeException(
String.format("Parameter '%s' is not known by operator '%s'", paramName, operatorName));
}
}
return parameters;
}
private void runOperator() throws Exception {
Map<String, String> parameterMap = getRawParameterMap();
String operatorName = commandLineArgs.getOperatorName();
Map<String, Object> parameters = convertParameterMap(operatorName, parameterMap);
Map<String, Product> sourceProducts = getSourceProductMap();
Product targetProduct = createOpProduct(operatorName, parameters, sourceProducts);
// write product only if Operator does not implement the Output interface
OperatorProductReader opProductReader = null;
if (targetProduct.getProductReader() instanceof OperatorProductReader) {
opProductReader = (OperatorProductReader) targetProduct.getProductReader();
}
Operator operator =
opProductReader != null ? opProductReader.getOperatorContext().getOperator() : null;
if (operator instanceof Output) {
final OperatorExecutor executor = OperatorExecutor.create(operator);
executor.execute(ProgressMonitor.NULL);
} else {
String filePath = commandLineArgs.getTargetFilePath();
String formatName = commandLineArgs.getTargetFormatName();
writeProduct(
targetProduct, filePath, formatName, commandLineArgs.isClearCacheAfterRowWrite());
}
VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
if (operator != null) {
OperatorSpi spi = operator.getSpi();
velocityContext.put("operator", operator);
velocityContext.put("operatorSpi", spi);
velocityContext.put(
"operatorMetadata", spi.getOperatorClass().getAnnotation(OperatorMetadata.class));
}
velocityContext.put("operatorName", operatorName);
velocityContext.put("parameters", parameters); // Check if we should use parameterMap here (nf)
velocityContext.put("sourceProduct", sourceProducts.get("sourceProduct"));
velocityContext.put(
"sourceProducts", sourceProducts); // Check if we should use an array here (nf)
velocityContext.put("targetProduct", targetProduct);
velocityContext.put("targetProducts", new Product[] {targetProduct});
}
private void initVelocityContext() throws Exception {
VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
velocityContext.put("system", System.getProperties());
velocityContext.put("softwareName", "BEAM gpt");
velocityContext.put("softwareVersion", System.getProperty("beam.version", ""));
velocityContext.put("commandLineArgs", commandLineArgs);
// Derived properties (shortcuts).
// Check if we really want them, if so, we have to maintain them in the future (nf)
File targetFile = new File(commandLineArgs.getTargetFilePath());
File parentFile = targetFile.getParentFile();
velocityContext.put("targetFile", targetFile);
velocityContext.put("targetDir", parentFile != null ? parentFile : new File("."));
velocityContext.put("targetBaseName", FileUtils.getFilenameWithoutExtension(targetFile));
velocityContext.put("targetName", targetFile.getName());
velocityContext.put("targetFormat", commandLineArgs.getTargetFormatName());
// Check if we also put the following into the context?
// Actually no, because this puts the ontext in an unknown state, because we don't know which
// are the key's names (nf)
// velocityContext.putAll(commandLineArgs.getParameterMap());
// velocityContext.putAll(commandLineArgs.getTargetFilePathMap());
// velocityContext.putAll(commandLineArgs.getSourceFilePathMap());
}
private void runGraph() throws Exception {
final OperatorSpiRegistry operatorSpiRegistry =
GPF.getDefaultInstance().getOperatorSpiRegistry();
Map<String, String> templateVariables = getRawParameterMap();
Map<String, String> sourceNodeIdMap = getSourceNodeIdMap();
templateVariables.putAll(sourceNodeIdMap);
// todo - use Velocity and the current Velocity context for reading the graph XML! (nf,
// 20120610)
Graph graph = readGraph(commandLineArgs.getGraphFilePath(), templateVariables);
Node lastNode = graph.getNode(graph.getNodeCount() - 1);
SortedMap<String, String> sourceFilePathsMap = commandLineArgs.getSourceFilePathMap();
// For each source path add a ReadOp to the graph
String readOperatorAlias = OperatorSpi.getOperatorAlias(ReadOp.class);
for (Entry<String, String> entry : sourceFilePathsMap.entrySet()) {
String sourceId = entry.getKey();
String sourceFilePath = entry.getValue();
String sourceNodeId = sourceNodeIdMap.get(sourceId);
if (graph.getNode(sourceNodeId) == null) {
DomElement configuration = new DefaultDomElement("parameters");
configuration.createChild("file").setValue(sourceFilePath);
Node sourceNode = new Node(sourceNodeId, readOperatorAlias);
sourceNode.setConfiguration(configuration);
graph.addNode(sourceNode);
}
}
final String operatorName = lastNode.getOperatorName();
final OperatorSpi lastOpSpi = operatorSpiRegistry.getOperatorSpi(operatorName);
if (lastOpSpi == null) {
throw new GraphException(
String.format("Unknown operator name '%s'. No SPI found.", operatorName));
}
if (!Output.class.isAssignableFrom(lastOpSpi.getOperatorClass())) {
// If the graph's last node does not implement Output, then add a WriteOp
String writeOperatorAlias = OperatorSpi.getOperatorAlias(WriteOp.class);
DomElement configuration = new DefaultDomElement("parameters");
configuration.createChild("file").setValue(commandLineArgs.getTargetFilePath());
configuration.createChild("formatName").setValue(commandLineArgs.getTargetFormatName());
configuration
.createChild("clearCacheAfterRowWrite")
.setValue(Boolean.toString(commandLineArgs.isClearCacheAfterRowWrite()));
Node targetNode = new Node(WRITE_OP_ID_PREFIX + lastNode.getId(), writeOperatorAlias);
targetNode.addSource(new NodeSource("source", lastNode.getId()));
targetNode.setConfiguration(configuration);
graph.addNode(targetNode);
}
executeGraph(graph);
VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
File graphFile = new File(commandLineArgs.getGraphFilePath());
velocityContext.put("graph", graph);
metadataResourceEngine.readResource("graphXml", graphFile.getPath());
}