// TODO - also use this scheme in the GPF GUIs (nf, 2012-03-02)
// See also [BEAM-1375] Allow gpt to use template variables in parameter files
private Map<String, String> getRawParameterMap() throws Exception {
Map<String, String> parameterMap;
String parameterFilePath = commandLineArgs.getParameterFilePath();
if (parameterFilePath != null) {
// put command line parameters in the Velocity context so that we can reference them in the
// parameters file
VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
velocityContext.put("parameters", commandLineArgs.getParameterMap());
Resource parameterFile =
metadataResourceEngine.readResource("parameterFile", parameterFilePath);
Map<String, String> configFilemap = parameterFile.getMap();
if (!parameterFile.isXml()) {
configFilemap.putAll(commandLineArgs.getParameterMap());
}
parameterMap = configFilemap;
} else {
parameterMap = new HashMap<String, String>();
}
// CLI parameters shall always overwrite file parameters
parameterMap.putAll(commandLineArgs.getParameterMap());
metadataResourceEngine.getVelocityContext().put("parameters", parameterMap);
return parameterMap;
}
void processMetadata() throws Exception {
VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
velocityContext.put(KEY_DATE_FORMAT, new SimpleDateFormat("yyyy-MM-dd"));
velocityContext.put(KEY_DATE, new Date());
HashMap<String, String> metadataPaths = cliHandler.fetchGlobalMetadataFiles();
for (String key : metadataPaths.keySet()) {
metadataResourceEngine.readResource(key, metadataPaths.get(key));
}
Map<String, String> sourcePaths = cliHandler.fetchSourceItemFiles();
for (String key : sourcePaths.keySet()) {
metadataResourceEngine.readRelatedResource(key, sourcePaths.get(key));
}
velocityContext.put(KEY_XPATH, new XPathHandler());
velocityContext.put(KEY_SOURCES, sourcePaths);
velocityContext.put(KEY_SYSTEM, System.getProperties());
velocityContext.put(KEY_ARGS, Arrays.asList(cliHandler.fetchArguments()));
Map<String, String> templatePaths = cliHandler.fetchTemplateFiles();
String outputItemPath = cliHandler.fetchTargetItemFile();
velocityContext.put(KEY_TARGET, outputItemPath);
for (String templateKey : templatePaths.keySet()) {
metadataResourceEngine.writeRelatedResource(templatePaths.get(templateKey), outputItemPath);
}
}
private void readMetadata(String path, boolean fail) throws Exception {
try {
metadataResourceEngine.readResource("metadata", path);
} catch (Exception e) {
if (fail) {
throw e;
}
final String message =
String.format("Failed to read metadata file '%s': %s", path, e.getMessage());
if (commandLineContext.fileExists(path)) {
logSevereProblem(message, e);
} else {
commandLineContext.getLogger().warning(message);
}
}
}
private void runGraph() throws Exception {
final OperatorSpiRegistry operatorSpiRegistry =
GPF.getDefaultInstance().getOperatorSpiRegistry();
Map<String, String> templateVariables = getRawParameterMap();
Map<String, String> sourceNodeIdMap = getSourceNodeIdMap();
templateVariables.putAll(sourceNodeIdMap);
// todo - use Velocity and the current Velocity context for reading the graph XML! (nf,
// 20120610)
Graph graph = readGraph(commandLineArgs.getGraphFilePath(), templateVariables);
Node lastNode = graph.getNode(graph.getNodeCount() - 1);
SortedMap<String, String> sourceFilePathsMap = commandLineArgs.getSourceFilePathMap();
// For each source path add a ReadOp to the graph
String readOperatorAlias = OperatorSpi.getOperatorAlias(ReadOp.class);
for (Entry<String, String> entry : sourceFilePathsMap.entrySet()) {
String sourceId = entry.getKey();
String sourceFilePath = entry.getValue();
String sourceNodeId = sourceNodeIdMap.get(sourceId);
if (graph.getNode(sourceNodeId) == null) {
DomElement configuration = new DefaultDomElement("parameters");
configuration.createChild("file").setValue(sourceFilePath);
Node sourceNode = new Node(sourceNodeId, readOperatorAlias);
sourceNode.setConfiguration(configuration);
graph.addNode(sourceNode);
}
}
final String operatorName = lastNode.getOperatorName();
final OperatorSpi lastOpSpi = operatorSpiRegistry.getOperatorSpi(operatorName);
if (lastOpSpi == null) {
throw new GraphException(
String.format("Unknown operator name '%s'. No SPI found.", operatorName));
}
if (!Output.class.isAssignableFrom(lastOpSpi.getOperatorClass())) {
// If the graph's last node does not implement Output, then add a WriteOp
String writeOperatorAlias = OperatorSpi.getOperatorAlias(WriteOp.class);
DomElement configuration = new DefaultDomElement("parameters");
configuration.createChild("file").setValue(commandLineArgs.getTargetFilePath());
configuration.createChild("formatName").setValue(commandLineArgs.getTargetFormatName());
configuration
.createChild("clearCacheAfterRowWrite")
.setValue(Boolean.toString(commandLineArgs.isClearCacheAfterRowWrite()));
Node targetNode = new Node(WRITE_OP_ID_PREFIX + lastNode.getId(), writeOperatorAlias);
targetNode.addSource(new NodeSource("source", lastNode.getId()));
targetNode.setConfiguration(configuration);
graph.addNode(targetNode);
}
executeGraph(graph);
VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
File graphFile = new File(commandLineArgs.getGraphFilePath());
velocityContext.put("graph", graph);
metadataResourceEngine.readResource("graphXml", graphFile.getPath());
}