@Override
public void graphProcessingStopped(GraphContext graphContext) {
VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
velocityContext.put("graph", graphContext.getGraph());
Product[] outputProducts = graphContext.getOutputProducts();
if (outputProducts.length >= 1) {
velocityContext.put("targetProduct", outputProducts[0]);
}
velocityContext.put("targetProducts", outputProducts);
Product sourceProduct = null;
Map<String, Product> sourceProducts = new HashMap<String, Product>();
for (Node node : graphContext.getGraph().getNodes()) {
final NodeContext nodeContext = graphContext.getNodeContext(node);
if (nodeContext.getOperator() instanceof ReadOp) {
final Product product = nodeContext.getOperator().getTargetProduct();
if (sourceProduct == null) {
sourceProduct = product;
}
if (node.getId().startsWith(READ_OP_ID_PREFIX)) {
final String sourceId = node.getId().substring(READ_OP_ID_PREFIX.length());
sourceProducts.put(sourceId, product);
}
}
}
velocityContext.put("sourceProduct", sourceProduct);
velocityContext.put("sourceProducts", sourceProducts);
}
private void runGraph() throws Exception {
final OperatorSpiRegistry operatorSpiRegistry =
GPF.getDefaultInstance().getOperatorSpiRegistry();
Map<String, String> templateVariables = getRawParameterMap();
Map<String, String> sourceNodeIdMap = getSourceNodeIdMap();
templateVariables.putAll(sourceNodeIdMap);
// todo - use Velocity and the current Velocity context for reading the graph XML! (nf,
// 20120610)
Graph graph = readGraph(commandLineArgs.getGraphFilePath(), templateVariables);
Node lastNode = graph.getNode(graph.getNodeCount() - 1);
SortedMap<String, String> sourceFilePathsMap = commandLineArgs.getSourceFilePathMap();
// For each source path add a ReadOp to the graph
String readOperatorAlias = OperatorSpi.getOperatorAlias(ReadOp.class);
for (Entry<String, String> entry : sourceFilePathsMap.entrySet()) {
String sourceId = entry.getKey();
String sourceFilePath = entry.getValue();
String sourceNodeId = sourceNodeIdMap.get(sourceId);
if (graph.getNode(sourceNodeId) == null) {
DomElement configuration = new DefaultDomElement("parameters");
configuration.createChild("file").setValue(sourceFilePath);
Node sourceNode = new Node(sourceNodeId, readOperatorAlias);
sourceNode.setConfiguration(configuration);
graph.addNode(sourceNode);
}
}
final String operatorName = lastNode.getOperatorName();
final OperatorSpi lastOpSpi = operatorSpiRegistry.getOperatorSpi(operatorName);
if (lastOpSpi == null) {
throw new GraphException(
String.format("Unknown operator name '%s'. No SPI found.", operatorName));
}
if (!Output.class.isAssignableFrom(lastOpSpi.getOperatorClass())) {
// If the graph's last node does not implement Output, then add a WriteOp
String writeOperatorAlias = OperatorSpi.getOperatorAlias(WriteOp.class);
DomElement configuration = new DefaultDomElement("parameters");
configuration.createChild("file").setValue(commandLineArgs.getTargetFilePath());
configuration.createChild("formatName").setValue(commandLineArgs.getTargetFormatName());
configuration
.createChild("clearCacheAfterRowWrite")
.setValue(Boolean.toString(commandLineArgs.isClearCacheAfterRowWrite()));
Node targetNode = new Node(WRITE_OP_ID_PREFIX + lastNode.getId(), writeOperatorAlias);
targetNode.addSource(new NodeSource("source", lastNode.getId()));
targetNode.setConfiguration(configuration);
graph.addNode(targetNode);
}
executeGraph(graph);
VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
File graphFile = new File(commandLineArgs.getGraphFilePath());
velocityContext.put("graph", graph);
metadataResourceEngine.readResource("graphXml", graphFile.getPath());
}
private void testGraph(
String[] args,
int expectedNodeCount,
String expectedLog,
String expectedSourceNodeId1,
String expectedSourceFilepath1,
String expectedSourceNodeId2,
String expectedSourceFilepath2,
String expectedTargetNodeId,
String expectedTargetFilepath,
String expectedTargetFormat,
String expectedThreshold,
String expectedExpression)
throws Exception {
clTool.run(args);
assertEquals(expectedLog, context.logString);
Graph executedGraph = context.executedGraph;
assertNotNull(executedGraph);
assertEquals(expectedNodeCount, executedGraph.getNodeCount());
Node node1 = executedGraph.getNode("node1");
assertEquals(expectedSourceNodeId1, node1.getSource(0).getSourceNodeId());
assertEquals(expectedThreshold, node1.getConfiguration().getChild("threshold").getValue());
Node node2 = executedGraph.getNode("node2");
assertEquals("node1", node2.getSource(0).getSourceNodeId());
assertEquals(expectedSourceNodeId2, node2.getSource(1).getSourceNodeId());
assertEquals(expectedExpression, node2.getConfiguration().getChild("expression").getValue());
if (expectedSourceFilepath1 != null) {
Node generatedReaderNode1 = executedGraph.getNode(expectedSourceNodeId1);
assertNotNull(generatedReaderNode1);
assertEquals(
expectedSourceFilepath1,
generatedReaderNode1.getConfiguration().getChild("file").getValue());
}
if (expectedSourceFilepath2 != null) {
Node generatedReaderNode2 = executedGraph.getNode(expectedSourceNodeId2);
assertNotNull(generatedReaderNode2);
assertEquals(
expectedSourceFilepath2,
generatedReaderNode2.getConfiguration().getChild("file").getValue());
}
Node generatedWriterNode = executedGraph.getNode(expectedTargetNodeId);
assertNotNull(generatedWriterNode);
assertEquals("node2", generatedWriterNode.getSource(0).getSourceNodeId());
DomElement parameters = generatedWriterNode.getConfiguration();
assertNotNull(parameters);
assertNotNull(expectedTargetFilepath, parameters.getChild("file").getValue());
assertNotNull(expectedTargetFormat, parameters.getChild("formatName").getValue());
}
