Exemplo n.º 1
0
  private void testGraph(
      String[] args,
      int expectedNodeCount,
      String expectedLog,
      String expectedSourceNodeId1,
      String expectedSourceFilepath1,
      String expectedSourceNodeId2,
      String expectedSourceFilepath2,
      String expectedTargetNodeId,
      String expectedTargetFilepath,
      String expectedTargetFormat,
      String expectedThreshold,
      String expectedExpression)
      throws Exception {
    clTool.run(args);

    assertEquals(expectedLog, context.logString);

    Graph executedGraph = context.executedGraph;
    assertNotNull(executedGraph);
    assertEquals(expectedNodeCount, executedGraph.getNodeCount());

    Node node1 = executedGraph.getNode("node1");
    assertEquals(expectedSourceNodeId1, node1.getSource(0).getSourceNodeId());
    assertEquals(expectedThreshold, node1.getConfiguration().getChild("threshold").getValue());

    Node node2 = executedGraph.getNode("node2");
    assertEquals("node1", node2.getSource(0).getSourceNodeId());
    assertEquals(expectedSourceNodeId2, node2.getSource(1).getSourceNodeId());
    assertEquals(expectedExpression, node2.getConfiguration().getChild("expression").getValue());

    if (expectedSourceFilepath1 != null) {
      Node generatedReaderNode1 = executedGraph.getNode(expectedSourceNodeId1);
      assertNotNull(generatedReaderNode1);
      assertEquals(
          expectedSourceFilepath1,
          generatedReaderNode1.getConfiguration().getChild("file").getValue());
    }

    if (expectedSourceFilepath2 != null) {
      Node generatedReaderNode2 = executedGraph.getNode(expectedSourceNodeId2);
      assertNotNull(generatedReaderNode2);
      assertEquals(
          expectedSourceFilepath2,
          generatedReaderNode2.getConfiguration().getChild("file").getValue());
    }

    Node generatedWriterNode = executedGraph.getNode(expectedTargetNodeId);
    assertNotNull(generatedWriterNode);
    assertEquals("node2", generatedWriterNode.getSource(0).getSourceNodeId());

    DomElement parameters = generatedWriterNode.getConfiguration();
    assertNotNull(parameters);
    assertNotNull(expectedTargetFilepath, parameters.getChild("file").getValue());
    assertNotNull(expectedTargetFormat, parameters.getChild("formatName").getValue());
  }
Exemplo n.º 2
0
  private void runGraph() throws Exception {
    final OperatorSpiRegistry operatorSpiRegistry =
        GPF.getDefaultInstance().getOperatorSpiRegistry();

    Map<String, String> templateVariables = getRawParameterMap();

    Map<String, String> sourceNodeIdMap = getSourceNodeIdMap();
    templateVariables.putAll(sourceNodeIdMap);
    // todo - use Velocity and the current Velocity context for reading the graph XML! (nf,
    // 20120610)
    Graph graph = readGraph(commandLineArgs.getGraphFilePath(), templateVariables);
    Node lastNode = graph.getNode(graph.getNodeCount() - 1);
    SortedMap<String, String> sourceFilePathsMap = commandLineArgs.getSourceFilePathMap();

    // For each source path add a ReadOp to the graph
    String readOperatorAlias = OperatorSpi.getOperatorAlias(ReadOp.class);
    for (Entry<String, String> entry : sourceFilePathsMap.entrySet()) {
      String sourceId = entry.getKey();
      String sourceFilePath = entry.getValue();
      String sourceNodeId = sourceNodeIdMap.get(sourceId);
      if (graph.getNode(sourceNodeId) == null) {

        DomElement configuration = new DefaultDomElement("parameters");
        configuration.createChild("file").setValue(sourceFilePath);

        Node sourceNode = new Node(sourceNodeId, readOperatorAlias);
        sourceNode.setConfiguration(configuration);

        graph.addNode(sourceNode);
      }
    }

    final String operatorName = lastNode.getOperatorName();
    final OperatorSpi lastOpSpi = operatorSpiRegistry.getOperatorSpi(operatorName);
    if (lastOpSpi == null) {
      throw new GraphException(
          String.format("Unknown operator name '%s'. No SPI found.", operatorName));
    }

    if (!Output.class.isAssignableFrom(lastOpSpi.getOperatorClass())) {

      // If the graph's last node does not implement Output, then add a WriteOp
      String writeOperatorAlias = OperatorSpi.getOperatorAlias(WriteOp.class);

      DomElement configuration = new DefaultDomElement("parameters");
      configuration.createChild("file").setValue(commandLineArgs.getTargetFilePath());
      configuration.createChild("formatName").setValue(commandLineArgs.getTargetFormatName());
      configuration
          .createChild("clearCacheAfterRowWrite")
          .setValue(Boolean.toString(commandLineArgs.isClearCacheAfterRowWrite()));

      Node targetNode = new Node(WRITE_OP_ID_PREFIX + lastNode.getId(), writeOperatorAlias);
      targetNode.addSource(new NodeSource("source", lastNode.getId()));
      targetNode.setConfiguration(configuration);

      graph.addNode(targetNode);
    }
    executeGraph(graph);
    VelocityContext velocityContext = metadataResourceEngine.getVelocityContext();
    File graphFile = new File(commandLineArgs.getGraphFilePath());
    velocityContext.put("graph", graph);

    metadataResourceEngine.readResource("graphXml", graphFile.getPath());
  }