private static void addExecutionVertices(
      Map<ExecutionGroupVertex, ManagementGroupVertex> groupMap, ExecutionGraph executionGraph) {

    ExecutionGraphIterator iterator = new ExecutionGraphIterator(executionGraph, true);
    final Map<ExecutionVertex, ManagementVertex> vertexMap =
        new HashMap<ExecutionVertex, ManagementVertex>();
    final Map<ExecutionGate, ManagementGate> gateMap = new HashMap<ExecutionGate, ManagementGate>();

    while (iterator.hasNext()) {

      final ExecutionVertex ev = iterator.next();
      final ManagementGroupVertex parent = groupMap.get(ev.getGroupVertex());

      final AbstractInstance instance = ev.getAllocatedResource().getInstance();
      final ManagementVertex managementVertex =
          new ManagementVertex(
              parent,
              ev.getID().toManagementVertexID(),
              (instance.getInstanceConnectionInfo() != null)
                  ? instance.getInstanceConnectionInfo().toString()
                  : instance.toString(),
              instance.getType().toString(),
              ev.getIndexInVertexGroup());
      managementVertex.setExecutionState(ev.getExecutionState());
      vertexMap.put(ev, managementVertex);

      for (int i = 0; i < ev.getNumberOfOutputGates(); i++) {
        final ExecutionGate outputGate = ev.getOutputGate(i);
        final ManagementGate managementGate =
            new ManagementGate(managementVertex, new ManagementGateID(), i, false);
        gateMap.put(outputGate, managementGate);
      }

      for (int i = 0; i < ev.getNumberOfInputGates(); i++) {
        final ExecutionGate inputGate = ev.getInputGate(i);
        final ManagementGate managementGate =
            new ManagementGate(managementVertex, new ManagementGateID(), i, true);
        gateMap.put(inputGate, managementGate);
      }
    }

    iterator = new ExecutionGraphIterator(executionGraph, true);

    // Setup connections
    while (iterator.hasNext()) {

      final ExecutionVertex source = iterator.next();

      for (int i = 0; i < source.getNumberOfOutputGates(); i++) {

        final ExecutionGate outputGate = source.getOutputGate(i);
        final ManagementGate manangementOutputGate = gateMap.get(outputGate);
        final ChannelType channelType = outputGate.getChannelType();

        for (int j = 0; j < outputGate.getNumberOfEdges(); j++) {

          final ExecutionEdge outputChannel = outputGate.getEdge(j);

          final ManagementGate managementInputGate = gateMap.get(outputChannel.getInputGate());

          final ManagementEdgeID sourceEdgeID =
              new ManagementEdgeID(outputChannel.getOutputChannelID());
          final ManagementEdgeID targetEdgeID =
              new ManagementEdgeID(outputChannel.getInputChannelID());
          new ManagementEdge(
              sourceEdgeID,
              targetEdgeID,
              manangementOutputGate,
              j,
              managementInputGate,
              outputChannel.getInputGateIndex(),
              channelType);
        }
      }
    }
  }
Пример #2
0
  void scheduleOrUpdateConsumers(List<List<ExecutionEdge>> allConsumers) {
    final int numConsumers = allConsumers.size();

    if (numConsumers > 1) {
      fail(
          new IllegalStateException(
              "Currently, only a single consumer group per partition is supported."));
    } else if (numConsumers == 0) {
      return;
    }

    for (ExecutionEdge edge : allConsumers.get(0)) {
      final ExecutionVertex consumerVertex = edge.getTarget();

      final Execution consumer = consumerVertex.getCurrentExecutionAttempt();
      final ExecutionState consumerState = consumer.getState();

      final IntermediateResultPartition partition = edge.getSource();

      // ----------------------------------------------------------------
      // Consumer is created => try to deploy and cache input channel
      // descriptors if there is a deployment race
      // ----------------------------------------------------------------
      if (consumerState == CREATED) {
        final Execution partitionExecution = partition.getProducer().getCurrentExecutionAttempt();

        consumerVertex.cachePartitionInfo(
            PartialInputChannelDeploymentDescriptor.fromEdge(partition, partitionExecution));

        // When deploying a consuming task, its task deployment descriptor will contain all
        // deployment information available at the respective time. It is possible that some
        // of the partitions to be consumed have not been created yet. These are updated
        // runtime via the update messages.
        //
        // TODO The current approach may send many update messages even though the consuming
        // task has already been deployed with all necessary information. We have to check
        // whether this is a problem and fix it, if it is.
        future(
            new Callable<Boolean>() {
              @Override
              public Boolean call() throws Exception {
                try {
                  consumerVertex.scheduleForExecution(
                      consumerVertex.getExecutionGraph().getScheduler(),
                      consumerVertex.getExecutionGraph().isQueuedSchedulingAllowed());
                } catch (Throwable t) {
                  fail(
                      new IllegalStateException(
                          "Could not schedule consumer " + "vertex " + consumerVertex, t));
                }

                return true;
              }
            },
            executionContext);

        // double check to resolve race conditions
        if (consumerVertex.getExecutionState() == RUNNING) {
          consumerVertex.sendPartitionInfos();
        }
      }
      // ----------------------------------------------------------------
      // Consumer is running => send update message now
      // ----------------------------------------------------------------
      else {
        if (consumerState == RUNNING) {
          final SimpleSlot consumerSlot = consumer.getAssignedResource();

          if (consumerSlot == null) {
            // The consumer has been reset concurrently
            continue;
          }

          final Instance consumerInstance = consumerSlot.getInstance();

          final ResultPartitionID partitionId =
              new ResultPartitionID(partition.getPartitionId(), attemptId);

          final Instance partitionInstance =
              partition.getProducer().getCurrentAssignedResource().getInstance();

          final ResultPartitionLocation partitionLocation;

          if (consumerInstance.equals(partitionInstance)) {
            // Consuming task is deployed to the same instance as the partition => local
            partitionLocation = ResultPartitionLocation.createLocal();
          } else {
            // Different instances => remote
            final ConnectionID connectionId =
                new ConnectionID(
                    partitionInstance.getInstanceConnectionInfo(),
                    partition.getIntermediateResult().getConnectionIndex());

            partitionLocation = ResultPartitionLocation.createRemote(connectionId);
          }

          final InputChannelDeploymentDescriptor descriptor =
              new InputChannelDeploymentDescriptor(partitionId, partitionLocation);

          final UpdatePartitionInfo updateTaskMessage =
              new UpdateTaskSinglePartitionInfo(
                  consumer.getAttemptId(), partition.getIntermediateResult().getId(), descriptor);

          sendUpdatePartitionInfoRpcCall(consumerSlot, updateTaskMessage);
        }
        // ----------------------------------------------------------------
        // Consumer is scheduled or deploying => cache input channel
        // deployment descriptors and send update message later
        // ----------------------------------------------------------------
        else if (consumerState == SCHEDULED || consumerState == DEPLOYING) {
          final Execution partitionExecution = partition.getProducer().getCurrentExecutionAttempt();

          consumerVertex.cachePartitionInfo(
              PartialInputChannelDeploymentDescriptor.fromEdge(partition, partitionExecution));

          // double check to resolve race conditions
          if (consumerVertex.getExecutionState() == RUNNING) {
            consumerVertex.sendPartitionInfos();
          }
        }
      }
    }
  }