Ejemplo n.º 1
0
  /**
   * Receives an AcknowledgeCheckpoint message and returns whether the message was associated with a
   * pending checkpoint.
   *
   * @param message Checkpoint ack from the task manager
   * @return Flag indicating whether the ack'd checkpoint was associated with a pending checkpoint.
   * @throws Exception If the checkpoint cannot be added to the completed checkpoint store.
   */
  public boolean receiveAcknowledgeMessage(AcknowledgeCheckpoint message) throws Exception {
    if (shutdown || message == null) {
      return false;
    }
    if (!job.equals(message.getJob())) {
      LOG.error("Received AcknowledgeCheckpoint message for wrong job: {}", message);
      return false;
    }

    final long checkpointId = message.getCheckpointId();

    CompletedCheckpoint completed = null;
    PendingCheckpoint checkpoint;

    // Flag indicating whether the ack message was for a known pending
    // checkpoint.
    boolean isPendingCheckpoint;

    synchronized (lock) {
      // we need to check inside the lock for being shutdown as well, otherwise we
      // get races and invalid error log messages
      if (shutdown) {
        return false;
      }

      checkpoint = pendingCheckpoints.get(checkpointId);

      if (checkpoint != null && !checkpoint.isDiscarded()) {
        isPendingCheckpoint = true;

        if (checkpoint.acknowledgeTask(
            message.getTaskExecutionId(),
            message.getState(),
            message.getStateSize(),
            null)) { // TODO: Give KV-state to the acknowledgeTask method
          if (checkpoint.isFullyAcknowledged()) {
            completed = checkpoint.toCompletedCheckpoint();

            completedCheckpointStore.addCheckpoint(completed);

            LOG.info(
                "Completed checkpoint "
                    + checkpointId
                    + " (in "
                    + completed.getDuration()
                    + " ms)");

            if (LOG.isDebugEnabled()) {
              StringBuilder builder = new StringBuilder();
              for (Map.Entry<JobVertexID, TaskState> entry : completed.getTaskStates().entrySet()) {
                builder
                    .append("JobVertexID: ")
                    .append(entry.getKey())
                    .append(" {")
                    .append(entry.getValue())
                    .append("}");
              }

              LOG.debug(builder.toString());
            }

            pendingCheckpoints.remove(checkpointId);
            rememberRecentCheckpointId(checkpointId);

            dropSubsumedCheckpoints(completed.getTimestamp());

            onFullyAcknowledgedCheckpoint(completed);

            triggerQueuedRequests();
          }
        } else {
          // checkpoint did not accept message
          LOG.error(
              "Received duplicate or invalid acknowledge message for checkpoint "
                  + checkpointId
                  + " , task "
                  + message.getTaskExecutionId());
        }
      } else if (checkpoint != null) {
        // this should not happen
        throw new IllegalStateException(
            "Received message for discarded but non-removed checkpoint " + checkpointId);
      } else {
        // message is for an unknown checkpoint, or comes too late (checkpoint disposed)
        if (recentPendingCheckpoints.contains(checkpointId)) {
          isPendingCheckpoint = true;
          LOG.warn("Received late message for now expired checkpoint attempt " + checkpointId);
        } else {
          isPendingCheckpoint = false;
        }
      }
    }

    // send the confirmation messages to the necessary targets. we do this here
    // to be outside the lock scope
    if (completed != null) {
      final long timestamp = completed.getTimestamp();

      for (ExecutionVertex ev : tasksToCommitTo) {
        Execution ee = ev.getCurrentExecutionAttempt();
        if (ee != null) {
          ExecutionAttemptID attemptId = ee.getAttemptId();
          NotifyCheckpointComplete notifyMessage =
              new NotifyCheckpointComplete(job, attemptId, checkpointId, timestamp);
          ev.sendMessageToCurrentExecution(notifyMessage, ee.getAttemptId());
        }
      }

      statsTracker.onCompletedCheckpoint(completed);
    }

    return isPendingCheckpoint;
  }
Ejemplo n.º 2
0
  void scheduleOrUpdateConsumers(List<List<ExecutionEdge>> allConsumers) {
    final int numConsumers = allConsumers.size();

    if (numConsumers > 1) {
      fail(
          new IllegalStateException(
              "Currently, only a single consumer group per partition is supported."));
    } else if (numConsumers == 0) {
      return;
    }

    for (ExecutionEdge edge : allConsumers.get(0)) {
      final ExecutionVertex consumerVertex = edge.getTarget();

      final Execution consumer = consumerVertex.getCurrentExecutionAttempt();
      final ExecutionState consumerState = consumer.getState();

      final IntermediateResultPartition partition = edge.getSource();

      // ----------------------------------------------------------------
      // Consumer is created => try to deploy and cache input channel
      // descriptors if there is a deployment race
      // ----------------------------------------------------------------
      if (consumerState == CREATED) {
        final Execution partitionExecution = partition.getProducer().getCurrentExecutionAttempt();

        consumerVertex.cachePartitionInfo(
            PartialInputChannelDeploymentDescriptor.fromEdge(partition, partitionExecution));

        // When deploying a consuming task, its task deployment descriptor will contain all
        // deployment information available at the respective time. It is possible that some
        // of the partitions to be consumed have not been created yet. These are updated
        // runtime via the update messages.
        //
        // TODO The current approach may send many update messages even though the consuming
        // task has already been deployed with all necessary information. We have to check
        // whether this is a problem and fix it, if it is.
        future(
            new Callable<Boolean>() {
              @Override
              public Boolean call() throws Exception {
                try {
                  consumerVertex.scheduleForExecution(
                      consumerVertex.getExecutionGraph().getScheduler(),
                      consumerVertex.getExecutionGraph().isQueuedSchedulingAllowed());
                } catch (Throwable t) {
                  fail(
                      new IllegalStateException(
                          "Could not schedule consumer " + "vertex " + consumerVertex, t));
                }

                return true;
              }
            },
            executionContext);

        // double check to resolve race conditions
        if (consumerVertex.getExecutionState() == RUNNING) {
          consumerVertex.sendPartitionInfos();
        }
      }
      // ----------------------------------------------------------------
      // Consumer is running => send update message now
      // ----------------------------------------------------------------
      else {
        if (consumerState == RUNNING) {
          final SimpleSlot consumerSlot = consumer.getAssignedResource();

          if (consumerSlot == null) {
            // The consumer has been reset concurrently
            continue;
          }

          final Instance consumerInstance = consumerSlot.getInstance();

          final ResultPartitionID partitionId =
              new ResultPartitionID(partition.getPartitionId(), attemptId);

          final Instance partitionInstance =
              partition.getProducer().getCurrentAssignedResource().getInstance();

          final ResultPartitionLocation partitionLocation;

          if (consumerInstance.equals(partitionInstance)) {
            // Consuming task is deployed to the same instance as the partition => local
            partitionLocation = ResultPartitionLocation.createLocal();
          } else {
            // Different instances => remote
            final ConnectionID connectionId =
                new ConnectionID(
                    partitionInstance.getInstanceConnectionInfo(),
                    partition.getIntermediateResult().getConnectionIndex());

            partitionLocation = ResultPartitionLocation.createRemote(connectionId);
          }

          final InputChannelDeploymentDescriptor descriptor =
              new InputChannelDeploymentDescriptor(partitionId, partitionLocation);

          final UpdatePartitionInfo updateTaskMessage =
              new UpdateTaskSinglePartitionInfo(
                  consumer.getAttemptId(), partition.getIntermediateResult().getId(), descriptor);

          sendUpdatePartitionInfoRpcCall(consumerSlot, updateTaskMessage);
        }
        // ----------------------------------------------------------------
        // Consumer is scheduled or deploying => cache input channel
        // deployment descriptors and send update message later
        // ----------------------------------------------------------------
        else if (consumerState == SCHEDULED || consumerState == DEPLOYING) {
          final Execution partitionExecution = partition.getProducer().getCurrentExecutionAttempt();

          consumerVertex.cachePartitionInfo(
              PartialInputChannelDeploymentDescriptor.fromEdge(partition, partitionExecution));

          // double check to resolve race conditions
          if (consumerVertex.getExecutionState() == RUNNING) {
            consumerVertex.sendPartitionInfos();
          }
        }
      }
    }
  }
Ejemplo n.º 3
0
  /**
   * Triggers a new checkpoint and uses the given timestamp as the checkpoint timestamp.
   *
   * @param timestamp The timestamp for the checkpoint.
   * @param nextCheckpointId The checkpoint ID to use for this checkpoint or <code>-1</code> if the
   *     checkpoint ID counter should be queried.
   */
  public boolean triggerCheckpoint(long timestamp, long nextCheckpointId) throws Exception {
    // make some eager pre-checks
    synchronized (lock) {
      // abort if the coordinator has been shutdown in the meantime
      if (shutdown) {
        return false;
      }

      // sanity check: there should never be more than one trigger request queued
      if (triggerRequestQueued) {
        LOG.warn("Trying to trigger another checkpoint while one was queued already");
        return false;
      }

      // if too many checkpoints are currently in progress, we need to mark that a request is queued
      if (pendingCheckpoints.size() >= maxConcurrentCheckpointAttempts) {
        triggerRequestQueued = true;
        if (currentPeriodicTrigger != null) {
          currentPeriodicTrigger.cancel();
          currentPeriodicTrigger = null;
        }
        return false;
      }

      // make sure the minimum interval between checkpoints has passed
      if (lastTriggeredCheckpoint + minPauseBetweenCheckpoints > timestamp) {
        if (currentPeriodicTrigger != null) {
          currentPeriodicTrigger.cancel();
          currentPeriodicTrigger = null;
        }
        ScheduledTrigger trigger = new ScheduledTrigger();
        timer.scheduleAtFixedRate(trigger, minPauseBetweenCheckpoints, baseInterval);
        return false;
      }
    }

    // first check if all tasks that we need to trigger are running.
    // if not, abort the checkpoint
    ExecutionAttemptID[] triggerIDs = new ExecutionAttemptID[tasksToTrigger.length];
    for (int i = 0; i < tasksToTrigger.length; i++) {
      Execution ee = tasksToTrigger[i].getCurrentExecutionAttempt();
      if (ee != null && ee.getState() == ExecutionState.RUNNING) {
        triggerIDs[i] = ee.getAttemptId();
      } else {
        LOG.info(
            "Checkpoint triggering task {} is not being executed at the moment. Aborting checkpoint.",
            tasksToTrigger[i].getSimpleName());
        return false;
      }
    }

    // next, check if all tasks that need to acknowledge the checkpoint are running.
    // if not, abort the checkpoint
    Map<ExecutionAttemptID, ExecutionVertex> ackTasks = new HashMap<>(tasksToWaitFor.length);

    for (ExecutionVertex ev : tasksToWaitFor) {
      Execution ee = ev.getCurrentExecutionAttempt();
      if (ee != null) {
        ackTasks.put(ee.getAttemptId(), ev);
      } else {
        LOG.info(
            "Checkpoint acknowledging task {} is not being executed at the moment. Aborting checkpoint.",
            ev.getSimpleName());
        return false;
      }
    }

    // we will actually trigger this checkpoint!

    lastTriggeredCheckpoint = timestamp;
    final long checkpointID;
    if (nextCheckpointId < 0) {
      try {
        // this must happen outside the locked scope, because it communicates
        // with external services (in HA mode) and may block for a while.
        checkpointID = checkpointIdCounter.getAndIncrement();
      } catch (Throwable t) {
        int numUnsuccessful = ++numUnsuccessfulCheckpointsTriggers;
        LOG.warn(
            "Failed to trigger checkpoint ("
                + numUnsuccessful
                + " consecutive failed attempts so far)",
            t);
        return false;
      }
    } else {
      checkpointID = nextCheckpointId;
    }

    LOG.info("Triggering checkpoint " + checkpointID + " @ " + timestamp);

    final PendingCheckpoint checkpoint =
        new PendingCheckpoint(job, checkpointID, timestamp, ackTasks);

    // schedule the timer that will clean up the expired checkpoints
    TimerTask canceller =
        new TimerTask() {
          @Override
          public void run() {
            try {
              synchronized (lock) {
                // only do the work if the checkpoint is not discarded anyways
                // note that checkpoint completion discards the pending checkpoint object
                if (!checkpoint.isDiscarded()) {
                  LOG.info("Checkpoint " + checkpointID + " expired before completing.");

                  checkpoint.discard(userClassLoader);
                  pendingCheckpoints.remove(checkpointID);
                  rememberRecentCheckpointId(checkpointID);

                  onCancelCheckpoint(checkpointID);

                  triggerQueuedRequests();
                }
              }
            } catch (Throwable t) {
              LOG.error("Exception while handling checkpoint timeout", t);
            }
          }
        };

    try {
      // re-acquire the lock
      synchronized (lock) {
        // since we released the lock in the meantime, we need to re-check
        // that the conditions still hold. this is clumsy, but it allows us to
        // release the lock in the meantime while calls to external services are
        // blocking progress, and still gives us early checks that skip work
        // if no checkpoint can happen anyways
        if (shutdown) {
          return false;
        } else if (triggerRequestQueued) {
          LOG.warn("Trying to trigger another checkpoint while one was queued already");
          return false;
        } else if (pendingCheckpoints.size() >= maxConcurrentCheckpointAttempts) {
          triggerRequestQueued = true;
          if (currentPeriodicTrigger != null) {
            currentPeriodicTrigger.cancel();
            currentPeriodicTrigger = null;
          }
          return false;
        }

        pendingCheckpoints.put(checkpointID, checkpoint);
        timer.schedule(canceller, checkpointTimeout);
      }
      // end of lock scope

      // send the messages to the tasks that trigger their checkpoint
      for (int i = 0; i < tasksToTrigger.length; i++) {
        ExecutionAttemptID id = triggerIDs[i];
        TriggerCheckpoint message = new TriggerCheckpoint(job, id, checkpointID, timestamp);
        tasksToTrigger[i].sendMessageToCurrentExecution(message, id);
      }

      numUnsuccessfulCheckpointsTriggers = 0;
      return true;
    } catch (Throwable t) {
      // guard the map against concurrent modifications
      synchronized (lock) {
        pendingCheckpoints.remove(checkpointID);
      }

      int numUnsuccessful = ++numUnsuccessfulCheckpointsTriggers;
      LOG.warn(
          "Failed to trigger checkpoint ("
              + numUnsuccessful
              + " consecutive failed attempts so far)",
          t);
      if (!checkpoint.isDiscarded()) {
        checkpoint.discard(userClassLoader);
      }
      return false;
    }
  }