Beispiel #1
0
  @Test
  public void testGenerateScopeCustom() {
    MetricRegistry registry = new MetricRegistry(new Configuration());

    TaskManagerScopeFormat tmFormat = new TaskManagerScopeFormat("abc");
    TaskManagerJobScopeFormat jmFormat = new TaskManagerJobScopeFormat("def", tmFormat);
    TaskScopeFormat taskFormat =
        new TaskScopeFormat("<tm_id>.<job_id>.<task_id>.<task_attempt_id>", jmFormat);

    JobID jid = new JobID();
    AbstractID vertexId = new AbstractID();
    AbstractID executionId = new AbstractID();

    TaskManagerMetricGroup tmGroup =
        new TaskManagerMetricGroup(registry, "theHostName", "test-tm-id");
    JobMetricGroup jmGroup = new JobMetricGroup(registry, tmGroup, jid, "myJobName");
    TaskMetricGroup taskGroup =
        new TaskMetricGroup(
            registry, jmGroup, taskFormat, vertexId, executionId, "aTaskName", 13, 2);

    assertArrayEquals(
        new String[] {"test-tm-id", jid.toString(), vertexId.toString(), executionId.toString()},
        taskGroup.getScopeComponents());

    assertEquals(
        String.format("test-tm-id.%s.%s.%s", jid, vertexId, executionId),
        taskGroup.getScopeString());
  }
  @Test
  public void testTriggerSavepointFailure() throws Exception {
    replaceStdOutAndStdErr();

    try {
      JobID jobId = new JobID();
      ActorGateway jobManager = mock(ActorGateway.class);

      Promise<Object> triggerResponse = new scala.concurrent.impl.Promise.DefaultPromise<>();

      when(jobManager.ask(
              Mockito.eq(new JobManagerMessages.TriggerSavepoint(jobId)),
              Mockito.any(FiniteDuration.class)))
          .thenReturn(triggerResponse.future());

      Exception testException = new Exception("expectedTestException");

      triggerResponse.success(new JobManagerMessages.TriggerSavepointFailure(jobId, testException));

      CliFrontend frontend = new MockCliFrontend(CliFrontendTestUtils.getConfigDir(), jobManager);

      String[] parameters = {jobId.toString()};
      int returnCode = frontend.savepoint(parameters);

      assertTrue(returnCode != 0);
      verify(jobManager, times(1))
          .ask(
              Mockito.eq(new JobManagerMessages.TriggerSavepoint(jobId)),
              Mockito.any(FiniteDuration.class));

      assertTrue(buffer.toString().contains("expectedTestException"));
    } finally {
      restoreStdOutAndStdErr();
    }
  }
  /**
   * Handles an incoming DELETE request from a BLOB client.
   *
   * @param inputStream The input stream to read the request from.
   * @param outputStream The output stream to write the response to.
   * @throws java.io.IOException Thrown if an I/O error occurs while reading the request data from
   *     the input stream.
   */
  private void delete(InputStream inputStream, OutputStream outputStream, byte[] buf)
      throws IOException {

    try {
      int type = inputStream.read();
      if (type < 0) {
        throw new EOFException("Premature end of DELETE request");
      }

      if (type == CONTENT_ADDRESSABLE) {
        BlobKey key = BlobKey.readFromInputStream(inputStream);
        File blobFile = this.blobServer.getStorageLocation(key);
        if (blobFile.exists() && !blobFile.delete()) {
          throw new IOException("Cannot delete BLOB file " + blobFile.getAbsolutePath());
        }
      } else if (type == NAME_ADDRESSABLE) {
        byte[] jidBytes = new byte[JobID.SIZE];
        readFully(inputStream, jidBytes, 0, JobID.SIZE, "JobID");
        JobID jobID = JobID.fromByteArray(jidBytes);

        String key = readKey(buf, inputStream);

        File blobFile = this.blobServer.getStorageLocation(jobID, key);
        if (blobFile.exists() && !blobFile.delete()) {
          throw new IOException("Cannot delete BLOB file " + blobFile.getAbsolutePath());
        }
      } else if (type == JOB_ID_SCOPE) {
        byte[] jidBytes = new byte[JobID.SIZE];
        readFully(inputStream, jidBytes, 0, JobID.SIZE, "JobID");
        JobID jobID = JobID.fromByteArray(jidBytes);

        blobServer.deleteJobDirectory(jobID);
      } else {
        throw new IOException("Unrecognized addressing type: " + type);
      }

      outputStream.write(RETURN_OKAY);
    } catch (Throwable t) {
      LOG.error("DELETE operation failed", t);
      try {
        writeErrorToStream(outputStream, t);
      } catch (IOException e) {
        // since we are in an exception case, it means not much that we could not send the error
        // ignore this
      }
      clientSocket.close();
    }
  }
 protected String internalJobId() {
   return jobID.toString();
 }
  /**
   * Receives an AcknowledgeCheckpoint message and returns whether the message was associated with a
   * pending checkpoint.
   *
   * @param message Checkpoint ack from the task manager
   * @return Flag indicating whether the ack'd checkpoint was associated with a pending checkpoint.
   * @throws Exception If the checkpoint cannot be added to the completed checkpoint store.
   */
  public boolean receiveAcknowledgeMessage(AcknowledgeCheckpoint message) throws Exception {
    if (shutdown || message == null) {
      return false;
    }
    if (!job.equals(message.getJob())) {
      LOG.error("Received AcknowledgeCheckpoint message for wrong job: {}", message);
      return false;
    }

    final long checkpointId = message.getCheckpointId();

    CompletedCheckpoint completed = null;
    PendingCheckpoint checkpoint;

    // Flag indicating whether the ack message was for a known pending
    // checkpoint.
    boolean isPendingCheckpoint;

    synchronized (lock) {
      // we need to check inside the lock for being shutdown as well, otherwise we
      // get races and invalid error log messages
      if (shutdown) {
        return false;
      }

      checkpoint = pendingCheckpoints.get(checkpointId);

      if (checkpoint != null && !checkpoint.isDiscarded()) {
        isPendingCheckpoint = true;

        if (checkpoint.acknowledgeTask(
            message.getTaskExecutionId(),
            message.getState(),
            message.getStateSize(),
            null)) { // TODO: Give KV-state to the acknowledgeTask method
          if (checkpoint.isFullyAcknowledged()) {
            completed = checkpoint.toCompletedCheckpoint();

            completedCheckpointStore.addCheckpoint(completed);

            LOG.info(
                "Completed checkpoint "
                    + checkpointId
                    + " (in "
                    + completed.getDuration()
                    + " ms)");

            if (LOG.isDebugEnabled()) {
              StringBuilder builder = new StringBuilder();
              for (Map.Entry<JobVertexID, TaskState> entry : completed.getTaskStates().entrySet()) {
                builder
                    .append("JobVertexID: ")
                    .append(entry.getKey())
                    .append(" {")
                    .append(entry.getValue())
                    .append("}");
              }

              LOG.debug(builder.toString());
            }

            pendingCheckpoints.remove(checkpointId);
            rememberRecentCheckpointId(checkpointId);

            dropSubsumedCheckpoints(completed.getTimestamp());

            onFullyAcknowledgedCheckpoint(completed);

            triggerQueuedRequests();
          }
        } else {
          // checkpoint did not accept message
          LOG.error(
              "Received duplicate or invalid acknowledge message for checkpoint "
                  + checkpointId
                  + " , task "
                  + message.getTaskExecutionId());
        }
      } else if (checkpoint != null) {
        // this should not happen
        throw new IllegalStateException(
            "Received message for discarded but non-removed checkpoint " + checkpointId);
      } else {
        // message is for an unknown checkpoint, or comes too late (checkpoint disposed)
        if (recentPendingCheckpoints.contains(checkpointId)) {
          isPendingCheckpoint = true;
          LOG.warn("Received late message for now expired checkpoint attempt " + checkpointId);
        } else {
          isPendingCheckpoint = false;
        }
      }
    }

    // send the confirmation messages to the necessary targets. we do this here
    // to be outside the lock scope
    if (completed != null) {
      final long timestamp = completed.getTimestamp();

      for (ExecutionVertex ev : tasksToCommitTo) {
        Execution ee = ev.getCurrentExecutionAttempt();
        if (ee != null) {
          ExecutionAttemptID attemptId = ee.getAttemptId();
          NotifyCheckpointComplete notifyMessage =
              new NotifyCheckpointComplete(job, attemptId, checkpointId, timestamp);
          ev.sendMessageToCurrentExecution(notifyMessage, ee.getAttemptId());
        }
      }

      statsTracker.onCompletedCheckpoint(completed);
    }

    return isPendingCheckpoint;
  }
  /**
   * Receives a {@link DeclineCheckpoint} message and returns whether the message was associated
   * with a pending checkpoint.
   *
   * @param message Checkpoint decline from the task manager
   * @return Flag indicating whether the declined checkpoint was associated with a pending
   *     checkpoint.
   */
  public boolean receiveDeclineMessage(DeclineCheckpoint message) throws Exception {
    if (shutdown || message == null) {
      return false;
    }
    if (!job.equals(message.getJob())) {
      LOG.error("Received DeclineCheckpoint message for wrong job: {}", message);
      return false;
    }

    final long checkpointId = message.getCheckpointId();

    PendingCheckpoint checkpoint;

    // Flag indicating whether the ack message was for a known pending
    // checkpoint.
    boolean isPendingCheckpoint;

    synchronized (lock) {
      // we need to check inside the lock for being shutdown as well, otherwise we
      // get races and invalid error log messages
      if (shutdown) {
        return false;
      }

      checkpoint = pendingCheckpoints.get(checkpointId);

      if (checkpoint != null && !checkpoint.isDiscarded()) {
        isPendingCheckpoint = true;

        LOG.info(
            "Discarding checkpoint "
                + checkpointId
                + " because of checkpoint decline from task "
                + message.getTaskExecutionId());

        pendingCheckpoints.remove(checkpointId);
        checkpoint.discard(userClassLoader);
        rememberRecentCheckpointId(checkpointId);

        boolean haveMoreRecentPending = false;
        Iterator<Map.Entry<Long, PendingCheckpoint>> entries =
            pendingCheckpoints.entrySet().iterator();
        while (entries.hasNext()) {
          PendingCheckpoint p = entries.next().getValue();
          if (!p.isDiscarded()
              && p.getCheckpointTimestamp() >= checkpoint.getCheckpointTimestamp()) {
            haveMoreRecentPending = true;
            break;
          }
        }
        if (!haveMoreRecentPending && !triggerRequestQueued) {
          LOG.info("Triggering new checkpoint because of discarded checkpoint " + checkpointId);
          triggerCheckpoint(System.currentTimeMillis());
        } else if (!haveMoreRecentPending) {
          LOG.info(
              "Promoting queued checkpoint request because of discarded checkpoint "
                  + checkpointId);
          triggerQueuedRequests();
        }
      } else if (checkpoint != null) {
        // this should not happen
        throw new IllegalStateException(
            "Received message for discarded but non-removed checkpoint " + checkpointId);
      } else {
        // message is for an unknown checkpoint, or comes too late (checkpoint disposed)
        if (recentPendingCheckpoints.contains(checkpointId)) {
          isPendingCheckpoint = true;
          LOG.info(
              "Received another decline checkpoint message for now expired checkpoint attempt "
                  + checkpointId);
        } else {
          isPendingCheckpoint = false;
        }
      }
    }

    return isPendingCheckpoint;
  }
  /**
   * Handles an incoming PUT request from a BLOB client.
   *
   * @param inputStream The input stream to read incoming data from.
   * @param outputStream The output stream to send data back to the client.
   * @param buf An auxiliary buffer for data serialization/deserialization.
   */
  private void put(InputStream inputStream, OutputStream outputStream, byte[] buf)
      throws IOException {
    JobID jobID = null;
    String key = null;
    MessageDigest md = null;

    File incomingFile = null;
    FileOutputStream fos = null;

    try {
      final int contentAddressable = inputStream.read();
      if (contentAddressable < 0) {
        throw new EOFException("Premature end of PUT request");
      }

      if (contentAddressable == NAME_ADDRESSABLE) {
        // Receive the job ID and key
        byte[] jidBytes = new byte[JobID.SIZE];
        readFully(inputStream, jidBytes, 0, JobID.SIZE, "JobID");
        jobID = JobID.fromByteArray(jidBytes);
        key = readKey(buf, inputStream);
      } else if (contentAddressable == CONTENT_ADDRESSABLE) {
        md = BlobUtils.createMessageDigest();
      } else {
        throw new IOException("Unknown type of BLOB addressing.");
      }

      if (LOG.isDebugEnabled()) {
        if (contentAddressable == NAME_ADDRESSABLE) {
          LOG.debug(String.format("Received PUT request for BLOB under %s / \"%s\"", jobID, key));
        } else {
          LOG.debug("Received PUT request for content addressable BLOB");
        }
      }

      incomingFile = blobServer.createTemporaryFilename();
      fos = new FileOutputStream(incomingFile);

      while (true) {
        final int bytesExpected = readLength(inputStream);
        if (bytesExpected == -1) {
          // done
          break;
        }
        if (bytesExpected > BUFFER_SIZE) {
          throw new IOException("Unexpected number of incoming bytes: " + bytesExpected);
        }

        readFully(inputStream, buf, 0, bytesExpected, "buffer");
        fos.write(buf, 0, bytesExpected);

        if (md != null) {
          md.update(buf, 0, bytesExpected);
        }
      }
      fos.close();

      if (contentAddressable == NAME_ADDRESSABLE) {
        File storageFile = this.blobServer.getStorageLocation(jobID, key);
        Files.move(incomingFile, storageFile);
        incomingFile = null;
        outputStream.write(RETURN_OKAY);
      } else {
        BlobKey blobKey = new BlobKey(md.digest());
        File storageFile = blobServer.getStorageLocation(blobKey);
        Files.move(incomingFile, storageFile);
        incomingFile = null;

        // Return computed key to client for validation
        outputStream.write(RETURN_OKAY);
        blobKey.writeToOutputStream(outputStream);
      }
    } catch (SocketException e) {
      // happens when the other side disconnects
      LOG.debug("Socket connection closed", e);
    } catch (Throwable t) {
      LOG.error("PUT operation failed", t);
      try {
        writeErrorToStream(outputStream, t);
      } catch (IOException e) {
        // since we are in an exception case, it means not much that we could not send the error
        // ignore this
      }
      clientSocket.close();
    } finally {
      if (fos != null) {
        try {
          fos.close();
        } catch (Throwable t) {
          LOG.warn("Cannot close stream to BLOB staging file", t);
        }
      }
      if (incomingFile != null) {
        if (!incomingFile.delete()) {
          LOG.warn("Cannot delete BLOB server staging file " + incomingFile.getAbsolutePath());
        }
      }
    }
  }
  /**
   * Handles an incoming GET request from a BLOB client.
   *
   * @param inputStream the input stream to read incoming data from
   * @param outputStream the output stream to send data back to the client
   * @param buf an auxiliary buffer for data serialization/deserialization
   * @throws IOException thrown if an I/O error occurs while reading/writing data from/to the
   *     respective streams
   */
  private void get(InputStream inputStream, OutputStream outputStream, byte[] buf)
      throws IOException {

    File blobFile;
    try {
      final int contentAddressable = inputStream.read();

      if (contentAddressable < 0) {
        throw new EOFException("Premature end of GET request");
      }
      if (contentAddressable == NAME_ADDRESSABLE) {
        // Receive the job ID and key
        byte[] jidBytes = new byte[JobID.SIZE];
        readFully(inputStream, jidBytes, 0, JobID.SIZE, "JobID");

        JobID jobID = JobID.fromByteArray(jidBytes);
        String key = readKey(buf, inputStream);
        blobFile = this.blobServer.getStorageLocation(jobID, key);
      } else if (contentAddressable == CONTENT_ADDRESSABLE) {
        final BlobKey key = BlobKey.readFromInputStream(inputStream);
        blobFile = blobServer.getStorageLocation(key);
      } else {
        throw new IOException("Unknown type of BLOB addressing.");
      }

      // Check if BLOB exists
      if (!blobFile.exists()) {
        throw new IOException("Cannot find required BLOB at " + blobFile.getAbsolutePath());
      }
      if (blobFile.length() > Integer.MAX_VALUE) {
        throw new IOException("BLOB size exceeds the maximum size (2 GB).");
      }

      outputStream.write(RETURN_OKAY);

      // up to here, an error can give a good message
    } catch (Throwable t) {
      LOG.error("GET operation failed", t);
      try {
        writeErrorToStream(outputStream, t);
      } catch (IOException e) {
        // since we are in an exception case, it means not much that we could not send the error
        // ignore this
      }
      clientSocket.close();
      return;
    }

    // from here on, we started sending data, so all we can do is close the connection when
    // something happens
    try {
      int blobLen = (int) blobFile.length();
      writeLength(blobLen, outputStream);

      FileInputStream fis = new FileInputStream(blobFile);
      try {
        int bytesRemaining = blobLen;
        while (bytesRemaining > 0) {
          int read = fis.read(buf);
          if (read < 0) {
            throw new IOException(
                "Premature end of BLOB file stream for " + blobFile.getAbsolutePath());
          }
          outputStream.write(buf, 0, read);
          bytesRemaining -= read;
        }
      } finally {
        fis.close();
      }
    } catch (SocketException e) {
      // happens when the other side disconnects
      LOG.debug("Socket connection closed", e);
    } catch (Throwable t) {
      LOG.error("GET operation failed", t);
      clientSocket.close();
    }
  }