コード例 #1
0
ファイル: YarnClientImpl.java プロジェクト: JoeChien23/hadoop
 private void addTimelineDelegationToken(ContainerLaunchContext clc)
     throws YarnException, IOException {
   Credentials credentials = new Credentials();
   DataInputByteBuffer dibb = new DataInputByteBuffer();
   ByteBuffer tokens = clc.getTokens();
   if (tokens != null) {
     dibb.reset(tokens);
     credentials.readTokenStorageStream(dibb);
     tokens.rewind();
   }
   // If the timeline delegation token is already in the CLC, no need to add
   // one more
   for (org.apache.hadoop.security.token.Token<? extends TokenIdentifier> token :
       credentials.getAllTokens()) {
     if (token.getKind().equals(TimelineDelegationTokenIdentifier.KIND_NAME)) {
       return;
     }
   }
   org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier>
       timelineDelegationToken = getTimelineDelegationToken();
   if (timelineDelegationToken == null) {
     return;
   }
   credentials.addToken(timelineService, timelineDelegationToken);
   if (LOG.isDebugEnabled()) {
     LOG.debug("Add timline delegation token into credentials: " + timelineDelegationToken);
   }
   DataOutputBuffer dob = new DataOutputBuffer();
   credentials.writeTokenStorageToStream(dob);
   tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
   clc.setTokens(tokens);
 }
コード例 #2
0
 private Credentials parseCredentials(ApplicationSubmissionContext application)
     throws IOException {
   Credentials credentials = new Credentials();
   DataInputByteBuffer dibb = new DataInputByteBuffer();
   ByteBuffer tokens = application.getAMContainerSpec().getTokens();
   if (tokens != null) {
     dibb.reset(tokens);
     credentials.readTokenStorageStream(dibb);
     tokens.rewind();
   }
   return credentials;
 }
コード例 #3
0
  private static void loadSecureStore() throws IOException {
    if (!UserGroupInformation.isSecurityEnabled()) {
      return;
    }

    File file = new File(Constants.Files.CREDENTIALS);
    if (file.exists()) {
      Credentials credentials = new Credentials();
      try (DataInputStream input = new DataInputStream(new FileInputStream(file))) {
        credentials.readTokenStorageStream(input);
      }

      UserGroupInformation.getCurrentUser().addCredentials(credentials);
      LOG.info("Secure store updated from {}", file);
    }
  }
コード例 #4
0
  private void loadRMAppState(RMState rmState) throws Exception {
    try {
      List<ApplicationAttemptState> attempts = new ArrayList<ApplicationAttemptState>();

      for (FileStatus appDir : fs.listStatus(rmAppRoot)) {
        for (FileStatus childNodeStatus : fs.listStatus(appDir.getPath())) {
          assert childNodeStatus.isFile();
          String childNodeName = childNodeStatus.getPath().getName();
          byte[] childData = readFile(childNodeStatus.getPath(), childNodeStatus.getLen());
          if (childNodeName.startsWith(ApplicationId.appIdStrPrefix)) {
            // application
            LOG.info("Loading application from node: " + childNodeName);
            ApplicationId appId = ConverterUtils.toApplicationId(childNodeName);
            ApplicationStateDataPBImpl appStateData =
                new ApplicationStateDataPBImpl(ApplicationStateDataProto.parseFrom(childData));
            ApplicationState appState =
                new ApplicationState(
                    appStateData.getSubmitTime(),
                    appStateData.getApplicationSubmissionContext(),
                    appStateData.getUser());
            // assert child node name is same as actual applicationId
            assert appId.equals(appState.context.getApplicationId());
            rmState.appState.put(appId, appState);
          } else if (childNodeName.startsWith(ApplicationAttemptId.appAttemptIdStrPrefix)) {
            // attempt
            LOG.info("Loading application attempt from node: " + childNodeName);
            ApplicationAttemptId attemptId = ConverterUtils.toApplicationAttemptId(childNodeName);
            ApplicationAttemptStateDataPBImpl attemptStateData =
                new ApplicationAttemptStateDataPBImpl(
                    ApplicationAttemptStateDataProto.parseFrom(childData));
            Credentials credentials = null;
            if (attemptStateData.getAppAttemptTokens() != null) {
              credentials = new Credentials();
              DataInputByteBuffer dibb = new DataInputByteBuffer();
              dibb.reset(attemptStateData.getAppAttemptTokens());
              credentials.readTokenStorageStream(dibb);
            }
            ApplicationAttemptState attemptState =
                new ApplicationAttemptState(
                    attemptId, attemptStateData.getMasterContainer(), credentials);

            // assert child node name is same as application attempt id
            assert attemptId.equals(attemptState.getAttemptId());
            attempts.add(attemptState);
          } else {
            LOG.info("Unknown child node with name: " + childNodeName);
          }
        }
      }

      // go through all attempts and add them to their apps, Ideally, each
      // attempt node must have a corresponding app node, because remove
      // directory operation remove both at the same time
      for (ApplicationAttemptState attemptState : attempts) {
        ApplicationId appId = attemptState.getAttemptId().getApplicationId();
        ApplicationState appState = rmState.appState.get(appId);
        assert appState != null;
        appState.attempts.put(attemptState.getAttemptId(), attemptState);
      }
    } catch (Exception e) {
      LOG.error("Failed to load state.", e);
      throw e;
    }
  }
コード例 #5
0
  @Test
  public void testAutomaticTimelineDelegationTokenLoading() throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
    SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
    TimelineDelegationTokenIdentifier timelineDT = new TimelineDelegationTokenIdentifier();
    final Token<TimelineDelegationTokenIdentifier> dToken =
        new Token<TimelineDelegationTokenIdentifier>(
            timelineDT.getBytes(), new byte[0], timelineDT.getKind(), new Text());
    // crate a mock client
    YarnClientImpl client =
        spy(
            new YarnClientImpl() {
              @Override
              protected void serviceInit(Configuration conf) throws Exception {
                if (getConfig()
                    .getBoolean(
                        YarnConfiguration.TIMELINE_SERVICE_ENABLED,
                        YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
                  timelineServiceEnabled = true;
                  timelineClient = mock(TimelineClient.class);
                  when(timelineClient.getDelegationToken(any(String.class))).thenReturn(dToken);
                  timelineClient.init(getConfig());
                  timelineService = TimelineUtils.buildTimelineTokenService(getConfig());
                }
                this.setConfig(conf);
              }

              @Override
              protected void serviceStart() throws Exception {
                rmClient = mock(ApplicationClientProtocol.class);
              }

              @Override
              protected void serviceStop() throws Exception {}

              @Override
              public ApplicationReport getApplicationReport(ApplicationId appId) {
                ApplicationReport report = mock(ApplicationReport.class);
                when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.SUBMITTED);
                return report;
              }

              @Override
              public boolean isSecurityEnabled() {
                return true;
              }
            });
    client.init(conf);
    client.start();
    try {
      // when i == 0, timeline DT already exists, no need to get one more
      // when i == 1, timeline DT doesn't exist, need to get one more
      for (int i = 0; i < 2; ++i) {
        ApplicationSubmissionContext context = mock(ApplicationSubmissionContext.class);
        ApplicationId applicationId = ApplicationId.newInstance(0, i + 1);
        when(context.getApplicationId()).thenReturn(applicationId);
        DataOutputBuffer dob = new DataOutputBuffer();
        Credentials credentials = new Credentials();
        if (i == 0) {
          credentials.addToken(client.timelineService, dToken);
        }
        credentials.writeTokenStorageToStream(dob);
        ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
        ContainerLaunchContext clc =
            ContainerLaunchContext.newInstance(null, null, null, null, tokens, null);
        when(context.getAMContainerSpec()).thenReturn(clc);
        client.submitApplication(context);
        if (i == 0) {
          // GetTimelineDelegationToken shouldn't be called
          verify(client, never()).getTimelineDelegationToken();
        }
        // In either way, token should be there
        credentials = new Credentials();
        DataInputByteBuffer dibb = new DataInputByteBuffer();
        tokens = clc.getTokens();
        if (tokens != null) {
          dibb.reset(tokens);
          credentials.readTokenStorageStream(dibb);
          tokens.rewind();
        }
        Collection<Token<? extends TokenIdentifier>> dTokens = credentials.getAllTokens();
        Assert.assertEquals(1, dTokens.size());
        Assert.assertEquals(dToken, dTokens.iterator().next());
      }
    } finally {
      client.stop();
    }
  }