Example #1
0
 public List<MaterialRevision> addDependencyRevisionModification(
     List<MaterialRevision> materialRevisions,
     DependencyMaterial dependencyMaterial,
     Pipeline... upstreams) {
   String stageName = CaseInsensitiveString.str(dependencyMaterial.getStageName());
   String label = upstreams[0].getLabel();
   List<Modification> modifications = new ArrayList<Modification>();
   for (Pipeline upstream : upstreams) {
     modifications.add(
         new Modification(
             new Date(),
             DependencyMaterialRevision.create(
                     CaseInsensitiveString.str(dependencyMaterial.getPipelineName()),
                     upstream.getCounter(),
                     label,
                     stageName,
                     upstream.findStage(stageName).getCounter())
                 .getRevision(),
             label,
             upstream.getId()));
   }
   MaterialRevision depRev =
       addRevisionsWithModifications(
           dependencyMaterial, modifications.toArray(new Modification[0]));
   materialRevisions.add(depRev);
   return Arrays.asList(depRev);
 }
  @Test
  public void
      shouldUpdateResultOfStageWhenJobCompletes_irrespectiveOfOtherThreadsPrimingStageCache()
          throws Exception {
    Pipeline assigned = preCondition.createPipelineWithFirstStageAssigned();

    Stage stage = assigned.findStage(preCondition.devStage);
    StageSummaryModel model =
        stageService.findStageSummaryByIdentifier(
            stage.getIdentifier(),
            new Username(new CaseInsensitiveString("foo")),
            new HttpLocalizedOperationResult());

    JobIdentifier identifier = stage.getFirstJob().getIdentifier();
    scheduleService.updateJobStatus(identifier, JobState.Building);
    scheduleService.jobCompleting(identifier, JobResult.Passed, "uuid");

    Stage stageLoadedByOtherFlows = stageDao.stageById(stage.getId()); // priming the cache

    scheduleService.updateJobStatus(stage.getFirstJob().getIdentifier(), JobState.Completed);

    StageSummaryModel reloadedModel =
        stageService.findStageSummaryByIdentifier(
            stage.getIdentifier(),
            new Username(new CaseInsensitiveString("foo")),
            new HttpLocalizedOperationResult());
    Stage reloadedStage = reloadedModel.getStage();
    assertThat(reloadedStage.getFirstJob().getState(), is(JobState.Completed));
    assertThat(
        reloadedStage.getCompletedByTransitionId(),
        is(reloadedStage.getFirstJob().getTransitions().byState(JobState.Completed).getId()));
    assertThat(reloadedStage.getResult(), is(StageResult.Passed));
    assertThat(reloadedStage.getState(), is(StageState.Passed));
  }
  @Test
  public void shouldUpdateResultOfStageWhenJobCompletes() throws Exception {
    //        ScheduleService service = new ScheduleService(goConfigService, pipelineService,
    // stageService, currentActivityService, schedulingChecker, pipelineScheduledTopic, pipelineDao,
    //                stageDao, stageOrderService, securityService, pipelineScheduleQueue,
    // jobInstanceService, jobInstanceDao, agentAssignment, environmentConfigService,
    //                pipelineLockService, serverHealthService, transactionTemplate, agentService);

    Pipeline assigned = preCondition.createPipelineWithFirstStageAssigned();

    Stage stage = assigned.findStage(preCondition.devStage);
    StageSummaryModel model =
        stageService.findStageSummaryByIdentifier(
            stage.getIdentifier(),
            new Username(new CaseInsensitiveString("foo")),
            new HttpLocalizedOperationResult());
    assertThat(model.getStage().getFirstJob().getState(), is(JobState.Assigned));
    scheduleService.updateJobStatus(stage.getFirstJob().getIdentifier(), JobState.Building);
    StageSummaryModel reloadedModel =
        stageService.findStageSummaryByIdentifier(
            stage.getIdentifier(),
            new Username(new CaseInsensitiveString("foo")),
            new HttpLocalizedOperationResult());
    assertThat(reloadedModel.getStage().getFirstJob().getState(), is(JobState.Building));
  }
  @Test
  public void shouldScheduleIfAgentMatchingResources() throws Exception {
    JobConfig plan =
        evolveConfig
            .findBy(new CaseInsensitiveString(STAGE_NAME))
            .jobConfigByInstanceName("unit", true);
    plan.addResource("some-resource");

    scheduleHelper.schedule(evolveConfig, modifySomeFiles(evolveConfig), DEFAULT_APPROVED_BY);

    AgentConfig agentConfig = AgentMother.localAgent();
    agentConfig.addResource(new Resource("some-resource"));

    buildAssignmentService.onTimer();
    Work work = buildAssignmentService.assignWorkToAgent(agent(agentConfig));
    assertThat(work, is(not((Work) BuildAssignmentService.NO_WORK)));

    Pipeline pipeline =
        pipelineDao.mostRecentPipeline(CaseInsensitiveString.str(evolveConfig.name()));
    JobInstance job = pipeline.findStage(STAGE_NAME).findJob("unit");

    JobPlan loadedPlan = jobInstanceDao.loadPlan(job.getId());
    assertThat(loadedPlan.getResources(), is((List<Resource>) plan.resources()));

    assertThat(job.getState(), is(JobState.Assigned));
    assertThat(job.getAgentUuid(), is(agentConfig.getUuid()));
  }
  @Test
  public void shouldReScheduleToCorrectAgent() throws Exception {
    JobConfig plan =
        evolveConfig
            .findBy(new CaseInsensitiveString(STAGE_NAME))
            .jobConfigByInstanceName("unit", true);
    plan.addResource("some-resource");

    scheduleHelper.schedule(evolveConfig, modifySomeFiles(evolveConfig), DEFAULT_APPROVED_BY);

    buildAssignmentService.onTimer();

    AgentConfig agentConfig = AgentMother.localAgent();
    agentConfig.addResource(new Resource("some-resource"));
    Work work = buildAssignmentService.assignWorkToAgent(agent(agentConfig));
    assertThat(work, is(not((Work) BuildAssignmentService.NO_WORK)));

    Pipeline pipeline =
        pipelineDao.mostRecentPipeline(CaseInsensitiveString.str(evolveConfig.name()));
    JobInstance job = pipeline.findStage(STAGE_NAME).findJob("unit");

    JobInstance runningJob = jobInstanceDao.buildByIdWithTransitions(job.getId());

    scheduleService.rescheduleJob(runningJob);

    pipeline = pipelineDao.mostRecentPipeline(CaseInsensitiveString.str(evolveConfig.name()));
    JobInstance rescheduledJob = pipeline.findStage(STAGE_NAME).findJob("unit");

    assertThat(rescheduledJob.getId(), not(runningJob.getId()));

    buildAssignmentService.onTimer();
    Work noResourcesWork =
        buildAssignmentService.assignWorkToAgent(
            agent(AgentMother.localAgentWithResources("WITHOUT_RESOURCES")));
    assertThat(noResourcesWork, is((Work) BuildAssignmentService.NO_WORK));

    buildAssignmentService.onTimer();
    Work correctAgentWork = buildAssignmentService.assignWorkToAgent(agent(agentConfig));
    assertThat(correctAgentWork, is(not((Work) BuildAssignmentService.NO_WORK)));
  }
  @Test
  public void shouldUpdateResultOfStageWhenJobCompletesOnTransactionCommitOnly() throws Exception {
    StageService stageService = mock(StageService.class);
    StageDao stageDao = mock(StageDao.class);
    SchedulingPerformanceLogger schedulingPerformanceLogger =
        mock(SchedulingPerformanceLogger.class);

    ScheduleService service =
        new ScheduleService(
            goConfigService,
            pipelineService,
            stageService,
            schedulingChecker,
            pipelineScheduledTopic,
            pipelineDao,
            stageDao,
            stageOrderService,
            securityService,
            pipelineScheduleQueue,
            jobInstanceService,
            jobInstanceDao,
            agentAssignment,
            environmentConfigService,
            pipelineLockService,
            serverHealthService,
            transactionTemplate,
            agentService,
            synchronizationManager,
            null,
            null,
            null,
            null,
            schedulingPerformanceLogger);

    Pipeline assigned = preCondition.createPipelineWithFirstStageAssigned();

    Stage stage = assigned.findStage(preCondition.devStage);

    when(stageService.stageById(stage.getId())).thenThrow(new RuntimeException("find fails"));

    try {
      service.updateJobStatus(stage.getFirstJob().getIdentifier(), JobState.Completed);
      fail("should have failed because stage lookup bombed");
    } catch (Exception e) {
      // ignore
    }
    verify(stageDao, never())
        .clearCachedAllStages(
            stage.getIdentifier().getPipelineName(),
            stage.getIdentifier().getPipelineCounter(),
            stage.getName());
  }
  @Test
  public void shouldNotScheduleIfAgentDoesNotHaveResources() throws Exception {
    JobConfig plan =
        evolveConfig
            .findBy(new CaseInsensitiveString(STAGE_NAME))
            .jobConfigByInstanceName("unit", true);
    plan.addResource("some-resource");

    scheduleHelper.schedule(evolveConfig, modifySomeFiles(evolveConfig), DEFAULT_APPROVED_BY);

    Work work = buildAssignmentService.assignWorkToAgent(agent(AgentMother.localAgent()));

    Pipeline pipeline =
        pipelineDao.mostRecentPipeline(CaseInsensitiveString.str(evolveConfig.name()));
    JobInstance job = pipeline.findStage(STAGE_NAME).findJob("unit");

    assertThat(work, is((Work) BuildAssignmentService.NO_WORK));
    assertThat(job.getState(), is(JobState.Scheduled));
    assertThat(job.getAgentUuid(), is(nullValue()));
  }
  /**
   * (uppest/2/uppest-stage/1) |------------------> upper-peer ------- |
   * ...................................|............................................... | . | . [
   * uppest-stage ............................|...................... {bar.zip
   * uppest/upper-peer/downer} V . uppest uppest-stage-2 ------> upper ------> downer ------>
   * downest {foo.zip uppest/upper/downer} (uppest/1/uppest-stage-2/1) uppest-stage-3 ]
   *
   * <p>.... :: fetch artifact call ---> :: material dependency
   */
  @Test
  public void shouldCreateWork_withAncestorFetchArtifactCalls_resolvedToRelevantStage()
      throws Exception {
    configHelper.addPipeline("uppest", "uppest-stage");
    configHelper.addStageToPipeline("uppest", "uppest-stage-2");
    PipelineConfig uppest = configHelper.addStageToPipeline("uppest", "uppest-stage-3");

    configHelper.addPipeline("upper", "upper-stage");
    DependencyMaterial upper_sMaterial =
        new DependencyMaterial(
            new CaseInsensitiveString("uppest"), new CaseInsensitiveString("uppest-stage-2"));
    PipelineConfig upper =
        configHelper.setMaterialConfigForPipeline("upper", upper_sMaterial.config());

    configHelper.addPipeline("upper-peer", "upper-peer-stage");
    DependencyMaterial upperPeer_sMaterial =
        new DependencyMaterial(
            new CaseInsensitiveString("uppest"), new CaseInsensitiveString("uppest-stage"));
    PipelineConfig upperPeer =
        configHelper.setMaterialConfigForPipeline("upper-peer", upperPeer_sMaterial.config());

    configHelper.addPipeline("downer", "downer-stage");
    DependencyMaterial downer_sUpperMaterial =
        new DependencyMaterial(
            new CaseInsensitiveString("upper"), new CaseInsensitiveString("upper-stage"));
    configHelper.setMaterialConfigForPipeline("downer", downer_sUpperMaterial.config());
    DependencyMaterial downer_sUpperPeerMaterial =
        new DependencyMaterial(
            new CaseInsensitiveString("upper-peer"), new CaseInsensitiveString("upper-peer-stage"));
    PipelineConfig downer =
        configHelper.addMaterialToPipeline("downer", downer_sUpperPeerMaterial.config());

    configHelper.addPipeline("downest", "downest-stage");
    DependencyMaterial downest_sMaterial =
        new DependencyMaterial(
            new CaseInsensitiveString("downer"), new CaseInsensitiveString("downer-stage"));
    configHelper.setMaterialConfigForPipeline("downest", downest_sMaterial.config());
    Tasks allFetchTasks = new Tasks();
    allFetchTasks.add(
        new FetchTask(
            new CaseInsensitiveString("uppest/upper/downer"),
            new CaseInsensitiveString("uppest-stage"),
            new CaseInsensitiveString("unit"),
            "foo.zip",
            "bar"));
    allFetchTasks.add(
        new FetchTask(
            new CaseInsensitiveString("uppest/upper-peer/downer"),
            new CaseInsensitiveString("uppest-stage"),
            new CaseInsensitiveString("unit"),
            "bar.zip",
            "baz"));
    configHelper.replaceAllJobsInStage(
        "downest",
        "downest-stage",
        new JobConfig(
            new CaseInsensitiveString("fetcher"),
            new Resources("fetcher"),
            new ArtifactPlans(),
            allFetchTasks));
    PipelineConfig downest =
        goConfigService
            .getCurrentConfig()
            .pipelineConfigByName(new CaseInsensitiveString("downest"));

    DefaultSchedulingContext defaultSchedulingCtx =
        new DefaultSchedulingContext(DEFAULT_APPROVED_BY);
    Pipeline uppestInstanceForUpper =
        instanceFactory.createPipelineInstance(
            uppest, modifySomeFiles(uppest), defaultSchedulingCtx, md5, new TimeProvider());
    dbHelper.savePipelineWithStagesAndMaterials(uppestInstanceForUpper);
    dbHelper.passStage(uppestInstanceForUpper.findStage("uppest-stage"));
    Stage upper_sMaterialStage =
        dbHelper.scheduleStage(
            uppestInstanceForUpper, uppest.getStage(new CaseInsensitiveString("uppest-stage-2")));
    dbHelper.passStage(upper_sMaterialStage);

    Pipeline uppestInstanceForUpperPeer =
        instanceFactory.createPipelineInstance(
            uppest,
            modifySomeFiles(uppest),
            new DefaultSchedulingContext("super-hero"),
            md5,
            new TimeProvider());
    dbHelper.savePipelineWithStagesAndMaterials(uppestInstanceForUpperPeer);
    Stage upperPeer_sMaterialStage = uppestInstanceForUpperPeer.findStage("uppest-stage");
    dbHelper.passStage(upperPeer_sMaterialStage);

    Pipeline upperInstance =
        instanceFactory.createPipelineInstance(
            upper,
            buildCauseForDependency(upper_sMaterial, upper_sMaterialStage),
            defaultSchedulingCtx,
            md5,
            new TimeProvider());
    dbHelper.savePipelineWithStagesAndMaterials(upperInstance);
    Stage downer_sUpperMaterialStage = upperInstance.findStage("upper-stage");
    dbHelper.passStage(downer_sUpperMaterialStage);

    Pipeline upperPeerInstance =
        instanceFactory.createPipelineInstance(
            upperPeer,
            buildCauseForDependency(upperPeer_sMaterial, upperPeer_sMaterialStage),
            defaultSchedulingCtx,
            md5,
            new TimeProvider());
    dbHelper.savePipelineWithStagesAndMaterials(upperPeerInstance);
    Stage downer_sUpperPeerMaterialStage = upperPeerInstance.findStage("upper-peer-stage");
    dbHelper.passStage(downer_sUpperPeerMaterialStage);

    MaterialRevisions downer_sMaterialRevisions =
        new MaterialRevisions(
            materialRevisionForDownstream(downer_sUpperMaterial, downer_sUpperMaterialStage),
            materialRevisionForDownstream(
                downer_sUpperPeerMaterial, downer_sUpperPeerMaterialStage));

    Pipeline downerInstance =
        instanceFactory.createPipelineInstance(
            downer,
            BuildCause.createManualForced(downer_sMaterialRevisions, loserUser),
            defaultSchedulingCtx,
            md5,
            new TimeProvider());
    dbHelper.savePipelineWithStagesAndMaterials(downerInstance);
    Stage downest_sMaterialStage = downerInstance.findStage("downer-stage");
    dbHelper.passStage(downest_sMaterialStage);

    Pipeline downestInstance =
        instanceFactory.createPipelineInstance(
            downest,
            buildCauseForDependency(downest_sMaterial, downest_sMaterialStage),
            defaultSchedulingCtx,
            md5,
            new TimeProvider());
    dbHelper.savePipelineWithStagesAndMaterials(downestInstance);

    buildAssignmentService.onTimer();
    AgentConfig agentConfig = AgentMother.localAgent();
    agentConfig.addResource(new Resource("fetcher"));
    BuildWork work = (BuildWork) buildAssignmentService.assignWorkToAgent(agent(agentConfig));

    List<Builder> builders = work.getAssignment().getBuilders();
    FetchArtifactBuilder fooZipFetch = (FetchArtifactBuilder) builders.get(0);
    assertThat(fooZipFetch.artifactLocator(), is("uppest/1/uppest-stage/latest/unit/foo.zip"));
    FetchArtifactBuilder barZipFetch = (FetchArtifactBuilder) builders.get(1);
    assertThat(barZipFetch.artifactLocator(), is("uppest/2/uppest-stage/1/unit/bar.zip"));
  }