public List<MaterialRevision> addDependencyRevisionModification(
List<MaterialRevision> materialRevisions,
DependencyMaterial dependencyMaterial,
Pipeline... upstreams) {
String stageName = CaseInsensitiveString.str(dependencyMaterial.getStageName());
String label = upstreams[0].getLabel();
List<Modification> modifications = new ArrayList<Modification>();
for (Pipeline upstream : upstreams) {
modifications.add(
new Modification(
new Date(),
DependencyMaterialRevision.create(
CaseInsensitiveString.str(dependencyMaterial.getPipelineName()),
upstream.getCounter(),
label,
stageName,
upstream.findStage(stageName).getCounter())
.getRevision(),
label,
upstream.getId()));
}
MaterialRevision depRev =
addRevisionsWithModifications(
dependencyMaterial, modifications.toArray(new Modification[0]));
materialRevisions.add(depRev);
return Arrays.asList(depRev);
}
@Test
public void
shouldUpdateResultOfStageWhenJobCompletes_irrespectiveOfOtherThreadsPrimingStageCache()
throws Exception {
Pipeline assigned = preCondition.createPipelineWithFirstStageAssigned();
Stage stage = assigned.findStage(preCondition.devStage);
StageSummaryModel model =
stageService.findStageSummaryByIdentifier(
stage.getIdentifier(),
new Username(new CaseInsensitiveString("foo")),
new HttpLocalizedOperationResult());
JobIdentifier identifier = stage.getFirstJob().getIdentifier();
scheduleService.updateJobStatus(identifier, JobState.Building);
scheduleService.jobCompleting(identifier, JobResult.Passed, "uuid");
Stage stageLoadedByOtherFlows = stageDao.stageById(stage.getId()); // priming the cache
scheduleService.updateJobStatus(stage.getFirstJob().getIdentifier(), JobState.Completed);
StageSummaryModel reloadedModel =
stageService.findStageSummaryByIdentifier(
stage.getIdentifier(),
new Username(new CaseInsensitiveString("foo")),
new HttpLocalizedOperationResult());
Stage reloadedStage = reloadedModel.getStage();
assertThat(reloadedStage.getFirstJob().getState(), is(JobState.Completed));
assertThat(
reloadedStage.getCompletedByTransitionId(),
is(reloadedStage.getFirstJob().getTransitions().byState(JobState.Completed).getId()));
assertThat(reloadedStage.getResult(), is(StageResult.Passed));
assertThat(reloadedStage.getState(), is(StageState.Passed));
}
@Test
public void shouldUpdateResultOfStageWhenJobCompletes() throws Exception {
// ScheduleService service = new ScheduleService(goConfigService, pipelineService,
// stageService, currentActivityService, schedulingChecker, pipelineScheduledTopic, pipelineDao,
// stageDao, stageOrderService, securityService, pipelineScheduleQueue,
// jobInstanceService, jobInstanceDao, agentAssignment, environmentConfigService,
// pipelineLockService, serverHealthService, transactionTemplate, agentService);
Pipeline assigned = preCondition.createPipelineWithFirstStageAssigned();
Stage stage = assigned.findStage(preCondition.devStage);
StageSummaryModel model =
stageService.findStageSummaryByIdentifier(
stage.getIdentifier(),
new Username(new CaseInsensitiveString("foo")),
new HttpLocalizedOperationResult());
assertThat(model.getStage().getFirstJob().getState(), is(JobState.Assigned));
scheduleService.updateJobStatus(stage.getFirstJob().getIdentifier(), JobState.Building);
StageSummaryModel reloadedModel =
stageService.findStageSummaryByIdentifier(
stage.getIdentifier(),
new Username(new CaseInsensitiveString("foo")),
new HttpLocalizedOperationResult());
assertThat(reloadedModel.getStage().getFirstJob().getState(), is(JobState.Building));
}
@Test
public void shouldScheduleIfAgentMatchingResources() throws Exception {
JobConfig plan =
evolveConfig
.findBy(new CaseInsensitiveString(STAGE_NAME))
.jobConfigByInstanceName("unit", true);
plan.addResource("some-resource");
scheduleHelper.schedule(evolveConfig, modifySomeFiles(evolveConfig), DEFAULT_APPROVED_BY);
AgentConfig agentConfig = AgentMother.localAgent();
agentConfig.addResource(new Resource("some-resource"));
buildAssignmentService.onTimer();
Work work = buildAssignmentService.assignWorkToAgent(agent(agentConfig));
assertThat(work, is(not((Work) BuildAssignmentService.NO_WORK)));
Pipeline pipeline =
pipelineDao.mostRecentPipeline(CaseInsensitiveString.str(evolveConfig.name()));
JobInstance job = pipeline.findStage(STAGE_NAME).findJob("unit");
JobPlan loadedPlan = jobInstanceDao.loadPlan(job.getId());
assertThat(loadedPlan.getResources(), is((List<Resource>) plan.resources()));
assertThat(job.getState(), is(JobState.Assigned));
assertThat(job.getAgentUuid(), is(agentConfig.getUuid()));
}
@Test
public void shouldReScheduleToCorrectAgent() throws Exception {
JobConfig plan =
evolveConfig
.findBy(new CaseInsensitiveString(STAGE_NAME))
.jobConfigByInstanceName("unit", true);
plan.addResource("some-resource");
scheduleHelper.schedule(evolveConfig, modifySomeFiles(evolveConfig), DEFAULT_APPROVED_BY);
buildAssignmentService.onTimer();
AgentConfig agentConfig = AgentMother.localAgent();
agentConfig.addResource(new Resource("some-resource"));
Work work = buildAssignmentService.assignWorkToAgent(agent(agentConfig));
assertThat(work, is(not((Work) BuildAssignmentService.NO_WORK)));
Pipeline pipeline =
pipelineDao.mostRecentPipeline(CaseInsensitiveString.str(evolveConfig.name()));
JobInstance job = pipeline.findStage(STAGE_NAME).findJob("unit");
JobInstance runningJob = jobInstanceDao.buildByIdWithTransitions(job.getId());
scheduleService.rescheduleJob(runningJob);
pipeline = pipelineDao.mostRecentPipeline(CaseInsensitiveString.str(evolveConfig.name()));
JobInstance rescheduledJob = pipeline.findStage(STAGE_NAME).findJob("unit");
assertThat(rescheduledJob.getId(), not(runningJob.getId()));
buildAssignmentService.onTimer();
Work noResourcesWork =
buildAssignmentService.assignWorkToAgent(
agent(AgentMother.localAgentWithResources("WITHOUT_RESOURCES")));
assertThat(noResourcesWork, is((Work) BuildAssignmentService.NO_WORK));
buildAssignmentService.onTimer();
Work correctAgentWork = buildAssignmentService.assignWorkToAgent(agent(agentConfig));
assertThat(correctAgentWork, is(not((Work) BuildAssignmentService.NO_WORK)));
}
@Test
public void shouldUpdateResultOfStageWhenJobCompletesOnTransactionCommitOnly() throws Exception {
StageService stageService = mock(StageService.class);
StageDao stageDao = mock(StageDao.class);
SchedulingPerformanceLogger schedulingPerformanceLogger =
mock(SchedulingPerformanceLogger.class);
ScheduleService service =
new ScheduleService(
goConfigService,
pipelineService,
stageService,
schedulingChecker,
pipelineScheduledTopic,
pipelineDao,
stageDao,
stageOrderService,
securityService,
pipelineScheduleQueue,
jobInstanceService,
jobInstanceDao,
agentAssignment,
environmentConfigService,
pipelineLockService,
serverHealthService,
transactionTemplate,
agentService,
synchronizationManager,
null,
null,
null,
null,
schedulingPerformanceLogger);
Pipeline assigned = preCondition.createPipelineWithFirstStageAssigned();
Stage stage = assigned.findStage(preCondition.devStage);
when(stageService.stageById(stage.getId())).thenThrow(new RuntimeException("find fails"));
try {
service.updateJobStatus(stage.getFirstJob().getIdentifier(), JobState.Completed);
fail("should have failed because stage lookup bombed");
} catch (Exception e) {
// ignore
}
verify(stageDao, never())
.clearCachedAllStages(
stage.getIdentifier().getPipelineName(),
stage.getIdentifier().getPipelineCounter(),
stage.getName());
}
@Test
public void shouldNotScheduleIfAgentDoesNotHaveResources() throws Exception {
JobConfig plan =
evolveConfig
.findBy(new CaseInsensitiveString(STAGE_NAME))
.jobConfigByInstanceName("unit", true);
plan.addResource("some-resource");
scheduleHelper.schedule(evolveConfig, modifySomeFiles(evolveConfig), DEFAULT_APPROVED_BY);
Work work = buildAssignmentService.assignWorkToAgent(agent(AgentMother.localAgent()));
Pipeline pipeline =
pipelineDao.mostRecentPipeline(CaseInsensitiveString.str(evolveConfig.name()));
JobInstance job = pipeline.findStage(STAGE_NAME).findJob("unit");
assertThat(work, is((Work) BuildAssignmentService.NO_WORK));
assertThat(job.getState(), is(JobState.Scheduled));
assertThat(job.getAgentUuid(), is(nullValue()));
}
/**
* (uppest/2/uppest-stage/1) |------------------> upper-peer ------- |
* ...................................|............................................... | . | . [
* uppest-stage ............................|...................... {bar.zip
* uppest/upper-peer/downer} V . uppest uppest-stage-2 ------> upper ------> downer ------>
* downest {foo.zip uppest/upper/downer} (uppest/1/uppest-stage-2/1) uppest-stage-3 ]
*
* <p>.... :: fetch artifact call ---> :: material dependency
*/
@Test
public void shouldCreateWork_withAncestorFetchArtifactCalls_resolvedToRelevantStage()
throws Exception {
configHelper.addPipeline("uppest", "uppest-stage");
configHelper.addStageToPipeline("uppest", "uppest-stage-2");
PipelineConfig uppest = configHelper.addStageToPipeline("uppest", "uppest-stage-3");
configHelper.addPipeline("upper", "upper-stage");
DependencyMaterial upper_sMaterial =
new DependencyMaterial(
new CaseInsensitiveString("uppest"), new CaseInsensitiveString("uppest-stage-2"));
PipelineConfig upper =
configHelper.setMaterialConfigForPipeline("upper", upper_sMaterial.config());
configHelper.addPipeline("upper-peer", "upper-peer-stage");
DependencyMaterial upperPeer_sMaterial =
new DependencyMaterial(
new CaseInsensitiveString("uppest"), new CaseInsensitiveString("uppest-stage"));
PipelineConfig upperPeer =
configHelper.setMaterialConfigForPipeline("upper-peer", upperPeer_sMaterial.config());
configHelper.addPipeline("downer", "downer-stage");
DependencyMaterial downer_sUpperMaterial =
new DependencyMaterial(
new CaseInsensitiveString("upper"), new CaseInsensitiveString("upper-stage"));
configHelper.setMaterialConfigForPipeline("downer", downer_sUpperMaterial.config());
DependencyMaterial downer_sUpperPeerMaterial =
new DependencyMaterial(
new CaseInsensitiveString("upper-peer"), new CaseInsensitiveString("upper-peer-stage"));
PipelineConfig downer =
configHelper.addMaterialToPipeline("downer", downer_sUpperPeerMaterial.config());
configHelper.addPipeline("downest", "downest-stage");
DependencyMaterial downest_sMaterial =
new DependencyMaterial(
new CaseInsensitiveString("downer"), new CaseInsensitiveString("downer-stage"));
configHelper.setMaterialConfigForPipeline("downest", downest_sMaterial.config());
Tasks allFetchTasks = new Tasks();
allFetchTasks.add(
new FetchTask(
new CaseInsensitiveString("uppest/upper/downer"),
new CaseInsensitiveString("uppest-stage"),
new CaseInsensitiveString("unit"),
"foo.zip",
"bar"));
allFetchTasks.add(
new FetchTask(
new CaseInsensitiveString("uppest/upper-peer/downer"),
new CaseInsensitiveString("uppest-stage"),
new CaseInsensitiveString("unit"),
"bar.zip",
"baz"));
configHelper.replaceAllJobsInStage(
"downest",
"downest-stage",
new JobConfig(
new CaseInsensitiveString("fetcher"),
new Resources("fetcher"),
new ArtifactPlans(),
allFetchTasks));
PipelineConfig downest =
goConfigService
.getCurrentConfig()
.pipelineConfigByName(new CaseInsensitiveString("downest"));
DefaultSchedulingContext defaultSchedulingCtx =
new DefaultSchedulingContext(DEFAULT_APPROVED_BY);
Pipeline uppestInstanceForUpper =
instanceFactory.createPipelineInstance(
uppest, modifySomeFiles(uppest), defaultSchedulingCtx, md5, new TimeProvider());
dbHelper.savePipelineWithStagesAndMaterials(uppestInstanceForUpper);
dbHelper.passStage(uppestInstanceForUpper.findStage("uppest-stage"));
Stage upper_sMaterialStage =
dbHelper.scheduleStage(
uppestInstanceForUpper, uppest.getStage(new CaseInsensitiveString("uppest-stage-2")));
dbHelper.passStage(upper_sMaterialStage);
Pipeline uppestInstanceForUpperPeer =
instanceFactory.createPipelineInstance(
uppest,
modifySomeFiles(uppest),
new DefaultSchedulingContext("super-hero"),
md5,
new TimeProvider());
dbHelper.savePipelineWithStagesAndMaterials(uppestInstanceForUpperPeer);
Stage upperPeer_sMaterialStage = uppestInstanceForUpperPeer.findStage("uppest-stage");
dbHelper.passStage(upperPeer_sMaterialStage);
Pipeline upperInstance =
instanceFactory.createPipelineInstance(
upper,
buildCauseForDependency(upper_sMaterial, upper_sMaterialStage),
defaultSchedulingCtx,
md5,
new TimeProvider());
dbHelper.savePipelineWithStagesAndMaterials(upperInstance);
Stage downer_sUpperMaterialStage = upperInstance.findStage("upper-stage");
dbHelper.passStage(downer_sUpperMaterialStage);
Pipeline upperPeerInstance =
instanceFactory.createPipelineInstance(
upperPeer,
buildCauseForDependency(upperPeer_sMaterial, upperPeer_sMaterialStage),
defaultSchedulingCtx,
md5,
new TimeProvider());
dbHelper.savePipelineWithStagesAndMaterials(upperPeerInstance);
Stage downer_sUpperPeerMaterialStage = upperPeerInstance.findStage("upper-peer-stage");
dbHelper.passStage(downer_sUpperPeerMaterialStage);
MaterialRevisions downer_sMaterialRevisions =
new MaterialRevisions(
materialRevisionForDownstream(downer_sUpperMaterial, downer_sUpperMaterialStage),
materialRevisionForDownstream(
downer_sUpperPeerMaterial, downer_sUpperPeerMaterialStage));
Pipeline downerInstance =
instanceFactory.createPipelineInstance(
downer,
BuildCause.createManualForced(downer_sMaterialRevisions, loserUser),
defaultSchedulingCtx,
md5,
new TimeProvider());
dbHelper.savePipelineWithStagesAndMaterials(downerInstance);
Stage downest_sMaterialStage = downerInstance.findStage("downer-stage");
dbHelper.passStage(downest_sMaterialStage);
Pipeline downestInstance =
instanceFactory.createPipelineInstance(
downest,
buildCauseForDependency(downest_sMaterial, downest_sMaterialStage),
defaultSchedulingCtx,
md5,
new TimeProvider());
dbHelper.savePipelineWithStagesAndMaterials(downestInstance);
buildAssignmentService.onTimer();
AgentConfig agentConfig = AgentMother.localAgent();
agentConfig.addResource(new Resource("fetcher"));
BuildWork work = (BuildWork) buildAssignmentService.assignWorkToAgent(agent(agentConfig));
List<Builder> builders = work.getAssignment().getBuilders();
FetchArtifactBuilder fooZipFetch = (FetchArtifactBuilder) builders.get(0);
assertThat(fooZipFetch.artifactLocator(), is("uppest/1/uppest-stage/latest/unit/foo.zip"));
FetchArtifactBuilder barZipFetch = (FetchArtifactBuilder) builders.get(1);
assertThat(barZipFetch.artifactLocator(), is("uppest/2/uppest-stage/1/unit/bar.zip"));
}