/*
* Yes, install tabix as a part of the workflow. It's not in the seqware_whitestar or seqware_whitestar_pancancer container, so
* install it here.
*/
private Job installTools(Job parent) {
Job installTabixJob = this.getWorkflow().createBashJob("install tools");
installTabixJob.setCommand("sudo apt-get install tabix libstring-random-perl -y ");
installTabixJob.addParent(parent);
return installTabixJob;
}
private Job statInputFiles(Job parent) {
Job statFiles = this.getWorkflow().createBashJob("stat downloaded input files");
String moveToFailed =
GitUtils.gitMoveCommand(
"running-jobs",
"failed-jobs",
this.JSONlocation + "/" + this.JSONrepoName + "/" + this.JSONfolderName,
this.JSONfileName,
this.gitMoveTestMode,
this.getWorkflowBaseDir() + "/scripts/");
String statFilesCMD = "( ";
for (VcfInfo vcfInfo : this.vcfs) {
boolean useGnosIdInPath =
vcfInfo.getOriginatingPipeline() != Pipeline.smufin
&& this.pipelineDownloadMethods.get(vcfInfo.getOriginatingPipeline())
!= DownloadMethod.filesystemCopy;
// Remember: smufin won't have a gnos ID so don't try to use that in the path for stat.
String prefix =
"stat /datastore/vcf/"
+ vcfInfo.getOriginatingPipeline().toString()
+ "/"
+ (useGnosIdInPath ? vcfInfo.getPipelineGnosID() + "/" : "");
statFilesCMD += prefix + vcfInfo.getFileName() + " && \\\n";
statFilesCMD += prefix + vcfInfo.getIndexFileName() + " && \\\n";
}
// stat all tumour BAMS
for (int i = 0; i < this.tumours.size(); i++) {
String prefix =
"stat /datastore/bam/"
+ BAMType.tumour.toString()
+ "/"
+ (!this.bamDownloadMethod.equals(DownloadMethod.filesystemCopy.toString())
? this.tumours.get(i).getTumourBamGnosID() + "/"
: "");
statFilesCMD += prefix + this.tumours.get(i).getTumourBAMFileName() + " && \\\n";
statFilesCMD += prefix + this.tumours.get(i).getTumourBamIndexFileName() + " && \\\n";
}
String normalPrefix =
"stat /datastore/bam/"
+ BAMType.normal.toString()
+ "/"
+ (!this.bamDownloadMethod.equals(DownloadMethod.filesystemCopy.toString())
? this.normalBamGnosID + "/"
: "");
statFilesCMD += normalPrefix + this.normalBAMFileName + " && \\\n";
statFilesCMD += normalPrefix + this.normalBamIndexFileName + " \\\n";
statFilesCMD += " ) || " + moveToFailed;
statFiles.setCommand(statFilesCMD);
statFiles.addParent(parent);
return statFiles;
}
/**
* Copy the credentials files from ~/.gnos to /datastore/credentials
*
* @param parentJob
* @return
*/
private Job copyCredentials(Job parentJob) {
Job copy = this.getWorkflow().createBashJob("copy /home/ubuntu/.gnos");
copy.setCommand(
"mkdir /datastore/credentials && cp -r /home/ubuntu/.gnos/* /datastore/credentials && ls -l /datastore/credentials");
copy.addParent(parentJob);
if (this.vcfDownloadMethod.equals(DownloadMethod.s3.toString())) {
Job s3Setup = this.getWorkflow().createBashJob("s3 credentials setup");
s3Setup.setCommand(
"mkdir ~/.aws && cp /datastore/credentials/aws_credentials ~/.aws/credentials");
s3Setup.addParent(copy);
return s3Setup;
} else {
return copy;
}
}
/** Build the workflow!! */
@Override
public void buildWorkflow() {
try {
this.init();
// Pull the repo.
Job configJob = GitUtils.gitConfig(this.getWorkflow(), this.GITname, this.GITemail);
Job copy = this.copyCredentials(configJob);
Job pullRepo =
GitUtils.pullRepo(
this.getWorkflow(),
this.GITPemFile,
this.JSONrepo,
this.JSONrepoName,
this.JSONlocation);
pullRepo.addParent(copy);
Job installTools = this.installTools(copy);
// indicate job is in downloading stage.
String pathToScripts = this.getWorkflowBaseDir() + "/scripts";
Job move2download =
GitUtils.gitMove(
"queued-jobs",
"downloading-jobs",
this.getWorkflow(),
this.JSONlocation,
this.JSONrepoName,
this.JSONfolderName,
this.GITname,
this.GITemail,
this.gitMoveTestMode,
this.JSONfileName,
pathToScripts,
installTools,
pullRepo);
Job move2running;
if (!skipDownload) {
move2running = doDownload(pathToScripts, move2download);
} else {
// If user is skipping download, then we will just move directly to runnning...
move2running =
GitUtils.gitMove(
"downloading-jobs",
"running-jobs",
this.getWorkflow(),
this.JSONlocation,
this.JSONrepoName,
this.JSONfolderName,
this.GITname,
this.GITemail,
this.gitMoveTestMode,
this.JSONfileName,
pathToScripts,
move2download);
}
Job statFiles = statInputFiles(move2running);
Job sangerPassFilter = this.passFilterWorkflow(Pipeline.sanger, statFiles);
Job broadPassFilter = this.passFilterWorkflow(Pipeline.broad, statFiles);
Job dkfzemblPassFilter = this.passFilterWorkflow(Pipeline.dkfz_embl, statFiles);
Job smufinPassFilter = this.passFilterWorkflow(Pipeline.smufin, statFiles);
// ...No, we're not going to filter the Muse SNV file.
// update all filenames to include ".pass-filtered."
Function<String, String> addPassFilteredSuffix =
(x) -> {
return x.replace(".vcf.gz", ".pass-filtered.vcf.gz");
};
for (VcfInfo vInfo : this.vcfs) {
// ...except for MUSE filenames.
if (vInfo.getOriginatingPipeline() != Pipeline.muse) {
vInfo.setFileName(addPassFilteredSuffix.apply(vInfo.getFileName()));
}
}
// OxoG will run after move2running. Move2running will run after all the jobs that perform
// input file downloads and file preprocessing have finished.
List<Job> preprocessIndelsJobs = new ArrayList<Job>(this.tumours.size() * 3);
for (int i = 0; i < this.tumours.size(); i++) {
String tumourAliquotID = tumours.get(i).getAliquotID();
final String vcfNotFoundToken = "VCF_NOT_FOUND";
BiFunction<String, Predicate<VcfInfo>, String> generateVcfName =
(s, p) ->
"/"
+ s
+ "/"
+ this.vcfs
.stream()
.filter(vcfMatchesTypePipelineTumour(isIndel, p, tumourAliquotID))
.map(m -> m.getFileName())
.findFirst()
.orElse(vcfNotFoundToken);
String sangerIndelVcfName =
generateVcfName.apply(this.sangerGnosID, CommonPredicates.isSanger);
if (!sangerIndelVcfName.endsWith(vcfNotFoundToken)) {
Job sangerPreprocessVCF =
this.preProcessIndelVCF(
sangerPassFilter,
Pipeline.sanger,
sangerIndelVcfName,
this.tumours.get(i).getAliquotID());
preprocessIndelsJobs.add(sangerPreprocessVCF);
}
String dkfzEmblIndelVcfName =
generateVcfName.apply(this.dkfzemblGnosID, CommonPredicates.isDkfzEmbl);
if (!dkfzEmblIndelVcfName.endsWith(vcfNotFoundToken)) {
Job dkfzEmblPreprocessVCF =
this.preProcessIndelVCF(
dkfzemblPassFilter,
Pipeline.dkfz_embl,
dkfzEmblIndelVcfName,
this.tumours.get(i).getAliquotID());
preprocessIndelsJobs.add(dkfzEmblPreprocessVCF);
}
String broadIndelVcfName =
generateVcfName.apply(this.broadGnosID, CommonPredicates.isBroad);
if (!broadIndelVcfName.endsWith(vcfNotFoundToken)) {
Job broadPreprocessVCF =
this.preProcessIndelVCF(
broadPassFilter,
Pipeline.broad,
broadIndelVcfName,
this.tumours.get(i).getAliquotID());
preprocessIndelsJobs.add(broadPreprocessVCF);
}
// smufin INDEL VCFs will be in /datastore/vcf/smufin - they will not be nested in a GNOS
// ID-named directory.
String smufinIndelVcfName = generateVcfName.apply("", CommonPredicates.isSmufin);
if (!smufinIndelVcfName.endsWith(vcfNotFoundToken)) {
Job smufinPreprocessVCF =
this.preProcessIndelVCF(
smufinPassFilter,
Pipeline.smufin,
smufinIndelVcfName,
this.tumours.get(i).getAliquotID());
preprocessIndelsJobs.add(smufinPreprocessVCF);
}
}
// TODO: This probably doesn't need to be a list anymore.
List<Job> combineVCFJobs = new ArrayList<Job>(this.tumours.size());
Job combineVCFJob =
this.combineVCFsByType(
preprocessIndelsJobs.toArray(new Job[preprocessIndelsJobs.size()]));
combineVCFJobs.add(combineVCFJob);
List<Job> oxogJobs = new ArrayList<Job>(this.tumours.size());
for (int i = 0; i < this.tumours.size(); i++) {
TumourInfo tInf = this.tumours.get(i);
Job oxoG;
if (i > 0) {
// OxoG jobs can put a heavy CPU load on the system (in bursts) so probably better to run
// them in sequence.
// If there is > 1 OxoG job (i.e. a multi-tumour donor), each OxoG job should have the
// prior OxoG job as its parent.
oxoG =
this.doOxoG(
tInf.getTumourBamGnosID() + "/" + tInf.getTumourBAMFileName(),
tInf.getAliquotID(),
oxogJobs.get(i - 1));
} else {
oxoG =
this.doOxoG(
tInf.getTumourBamGnosID() + "/" + tInf.getTumourBAMFileName(),
tInf.getAliquotID());
}
for (int j = 0; j < combineVCFJobs.size(); j++) {
oxoG.addParent(combineVCFJobs.get(j));
}
oxogJobs.add(oxoG);
}
String pathToNormalMinibam =
"/datastore/bam/normal/"
+ (DownloadMethod.valueOf(this.bamDownloadMethod) != DownloadMethod.filesystemCopy
? this.normalBamGnosID + "/"
: "")
+ this.normalBAMFileName;
Job normalVariantBam =
this.doVariantBam(
BAMType.normal,
pathToNormalMinibam,
this.normalBAMFileName,
this.normalAliquotID,
combineVCFJobs.toArray(new Job[combineVCFJobs.size()]));
List<Job> parentJobsToAnnotationJobs = new ArrayList<Job>(this.tumours.size());
// create a list of tumour variant-bam jobs.
List<Job> variantBamJobs = new ArrayList<Job>(this.tumours.size() + 1);
for (int i = 0; i < this.tumours.size(); i++) {
TumourInfo tInfo = this.tumours.get(i);
String pathToTumourMinibam =
"/datastore/bam/tumour/"
+ (DownloadMethod.valueOf(this.bamDownloadMethod) != DownloadMethod.filesystemCopy
? tInfo.getTumourBamGnosID() + "/"
: "")
+ tInfo.getTumourBAMFileName();
Job tumourVariantBam =
this.doVariantBam(
BAMType.tumour,
pathToTumourMinibam,
tInfo.getTumourBAMFileName(),
tInfo.getAliquotID(),
combineVCFJobs.toArray(new Job[combineVCFJobs.size()]));
variantBamJobs.add(tumourVariantBam);
}
variantBamJobs.add(normalVariantBam);
// Now that we've built our list of variantbam and oxog jobs, we can set up the proper
// parent-child relationships between them.
// The idea is to run 1 OxoG at the same time as 2 variantbam jobs.
for (int i = 2; i < Math.max(variantBamJobs.size(), variantBamJobs.size()); i += 2) {
variantBamJobs.get(i).addParent(variantBamJobs.get(i - 2));
if (i + 1 < variantBamJobs.size()) {
variantBamJobs.get(i + 1).addParent(variantBamJobs.get(i - 2));
}
}
Job minibamSanityCheck = this.getWorkflow().createBashJob("Check minibams");
String moveToFailed =
GitUtils.gitMoveCommand(
"running-jobs",
"failed-jobs",
this.JSONlocation + "/" + this.JSONrepoName + "/" + this.JSONfolderName,
this.JSONfileName,
this.gitMoveTestMode,
this.getWorkflowBaseDir() + "/scripts/");
// A list of all pass-filtered files that the minibams will be checked against.
// NOTE: muse will not have any pass-filtered files and smufin will only have indel (or
// SNV-from-indel) files.
String filesToCheck =
this.vcfs
.stream()
.filter(
p ->
p.getFileName().endsWith(".vcf.gz")
&& p.getFileName().contains("pass-filter"))
.map(
m ->
"/datastore/vcf/"
+ m.getOriginatingPipeline().toString()
+ "/"
+ (m.getOriginatingPipeline() != Pipeline.smufin
? m.getPipelineGnosID() + "/"
: "/")
+ m.getFileName())
.sorted()
.reduce("", (a, b) -> a += " " + b);
minibamSanityCheck.setCommand(
"(bash " + pathToScripts + "/check_minibams.sh " + filesToCheck + ") || " + moveToFailed);
variantBamJobs.stream().forEach(job -> minibamSanityCheck.addParent(job));
parentJobsToAnnotationJobs.add(minibamSanityCheck);
// set up parent jobs to annotation jobs
oxogJobs.stream().forEach(job -> parentJobsToAnnotationJobs.add(job));
List<Job> annotationJobs = new ArrayList<Job>();
if (!this.skipAnnotation) {
annotationJobs =
this.doAnnotations(
parentJobsToAnnotationJobs.toArray(new Job[parentJobsToAnnotationJobs.size()]));
}
// Now do the Upload. The parents jobs are the Annotation jobs, unless the user set
// skipAnnotations=true, so there will
// not be any annotation jobs. In that case Upload's parents are the jobs that would have been
// parents to Annotation.
Job[] parentsToUpload =
(annotationJobs != null && annotationJobs.size() > 0)
? annotationJobs.toArray(new Job[annotationJobs.size()])
: parentJobsToAnnotationJobs.toArray(new Job[parentJobsToAnnotationJobs.size()]);
// indicate job is in uploading stage.
Job move2uploading = this.gitMove("running-jobs", "uploading-jobs", parentsToUpload);
Job uploadResults = doUpload(move2uploading);
// indicate job is complete.
this.gitMove("uploading-jobs", "completed-jobs", uploadResults);
// System.out.println(this.filesForUpload);
} catch (Exception e) {
throw new RuntimeException("Exception caught: " + e.getMessage(), e);
}
}
/**
* Runs the variant program inside the Broad's OxoG container to produce a mini-BAM for a given
* BAM.
*
* @param parent
* @param bamType - The type of BAM file to use. Determines the name of the output file.
* @param bamPath - The path to the input BAM file.
* @param tumourBAMFileName - Name of the BAM file. Only used if bamType == BAMType.tumour.
* @param aliquotID
* @return
*/
private Job doVariantBam(
BAMType bamType, String bamPath, String tumourBAMFileName, String aliquotID, Job... parents) {
Job runVariantbam;
if (!this.skipVariantBam) {
VariantBamJobGenerator variantBamJobGenerator =
new VariantBamJobGenerator(
this.JSONlocation, this.JSONrepoName, this.JSONfolderName, this.JSONfileName);
variantBamJobGenerator.setGitMoveTestMode(this.gitMoveTestMode);
variantBamJobGenerator.setIndelPadding(String.valueOf(this.indelPadding));
variantBamJobGenerator.setSnvPadding(String.valueOf(this.snvPadding));
variantBamJobGenerator.setSvPadding(String.valueOf(this.svPadding));
variantBamJobGenerator.setGitMoveTestMode(this.gitMoveTestMode);
variantBamJobGenerator.setAliquotID(aliquotID);
variantBamJobGenerator.setSnvVcf(
mergedVcfs.stream().filter(isSnv).findFirst().get().getFileName());
variantBamJobGenerator.setSvVcf(
mergedVcfs.stream().filter(isSv).findFirst().get().getFileName());
variantBamJobGenerator.setIndelVcf(
mergedVcfs.stream().filter(isIndel).findFirst().get().getFileName());
UpdateBamForUpload<String, String, Boolean> updateFilesForUpload =
(path, id, isLink) -> {
if (id == null || id.trim().equals("")) {
// only update the normalMinibamPath with the path to the actual BAM.
// If you get a .bai file here, add it to filesForUpload,
// but don't do anything else.
// Also, if what was passed in has been indicated as a symlink, just add
// to filesForUpload but do nothing else.
if (path.endsWith(".bam") && !isLink) {
this.normalMinibamPath = path;
}
this.filesForUpload.add(path);
} else {
for (TumourInfo tInfo : this.tumours) {
if (tInfo.getAliquotID().equals(id)) {
// Set the tumour minibam path only to the BAM file.
// If you get a .bai file here, add it to filesForUpload,
// but don't do anything else.
// Also, if what was passed in has been indicated as a symlink, just add
// to filesForUpload but do nothing else.
if (path.endsWith(".bam") && !isLink) {
tInfo.setTumourMinibamPath(path);
}
filesForUpload.add(path);
}
}
}
};
String bamName = (bamType == BAMType.normal ? this.normalBAMFileName : tumourBAMFileName);
runVariantbam =
variantBamJobGenerator.doVariantBam(
this, bamType, bamName, bamPath, tumourBAMFileName, updateFilesForUpload, parents);
} else {
runVariantbam =
this.getWorkflow()
.createBashJob(
"run "
+ bamType
+ (bamType == BAMType.tumour ? "_" + aliquotID + "_" : "")
+ " variantbam");
Arrays.stream(parents).forEach(parent -> runVariantbam.addParent(parent));
}
return runVariantbam;
}