/**
* Test the reads according to an independently derived context.
*
* @param view
* @param range
* @param reads
*/
@Override
protected void testReadsInContext(
LocusView view, List<GenomeLoc> range, List<GATKSAMRecord> reads) {
AllLocusView allLocusView = (AllLocusView) view;
// TODO: Should skip over loci not in the given range.
GenomeLoc firstLoc = range.get(0);
GenomeLoc lastLoc = range.get(range.size() - 1);
GenomeLoc bounds =
genomeLocParser.createGenomeLoc(
firstLoc.getContig(), firstLoc.getStart(), lastLoc.getStop());
for (int i = bounds.getStart(); i <= bounds.getStop(); i++) {
GenomeLoc site = genomeLocParser.createGenomeLoc("chr1", i);
AlignmentContext locusContext = allLocusView.next();
Assert.assertEquals(locusContext.getLocation(), site, "Locus context location is incorrect");
int expectedReadsAtSite = 0;
for (GATKSAMRecord read : reads) {
if (genomeLocParser.createGenomeLoc(read).containsP(locusContext.getLocation())) {
Assert.assertTrue(
locusContext.getReads().contains(read),
"Target locus context does not contain reads");
expectedReadsAtSite++;
}
}
Assert.assertEquals(
locusContext.getReads().size(),
expectedReadsAtSite,
"Found wrong number of reads at site");
}
}
/**
* return a deep copy of this collection.
*
* @return a new GenomeLocSortedSet, identical to the current GenomeLocSortedSet.
*/
public GenomeLocSortedSet clone() {
GenomeLocSortedSet ret = new GenomeLocSortedSet(genomeLocParser);
for (GenomeLoc loc : this.mArray) {
// ensure a deep copy
ret.mArray.add(
genomeLocParser.createGenomeLoc(loc.getContig(), loc.getStart(), loc.getStop()));
}
return ret;
}
private GenomeLoc createIntervalAfter(GenomeLoc interval) {
int contigLimit =
getToolkit()
.getSAMFileHeader()
.getSequenceDictionary()
.getSequence(interval.getContigIndex())
.getSequenceLength();
int start = Math.min(interval.getStop() + 1, contigLimit);
int stop = Math.min(interval.getStop() + expandInterval, contigLimit);
return parser.createGenomeLoc(interval.getContig(), interval.getContigIndex(), start, stop);
}
public Iterable<Shard> createShardsOverIntervals(
final SAMDataSource readsDataSource,
final GenomeLocSortedSet intervals,
final int maxShardSize) {
List<Shard> shards = new ArrayList<Shard>();
for (GenomeLoc interval : intervals) {
while (interval.size() > maxShardSize) {
shards.add(
new LocusShard(
intervals.getGenomeLocParser(),
readsDataSource,
Collections.singletonList(
intervals
.getGenomeLocParser()
.createGenomeLoc(
interval.getContig(),
interval.getStart(),
interval.getStart() + maxShardSize - 1)),
null));
interval =
intervals
.getGenomeLocParser()
.createGenomeLoc(
interval.getContig(), interval.getStart() + maxShardSize, interval.getStop());
}
shards.add(
new LocusShard(
intervals.getGenomeLocParser(),
readsDataSource,
Collections.singletonList(interval),
null));
}
return shards;
}
private boolean overlapsKnownCNV(VariantContext cnv) {
if (knownCNVs != null) {
final GenomeLoc loc =
getWalker().getToolkit().getGenomeLocParser().createGenomeLoc(cnv, true);
IntervalTree<GenomeLoc> intervalTree = knownCNVs.get(loc.getContig());
final Iterator<IntervalTree.Node<GenomeLoc>> nodeIt =
intervalTree.overlappers(loc.getStart(), loc.getStop());
while (nodeIt.hasNext()) {
final double overlapP = loc.reciprocialOverlapFraction(nodeIt.next().getValue());
if (overlapP > MIN_CNV_OVERLAP) return true;
}
}
return false;
}
/**
* Utility routine that prints out process information (including timing) every N records or every
* M seconds, for N and M set in global variables.
*
* <p>Synchronized to ensure that even with multiple threads calling notifyOfProgress we still get
* one clean stream of meter logs.
*
* <p>Note this thread doesn't actually print progress, unless must print is true, but just
* registers the progress itself. A separate printing daemon periodically polls the meter to print
* out progress
*
* @param loc Current location, can be null if you are at the end of the processing unit
* @param nTotalRecordsProcessed the total number of records we've processed
*/
public synchronized void notifyOfProgress(
final GenomeLoc loc, final long nTotalRecordsProcessed) {
if (nTotalRecordsProcessed < 0)
throw new IllegalArgumentException("nTotalRecordsProcessed must be >= 0");
// weird comparison to ensure that loc == null (in unmapped reads) is keep before maxGenomeLoc
// == null (on startup)
this.maxGenomeLoc = loc == null ? loc : (maxGenomeLoc == null ? loc : loc.max(maxGenomeLoc));
this.nTotalRecordsProcessed = Math.max(this.nTotalRecordsProcessed, nTotalRecordsProcessed);
// a pretty name for our position
this.positionMessage =
maxGenomeLoc == null
? "unmapped reads"
: String.format("%s:%d", maxGenomeLoc.getContig(), maxGenomeLoc.getStart());
}
public final Map<String, IntervalTree<GenomeLoc>> createIntervalTreeByContig(
final IntervalBinding<Feature> intervals) {
final Map<String, IntervalTree<GenomeLoc>> byContig =
new HashMap<String, IntervalTree<GenomeLoc>>();
final List<GenomeLoc> locs = intervals.getIntervals(getToolkit());
// set up the map from contig -> interval tree
for (final String contig : getContigNames())
byContig.put(contig, new IntervalTree<GenomeLoc>());
for (final GenomeLoc loc : locs) {
byContig.get(loc.getContig()).put(loc.getStart(), loc.getStop(), loc);
}
return byContig;
}
/**
* Returns a new GenomeLoc that represents the region between the endpoints of this and that.
* Requires that this and that GenomeLoc are both mapped.
*/
@Requires({"that != null", "isUnmapped(this) == isUnmapped(that)"})
@Ensures("result != null")
public GenomeLoc endpointSpan(GenomeLoc that) throws ReviewedStingException {
if (GenomeLoc.isUnmapped(this) || GenomeLoc.isUnmapped(that)) {
throw new ReviewedStingException("Cannot get endpoint span for unmerged genome locs");
}
if (!this.getContig().equals(that.getContig())) {
throw new ReviewedStingException(
"Cannot get endpoint span for genome locs on different contigs");
}
return new GenomeLoc(
getContig(),
this.contigIndex,
Math.min(getStart(), that.getStart()),
Math.max(getStop(), that.getStop()));
}
/**
* Determines what is the position of the read in relation to the interval. Note: This function
* uses the UNCLIPPED ENDS of the reads for the comparison.
*
* @param read the read
* @param interval the interval
* @return the overlap type as described by ReadAndIntervalOverlap enum (see above)
*/
public static ReadAndIntervalOverlap getReadAndIntervalOverlapType(
GATKSAMRecord read, GenomeLoc interval) {
int sStart = read.getSoftStart();
int sStop = read.getSoftEnd();
int uStart = read.getUnclippedStart();
int uStop = read.getUnclippedEnd();
if (!read.getReferenceName().equals(interval.getContig()))
return ReadAndIntervalOverlap.NO_OVERLAP_CONTIG;
else if (uStop < interval.getStart()) return ReadAndIntervalOverlap.NO_OVERLAP_LEFT;
else if (uStart > interval.getStop()) return ReadAndIntervalOverlap.NO_OVERLAP_RIGHT;
else if (sStop < interval.getStart()) return ReadAndIntervalOverlap.NO_OVERLAP_HARDCLIPPED_LEFT;
else if (sStart > interval.getStop())
return ReadAndIntervalOverlap.NO_OVERLAP_HARDCLIPPED_RIGHT;
else if ((sStart >= interval.getStart()) && (sStop <= interval.getStop()))
return ReadAndIntervalOverlap.OVERLAP_CONTAINED;
else if ((sStart < interval.getStart()) && (sStop > interval.getStop()))
return ReadAndIntervalOverlap.OVERLAP_LEFT_AND_RIGHT;
else if ((sStart < interval.getStart())) return ReadAndIntervalOverlap.OVERLAP_LEFT;
else return ReadAndIntervalOverlap.OVERLAP_RIGHT;
}
public void writeBeagleOutput(
VariantContext preferredVC, VariantContext otherVC, boolean isValidationSite, double prior) {
GenomeLoc currentLoc =
VariantContextUtils.getLocation(getToolkit().getGenomeLocParser(), preferredVC);
StringBuffer beagleOut = new StringBuffer();
String marker = String.format("%s:%d ", currentLoc.getContig(), currentLoc.getStart());
beagleOut.append(marker);
if (markers != null)
markers.append(marker).append("\t").append(Integer.toString(markerCounter++)).append("\t");
for (Allele allele : preferredVC.getAlleles()) {
String bglPrintString;
if (allele.isNoCall() || allele.isNull()) bglPrintString = "-";
else bglPrintString = allele.getBaseString(); // get rid of * in case of reference allele
beagleOut.append(String.format("%s ", bglPrintString));
if (markers != null) markers.append(bglPrintString).append("\t");
}
if (markers != null) markers.append("\n");
GenotypesContext preferredGenotypes = preferredVC.getGenotypes();
GenotypesContext otherGenotypes = goodSite(otherVC) ? otherVC.getGenotypes() : null;
for (String sample : samples) {
boolean isMaleOnChrX = CHECK_IS_MALE_ON_CHR_X && getSample(sample).getGender() == Gender.MALE;
Genotype genotype;
boolean isValidation;
// use sample as key into genotypes structure
if (preferredGenotypes.containsSample(sample)) {
genotype = preferredGenotypes.get(sample);
isValidation = isValidationSite;
} else if (otherGenotypes != null && otherGenotypes.containsSample(sample)) {
genotype = otherGenotypes.get(sample);
isValidation = !isValidationSite;
} else {
// there is magically no genotype for this sample.
throw new StingException(
"Sample "
+ sample
+ " arose with no genotype in variant or validation VCF. This should never happen.");
}
/*
* Use likelihoods if: is validation, prior is negative; or: is not validation, has genotype key
*/
double[] log10Likelihoods = null;
if ((isValidation && prior < 0.0) || genotype.hasLikelihoods()) {
log10Likelihoods = genotype.getLikelihoods().getAsVector();
// see if we need to randomly mask out genotype in this position.
if (GenomeAnalysisEngine.getRandomGenerator().nextDouble() <= insertedNoCallRate) {
// we are masking out this genotype
log10Likelihoods =
isMaleOnChrX ? HAPLOID_FLAT_LOG10_LIKELIHOODS : DIPLOID_FLAT_LOG10_LIKELIHOODS;
}
if (isMaleOnChrX) {
log10Likelihoods[1] = -255; // todo -- warning this is dangerous for multi-allele case
}
}
/** otherwise, use the prior uniformly */
else if (!isValidation && genotype.isCalled() && !genotype.hasLikelihoods()) {
// hack to deal with input VCFs with no genotype likelihoods. Just assume the called
// genotype
// is confident. This is useful for Hapmap and 1KG release VCFs.
double AA = (1.0 - prior) / 2.0;
double AB = (1.0 - prior) / 2.0;
double BB = (1.0 - prior) / 2.0;
if (genotype.isHomRef()) {
AA = prior;
} else if (genotype.isHet()) {
AB = prior;
} else if (genotype.isHomVar()) {
BB = prior;
}
log10Likelihoods = MathUtils.toLog10(new double[] {AA, isMaleOnChrX ? 0.0 : AB, BB});
} else {
log10Likelihoods =
isMaleOnChrX ? HAPLOID_FLAT_LOG10_LIKELIHOODS : DIPLOID_FLAT_LOG10_LIKELIHOODS;
}
writeSampleLikelihoods(beagleOut, preferredVC, log10Likelihoods);
}
beagleWriter.println(beagleOut.toString());
}
/**
* Main entry function to calculate genotypes of a given VC with corresponding GL's
*
* @param tracker Tracker
* @param refContext Reference context
* @param rawContext Raw context
* @param stratifiedContexts Stratified alignment contexts
* @param vc Input VC
* @param model GL calculation model
* @param inheritAttributesFromInputVC Output VC will contain attributes inherited from input vc
* @return VC with assigned genotypes
*/
public VariantCallContext calculateGenotypes(
final RefMetaDataTracker tracker,
final ReferenceContext refContext,
final AlignmentContext rawContext,
Map<String, AlignmentContext> stratifiedContexts,
final VariantContext vc,
final GenotypeLikelihoodsCalculationModel.Model model,
final boolean inheritAttributesFromInputVC,
final Map<String, org.broadinstitute.sting.utils.genotyper.PerReadAlleleLikelihoodMap>
perReadAlleleLikelihoodMap) {
boolean limitedContext =
tracker == null || refContext == null || rawContext == null || stratifiedContexts == null;
// initialize the data for this thread if that hasn't been done yet
if (afcm.get() == null) {
afcm.set(AFCalcFactory.createAFCalc(UAC, N, logger));
}
// estimate our confidence in a reference call and return
if (vc.getNSamples() == 0) {
if (limitedContext) return null;
return (UAC.OutputMode != OUTPUT_MODE.EMIT_ALL_SITES
? estimateReferenceConfidence(vc, stratifiedContexts, getTheta(model), false, 1.0)
: generateEmptyContext(tracker, refContext, stratifiedContexts, rawContext));
}
AFCalcResult AFresult = afcm.get().getLog10PNonRef(vc, getAlleleFrequencyPriors(model));
// is the most likely frequency conformation AC=0 for all alternate alleles?
boolean bestGuessIsRef = true;
// determine which alternate alleles have AF>0
final List<Allele> myAlleles = new ArrayList<Allele>(vc.getAlleles().size());
final List<Integer> alleleCountsofMLE = new ArrayList<Integer>(vc.getAlleles().size());
myAlleles.add(vc.getReference());
for (int i = 0; i < AFresult.getAllelesUsedInGenotyping().size(); i++) {
final Allele alternateAllele = AFresult.getAllelesUsedInGenotyping().get(i);
if (alternateAllele.isReference()) continue;
// we are non-ref if the probability of being non-ref > the emit confidence.
// the emit confidence is phred-scaled, say 30 => 10^-3.
// the posterior AF > 0 is log10: -5 => 10^-5
// we are non-ref if 10^-5 < 10^-3 => -5 < -3
final boolean isNonRef =
AFresult.isPolymorphic(alternateAllele, UAC.STANDARD_CONFIDENCE_FOR_EMITTING / -10.0);
// if the most likely AC is not 0, then this is a good alternate allele to use
if (isNonRef) {
myAlleles.add(alternateAllele);
alleleCountsofMLE.add(AFresult.getAlleleCountAtMLE(alternateAllele));
bestGuessIsRef = false;
}
// if in GENOTYPE_GIVEN_ALLELES mode, we still want to allow the use of a poor allele
else if (UAC.GenotypingMode
== GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES) {
myAlleles.add(alternateAllele);
alleleCountsofMLE.add(AFresult.getAlleleCountAtMLE(alternateAllele));
}
}
final double PoFGT0 = Math.pow(10, AFresult.getLog10PosteriorOfAFGT0());
// note the math.abs is necessary because -10 * 0.0 => -0.0 which isn't nice
final double phredScaledConfidence =
Math.abs(
!bestGuessIsRef
|| UAC.GenotypingMode
== GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE
.GENOTYPE_GIVEN_ALLELES
? -10 * AFresult.getLog10PosteriorOfAFEq0()
: -10 * AFresult.getLog10PosteriorOfAFGT0());
// return a null call if we don't pass the confidence cutoff or the most likely allele frequency
// is zero
if (UAC.OutputMode != OUTPUT_MODE.EMIT_ALL_SITES
&& !passesEmitThreshold(phredScaledConfidence, bestGuessIsRef)) {
// technically, at this point our confidence in a reference call isn't accurately estimated
// because it didn't take into account samples with no data, so let's get a better estimate
return limitedContext
? null
: estimateReferenceConfidence(vc, stratifiedContexts, getTheta(model), true, PoFGT0);
}
// start constructing the resulting VC
final GenomeLoc loc = genomeLocParser.createGenomeLoc(vc);
final VariantContextBuilder builder =
new VariantContextBuilder(
"UG_call", loc.getContig(), loc.getStart(), loc.getStop(), myAlleles);
builder.log10PError(phredScaledConfidence / -10.0);
if (!passesCallThreshold(phredScaledConfidence)) builder.filters(filter);
// create the genotypes
final GenotypesContext genotypes = afcm.get().subsetAlleles(vc, myAlleles, true, ploidy);
builder.genotypes(genotypes);
// print out stats if we have a writer
if (verboseWriter != null && !limitedContext)
printVerboseData(refContext.getLocus().toString(), vc, PoFGT0, phredScaledConfidence, model);
// *** note that calculating strand bias involves overwriting data structures, so we do that
// last
final HashMap<String, Object> attributes = new HashMap<String, Object>();
// inherit attributed from input vc if requested
if (inheritAttributesFromInputVC) attributes.putAll(vc.getAttributes());
// if the site was downsampled, record that fact
if (!limitedContext && rawContext.hasPileupBeenDownsampled())
attributes.put(VCFConstants.DOWNSAMPLED_KEY, true);
if (UAC.ANNOTATE_NUMBER_OF_ALLELES_DISCOVERED)
attributes.put(NUMBER_OF_DISCOVERED_ALLELES_KEY, vc.getAlternateAlleles().size());
// add the MLE AC and AF annotations
if (alleleCountsofMLE.size() > 0) {
attributes.put(VCFConstants.MLE_ALLELE_COUNT_KEY, alleleCountsofMLE);
final int AN = builder.make().getCalledChrCount();
final ArrayList<Double> MLEfrequencies = new ArrayList<Double>(alleleCountsofMLE.size());
// the MLEAC is allowed to be larger than the AN (e.g. in the case of all PLs being 0, the GT
// is ./. but the exact model may arbitrarily choose an AC>1)
for (int AC : alleleCountsofMLE) MLEfrequencies.add(Math.min(1.0, (double) AC / (double) AN));
attributes.put(VCFConstants.MLE_ALLELE_FREQUENCY_KEY, MLEfrequencies);
}
if (UAC.COMPUTE_SLOD && !limitedContext && !bestGuessIsRef) {
// final boolean DEBUG_SLOD = false;
// the overall lod
// double overallLog10PofNull = AFresult.log10AlleleFrequencyPosteriors[0];
double overallLog10PofF = AFresult.getLog10LikelihoodOfAFGT0();
// if ( DEBUG_SLOD ) System.out.println("overallLog10PofF=" + overallLog10PofF);
List<Allele> allAllelesToUse = builder.make().getAlleles();
// the forward lod
VariantContext vcForward =
calculateLikelihoods(
tracker,
refContext,
stratifiedContexts,
AlignmentContextUtils.ReadOrientation.FORWARD,
allAllelesToUse,
false,
model,
perReadAlleleLikelihoodMap);
AFresult = afcm.get().getLog10PNonRef(vcForward, getAlleleFrequencyPriors(model));
// double[] normalizedLog10Posteriors =
// MathUtils.normalizeFromLog10(AFresult.log10AlleleFrequencyPosteriors, true);
double forwardLog10PofNull = AFresult.getLog10LikelihoodOfAFEq0();
double forwardLog10PofF = AFresult.getLog10LikelihoodOfAFGT0();
// if ( DEBUG_SLOD ) System.out.println("forwardLog10PofNull=" + forwardLog10PofNull + ",
// forwardLog10PofF=" + forwardLog10PofF);
// the reverse lod
VariantContext vcReverse =
calculateLikelihoods(
tracker,
refContext,
stratifiedContexts,
AlignmentContextUtils.ReadOrientation.REVERSE,
allAllelesToUse,
false,
model,
perReadAlleleLikelihoodMap);
AFresult = afcm.get().getLog10PNonRef(vcReverse, getAlleleFrequencyPriors(model));
// normalizedLog10Posteriors =
// MathUtils.normalizeFromLog10(AFresult.log10AlleleFrequencyPosteriors, true);
double reverseLog10PofNull = AFresult.getLog10LikelihoodOfAFEq0();
double reverseLog10PofF = AFresult.getLog10LikelihoodOfAFGT0();
// if ( DEBUG_SLOD ) System.out.println("reverseLog10PofNull=" + reverseLog10PofNull + ",
// reverseLog10PofF=" + reverseLog10PofF);
double forwardLod = forwardLog10PofF + reverseLog10PofNull - overallLog10PofF;
double reverseLod = reverseLog10PofF + forwardLog10PofNull - overallLog10PofF;
// if ( DEBUG_SLOD ) System.out.println("forward lod=" + forwardLod + ", reverse lod=" +
// reverseLod);
// strand score is max bias between forward and reverse strands
double strandScore = Math.max(forwardLod, reverseLod);
// rescale by a factor of 10
strandScore *= 10.0;
// logger.debug(String.format("SLOD=%f", strandScore));
if (!Double.isNaN(strandScore)) attributes.put("SB", strandScore);
}
// finish constructing the resulting VC
builder.attributes(attributes);
VariantContext vcCall = builder.make();
// if we are subsetting alleles (either because there were too many or because some were not
// polymorphic)
// then we may need to trim the alleles (because the original VariantContext may have had to pad
// at the end).
if (myAlleles.size() != vc.getAlleles().size()
&& !limitedContext) // limitedContext callers need to handle allele trimming on their own to
// keep their perReadAlleleLikelihoodMap alleles in sync
vcCall = VariantContextUtils.reverseTrimAlleles(vcCall);
if (annotationEngine != null
&& !limitedContext) { // limitedContext callers need to handle annotations on their own by
// calling their own annotationEngine
// Note: we want to use the *unfiltered* and *unBAQed* context for the annotations
final ReadBackedPileup pileup = rawContext.getBasePileup();
stratifiedContexts = AlignmentContextUtils.splitContextBySampleName(pileup);
vcCall =
annotationEngine.annotateContext(
tracker, refContext, stratifiedContexts, vcCall, perReadAlleleLikelihoodMap);
}
return new VariantCallContext(vcCall, confidentlyCalled(phredScaledConfidence, PoFGT0));
}
/**
* Merges VariantContexts into a single hybrid. Takes genotypes for common samples in priority
* order, if provided. If uniqifySamples is true, the priority order is ignored and names are
* created by concatenating the VC name with the sample name
*
* @param genomeLocParser loc parser
* @param unsortedVCs collection of unsorted VCs
* @param priorityListOfVCs priority list detailing the order in which we should grab the VCs
* @param filteredRecordMergeType merge type for filtered records
* @param genotypeMergeOptions merge option for genotypes
* @param annotateOrigin should we annotate the set it came from?
* @param printMessages should we print messages?
* @param setKey the key name of the set
* @param filteredAreUncalled are filtered records uncalled?
* @param mergeInfoWithMaxAC should we merge in info from the VC with maximum allele count?
* @return new VariantContext representing the merge of unsortedVCs
*/
public static VariantContext simpleMerge(
final GenomeLocParser genomeLocParser,
final Collection<VariantContext> unsortedVCs,
final List<String> priorityListOfVCs,
final FilteredRecordMergeType filteredRecordMergeType,
final GenotypeMergeType genotypeMergeOptions,
final boolean annotateOrigin,
final boolean printMessages,
final String setKey,
final boolean filteredAreUncalled,
final boolean mergeInfoWithMaxAC) {
if (unsortedVCs == null || unsortedVCs.size() == 0) return null;
if (annotateOrigin && priorityListOfVCs == null)
throw new IllegalArgumentException(
"Cannot merge calls and annotate their origins without a complete priority list of VariantContexts");
if (genotypeMergeOptions == GenotypeMergeType.REQUIRE_UNIQUE)
verifyUniqueSampleNames(unsortedVCs);
List<VariantContext> prepaddedVCs =
sortVariantContextsByPriority(unsortedVCs, priorityListOfVCs, genotypeMergeOptions);
// Make sure all variant contexts are padded with reference base in case of indels if necessary
List<VariantContext> VCs = new ArrayList<VariantContext>();
for (VariantContext vc : prepaddedVCs) {
// also a reasonable place to remove filtered calls, if needed
if (!filteredAreUncalled || vc.isNotFiltered())
VCs.add(createVariantContextWithPaddedAlleles(vc, false));
}
if (VCs.size() == 0) // everything is filtered out and we're filteredAreUncalled
return null;
// establish the baseline info from the first VC
final VariantContext first = VCs.get(0);
final String name = first.getSource();
final Allele refAllele = determineReferenceAllele(VCs);
final Set<Allele> alleles = new LinkedHashSet<Allele>();
final Set<String> filters = new TreeSet<String>();
final Map<String, Object> attributes = new TreeMap<String, Object>();
final Set<String> inconsistentAttributes = new HashSet<String>();
final Set<String> variantSources =
new HashSet<
String>(); // contains the set of sources we found in our set of VCs that are variant
final Set<String> rsIDs = new LinkedHashSet<String>(1); // most of the time there's one id
GenomeLoc loc = getLocation(genomeLocParser, first);
int depth = 0;
int maxAC = -1;
final Map<String, Object> attributesWithMaxAC = new TreeMap<String, Object>();
double log10PError = 1;
VariantContext vcWithMaxAC = null;
GenotypesContext genotypes = GenotypesContext.create();
// counting the number of filtered and variant VCs
int nFiltered = 0;
boolean remapped = false;
// cycle through and add info from the other VCs, making sure the loc/reference matches
for (VariantContext vc : VCs) {
if (loc.getStart() != vc.getStart()) // || !first.getReference().equals(vc.getReference()) )
throw new ReviewedStingException(
"BUG: attempting to merge VariantContexts with different start sites: first="
+ first.toString()
+ " second="
+ vc.toString());
if (getLocation(genomeLocParser, vc).size() > loc.size())
loc = getLocation(genomeLocParser, vc); // get the longest location
nFiltered += vc.isFiltered() ? 1 : 0;
if (vc.isVariant()) variantSources.add(vc.getSource());
AlleleMapper alleleMapping = resolveIncompatibleAlleles(refAllele, vc, alleles);
remapped = remapped || alleleMapping.needsRemapping();
alleles.addAll(alleleMapping.values());
mergeGenotypes(
genotypes, vc, alleleMapping, genotypeMergeOptions == GenotypeMergeType.UNIQUIFY);
log10PError = Math.min(log10PError, vc.isVariant() ? vc.getLog10PError() : 1);
filters.addAll(vc.getFilters());
//
// add attributes
//
// special case DP (add it up) and ID (just preserve it)
//
if (vc.hasAttribute(VCFConstants.DEPTH_KEY))
depth += vc.getAttributeAsInt(VCFConstants.DEPTH_KEY, 0);
if (vc.hasID()) rsIDs.add(vc.getID());
if (mergeInfoWithMaxAC && vc.hasAttribute(VCFConstants.ALLELE_COUNT_KEY)) {
String rawAlleleCounts = vc.getAttributeAsString(VCFConstants.ALLELE_COUNT_KEY, null);
// lets see if the string contains a , separator
if (rawAlleleCounts.contains(VCFConstants.INFO_FIELD_ARRAY_SEPARATOR)) {
List<String> alleleCountArray =
Arrays.asList(
rawAlleleCounts
.substring(1, rawAlleleCounts.length() - 1)
.split(VCFConstants.INFO_FIELD_ARRAY_SEPARATOR));
for (String alleleCount : alleleCountArray) {
final int ac = Integer.valueOf(alleleCount.trim());
if (ac > maxAC) {
maxAC = ac;
vcWithMaxAC = vc;
}
}
} else {
final int ac = Integer.valueOf(rawAlleleCounts);
if (ac > maxAC) {
maxAC = ac;
vcWithMaxAC = vc;
}
}
}
for (Map.Entry<String, Object> p : vc.getAttributes().entrySet()) {
String key = p.getKey();
// if we don't like the key already, don't go anywhere
if (!inconsistentAttributes.contains(key)) {
boolean alreadyFound = attributes.containsKey(key);
Object boundValue = attributes.get(key);
boolean boundIsMissingValue =
alreadyFound && boundValue.equals(VCFConstants.MISSING_VALUE_v4);
if (alreadyFound && !boundValue.equals(p.getValue()) && !boundIsMissingValue) {
// we found the value but we're inconsistent, put it in the exclude list
// System.out.printf("Inconsistent INFO values: %s => %s and %s%n", key, boundValue,
// p.getValue());
inconsistentAttributes.add(key);
attributes.remove(key);
} else if (!alreadyFound || boundIsMissingValue) { // no value
// if ( vc != first ) System.out.printf("Adding key %s => %s%n", p.getKey(),
// p.getValue());
attributes.put(key, p.getValue());
}
}
}
}
// if we have more alternate alleles in the merged VC than in one or more of the
// original VCs, we need to strip out the GL/PLs (because they are no longer accurate), as well
// as allele-dependent attributes like AC,AF
for (VariantContext vc : VCs) {
if (vc.alleles.size() == 1) continue;
if (hasPLIncompatibleAlleles(alleles, vc.alleles)) {
if (!genotypes.isEmpty())
logger.warn(
String.format(
"Stripping PLs at %s due incompatible alleles merged=%s vs. single=%s",
genomeLocParser.createGenomeLoc(vc), alleles, vc.alleles));
genotypes = stripPLs(genotypes);
// this will remove stale AC,AF attributed from vc
calculateChromosomeCounts(vc, attributes, true);
break;
}
}
// take the VC with the maxAC and pull the attributes into a modifiable map
if (mergeInfoWithMaxAC && vcWithMaxAC != null) {
attributesWithMaxAC.putAll(vcWithMaxAC.getAttributes());
}
// if at least one record was unfiltered and we want a union, clear all of the filters
if ((filteredRecordMergeType == FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED
&& nFiltered != VCs.size())
|| filteredRecordMergeType == FilteredRecordMergeType.KEEP_UNCONDITIONAL) filters.clear();
if (annotateOrigin) { // we care about where the call came from
String setValue;
if (nFiltered == 0
&& variantSources.size() == priorityListOfVCs.size()) // nothing was unfiltered
setValue = MERGE_INTERSECTION;
else if (nFiltered == VCs.size()) // everything was filtered out
setValue = MERGE_FILTER_IN_ALL;
else if (variantSources.isEmpty()) // everyone was reference
setValue = MERGE_REF_IN_ALL;
else {
LinkedHashSet<String> s = new LinkedHashSet<String>();
for (VariantContext vc : VCs)
if (vc.isVariant())
s.add(vc.isFiltered() ? MERGE_FILTER_PREFIX + vc.getSource() : vc.getSource());
setValue = Utils.join("-", s);
}
if (setKey != null) {
attributes.put(setKey, setValue);
if (mergeInfoWithMaxAC && vcWithMaxAC != null) {
attributesWithMaxAC.put(setKey, vcWithMaxAC.getSource());
}
}
}
if (depth > 0) attributes.put(VCFConstants.DEPTH_KEY, String.valueOf(depth));
final String ID = rsIDs.isEmpty() ? VCFConstants.EMPTY_ID_FIELD : Utils.join(",", rsIDs);
final VariantContextBuilder builder = new VariantContextBuilder().source(name).id(ID);
builder.loc(loc.getContig(), loc.getStart(), loc.getStop());
builder.alleles(alleles);
builder.genotypes(genotypes);
builder.log10PError(log10PError);
builder.filters(filters).attributes(mergeInfoWithMaxAC ? attributesWithMaxAC : attributes);
// Trim the padded bases of all alleles if necessary
VariantContext merged = createVariantContextWithTrimmedAlleles(builder.make());
if (printMessages && remapped) System.out.printf("Remapped => %s%n", merged);
return merged;
}
@Override
public T traverse(
final ActiveRegionWalker<M, T> walker, final LocusShardDataProvider dataProvider, T sum) {
logger.debug(String.format("TraverseActiveRegion.traverse: Shard is %s", dataProvider));
final LocusView locusView = getLocusView(walker, dataProvider);
final GenomeLocSortedSet initialIntervals = engine.getIntervals();
final LocusReferenceView referenceView = new LocusReferenceView(walker, dataProvider);
final int activeRegionExtension =
walker.getClass().getAnnotation(ActiveRegionExtension.class).extension();
final int maxRegionSize =
walker.getClass().getAnnotation(ActiveRegionExtension.class).maxRegion();
if (locusView
.hasNext()) { // trivial optimization to avoid unnecessary processing when there's nothing
// here at all
int minStart = Integer.MAX_VALUE;
ActivityProfile profile =
new ActivityProfile(engine.getGenomeLocParser(), walker.hasPresetActiveRegions());
ReferenceOrderedView referenceOrderedDataView =
getReferenceOrderedView(walker, dataProvider, locusView);
// We keep processing while the next reference location is within the interval
GenomeLoc prevLoc = null;
while (locusView.hasNext()) {
final AlignmentContext locus = locusView.next();
GenomeLoc location = locus.getLocation();
if (prevLoc != null) {
// fill in the active / inactive labels from the stop of the previous location to the
// start of this location
// TODO refactor to separate function
for (int iii = prevLoc.getStop() + 1; iii < location.getStart(); iii++) {
final GenomeLoc fakeLoc =
engine.getGenomeLocParser().createGenomeLoc(prevLoc.getContig(), iii, iii);
if (initialIntervals == null || initialIntervals.overlaps(fakeLoc)) {
profile.add(
fakeLoc,
new ActivityProfileResult(
walker.hasPresetActiveRegions()
&& walker.presetActiveRegions.overlaps(fakeLoc)
? 1.0
: 0.0));
}
}
}
dataProvider.getShard().getReadMetrics().incrementNumIterations();
// create reference context. Note that if we have a pileup of "extended events", the context
// will
// hold the (longest) stretch of deleted reference bases (if deletions are present in the
// pileup).
final ReferenceContext refContext = referenceView.getReferenceContext(location);
// Iterate forward to get all reference ordered data covering this location
final RefMetaDataTracker tracker =
referenceOrderedDataView.getReferenceOrderedDataAtLocus(
locus.getLocation(), refContext);
// Call the walkers isActive function for this locus and add them to the list to be
// integrated later
if (initialIntervals == null || initialIntervals.overlaps(location)) {
profile.add(location, walkerActiveProb(walker, tracker, refContext, locus, location));
}
// Grab all the previously unseen reads from this pileup and add them to the massive read
// list
for (final PileupElement p : locus.getBasePileup()) {
final GATKSAMRecord read = p.getRead();
if (!myReads.contains(read)) {
myReads.add(read);
}
// If this is the last pileup for this shard calculate the minimum alignment start so that
// we know
// which active regions in the work queue are now safe to process
minStart = Math.min(minStart, read.getAlignmentStart());
}
prevLoc = location;
printProgress(locus.getLocation());
}
updateCumulativeMetrics(dataProvider.getShard());
// Take the individual isActive calls and integrate them into contiguous active regions and
// add these blocks of work to the work queue
// band-pass filter the list of isActive probabilities and turn into active regions
final ActivityProfile bandPassFiltered = profile.bandPassFilter();
final List<ActiveRegion> activeRegions =
bandPassFiltered.createActiveRegions(activeRegionExtension, maxRegionSize);
// add active regions to queue of regions to process
// first check if can merge active regions over shard boundaries
if (!activeRegions.isEmpty()) {
if (!workQueue.isEmpty()) {
final ActiveRegion last = workQueue.getLast();
final ActiveRegion first = activeRegions.get(0);
if (last.isActive == first.isActive
&& last.getLocation().contiguousP(first.getLocation())
&& last.getLocation().size() + first.getLocation().size() <= maxRegionSize) {
workQueue.removeLast();
activeRegions.remove(first);
workQueue.add(
new ActiveRegion(
last.getLocation().union(first.getLocation()),
first.isActive,
this.engine.getGenomeLocParser(),
activeRegionExtension));
}
}
workQueue.addAll(activeRegions);
}
logger.debug(
"Integrated "
+ profile.size()
+ " isActive calls into "
+ activeRegions.size()
+ " regions.");
// now go and process all of the active regions
sum = processActiveRegions(walker, sum, minStart, dataProvider.getLocus().getContig());
}
return sum;
}
/**
* Creates an iterator that can traverse over the entire reference specified in the given
* ShardDataProvider.
*
* @param completeLocus Data provider to use as a backing source. Provider must have a reference
* (hasReference() == true).
*/
public GenomeLocusIterator(GenomeLocParser parser, GenomeLoc completeLocus) {
this.parser = parser;
this.completeLocus = completeLocus;
this.currentLocus = parser.createGenomeLoc(completeLocus.getContig(), completeLocus.getStart());
}
public Datum map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
GenomeLoc cur = context.getLocation();
if (verbose && showSkipped) {
for (long i = context.getSkippedBases(); i >= 0; i--) {
SAMSequenceDictionary dictionary =
getToolkit().getReferenceDataSource().getReference().getSequenceDictionary();
SAMSequenceRecord contig = dictionary.getSequence(cur.getContig());
if (cur.getStop() < contig.getSequenceLength())
cur = getToolkit().getGenomeLocParser().incPos(cur, 1);
else
cur =
getToolkit()
.getGenomeLocParser()
.createGenomeLoc(
dictionary.getSequence(contig.getSequenceIndex() + 1).getSequenceName(),
1,
1);
out.printf("%s: skipped%n", cur);
}
}
long nRodsHere = 0;
long nTotalBases = 0;
if (ref == null) {
// we're getting the last skipped update
if (verbose)
out.printf(
"Last position was %s: skipping %d bases%n",
context.getLocation(), context.getSkippedBases());
nRodsHere = -1; // don't update this
nTotalBases = context.getSkippedBases();
} else {
Collection<RODRecordList> rods = new LinkedList<RODRecordList>();
for (RODRecordList rod : tracker.getBoundRodTracks()) {
// System.out.printf("Considering rod %s%n", rod);
if (rod.getLocation().getStart() == context.getLocation().getStart()
&& !rod.getName().equals("interval")) {
// only consider the first element
// System.out.printf("adding it%n");
rods.add(rod);
}
}
nRodsHere = rods.size();
if (nRodsHere > 0) {
if (verbose) {
List<String> names = new ArrayList<String>();
for (RODRecordList rod : rods) {
names.add(rod.getName());
}
// System.out.printf("context is %s", context.getSkippedBases());
out.printf(
"At %s: found %d rod(s) [%s] after skipping %d bases%n",
context.getLocation(), nRodsHere, Utils.join(",", names), context.getSkippedBases());
}
}
nTotalBases = context.getSkippedBases() + 1;
}
return new Datum(nRodsHere, context.getSkippedBases(), nTotalBases);
}
/**
* Read in a list of ExactCall objects from reader, keeping only those with starts in startsToKeep
* or all sites (if this is empty)
*
* @param reader a just-opened reader sitting at the start of the file
* @param startsToKeep a list of start position of the calls to keep, or empty if all calls should
* be kept
* @param parser a genome loc parser to create genome locs
* @return a list of ExactCall objects in reader
* @throws IOException
*/
public static List<ExactCall> readExactLog(
final BufferedReader reader, final List<Integer> startsToKeep, GenomeLocParser parser)
throws IOException {
if (reader == null) throw new IllegalArgumentException("reader cannot be null");
if (startsToKeep == null) throw new IllegalArgumentException("startsToKeep cannot be null");
if (parser == null) throw new IllegalArgumentException("GenomeLocParser cannot be null");
List<ExactCall> calls = new LinkedList<ExactCall>();
// skip the header line
reader.readLine();
// skip the first "type" line
reader.readLine();
while (true) {
final VariantContextBuilder builder = new VariantContextBuilder();
final List<Allele> alleles = new ArrayList<Allele>();
final List<Genotype> genotypes = new ArrayList<Genotype>();
final double[] posteriors = new double[2];
final double[] priors = MathUtils.normalizeFromLog10(new double[] {0.5, 0.5}, true);
final List<Integer> mle = new ArrayList<Integer>();
final Map<Allele, Double> log10pNonRefByAllele = new HashMap<Allele, Double>();
long runtimeNano = -1;
GenomeLoc currentLoc = null;
while (true) {
final String line = reader.readLine();
if (line == null) return calls;
final String[] parts = line.split("\t");
final GenomeLoc lineLoc = parser.parseGenomeLoc(parts[0]);
final String variable = parts[1];
final String key = parts[2];
final String value = parts[3];
if (currentLoc == null) currentLoc = lineLoc;
if (variable.equals("type")) {
if (startsToKeep.isEmpty() || startsToKeep.contains(currentLoc.getStart())) {
builder.alleles(alleles);
final int stop = currentLoc.getStart() + alleles.get(0).length() - 1;
builder.chr(currentLoc.getContig()).start(currentLoc.getStart()).stop(stop);
builder.genotypes(genotypes);
final int[] mleInts = ArrayUtils.toPrimitive(mle.toArray(new Integer[] {}));
final AFCalcResult result =
new AFCalcResult(mleInts, 1, alleles, posteriors, priors, log10pNonRefByAllele);
calls.add(new ExactCall(builder.make(), runtimeNano, result));
}
break;
} else if (variable.equals("allele")) {
final boolean isRef = key.equals("0");
alleles.add(Allele.create(value, isRef));
} else if (variable.equals("PL")) {
final GenotypeBuilder gb = new GenotypeBuilder(key);
gb.PL(GenotypeLikelihoods.fromPLField(value).getAsPLs());
genotypes.add(gb.make());
} else if (variable.equals("log10PosteriorOfAFEq0")) {
posteriors[0] = Double.valueOf(value);
} else if (variable.equals("log10PosteriorOfAFGt0")) {
posteriors[1] = Double.valueOf(value);
} else if (variable.equals("MLE")) {
mle.add(Integer.valueOf(value));
} else if (variable.equals("pNonRefByAllele")) {
final Allele a = Allele.create(key);
log10pNonRefByAllele.put(a, Double.valueOf(value));
} else if (variable.equals("runtime.nano")) {
runtimeNano = Long.valueOf(value);
} else {
// nothing to do
}
}
}
}
private GenomeLoc createIntervalBefore(GenomeLoc interval) {
int start = Math.max(interval.getStart() - expandInterval, 0);
int stop = Math.max(interval.getStart() - 1, 0);
return parser.createGenomeLoc(interval.getContig(), interval.getContigIndex(), start, stop);
}