/**
   * Performs allele biased down-sampling on a pileup and computes the list of elements to remove
   *
   * @param reads original list of records
   * @param numElementsToRemove the number of records to remove
   * @return the list of pileup elements TO REMOVE
   */
  protected static List<GATKSAMRecord> downsampleElements(
      final List<GATKSAMRecord> reads, final int numElementsToRemove) {
    // are there no elements to remove?
    if (numElementsToRemove == 0) return Collections.<GATKSAMRecord>emptyList();

    final ArrayList<GATKSAMRecord> elementsToRemove =
        new ArrayList<GATKSAMRecord>(numElementsToRemove);
    final int originalElementCount = reads.size();

    // should we remove all of the elements?
    if (numElementsToRemove >= originalElementCount) {
      elementsToRemove.addAll(reads);
      return elementsToRemove;
    }

    // create a bitset describing which elements to remove
    final BitSet itemsToRemove = new BitSet(originalElementCount);
    for (final Integer selectedIndex :
        MathUtils.sampleIndicesWithoutReplacement(originalElementCount, numElementsToRemove)) {
      itemsToRemove.set(selectedIndex);
    }

    int currentBitSetIndex = 0;
    for (final GATKSAMRecord read : reads) {
      if (itemsToRemove.get(currentBitSetIndex++)) elementsToRemove.add(read);
    }

    return elementsToRemove;
  }
  /**
   * P(somatic | D) = P(somatic) * P(D | somatic) = P(somatic) * P(D | normals are ref) * P(D |
   * tumors are non-ref)
   *
   * <p>P(! somatic | D) = P(! somatic) * P(D | ! somatic) = P(! somatic) * * ( P(D | normals are
   * non-ref) * P(D | tumors are non-ref) [germline] + P(D | normals are ref) * P(D | tumors are
   * ref)) [no-variant at all]
   *
   * @param vc
   * @return
   */
  private double calcLog10pSomatic(final VariantContext vc) {
    // walk over tumors
    double log10pNonRefInTumors = log10pNonRefInSamples(vc, tumorSample);
    double log10pRefInTumors = log10pRefInSamples(vc, tumorSample);

    // walk over normals
    double log10pNonRefInNormals = log10pNonRefInSamples(vc, normalSample);
    double log10pRefInNormals = log10pRefInSamples(vc, normalSample);

    // priors
    double log10pSomaticPrior = QualityUtils.qualToErrorProbLog10(somaticPriorQ);
    double log10pNotSomaticPrior = Math.log10(1 - QualityUtils.qualToErrorProb(somaticPriorQ));

    double log10pNotSomaticGermline = log10pNonRefInNormals + log10pNonRefInTumors;
    double log10pNotSomaticNoVariant = log10pRefInNormals + log10pRefInTumors;

    double log10pNotSomatic =
        log10pNotSomaticPrior
            + MathUtils.log10sumLog10(
                new double[] {log10pNotSomaticGermline, log10pNotSomaticNoVariant});
    double log10pSomatic = log10pSomaticPrior + log10pNonRefInTumors + log10pRefInNormals;
    double lod = log10pSomatic - log10pNotSomatic;

    return Double.isInfinite(lod) ? -10000 : lod;
  }
  public double[] computeReadHaplotypeLikelihoods(
      ReadBackedPileup pileup, HashMap<Allele, Haplotype> haplotypesInVC) {
    double[][] haplotypeLikehoodMatrix = new double[haplotypesInVC.size()][haplotypesInVC.size()];
    double readLikelihoods[][] = new double[pileup.getReads().size()][haplotypesInVC.size()];
    int i = 0;
    for (GATKSAMRecord read : pileup.getReads()) {
      if (ReadUtils.is454Read(read)) {
        continue;
      }
      // for each read/haplotype combination, compute likelihoods, ie -10*log10(Pr(R | Hi))
      // = sum_j(-10*log10(Pr(R_j | Hi) since reads are assumed to be independent
      int j = 0;
      for (Map.Entry<Allele, Haplotype> a : haplotypesInVC.entrySet()) {
        readLikelihoods[i][j] = computeReadLikelihoodGivenHaplotype(a.getValue(), read);
        if (DEBUG) {
          System.out.print(read.getReadName() + " ");

          System.out.format(
              "%d %d S:%d US:%d E:%d UE:%d C:%s %3.4f\n",
              i,
              j,
              read.getAlignmentStart(),
              read.getUnclippedStart(),
              read.getAlignmentEnd(),
              read.getUnclippedEnd(),
              read.getCigarString(),
              readLikelihoods[i][j]);
        }
        j++;
      }
      i++;
    }

    for (i = 0; i < haplotypesInVC.size(); i++) {
      for (int j = i; j < haplotypesInVC.size(); j++) {
        // combine likelihoods of haplotypeLikelihoods[i], haplotypeLikelihoods[j]
        // L(Hi, Hj) = sum_reads ( Pr(R|Hi)/2 + Pr(R|Hj)/2)
        // readLikelihoods[k][j] has log10(Pr(R_k) | H[j] )
        double[] readLikelihood = new double[2]; // diploid sample
        for (int readIdx = 0; readIdx < pileup.getReads().size(); readIdx++) {
          readLikelihood[0] = -readLikelihoods[readIdx][i] / 10;
          readLikelihood[1] = -readLikelihoods[readIdx][j] / 10;

          // Compute log10(10^x1/2 + 10^x2/2) = log10(10^x1+x0^x2)-log10(2)
          // First term is approximated by Jacobian log with table lookup.
          // Second term is a constant added to both likelihoods so will be ignored
          haplotypeLikehoodMatrix[i][j] +=
              MathUtils.approximateLog10SumLog10(readLikelihood[0], readLikelihood[1]);
        }
      }
    }

    return getHaplotypeLikelihoods(haplotypeLikehoodMatrix);
  }
  private double log10PLFromSamples(
      final VariantContext vc, final String sample, boolean calcRefP) {

    Genotype g = vc.getGenotype(sample);
    double log10pSample = -1000;
    if (!g.isNoCall()) {
      final double[] gLikelihoods = MathUtils.normalizeFromLog10(g.getLikelihoods().getAsVector());
      log10pSample = Math.log10(calcRefP ? gLikelihoods[0] : 1 - gLikelihoods[0]);
      log10pSample = Double.isInfinite(log10pSample) ? -10000 : log10pSample;
    }
    return log10pSample;
  }
  /**
   * Computes an allele biased version of the given pileup
   *
   * @param pileup the original pileup
   * @param downsamplingFraction the fraction of total reads to remove per allele
   * @return allele biased pileup
   */
  public static ReadBackedPileup createAlleleBiasedBasePileup(
      final ReadBackedPileup pileup, final double downsamplingFraction) {
    // special case removal of all or no reads
    if (downsamplingFraction <= 0.0) return pileup;
    if (downsamplingFraction >= 1.0)
      return new ReadBackedPileupImpl(pileup.getLocation(), new ArrayList<PileupElement>());

    final PileupElementList[] alleleStratifiedElements = new PileupElementList[4];
    for (int i = 0; i < 4; i++) alleleStratifiedElements[i] = new PileupElementList();

    // start by stratifying the reads by the alleles they represent at this position
    for (final PileupElement pe : pileup) {
      final int baseIndex = BaseUtils.simpleBaseToBaseIndex(pe.getBase());
      if (baseIndex != -1) alleleStratifiedElements[baseIndex].add(pe);
    }

    // make a listing of allele counts and calculate the total count
    final int[] alleleCounts = calculateAlleleCounts(alleleStratifiedElements);
    final int totalAlleleCount = (int) MathUtils.sum(alleleCounts);

    // do smart down-sampling
    final int numReadsToRemove = (int) (totalAlleleCount * downsamplingFraction); // floor
    final int[] targetAlleleCounts = runSmartDownsampling(alleleCounts, numReadsToRemove);

    final HashSet<PileupElement> readsToRemove = new HashSet<PileupElement>(numReadsToRemove);
    for (int i = 0; i < 4; i++) {
      final PileupElementList alleleList = alleleStratifiedElements[i];
      // if we don't need to remove any reads, then don't
      if (alleleCounts[i] > targetAlleleCounts[i])
        readsToRemove.addAll(
            downsampleElements(
                alleleList, alleleCounts[i], alleleCounts[i] - targetAlleleCounts[i]));
    }

    // we need to keep the reads sorted because the FragmentUtils code will expect them in
    // coordinate order and will fail otherwise
    final List<PileupElement> readsToKeep =
        new ArrayList<PileupElement>(totalAlleleCount - numReadsToRemove);
    for (final PileupElement pe : pileup) {
      if (!readsToRemove.contains(pe)) {
        readsToKeep.add(pe);
      }
    }

    return new ReadBackedPileupImpl(
        pileup.getLocation(), new ArrayList<PileupElement>(readsToKeep));
  }
  /**
   * Get the most likely alleles estimated across all reads in this object
   *
   * <p>Takes the most likely two alleles according to their diploid genotype likelihoods. That is,
   * for each allele i and j we compute p(D | i,j) where D is the read likelihoods. We track the
   * maximum i,j likelihood and return an object that contains the alleles i and j as well as the
   * max likelihood.
   *
   * <p>Note that the second most likely diploid genotype is not tracked so the resulting
   * MostLikelyAllele doesn't have a meaningful get best likelihood.
   *
   * @return a MostLikelyAllele object, or null if this map is empty
   */
  public MostLikelyAllele getMostLikelyDiploidAlleles() {
    if (isEmpty()) return null;

    int hap1 = 0;
    int hap2 = 0;
    double maxElement = Double.NEGATIVE_INFINITY;
    for (int iii = 0; iii < alleles.size(); iii++) {
      final Allele iii_allele = alleles.get(iii);
      for (int jjj = 0; jjj <= iii; jjj++) {
        final Allele jjj_allele = alleles.get(jjj);

        double haplotypeLikelihood = 0.0;
        for (final Map.Entry<GATKSAMRecord, Map<Allele, Double>> entry :
            likelihoodReadMap.entrySet()) {
          // Compute log10(10^x1/2 + 10^x2/2) = log10(10^x1+10^x2)-log10(2)
          final double likelihood_iii = entry.getValue().get(iii_allele);
          final double likelihood_jjj = entry.getValue().get(jjj_allele);
          haplotypeLikelihood +=
              MathUtils.approximateLog10SumLog10(likelihood_iii, likelihood_jjj)
                  + MathUtils.LOG_ONE_HALF;

          // fast exit.  If this diploid pair is already worse than the max, just stop and look at
          // the next pair
          if (haplotypeLikelihood < maxElement) break;
        }

        // keep track of the max element and associated indices
        if (haplotypeLikelihood > maxElement) {
          hap1 = iii;
          hap2 = jjj;
          maxElement = haplotypeLikelihood;
        }
      }
    }

    if (maxElement == Double.NEGATIVE_INFINITY)
      throw new IllegalStateException(
          "max likelihood is " + maxElement + " indicating something has gone wrong");

    return new MostLikelyAllele(alleles.get(hap1), alleles.get(hap2), maxElement, maxElement);
  }
  @Override
  public CallableBaseState map(
      RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
    CalledState state;

    if (BaseUtils.isNBase(ref.getBase())) {
      state = CalledState.REF_N;
    } else {
      // count up the depths of all and QC+ bases
      int rawDepth = 0, QCDepth = 0, lowMAPQDepth = 0;
      for (PileupElement e : context.getBasePileup()) {
        rawDepth++;

        if (e.getMappingQual() <= maxLowMAPQ) lowMAPQDepth++;

        if (e.getMappingQual() >= minMappingQuality
            && (e.getQual() >= minBaseQuality || e.isDeletion())) {
          QCDepth++;
        }
      }

      // System.out.printf("%s rawdepth = %d QCDepth = %d lowMAPQ = %d%n", context.getLocation(),
      // rawDepth, QCDepth, lowMAPQDepth);
      if (rawDepth == 0) {
        state = CalledState.NO_COVERAGE;
      } else if (rawDepth >= minDepthLowMAPQ
          && MathUtils.ratio(lowMAPQDepth, rawDepth) >= maxLowMAPQFraction) {
        state = CalledState.POOR_MAPPING_QUALITY;
      } else if (QCDepth < minDepth) {
        state = CalledState.LOW_COVERAGE;
      } else if (rawDepth >= maxDepth && maxDepth != -1) {
        state = CalledState.EXCESSIVE_COVERAGE;
      } else {
        state = CalledState.CALLABLE;
      }
    }

    return new CallableBaseState(getToolkit().getGenomeLocParser(), context.getLocation(), state);
  }
  public double computeReadLikelihoodGivenHaplotype(Haplotype haplotype, SAMRecord read) {

    long numStartClippedBases = 0;
    long numEndClippedBases = 0;

    byte[] unclippedReadQuals = read.getBaseQualities();
    byte[] unclippedReadBases = read.getReadBases();

    // Do a stricter base clipping than provided by CIGAR string, since this one may be too
    // conservative,
    // and may leave a string of Q2 bases still hanging off the reads.
    for (int i = 0; i < read.getReadLength(); i++) {
      if (unclippedReadQuals[i] < BASE_QUAL_THRESHOLD) numStartClippedBases++;
      else break;
    }
    for (int i = read.getReadLength() - 1; i >= 0; i--) {
      if (unclippedReadQuals[i] < BASE_QUAL_THRESHOLD) numEndClippedBases++;
      else break;
    }
    // System.out.format("numstart: %d numend: %d\n", numStartClippedBases, numEndClippedBases);
    if (numStartClippedBases + numEndClippedBases >= read.getReadLength()) {
      return 0; /// Double.POSITIVE_INFINITY;
    }
    byte[] readBases =
        Arrays.copyOfRange(
            unclippedReadBases,
            (int) numStartClippedBases,
            (int) (read.getReadBases().length - numEndClippedBases));

    byte[] readQuals =
        Arrays.copyOfRange(
            unclippedReadQuals,
            (int) numStartClippedBases,
            (int) (read.getReadBases().length - numEndClippedBases));

    int readLength = readBases.length;

    // initialize path metric and traceback memories for Viterbi computation
    pathMetricArray = new double[readLength + 1][PATH_METRIC_TABLE_LENGTH];
    bestStateIndexArray = new int[readLength + 1][PATH_METRIC_TABLE_LENGTH];

    for (int k = 1; k < PATH_METRIC_TABLE_LENGTH; k++) pathMetricArray[0][k] = 0;

    /*

     if (doSimpleCalculationModel) {

        // No Viterbi algorithm - assume no sequencing indel artifacts,

        // so we can collapse computations and pr(read | haplotype) is just probability of observing overlap
        // of read with haplotype.
        int haplotypeIndex = initialIndexInHaplotype;
        double c =  0.0;//deletionErrorProbabilities[1] +logOneMinusInsertionStartProbability;
        // compute likelihood of portion of base to the left of the haplotype
        for (int indR=readStartIdx-1; indR >= 0; indR--) {
            byte readBase = readBases[indR];
            byte readQual = readQuals[indR];
            if (readQual <= 2)
                continue;
            double pBaseRead = getProbabilityOfReadBaseGivenXandI((byte)0, readBase, readQual, LEFT_ALIGN_INDEX, 0);

            // pBaseRead has -10*log10(Prob(base[i]|haplotype[i])
            pRead += pBaseRead;

        }
        //System.out.format("\nSt: %d Pre-Likelihood:%f\n",readStartIdx, pRead);

        for (int indR=readStartIdx; indR < readBases.length; indR++) {
            byte readBase = readBases[indR];
            byte readQual = readQuals[indR];

            byte haplotypeBase;
            if (haplotypeIndex < RIGHT_ALIGN_INDEX)
                haplotypeBase = haplotype.getBases()[haplotypeIndex];
            else
                haplotypeBase = (byte)0; // dummy

            double pBaseRead = getProbabilityOfReadBaseGivenXandI(haplotypeBase, readBase, readQual, haplotypeIndex, 0);
            if (haplotypeBase != 0)
                pBaseRead += c;

            // pBaseRead has -10*log10(Prob(base[i]|haplotype[i])
            if (readQual > 3)
                pRead += pBaseRead;
            haplotypeIndex++;
            if (haplotypeIndex >= haplotype.getBases().length)
                haplotypeIndex = RIGHT_ALIGN_INDEX;
            //System.out.format("H:%c R:%c RQ:%d HI:%d %4.5f %4.5f\n", haplotypeBase, readBase, (int)readQual, haplotypeIndex, pBaseRead, pRead);
         }
        //System.out.format("\nSt: %d Post-Likelihood:%f\n",readStartIdx, pRead);

        if (DEBUG) {
            System.out.println(read.getReadName());
            System.out.print("Haplotype:");

            for (int k=0; k <haplotype.getBases().length; k++) {
                System.out.format("%c ", haplotype.getBases()[k]);
            }
            System.out.println();

            System.out.print("Read bases: ");
            for (int k=0; k <readBases.length; k++) {
                System.out.format("%c ", readBases[k]);
            }
            System.out.format("\nLikelihood:%f\n",pRead);

        }

        if (read.getReadName().contains("106880")) {

            System.out.println("aca");

            System.out.println("Haplotype:");

            for (int k=initialIndexInHaplotype; k <haplotype.getBases().length; k++) {
                System.out.format("%c ", haplotype.getBases()[k]);
            }
            System.out.println();

            System.out.println("Read bases: ");
            for (int k=readStartIdx; k <readBases.length; k++) {
                System.out.format("%c ", readBases[k]);
            }

        }
        return pRead;

    }
            */

    // Update path metric computations based on branch metric (Add/Compare/Select operations)
    // do forward direction first, ie from anchor to end of read
    // outer loop
    for (int indR = 0; indR < readLength; indR++) {
      byte readBase = readBases[indR];
      byte readQual = readQuals[indR];

      for (int indX = LEFT_ALIGN_INDEX; indX <= RIGHT_ALIGN_INDEX; indX++) {

        byte haplotypeBase;
        if (indX > LEFT_ALIGN_INDEX && indX < RIGHT_ALIGN_INDEX)
          haplotypeBase = haplotype.getBases()[indX - 1];
        else haplotypeBase = readBase;

        updatePathMetrics(haplotypeBase, indX, indR, readBase, readQual);
      }
    }

    // for debugging only: compute backtracking to find optimal route through trellis. Since I'm
    // only interested
    // in log-likelihood of best state, this isn't really necessary.
    double bestMetric = MathUtils.arrayMin(pathMetricArray[readLength]);

    if (DEBUG) {

      System.out.println(read.getReadName());
      System.out.print("Haplotype:");

      for (int k = 0; k < haplotype.getBases().length; k++) {
        System.out.format("%c ", haplotype.getBases()[k]);
      }
      System.out.println();

      System.out.print("Read bases: ");
      for (int k = 0; k < readBases.length; k++) {
        System.out.format("%c ", readBases[k]);
      }
      System.out.println();

      System.out.print("Read quals: ");
      for (int k = 0; k < readQuals.length; k++) {
        System.out.format("%d ", (int) readQuals[k]);
      }
      System.out.println();

      // start from last position of read, go backwards to find optimal alignment
      int[] bestIndexArray = new int[readLength];
      int bestIndex = MathUtils.minElementIndex(pathMetricArray[readLength]);
      bestIndexArray[readLength - 1] = bestIndex;

      for (int k = readLength - 2; k >= 0; k--) {
        bestIndex = bestStateIndexArray[k][bestIndex];
        bestIndexArray[k] = bestIndex;
      }

      System.out.print("Alignment: ");
      for (int k = 0; k < readBases.length; k++) {
        System.out.format("%d ", bestIndexArray[k]);
      }
      System.out.println();
    }
    // now just take optimum along all path metrics: that's the log likelihood of best alignment
    if (DEBUG) System.out.format("Likelihood: %5.4f\n", bestMetric);
    return bestMetric;
  }
Exemple #9
0
  public final List<Sample> parse(
      Reader reader, EnumSet<MissingPedField> missingFields, SampleDB sampleDB) {
    final List<String> lines = new XReadLines(reader).readLines();

    // What are the record offsets?
    final int familyPos = missingFields.contains(MissingPedField.NO_FAMILY_ID) ? -1 : 0;
    final int samplePos = familyPos + 1;
    final int paternalPos = missingFields.contains(MissingPedField.NO_PARENTS) ? -1 : samplePos + 1;
    final int maternalPos =
        missingFields.contains(MissingPedField.NO_PARENTS) ? -1 : paternalPos + 1;
    final int sexPos =
        missingFields.contains(MissingPedField.NO_SEX) ? -1 : Math.max(maternalPos, samplePos) + 1;
    final int phenotypePos =
        missingFields.contains(MissingPedField.NO_PHENOTYPE)
            ? -1
            : Math.max(sexPos, Math.max(maternalPos, samplePos)) + 1;
    final int nExpectedFields =
        MathUtils.arrayMaxInt(
                Arrays.asList(samplePos, paternalPos, maternalPos, sexPos, phenotypePos))
            + 1;

    // go through once and determine properties
    int lineNo = 1;
    boolean isQT = false;
    final List<String[]> splits = new ArrayList<String[]>(lines.size());
    for (final String line : lines) {
      if (line.startsWith(commentMarker)) continue;
      if (line.trim().equals("")) continue;

      final String[] parts = line.split("\\s+");

      if (parts.length != nExpectedFields)
        throw new UserException.MalformedFile(
            reader.toString(), "Bad PED line " + lineNo + ": wrong number of fields");

      if (phenotypePos != -1) {
        isQT = isQT || !CATAGORICAL_TRAIT_VALUES.contains(parts[phenotypePos]);
      }

      splits.add(parts);
      lineNo++;
    }
    logger.info("Phenotype is other? " + isQT);

    // now go through and parse each record
    lineNo = 1;
    final List<Sample> samples = new ArrayList<Sample>(splits.size());
    for (final String[] parts : splits) {
      String familyID = null, individualID, paternalID = null, maternalID = null;
      Gender sex = Gender.UNKNOWN;
      String quantitativePhenotype = Sample.UNSET_QT;
      Affection affection = Affection.UNKNOWN;

      if (familyPos != -1) familyID = maybeMissing(parts[familyPos]);
      individualID = parts[samplePos];
      if (paternalPos != -1) paternalID = maybeMissing(parts[paternalPos]);
      if (maternalPos != -1) maternalID = maybeMissing(parts[maternalPos]);

      if (sexPos != -1) {
        if (parts[sexPos].equals(SEX_MALE)) sex = Gender.MALE;
        else if (parts[sexPos].equals(SEX_FEMALE)) sex = Gender.FEMALE;
        else sex = Gender.UNKNOWN;
      }

      if (phenotypePos != -1) {
        if (isQT) {
          if (parts[phenotypePos].equals(MISSING_VALUE1)) affection = Affection.UNKNOWN;
          else {
            affection = Affection.OTHER;
            quantitativePhenotype = parts[phenotypePos];
          }
        } else {
          if (parts[phenotypePos].equals(MISSING_VALUE1)) affection = Affection.UNKNOWN;
          else if (parts[phenotypePos].equals(MISSING_VALUE2)) affection = Affection.UNKNOWN;
          else if (parts[phenotypePos].equals(PHENOTYPE_UNAFFECTED))
            affection = Affection.UNAFFECTED;
          else if (parts[phenotypePos].equals(PHENOTYPE_AFFECTED)) affection = Affection.AFFECTED;
          else
            throw new ReviewedGATKException(
                "Unexpected phenotype type " + parts[phenotypePos] + " at line " + lineNo);
        }
      }

      final Sample s =
          new Sample(
              individualID,
              sampleDB,
              familyID,
              paternalID,
              maternalID,
              sex,
              affection,
              quantitativePhenotype);
      samples.add(s);
      sampleDB.addSample(s);
      lineNo++;
    }

    for (final Sample sample : new ArrayList<Sample>(samples)) {
      Sample dad =
          maybeAddImplicitSample(
              sampleDB, sample.getPaternalID(), sample.getFamilyID(), Gender.MALE);
      if (dad != null) samples.add(dad);

      Sample mom =
          maybeAddImplicitSample(
              sampleDB, sample.getMaternalID(), sample.getFamilyID(), Gender.FEMALE);
      if (mom != null) samples.add(mom);
    }

    return samples;
  }