Esempio n. 1
0
  /** Initialize the stratifications, evaluations, evaluation contexts, and reporting object */
  public void initialize() {
    // Just list the modules, and exit quickly.
    if (LIST) {
      variantEvalUtils.listModulesAndExit();
    }

    // maintain the full list of comps
    comps.addAll(compsProvided);
    if (dbsnp.dbsnp.isBound()) {
      comps.add(dbsnp.dbsnp);
      knowns.add(dbsnp.dbsnp);
    }

    // Add a dummy comp track if none exists
    if (comps.size() == 0)
      comps.add(
          new RodBinding<VariantContext>(VariantContext.class, "none", "UNBOUND", "", new Tags()));

    // Set up set of additional knowns
    for (RodBinding<VariantContext> compRod : comps) {
      if (KNOWN_NAMES.contains(compRod.getName())) knowns.add(compRod);
    }

    // Now that we have all the rods categorized, determine the sample list from the eval rods.
    Map<String, VCFHeader> vcfRods = GATKVCFUtils.getVCFHeadersFromRods(getToolkit(), evals);
    Set<String> vcfSamples =
        SampleUtils.getSampleList(
            vcfRods, GATKVariantContextUtils.GenotypeMergeType.REQUIRE_UNIQUE);

    // Load the sample list, using an intermediate tree set to sort the samples
    final Set<String> allSampleNames = SampleUtils.getSamplesFromCommandLineInput(vcfSamples);
    sampleNamesForEvaluation.addAll(
        new TreeSet<String>(
            SampleUtils.getSamplesFromCommandLineInput(vcfSamples, SAMPLE_EXPRESSIONS)));
    isSubsettingSamples = !sampleNamesForEvaluation.containsAll(allSampleNames);

    if (Arrays.asList(STRATIFICATIONS_TO_USE).contains("Sample")) {
      sampleNamesForStratification.addAll(sampleNamesForEvaluation);
    }
    sampleNamesForStratification.add(ALL_SAMPLE_NAME);

    // Initialize select expressions
    for (VariantContextUtils.JexlVCMatchExp jexl :
        VariantContextUtils.initializeMatchExps(SELECT_NAMES, SELECT_EXPS)) {
      SortableJexlVCMatchExp sjexl = new SortableJexlVCMatchExp(jexl.name, jexl.exp);
      jexlExpressions.add(sjexl);
    }

    // Initialize the set of stratifications and evaluations to use
    // The list of stratifiers and evaluators to use
    final List<VariantStratifier> stratificationObjects =
        variantEvalUtils.initializeStratificationObjects(
            NO_STANDARD_STRATIFICATIONS, STRATIFICATIONS_TO_USE);
    final Set<Class<? extends VariantEvaluator>> evaluationClasses =
        variantEvalUtils.initializeEvaluationObjects(NO_STANDARD_MODULES, MODULES_TO_USE);

    checkForIncompatibleEvaluatorsAndStratifiers(stratificationObjects, evaluationClasses);

    for (VariantStratifier vs : stratificationObjects) {
      if (vs.getName().equals("Filter")) byFilterIsEnabled = true;
      else if (vs.getName().equals("Sample")) perSampleIsEnabled = true;
    }

    if (intervalsFile != null) {
      boolean fail = true;
      for (final VariantStratifier vs : stratificationObjects) {
        if (vs.getClass().equals(IntervalStratification.class)) fail = false;
      }
      if (fail)
        throw new UserException.BadArgumentValue(
            "ST", "stratIntervals argument provided but -ST IntervalStratification not provided");
    }

    // Initialize the evaluation contexts
    createStratificationStates(stratificationObjects, evaluationClasses);

    // Load ancestral alignments
    if (ancestralAlignmentsFile != null) {
      try {
        ancestralAlignments = new IndexedFastaSequenceFile(ancestralAlignmentsFile);
      } catch (FileNotFoundException e) {
        throw new ReviewedStingException(
            String.format(
                "The ancestral alignments file, '%s', could not be found",
                ancestralAlignmentsFile.getAbsolutePath()));
      }
    }

    // initialize CNVs
    if (knownCNVsFile != null) {
      knownCNVsByContig = createIntervalTreeByContig(knownCNVsFile);
    }
  }
Esempio n. 2
0
public class ReadMetricsUnitTest extends BaseTest {

  @Test
  public void testReadsSeenDoNotOverflowInt() {

    final ReadMetrics metrics = new ReadMetrics();

    final long moreThanMaxInt = ((long) Integer.MAX_VALUE) + 1L;

    for (long i = 0L; i < moreThanMaxInt; i++) {
      metrics.incrementNumReadsSeen();
    }

    Assert.assertEquals(metrics.getNumReadsSeen(), moreThanMaxInt);
    Assert.assertTrue(metrics.getNumReadsSeen() > (long) Integer.MAX_VALUE);

    logger.warn(String.format("%d %d %d", Integer.MAX_VALUE, moreThanMaxInt, Long.MAX_VALUE));
  }

  // Test the accuracy of the read metrics

  private IndexedFastaSequenceFile reference;
  private SAMSequenceDictionary dictionary;
  private SAMFileHeader header;
  private GATKSAMReadGroupRecord readGroup;
  private GenomeLocParser genomeLocParser;
  private File testBAM;

  private static final int numReadsPerContig = 250000;
  private static final List<String> contigs = Arrays.asList("1", "2", "3");

  @BeforeClass
  private void init() throws IOException {
    reference = new CachingIndexedFastaSequenceFile(new File(b37KGReference));
    dictionary = reference.getSequenceDictionary();
    genomeLocParser = new GenomeLocParser(dictionary);
    header = ArtificialSAMUtils.createDefaultReadGroup(new SAMFileHeader(), "test", "test");
    header.setSequenceDictionary(dictionary);
    header.setSortOrder(SAMFileHeader.SortOrder.coordinate);
    readGroup = new GATKSAMReadGroupRecord(header.getReadGroup("test"));

    final List<GATKSAMRecord> reads = new ArrayList<>();
    for (final String contig : contigs) {
      for (int i = 1; i <= numReadsPerContig; i++) {
        reads.add(buildSAMRecord("read" + contig + "_" + i, contig, i));
      }
    }

    createBAM(reads);
  }

  private void createBAM(final List<GATKSAMRecord> reads) throws IOException {
    testBAM = File.createTempFile("TraverseActiveRegionsUnitTest", ".bam");
    testBAM.deleteOnExit();

    SAMFileWriter out =
        new SAMFileWriterFactory()
            .setCreateIndex(true)
            .makeBAMWriter(reads.get(0).getHeader(), true, testBAM);
    for (GATKSAMRecord read : reads) {
      out.addAlignment(read);
    }
    out.close();

    new File(testBAM.getAbsolutePath().replace(".bam", ".bai")).deleteOnExit();
    new File(testBAM.getAbsolutePath() + ".bai").deleteOnExit();
  }

  // copied from LocusViewTemplate
  protected GATKSAMRecord buildSAMRecord(
      final String readName, final String contig, final int alignmentStart) {
    GATKSAMRecord record = new GATKSAMRecord(header);

    record.setReadName(readName);
    record.setReferenceIndex(dictionary.getSequenceIndex(contig));
    record.setAlignmentStart(alignmentStart);

    record.setCigarString("1M");
    record.setReadString("A");
    record.setBaseQualityString("A");
    record.setReadGroup(readGroup);

    return record;
  }

  @Test
  public void testCountsFromReadTraversal() {
    final GenomeAnalysisEngine engine = new GenomeAnalysisEngine();
    engine.setGenomeLocParser(genomeLocParser);

    final Collection<SAMReaderID> samFiles = new ArrayList<>();
    final SAMReaderID readerID = new SAMReaderID(testBAM, new Tags());
    samFiles.add(readerID);

    final SAMDataSource dataSource =
        new SAMDataSource(
            samFiles,
            new ThreadAllocation(),
            null,
            genomeLocParser,
            false,
            SAMFileReader.ValidationStringency.STRICT,
            null,
            null,
            new ValidationExclusion(),
            new ArrayList<ReadFilter>(),
            new ArrayList<ReadTransformer>(),
            false,
            (byte) 30,
            false,
            true);

    engine.setReadsDataSource(dataSource);

    final TraverseReadsNano traverseReadsNano = new TraverseReadsNano(1);
    final DummyReadWalker walker = new DummyReadWalker();
    traverseReadsNano.initialize(engine, walker, null);

    for (final Shard shard : dataSource.createShardIteratorOverAllReads(new ReadShardBalancer())) {
      final ReadShardDataProvider dataProvider =
          new ReadShardDataProvider(
              shard,
              engine.getGenomeLocParser(),
              dataSource.seek(shard),
              reference,
              new ArrayList<ReferenceOrderedDataSource>());
      traverseReadsNano.traverse(walker, dataProvider, 0);
      dataProvider.close();
    }

    Assert.assertEquals(
        engine.getCumulativeMetrics().getNumReadsSeen(), contigs.size() * numReadsPerContig);
    Assert.assertEquals(
        engine.getCumulativeMetrics().getNumIterations(), contigs.size() * numReadsPerContig);
  }

  @Test
  public void testCountsFromLocusTraversal() {
    final GenomeAnalysisEngine engine = new GenomeAnalysisEngine();
    engine.setGenomeLocParser(genomeLocParser);

    final Collection<SAMReaderID> samFiles = new ArrayList<>();
    final SAMReaderID readerID = new SAMReaderID(testBAM, new Tags());
    samFiles.add(readerID);

    final SAMDataSource dataSource =
        new SAMDataSource(
            samFiles,
            new ThreadAllocation(),
            null,
            genomeLocParser,
            false,
            SAMFileReader.ValidationStringency.STRICT,
            null,
            null,
            new ValidationExclusion(),
            new ArrayList<ReadFilter>(),
            new ArrayList<ReadTransformer>(),
            false,
            (byte) 30,
            false,
            true);

    engine.setReadsDataSource(dataSource);
    final Set<String> samples = SampleUtils.getSAMFileSamples(dataSource.getHeader());

    final TraverseLociNano traverseLociNano = new TraverseLociNano(1);
    final DummyLocusWalker walker = new DummyLocusWalker();
    traverseLociNano.initialize(engine, walker, null);

    for (final Shard shard : dataSource.createShardIteratorOverAllReads(new LocusShardBalancer())) {
      final WindowMaker windowMaker =
          new WindowMaker(
              shard, genomeLocParser, dataSource.seek(shard), shard.getGenomeLocs(), samples);
      for (WindowMaker.WindowMakerIterator window : windowMaker) {
        final LocusShardDataProvider dataProvider =
            new LocusShardDataProvider(
                shard,
                shard.getReadProperties(),
                genomeLocParser,
                window.getLocus(),
                window,
                reference,
                new ArrayList<ReferenceOrderedDataSource>());
        traverseLociNano.traverse(walker, dataProvider, 0);
        dataProvider.close();
      }
      windowMaker.close();
    }

    // dataSource.close();
    Assert.assertEquals(
        engine.getCumulativeMetrics().getNumReadsSeen(), contigs.size() * numReadsPerContig);
    Assert.assertEquals(
        engine.getCumulativeMetrics().getNumIterations(), contigs.size() * numReadsPerContig);
  }

  @Test
  public void testCountsFromActiveRegionTraversal() {
    final GenomeAnalysisEngine engine = new GenomeAnalysisEngine();
    engine.setGenomeLocParser(genomeLocParser);

    final Collection<SAMReaderID> samFiles = new ArrayList<>();
    final SAMReaderID readerID = new SAMReaderID(testBAM, new Tags());
    samFiles.add(readerID);

    final SAMDataSource dataSource =
        new SAMDataSource(
            samFiles,
            new ThreadAllocation(),
            null,
            genomeLocParser,
            false,
            SAMFileReader.ValidationStringency.STRICT,
            null,
            null,
            new ValidationExclusion(),
            new ArrayList<ReadFilter>(),
            new ArrayList<ReadTransformer>(),
            false,
            (byte) 30,
            false,
            true);

    engine.setReadsDataSource(dataSource);
    final Set<String> samples = SampleUtils.getSAMFileSamples(dataSource.getHeader());

    final List<GenomeLoc> intervals = new ArrayList<>(contigs.size());
    for (final String contig : contigs)
      intervals.add(genomeLocParser.createGenomeLoc(contig, 1, numReadsPerContig));

    final TraverseActiveRegions traverseActiveRegions = new TraverseActiveRegions();
    final DummyActiveRegionWalker walker = new DummyActiveRegionWalker();
    traverseActiveRegions.initialize(engine, walker, null);

    for (final Shard shard :
        dataSource.createShardIteratorOverIntervals(
            new GenomeLocSortedSet(genomeLocParser, intervals), new ActiveRegionShardBalancer())) {
      final WindowMaker windowMaker =
          new WindowMaker(
              shard, genomeLocParser, dataSource.seek(shard), shard.getGenomeLocs(), samples);
      for (WindowMaker.WindowMakerIterator window : windowMaker) {
        final LocusShardDataProvider dataProvider =
            new LocusShardDataProvider(
                shard,
                shard.getReadProperties(),
                genomeLocParser,
                window.getLocus(),
                window,
                reference,
                new ArrayList<ReferenceOrderedDataSource>());
        traverseActiveRegions.traverse(walker, dataProvider, 0);
        dataProvider.close();
      }
      windowMaker.close();
    }

    Assert.assertEquals(
        engine.getCumulativeMetrics().getNumReadsSeen(), contigs.size() * numReadsPerContig);
    Assert.assertEquals(
        engine.getCumulativeMetrics().getNumIterations(), contigs.size() * numReadsPerContig);
  }

  @Test
  public void testFilteredCounts() {
    final GenomeAnalysisEngine engine = new GenomeAnalysisEngine();
    engine.setGenomeLocParser(genomeLocParser);

    final Collection<SAMReaderID> samFiles = new ArrayList<>();
    final SAMReaderID readerID = new SAMReaderID(testBAM, new Tags());
    samFiles.add(readerID);

    final List<ReadFilter> filters = new ArrayList<>();
    filters.add(new EveryTenthReadFilter());

    final SAMDataSource dataSource =
        new SAMDataSource(
            samFiles,
            new ThreadAllocation(),
            null,
            genomeLocParser,
            false,
            SAMFileReader.ValidationStringency.STRICT,
            null,
            null,
            new ValidationExclusion(),
            filters,
            new ArrayList<ReadTransformer>(),
            false,
            (byte) 30,
            false,
            true);

    engine.setReadsDataSource(dataSource);

    final TraverseReadsNano traverseReadsNano = new TraverseReadsNano(1);
    final DummyReadWalker walker = new DummyReadWalker();
    traverseReadsNano.initialize(engine, walker, null);

    for (final Shard shard : dataSource.createShardIteratorOverAllReads(new ReadShardBalancer())) {
      final ReadShardDataProvider dataProvider =
          new ReadShardDataProvider(
              shard,
              engine.getGenomeLocParser(),
              dataSource.seek(shard),
              reference,
              new ArrayList<ReferenceOrderedDataSource>());
      traverseReadsNano.traverse(walker, dataProvider, 0);
      dataProvider.close();
    }

    Assert.assertEquals(
        (long)
            engine
                .getCumulativeMetrics()
                .getCountsByFilter()
                .get(EveryTenthReadFilter.class.getSimpleName()),
        contigs.size() * numReadsPerContig / 10);
  }

  class DummyLocusWalker extends LocusWalker<Integer, Integer> {
    @Override
    public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
      return 0;
    }

    @Override
    public Integer reduceInit() {
      return 0;
    }

    @Override
    public Integer reduce(Integer value, Integer sum) {
      return 0;
    }
  }

  class DummyReadWalker extends ReadWalker<Integer, Integer> {
    @Override
    public Integer map(
        ReferenceContext ref, GATKSAMRecord read, RefMetaDataTracker metaDataTracker) {
      return 0;
    }

    @Override
    public Integer reduceInit() {
      return 0;
    }

    @Override
    public Integer reduce(Integer value, Integer sum) {
      return 0;
    }
  }

  class DummyActiveRegionWalker extends ActiveRegionWalker<Integer, Integer> {
    @Override
    public ActivityProfileState isActive(
        RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
      return new ActivityProfileState(ref.getLocus(), 0.0);
    }

    @Override
    public Integer map(ActiveRegion activeRegion, RefMetaDataTracker metaDataTracker) {
      return 0;
    }

    @Override
    public Integer reduceInit() {
      return 0;
    }

    @Override
    public Integer reduce(Integer value, Integer sum) {
      return 0;
    }
  }

  private final class EveryTenthReadFilter extends ReadFilter {

    private int myCounter = 0;

    @Override
    public boolean filterOut(final SAMRecord record) {
      if (++myCounter == 10) {
        myCounter = 0;
        return true;
      }

      return false;
    }
  }
}