@Test
public void testSamReaderFromURL() throws IOException {
final String samFilePath = new File(TEST_DATA_DIR, "unsorted.sam").getAbsolutePath();
final URL samURL = new URL("file://" + samFilePath);
final SamReaderFactory factory =
SamReaderFactory.makeDefault().validationStringency(ValidationStringency.SILENT);
final SamReader reader = factory.open(SamInputResource.of(samURL));
Assert.assertEquals(countRecords(reader), 10);
}
@Test
public void testSamReaderFromSeekableStream() throws IOException {
// even though a SAM isn't indexable, make sure we can open one
// using a seekable stream
final File samFile = new File(TEST_DATA_DIR, "unsorted.sam");
final SamReaderFactory factory =
SamReaderFactory.makeDefault().validationStringency(ValidationStringency.SILENT);
final SamReader reader = factory.open(SamInputResource.of(new SeekableFileStream(samFile)));
Assert.assertEquals(countRecords(reader), 10);
}
@Test(expectedExceptions = SAMFormatException.class)
public void testSamReaderFromMalformedSeekableStream() throws IOException {
// use a bogus (.bai file) to force SamReaderFactory to fall through to the
// fallback code that assumes a SAM File when it can't determine the
// format of the input, to ensure that it results in a SAMFormatException
final File samFile = new File(TEST_DATA_DIR, "cram_with_bai_index.cram.bai");
final SamReaderFactory factory =
SamReaderFactory.makeDefault().validationStringency(ValidationStringency.SILENT);
final SamReader reader = factory.open(SamInputResource.of(new SeekableFileStream(samFile)));
countRecords(reader);
}
@Test(dataProvider = "composeAllPermutationsOfSamInputResource")
public void queryInputResourcePermutation(final SamInputResource resource) throws IOException {
final SamReader reader = SamReaderFactory.makeDefault().open(resource);
LOG.info(String.format("Query from %s ...", resource));
if (reader.hasIndex()) {
final StopWatch stopWatch = new StopWatch();
stopWatch.start();
final SAMRecordIterator q1 = reader.query("chr1", 500000, 100000000, true);
observedRecordOrdering1.add(Iterables.slurp(q1));
q1.close();
final SAMRecordIterator q20 = reader.query("chr20", 1, 1000000, true);
observedRecordOrdering20.add(Iterables.slurp(q20));
q20.close();
final SAMRecordIterator q3 = reader.query("chr3", 1, 10000000, true);
observedRecordOrdering3.add(Iterables.slurp(q3));
q3.close();
stopWatch.stop();
LOG.info(String.format("Finished queries in %sms", stopWatch.getElapsedTime()));
Assert.assertEquals(
observedRecordOrdering1.size(), 1, "read different records for chromosome 1");
Assert.assertEquals(
observedRecordOrdering20.size(), 1, "read different records for chromosome 20");
Assert.assertEquals(
observedRecordOrdering3.size(), 1, "read different records for chromosome 3");
} else if (resource.indexMaybe() != null) {
LOG.warn("Resource has an index source, but is not indexed: " + resource);
} else {
LOG.info("Skipping query operation: no index.");
}
reader.close();
}
@Test(
description =
"Read and then write SAM to verify header attribute ordering does not change depending on JVM version")
public void samRoundTrip() throws Exception {
final File input = new File(TEST_DATA_DIR, "roundtrip.sam");
final SamReader reader = SamReaderFactory.makeDefault().open(input);
final File outputFile = File.createTempFile("roundtrip-out", ".sam");
outputFile.delete();
outputFile.deleteOnExit();
FileOutputStream os = new FileOutputStream(outputFile);
final SAMFileWriterFactory factory = new SAMFileWriterFactory();
final SAMFileWriter writer = factory.makeSAMWriter(reader.getFileHeader(), false, os);
for (SAMRecord rec : reader) {
writer.addAlignment(rec);
}
writer.close();
os.close();
InputStream is = new FileInputStream(input);
String originalsam = IOUtil.readFully(is);
is.close();
is = new FileInputStream(outputFile);
String writtensam = IOUtil.readFully(is);
is.close();
Assert.assertEquals(writtensam, originalsam);
}
@Test(dataProvider = "variousFormatReaderTestCases")
public void variousFormatReaderTest(final String inputFile) throws IOException {
final File input = new File(TEST_DATA_DIR, inputFile);
final SamReader reader = SamReaderFactory.makeDefault().open(input);
for (final SAMRecord ignored : reader) {}
reader.close();
}
@Test
public void testBamIntegers() throws IOException {
final List<String> errorMessages = new ArrayList<String>();
final SamReader bamReader = SamReaderFactory.makeDefault().open(BAM_INPUT);
final File bamOutput = File.createTempFile("test", ".bam");
final File samOutput = File.createTempFile("test", ".sam");
final SAMFileWriter samWriter =
new SAMFileWriterFactory().makeWriter(bamReader.getFileHeader(), true, samOutput, null);
final SAMFileWriter bamWriter =
new SAMFileWriterFactory().makeWriter(bamReader.getFileHeader(), true, bamOutput, null);
final SAMRecordIterator iterator = bamReader.iterator();
while (iterator.hasNext()) {
try {
final SAMRecord rec = iterator.next();
samWriter.addAlignment(rec);
bamWriter.addAlignment(rec);
} catch (final Throwable e) {
System.out.println(e.getMessage());
errorMessages.add(e.getMessage());
}
}
CloserUtil.close(bamReader);
samWriter.close();
bamWriter.close();
Assert.assertEquals(errorMessages.size(), 0);
bamOutput.deleteOnExit();
samOutput.deleteOnExit();
}
@Test(description = "Write SAM records with null SAMFileHeader")
public void samNullHeaderRoundTrip() throws Exception {
final File input = new File(TEST_DATA_DIR, "roundtrip.sam");
final SamReader reader = SamReaderFactory.makeDefault().open(input);
final File outputFile = File.createTempFile("nullheader-out", ".sam");
outputFile.delete();
outputFile.deleteOnExit();
FileOutputStream os = new FileOutputStream(outputFile);
final SAMFileWriterFactory factory = new SAMFileWriterFactory();
final SAMFileWriter writer = factory.makeSAMWriter(reader.getFileHeader(), false, os);
for (SAMRecord rec : reader) {
rec.setHeader(null);
writer.addAlignment(rec);
}
writer.close();
os.close();
InputStream is = new FileInputStream(input);
String originalsam = IOUtil.readFully(is);
is.close();
is = new FileInputStream(outputFile);
String writtensam = IOUtil.readFully(is);
is.close();
Assert.assertEquals(writtensam, originalsam);
}
@Test
public void testWrap() throws IOException {
final Path input = Paths.get(TEST_DATA_DIR.getPath(), "noheader.sam");
final SamReader wrappedReader =
SamReaderFactory.makeDefault().setPathWrapper(SamReaderFactoryTest::addHeader).open(input);
int records = countRecords(wrappedReader);
Assert.assertEquals(10, records);
}
@Test
public void openPath() throws IOException {
final Path path = localBam.toPath();
final List<SAMRecord> records;
final SAMFileHeader fileHeader;
try (final SamReader reader = SamReaderFactory.makeDefault().open(path)) {
LOG.info(String.format("Reading from %s ...", path));
records = Iterables.slurp(reader);
fileHeader = reader.getFileHeader();
reader.close();
}
try (final SamReader fileReader = SamReaderFactory.makeDefault().open(localBam)) {
final List<SAMRecord> expectedRecords = Iterables.slurp(fileReader);
final SAMFileHeader expectedFileHeader = fileReader.getFileHeader();
Assert.assertEquals(records, expectedRecords);
Assert.assertEquals(fileHeader, expectedFileHeader);
}
}
public SamReader getSamReader(ResourceLocator locator, boolean requireIndex) throws IOException {
if (requireIndex) {
final SamReaderFactory factory =
SamReaderFactory.makeDefault().validationStringency(ValidationStringency.SILENT);
SeekableStream indexStream = getIndexStream(locator.getBamIndexPath());
this.indexed = true;
SeekableStream ss =
new IGVSeekableBufferedStream(
IGVSeekableStreamFactory.getInstance().getStreamFor(url), 128000);
SamInputResource resource = SamInputResource.of(ss).index(indexStream);
return factory.open(resource);
} else {
InputStream is = HttpUtils.getInstance().openConnectionStream(url);
return new SAMFileReader(new BufferedInputStream(is));
}
}
@Test(dataProvider = "variousFormatReaderTestCases")
public void samRecordFactoryTest(final String inputFile) throws IOException {
final File input = new File(TEST_DATA_DIR, inputFile);
final SAMRecordFactoryTester recordFactory = new SAMRecordFactoryTester();
final SamReaderFactory readerFactory =
SamReaderFactory.makeDefault().samRecordFactory(recordFactory);
final SamReader reader = readerFactory.open(input);
int i = 0;
for (final SAMRecord ignored : reader) {
++i;
}
reader.close();
Assert.assertTrue(i > 0);
if (inputFile.endsWith(".sam") || inputFile.endsWith(".sam.gz"))
Assert.assertEquals(recordFactory.samRecordsCreated, i);
else if (inputFile.endsWith(".bam")) Assert.assertEquals(recordFactory.bamRecordsCreated, i);
}
@Test
public void checkHasIndexForStreamingPathBamWithFileIndex() throws IOException {
InputResource bam = new NeverFilePathInputResource(localBam.toPath());
InputResource index = new FileInputResource(localBamIndex);
// ensure that the index is being used, not checked in queryInputResourcePermutation
try (final SamReader reader =
SamReaderFactory.makeDefault().open(new SamInputResource(bam, index))) {
Assert.assertTrue(reader.hasIndex());
}
}
// See https://github.com/samtools/htsjdk/issues/76
@Test(dataProvider = "queryIntervalIssue76TestCases")
public void queryIntervalIssue76(
final String sequenceName, final int start, final int end, final int expectedCount)
throws IOException {
final File input = new File(TEST_DATA_DIR, "issue76.bam");
final SamReader reader = SamReaderFactory.makeDefault().open(input);
final QueryInterval interval =
new QueryInterval(
reader.getFileHeader().getSequence(sequenceName).getSequenceIndex(), start, end);
Assert.assertEquals(countRecordsInQueryInterval(reader, interval), expectedCount);
reader.close();
}
private void getCRAMReaderFromInputResource(
final BiFunction<URL, URL, SamInputResource> getInputResource,
final boolean hasIndex,
final int expectedCount)
throws IOException {
final String cramFilePath =
new File(TEST_DATA_DIR, "cram_with_bai_index.cram").getAbsolutePath();
final String cramIndexPath =
new File(TEST_DATA_DIR, "cram_with_bai_index.cram.bai").getAbsolutePath();
final URL cramURL = new URL("file://" + cramFilePath);
final URL indexURL = new URL("file://" + cramIndexPath);
final SamReaderFactory factory =
SamReaderFactory.makeDefault()
.referenceSource(new ReferenceSource(new File(TEST_DATA_DIR, "hg19mini.fasta")))
.validationStringency(ValidationStringency.SILENT);
final SamReader reader = factory.open(getInputResource.apply(cramURL, indexURL));
int count =
hasIndex
? countRecordsInQueryInterval(reader, new QueryInterval(1, 10, 1000))
: countRecords(reader);
Assert.assertEquals(count, expectedCount);
}
private void runLocalRemoteTest(
final URL bamURL,
final File bamFile,
final String sequence,
final int startPos,
final int endPos,
final boolean contained) {
verbose("Testing query " + sequence + ":" + startPos + "-" + endPos + " ...");
final SamReader reader1 =
SamReaderFactory.makeDefault()
.disable(SamReaderFactory.Option.EAGERLY_DECODE)
.open(SamInputResource.of(bamFile).index(BAM_INDEX_FILE));
final SamReader reader2 =
SamReaderFactory.makeDefault()
.disable(SamReaderFactory.Option.EAGERLY_DECODE)
.open(SamInputResource.of(bamURL).index(BAM_INDEX_FILE));
final Iterator<SAMRecord> iter1 = reader1.query(sequence, startPos, endPos, contained);
final Iterator<SAMRecord> iter2 = reader2.query(sequence, startPos, endPos, contained);
final List<SAMRecord> records1 = new ArrayList<SAMRecord>();
final List<SAMRecord> records2 = new ArrayList<SAMRecord>();
while (iter1.hasNext()) {
records1.add(iter1.next());
}
while (iter2.hasNext()) {
records2.add(iter2.next());
}
assertTrue(records1.size() > 0);
assertEquals(records1.size(), records2.size());
for (int i = 0; i < records1.size(); i++) {
// System.out.println(records1.get(i).format());
assertEquals(records1.get(i).getSAMString(), records2.get(i).getSAMString());
}
}
private List<String> getReferenceNames(final URL bamFile) throws IOException {
final SamReader reader =
SamReaderFactory.makeDefault().open(SamInputResource.of(bamFile.openStream()));
final List<String> result = new ArrayList<String>();
final List<SAMSequenceRecord> seqRecords =
reader.getFileHeader().getSequenceDictionary().getSequences();
for (final SAMSequenceRecord seqRecord : seqRecords) {
if (seqRecord.getSequenceName() != null) {
result.add(seqRecord.getSequenceName());
}
}
reader.close();
return result;
}
@Test(dataProvider = "composeAllPermutationsOfSamInputResource")
public void exhaustInputResourcePermutation(final SamInputResource resource) throws IOException {
final SamReader reader = SamReaderFactory.makeDefault().open(resource);
LOG.info(String.format("Reading from %s ...", resource));
final List<SAMRecord> slurped = Iterables.slurp(reader);
final SAMFileHeader fileHeader = reader.getFileHeader();
reader.hasIndex();
reader.indexing().hasBrowseableIndex();
reader.close();
/* Ensure all tests have read the same records in the same order or, if this is the first test, set it as the template. */
observedHeaders.add(fileHeader);
observedRecordOrdering.add(slurped);
Assert.assertEquals(observedHeaders.size(), 1, "read different headers than other testcases");
Assert.assertEquals(
observedRecordOrdering.size(), 1, "read different records than other testcases");
}
@Test
public void customReaderFactoryTest() throws IOException {
try {
CustomReaderFactory.setInstance(
new CustomReaderFactory(
"https://www.googleapis.com/genomics/v1beta/reads/,"
+ "htsjdk.samtools.SamReaderFactoryTest$TestReaderFactory"));
final SamReader reader =
SamReaderFactory.makeDefault()
.open(
SamInputResource.of(
"https://www.googleapis.com/genomics/v1beta/reads/?uncompressed.sam"));
int i = 0;
for (@SuppressWarnings("unused") final SAMRecord ignored : reader) {
++i;
}
reader.close();
Assert.assertTrue(i > 0);
} finally {
CustomReaderFactory.resetToDefaultInstance();
}
}
@Override
public SamReader open(URL url) {
final File file = new File(TEST_DATA_DIR, url.getQuery());
LOG.info("Opening customr reader for " + file.toString());
return SamReaderFactory.makeDefault().open(file);
}
private int runQueryTest(
final URL bamURL,
final String sequence,
final int startPos,
final int endPos,
final boolean contained) {
verbose("Testing query " + sequence + ":" + startPos + "-" + endPos + " ...");
final SamReader reader1 =
SamReaderFactory.makeDefault()
.disable(SamReaderFactory.Option.EAGERLY_DECODE)
.open(SamInputResource.of(bamURL).index(BAM_INDEX_FILE));
final SamReader reader2 =
SamReaderFactory.makeDefault()
.disable(SamReaderFactory.Option.EAGERLY_DECODE)
.open(SamInputResource.of(bamURL).index(BAM_INDEX_FILE));
final Iterator<SAMRecord> iter1 = reader1.query(sequence, startPos, endPos, contained);
final Iterator<SAMRecord> iter2 = reader2.iterator();
// Compare ordered iterators.
// Confirm that iter1 is a subset of iter2 that properly filters.
SAMRecord record1 = null;
SAMRecord record2 = null;
int count1 = 0;
int count2 = 0;
int beforeCount = 0;
int afterCount = 0;
while (true) {
if (record1 == null && iter1.hasNext()) {
record1 = iter1.next();
count1++;
}
if (record2 == null && iter2.hasNext()) {
record2 = iter2.next();
count2++;
}
// System.out.println("Iteration:");
// System.out.println(" Record1 = " + ((record1 == null) ? "null" : record1.format()));
// System.out.println(" Record2 = " + ((record2 == null) ? "null" : record2.format()));
if (record1 == null && record2 == null) {
break;
}
if (record1 == null) {
checkPassesFilter(false, record2, sequence, startPos, endPos, contained);
record2 = null;
afterCount++;
continue;
}
assertNotNull(record2);
final int ordering = compareCoordinates(record1, record2);
if (ordering > 0) {
checkPassesFilter(false, record2, sequence, startPos, endPos, contained);
record2 = null;
beforeCount++;
continue;
}
assertTrue(ordering == 0);
checkPassesFilter(true, record1, sequence, startPos, endPos, contained);
checkPassesFilter(true, record2, sequence, startPos, endPos, contained);
assertEquals(record1.getReadName(), record2.getReadName());
assertEquals(record1.getReadString(), record2.getReadString());
record1 = null;
record2 = null;
}
CloserUtil.close(reader1);
CloserUtil.close(reader2);
verbose("Checked " + count1 + " records against " + count2 + " records.");
verbose("Found " + (count2 - beforeCount - afterCount) + " records matching.");
verbose("Found " + beforeCount + " records before.");
verbose("Found " + afterCount + " records after.");
return count1;
}
