public static void align(
Graph graph,
SAMRecord rec,
Node recNode,
ReferenceSequence sequence,
SAMProgramRecord programRecord,
int offset,
AlleleCoverageCutoffs alleleCoverageCutoffs,
boolean correctBases,
boolean useSequenceQualities,
int MAXIMUM_TOTAL_COVERAGE,
int MAX_HEAP_SIZE)
throws Exception {
int i;
AlignHeapNode curAlignHeapNode = null;
AlignHeapNode nextAlignHeapNode = null;
AlignHeapNode bestAlignHeapNode = null;
AlignHeap heap = null;
String read = null; // could be cs
String readBases = null; // always nt
String qualities = null; // could be cq
SRMAUtil.Space space = SRMAUtil.Space.NTSPACE;
ListIterator<NodeRecord> iter = null;
AlignHeapNodeComparator comp = null;
int alignmentStart = -1;
int numStartNodesAdded = 0;
boolean strand = rec.getReadNegativeStrandFlag(); // false -> forward, true -> reverse
String softClipStartBases = null;
String softClipStartQualities = null;
String softClipEndBases = null;
String softClipEndQualities = null;
// Debugging stuff
String readName = rec.getReadName();
assert SRMAUtil.Space.COLORSPACE != space;
// Get space
read = (String) rec.getAttribute("CS");
if (null == read) {
// Use base space
space = SRMAUtil.Space.NTSPACE;
} else {
// assumes CS and CQ are always in sequencing order
space = SRMAUtil.Space.COLORSPACE;
}
// Get read and qualities
if (space == SRMAUtil.Space.NTSPACE) {
byte tmpRead[] = rec.getReadString().getBytes();
byte tmpQualities[] = rec.getBaseQualityString().getBytes();
// Reverse once
if (strand) { // reverse
SAMRecordUtil.reverseArray(tmpRead);
SAMRecordUtil.reverseArray(tmpQualities);
}
read = new String(tmpRead);
readBases = new String(tmpRead);
qualities = new String(tmpQualities);
// Reverse again
if (strand) { // reverse
SAMRecordUtil.reverseArray(tmpRead);
SAMRecordUtil.reverseArray(tmpQualities);
}
} else {
byte tmpRead[] = rec.getReadString().getBytes();
// Reverse once
if (strand) { // reverse
SAMRecordUtil.reverseArray(tmpRead);
}
readBases = new String(tmpRead);
// Reverse again
if (strand) { // reverse
SAMRecordUtil.reverseArray(tmpRead);
}
read = SRMAUtil.normalizeColorSpaceRead(read);
qualities = (String) rec.getAttribute("CQ");
// Some aligners include a quality value for the adapter. A quality value
// IMHO should not be given for an unobserved (assumed) peice of data. Trim
// the first quality in this case
if (qualities.length() == 1 + read.length()) { // trim the first quality
qualities = qualities.substring(1);
}
}
// Reverse back
if (readBases.length() <= 0) {
throw new Exception("Error. The current alignment has no bases.");
}
if (read.length() <= 0) {
throw new Exception("Error. The current alignment has no bases.");
}
if (qualities.length() <= 0) {
throw new Exception("Error. The current alignment has no qualities.");
}
if (readBases.length() != read.length()) {
if (space == SRMAUtil.Space.COLORSPACE) {
throw new Exception(
"Error. The current alignment's read bases length does not match the length of the colors in the CS tag ["
+ rec.getReadName()
+ "].");
} else {
throw new Exception("Error. Internal error: readBases.length() != read.length()");
}
}
// Deal with soft-clipping
// - save the soft clipped sequence for latter
{
List<CigarElement> cigarElements = null;
cigarElements = rec.getCigar().getCigarElements();
CigarElement e1 = cigarElements.get(0); // first
CigarElement e2 = cigarElements.get(cigarElements.size() - 1); // last
// Soft-clipped
if (CigarOperator.S == e1.getOperator()) {
if (space == SRMAUtil.Space.COLORSPACE) {
throw new Exception(
"Error. Soft clipping with color-space data not currently supported.");
}
int l = e1.getLength();
if (strand) { // reverse
softClipStartBases = readBases.substring(readBases.length() - l);
softClipStartQualities = qualities.substring(qualities.length() - l);
readBases = readBases.substring(0, readBases.length() - l);
read = read.substring(0, read.length() - l);
qualities = qualities.substring(0, qualities.length() - l);
} else {
softClipStartBases = readBases.substring(0, l - 1);
softClipStartQualities = qualities.substring(0, l - 1);
readBases = readBases.substring(l);
read = read.substring(l);
qualities = qualities.substring(l);
}
}
if (CigarOperator.S == e2.getOperator()) {
if (space == SRMAUtil.Space.COLORSPACE) {
throw new Exception(
"Error. Soft clipping with color-space data not currently supported.");
}
int l = e2.getLength();
if (strand) { // reverse
softClipEndBases = readBases.substring(0, l - 1);
softClipEndQualities = qualities.substring(0, l - 1);
readBases = readBases.substring(l);
read = read.substring(l);
qualities = qualities.substring(l);
} else {
softClipEndBases = readBases.substring(readBases.length() - l);
softClipEndQualities = qualities.substring(qualities.length() - l);
readBases = readBases.substring(0, readBases.length() - l);
read = read.substring(0, read.length() - l);
qualities = qualities.substring(0, qualities.length() - l);
}
}
}
// Remove mate pair information
Align.removeMateInfo(rec);
comp =
new AlignHeapNodeComparator(
(strand) ? AlignHeap.HeapType.MAXHEAP : AlignHeap.HeapType.MINHEAP);
// Bound by original alignment if possible
bestAlignHeapNode =
Align.boundWithOriginalAlignment(
rec,
graph,
recNode,
comp,
strand,
read,
qualities,
readBases,
space,
sequence,
alleleCoverageCutoffs,
useSequenceQualities,
MAXIMUM_TOTAL_COVERAGE,
MAX_HEAP_SIZE);
/*
System.err.println("readName="+rec.getReadName());
if(null != bestAlignHeapNode) {
System.err.println("\nFOUND BEST:" + rec.toString());
}
else {
System.err.println("\nNOT FOUND (BEST): " + rec.toString());
}
Align.updateSAM(rec, programRecord, bestAlignHeapNode, space, read, qualities, softClipStartBases, softClipStartQualities, softClipEndBases, softClipEndQualities, strand, correctBases);
return;
*/
heap = new AlignHeap((strand) ? AlignHeap.HeapType.MAXHEAP : AlignHeap.HeapType.MINHEAP);
// Add start nodes
if (strand) { // reverse
alignmentStart = rec.getAlignmentEnd();
for (i = alignmentStart + offset; alignmentStart - offset <= i; i--) {
int position = graph.getPriorityQueueIndexAtPositionOrBefore(i);
PriorityQueue<Node> startNodeQueue = graph.getPriorityQueue(position);
if (0 != position && null != startNodeQueue) {
Iterator<Node> startNodeQueueIter = startNodeQueue.iterator();
while (startNodeQueueIter.hasNext()) {
Node startNode = startNodeQueueIter.next();
int f = passFilters(graph, startNode, alleleCoverageCutoffs, MAXIMUM_TOTAL_COVERAGE);
if (0 == f) {
heap.add(
new AlignHeapNode(
null,
startNode,
startNode.coverage,
read.charAt(0),
qualities.charAt(0),
useSequenceQualities,
space));
} else if (f < 0) {
return;
}
if (startNode.position < i) {
i = startNode.position;
}
numStartNodesAdded++;
}
}
}
} else {
alignmentStart = rec.getAlignmentStart();
for (i = alignmentStart - offset; i <= alignmentStart + offset; i++) {
int position = graph.getPriorityQueueIndexAtPositionOrGreater(i);
PriorityQueue<Node> startNodeQueue = graph.getPriorityQueue(position);
if (0 != position && null != startNodeQueue) {
Iterator<Node> startNodeQueueIter = startNodeQueue.iterator();
while (startNodeQueueIter.hasNext()) {
Node startNode = startNodeQueueIter.next();
int f = passFilters(graph, startNode, alleleCoverageCutoffs, MAXIMUM_TOTAL_COVERAGE);
if (0 == f) {
heap.add(
new AlignHeapNode(
null,
startNode,
startNode.coverage,
read.charAt(0),
qualities.charAt(0),
useSequenceQualities,
space));
} else if (f < 0) {
return;
}
if (i < startNode.position) {
i = startNode.position;
}
numStartNodesAdded++;
}
}
}
}
if (numStartNodesAdded == 0) {
throw new Exception("Did not add any start nodes!");
}
// Get first node off the heap
curAlignHeapNode = heap.poll();
while (null != curAlignHeapNode) {
if (MAX_HEAP_SIZE <= heap.size()) {
// too many to consider
return;
}
// System.err.println("strand:" + strand + "\tsize:" + heap.size() + "\talignmentStart:" +
// alignmentStart + "\toffset:" + offset + "\treadOffset:" + curAlignHeapNode.readOffset);
// System.err.print("size:" + heap.size() + ":" + curAlignHeapNode.readOffset + ":" +
// curAlignHeapNode.score + ":" + curAlignHeapNode.alleleCoverageSum + ":" +
// curAlignHeapNode.startPosition + "\t");
// curAlignHeapNode.node.print(System.err);
// System.err.print("\rposition:" + curAlignHeapNode.node.position + "\treadOffset:" +
// curAlignHeapNode.readOffset);
// Remove all non-insertions with the same contig/pos/read-offset/type/base and lower score
nextAlignHeapNode = heap.peek();
while (Node.INSERTION != curAlignHeapNode.node.type
&& null != nextAlignHeapNode
&& 0 == comp.compare(curAlignHeapNode, nextAlignHeapNode)) {
if (curAlignHeapNode.score < nextAlignHeapNode.score
|| (curAlignHeapNode.score == nextAlignHeapNode.score
&& curAlignHeapNode.alleleCoverageSum < nextAlignHeapNode.alleleCoverageSum)) {
// Update current node
curAlignHeapNode = heap.poll();
} else {
// Ignore next node
heap.poll();
}
nextAlignHeapNode = heap.peek();
}
nextAlignHeapNode = null;
// Check if the alignment is complete
if (curAlignHeapNode.readOffset == read.length() - 1) {
// All read bases examined, store if has the best alignment.
// System.err.print(curAlignHeapNode.alleleCoverageSum + ":" + curAlignHeapNode.score +
// ":");
// System.err.print(curAlignHeapNode.startPosition + ":");
// curAlignHeapNode.node.print(System.err);
if (null == bestAlignHeapNode
|| bestAlignHeapNode.score < curAlignHeapNode.score
|| (bestAlignHeapNode.score == curAlignHeapNode.score
&& bestAlignHeapNode.alleleCoverageSum < curAlignHeapNode.alleleCoverageSum)) {
bestAlignHeapNode = curAlignHeapNode;
}
} else if (null != bestAlignHeapNode && curAlignHeapNode.score < bestAlignHeapNode.score) {
// ignore, under the assumption that scores can only become more negative.
} else {
if (strand) { // reverse
// Go to all the "prev" nodes
iter = curAlignHeapNode.node.prev.listIterator();
} else { // forward
// Go to all "next" nodes
iter = curAlignHeapNode.node.next.listIterator();
}
while (iter.hasNext()) {
NodeRecord next = iter.next();
int f =
passFilters(
graph, next.node, next.coverage, alleleCoverageCutoffs, MAXIMUM_TOTAL_COVERAGE);
if (0 == f) {
heap.add(
new AlignHeapNode(
curAlignHeapNode,
next.node,
next.coverage,
read.charAt(curAlignHeapNode.readOffset + 1),
qualities.charAt(curAlignHeapNode.readOffset + 1),
useSequenceQualities,
space));
} else if (f < 0) {
return;
}
}
iter = null;
}
// Get next node
curAlignHeapNode = heap.poll();
}
// Recover alignment
Align.updateSAM(
rec,
sequence,
programRecord,
bestAlignHeapNode,
space,
read,
qualities,
softClipStartBases,
softClipStartQualities,
softClipEndBases,
softClipEndQualities,
strand,
correctBases);
}
private int runQueryTest(
final URL bamURL,
final String sequence,
final int startPos,
final int endPos,
final boolean contained) {
verbose("Testing query " + sequence + ":" + startPos + "-" + endPos + " ...");
final SamReader reader1 =
SamReaderFactory.makeDefault()
.disable(SamReaderFactory.Option.EAGERLY_DECODE)
.open(SamInputResource.of(bamURL).index(BAM_INDEX_FILE));
final SamReader reader2 =
SamReaderFactory.makeDefault()
.disable(SamReaderFactory.Option.EAGERLY_DECODE)
.open(SamInputResource.of(bamURL).index(BAM_INDEX_FILE));
final Iterator<SAMRecord> iter1 = reader1.query(sequence, startPos, endPos, contained);
final Iterator<SAMRecord> iter2 = reader2.iterator();
// Compare ordered iterators.
// Confirm that iter1 is a subset of iter2 that properly filters.
SAMRecord record1 = null;
SAMRecord record2 = null;
int count1 = 0;
int count2 = 0;
int beforeCount = 0;
int afterCount = 0;
while (true) {
if (record1 == null && iter1.hasNext()) {
record1 = iter1.next();
count1++;
}
if (record2 == null && iter2.hasNext()) {
record2 = iter2.next();
count2++;
}
// System.out.println("Iteration:");
// System.out.println(" Record1 = " + ((record1 == null) ? "null" : record1.format()));
// System.out.println(" Record2 = " + ((record2 == null) ? "null" : record2.format()));
if (record1 == null && record2 == null) {
break;
}
if (record1 == null) {
checkPassesFilter(false, record2, sequence, startPos, endPos, contained);
record2 = null;
afterCount++;
continue;
}
assertNotNull(record2);
final int ordering = compareCoordinates(record1, record2);
if (ordering > 0) {
checkPassesFilter(false, record2, sequence, startPos, endPos, contained);
record2 = null;
beforeCount++;
continue;
}
assertTrue(ordering == 0);
checkPassesFilter(true, record1, sequence, startPos, endPos, contained);
checkPassesFilter(true, record2, sequence, startPos, endPos, contained);
assertEquals(record1.getReadName(), record2.getReadName());
assertEquals(record1.getReadString(), record2.getReadString());
record1 = null;
record2 = null;
}
CloserUtil.close(reader1);
CloserUtil.close(reader2);
verbose("Checked " + count1 + " records against " + count2 + " records.");
verbose("Found " + (count2 - beforeCount - afterCount) + " records matching.");
verbose("Found " + beforeCount + " records before.");
verbose("Found " + afterCount + " records after.");
return count1;
}
@Override
public void execute() {
log.info("Initializing kmer code map...");
Map<Character, Integer> kmerCodeIndices = new HashMap<Character, Integer>();
kmerCodeIndices.put('0', 1);
kmerCodeIndices.put('A', 3);
kmerCodeIndices.put('B', 4);
kmerCodeIndices.put('C', 5);
kmerCodeIndices.put('_', 6);
kmerCodeIndices.put('.', 7);
kmerCodeIndices.put('1', 9);
Map<Character, String> kmerCodeNames = new LinkedHashMap<Character, String>();
kmerCodeNames.put('0', "ref0");
kmerCodeNames.put('A', "repetitive");
kmerCodeNames.put('B', "both");
kmerCodeNames.put('C', "lowcoverage");
kmerCodeNames.put('_', "lowconfidence");
kmerCodeNames.put('.', "novel");
kmerCodeNames.put('1', "ref1");
if (KMER_CODE_NAMES != null) {
for (Character c : kmerCodeNames.keySet()) {
String cStr = String.valueOf(c);
if (KMER_CODE_NAMES.containsKey(cStr)) {
kmerCodeNames.put(c, KMER_CODE_NAMES.get(cStr));
}
}
}
for (Character c : kmerCodeNames.keySet()) {
log.info(" {} {}: {}", c, kmerCodeIndices.get(c), kmerCodeNames.get(c));
}
log.info("Loading annotated contigs...");
Map<String, Map<String, String>> annotatedContigs = new HashMap<String, Map<String, String>>();
int kmerSize = 0;
if (ANN.length() > 0) {
TableReader tr = new TableReader(ANN);
for (Map<String, String> te : tr) {
String contigName = te.get("contigName");
if (kmerSize == 0) {
kmerSize = te.get("seq").length() - te.get("kmerOrigin").length() + 1;
}
annotatedContigs.put(contigName, te);
String[] ref0ToCanonicalExact =
(te.get("ref0ToCanonicalExact").equals("NA")
|| te.get("ref0ToCanonicalExact").equals("*:0-0")
? "NA:0-0"
: te.get("ref0ToCanonicalExact"))
.split("[:-]");
String[] ref1ToCanonicalExact =
(te.get("ref1ToCanonicalExact").equals("NA")
|| te.get("ref1ToCanonicalExact").equals("*:0-0")
? "NA:0-0"
: te.get("ref1ToCanonicalExact"))
.split("[:-]");
cout.println(
te.get("sampleName")
+ "_"
+ te.get("accession")
+ "_"
+ contigName
+ " "
+ ref0ToCanonicalExact[0]
+ " "
+ ref0ToCanonicalExact[1]
+ " "
+ ref0ToCanonicalExact[2]
+ " radius1=0.8r");
cout.println(
te.get("sampleName")
+ "_"
+ te.get("accession")
+ "_"
+ contigName
+ " "
+ ref1ToCanonicalExact[0]
+ " "
+ ref1ToCanonicalExact[1]
+ " "
+ ref1ToCanonicalExact[2]
+ " radius2=0.6r");
}
}
log.info(" contigs: {}", annotatedContigs.size());
log.info(" kmer size: {}", kmerSize);
log.info("Computing kmer inheritance information...");
SAMFileHeader sfh = CONTIGS.getFileHeader();
for (Character c : kmerCodeNames.keySet()) {
SAMReadGroupRecord rgr = new SAMReadGroupRecord(kmerCodeNames.get(c));
rgr.setSample(kmerCodeNames.get(c));
sfh.addReadGroup(rgr);
}
SAMFileWriterFactory sfwf = new SAMFileWriterFactory();
sfwf.setCreateIndex(true);
SAMFileWriter sfw = sfwf.makeBAMWriter(sfh, false, bout);
TableWriter tw = new TableWriter(sout);
Set<IGVEntry> igvEntries = new TreeSet<IGVEntry>();
int numContigs = 0;
for (SAMRecord contig : CONTIGS) {
if (CONTIG_NAMES == null
|| CONTIG_NAMES.isEmpty()
|| CONTIG_NAMES.contains(contig.getReadName())) {
Map<String, String> te = annotatedContigs.get(contig.getReadName());
if (annotatedContigs.containsKey(contig.getReadName())) {
String seq = contig.getReadString();
// log.debug(" te: {}", te);
String annSeq = te.get("seq");
String kmerOrigin = te.get("kmerOrigin");
Map<CortexKmer, Character> kmerCodes = new HashMap<CortexKmer, Character>();
for (int i = 0; i < kmerOrigin.length(); i++) {
CortexKmer kmer = new CortexKmer(annSeq.substring(i, i + kmerSize));
Character code = kmerOrigin.charAt(i);
kmerCodes.put(kmer, code);
}
Map<Character, Integer> kmerStats = new HashMap<Character, Integer>();
for (Character c : kmerCodeNames.keySet()) {
kmerStats.put(c, 0);
}
boolean changed = false;
// We want to be able to examine soft-clipped regions as well.
List<CigarElement> ces = new ArrayList<CigarElement>();
for (CigarElement ce : contig.getCigar().getCigarElements()) {
if (ce.getOperator().equals(CigarOperator.S)) {
ces.add(new CigarElement(ce.getLength(), CigarOperator.M));
changed = true;
} else {
ces.add(ce);
}
}
if (changed) {
CigarElement firstCe = contig.getCigar().getCigarElements().get(0);
if (firstCe.getOperator().equals(CigarOperator.S)) {
contig.setAlignmentStart(contig.getAlignmentStart() - firstCe.getLength());
}
contig.setCigar(new Cigar(ces));
}
for (AlignmentBlock ab : contig.getAlignmentBlocks()) {
for (int i = ab.getReadStart() - 1; i < ab.getReadStart() + ab.getLength(); i++) {
if (i + kmerSize < seq.length()) {
CortexKmer kmer = new CortexKmer(seq.substring(i, i + kmerSize));
SAMRecord skmer = new SAMRecord(CONTIGS.getFileHeader());
skmer.setReadBases(seq.substring(i, i + kmerSize).getBytes());
List<CigarElement> cigarElements = new ArrayList<CigarElement>();
cigarElements.add(new CigarElement(kmerSize, CigarOperator.M));
Cigar cigar = new Cigar(cigarElements);
skmer.setReadName(contig.getReadName() + "." + kmer.getKmerAsString());
skmer.setReferenceName(contig.getReferenceName());
skmer.setCigar(cigar);
skmer.setReadPairedFlag(false);
skmer.setDuplicateReadFlag(false);
skmer.setMateNegativeStrandFlag(false);
skmer.setAlignmentStart(ab.getReferenceStart() - ab.getReadStart() + 1 + i);
skmer.setAttribute("RG", "none");
skmer.setMappingQuality(0);
Character c = kmerCodes.get(kmer);
String codeName = kmerCodeNames.get(c);
String parentReadGroupId = null;
String sampleReadGroupId = null;
for (SAMReadGroupRecord rgr : sfh.getReadGroups()) {
if (rgr.getSample().equals(codeName)) {
parentReadGroupId = rgr.getReadGroupId();
}
if (rgr.getSample().equals(contig.getReadGroup().getSample())) {
sampleReadGroupId = rgr.getReadGroupId();
}
}
skmer.setAttribute(
"RG", parentReadGroupId != null ? parentReadGroupId : sampleReadGroupId);
skmer.setMappingQuality(99);
sfw.addAlignment(skmer);
kmerStats.put(c, kmerStats.get(c) + 1);
IGVEntry igvEntry = new IGVEntry();
igvEntry.chromosome = contig.getReferenceName();
igvEntry.start = ab.getReferenceStart() - ab.getReadStart() + i;
igvEntry.parentageName = kmerCodeNames.get(c);
igvEntry.parentage = kmerCodeIndices.get(c);
igvEntries.add(igvEntry);
}
}
}
if (!contig.isSecondaryOrSupplementary()) {
beout.println(
contig.getReferenceName()
+ "\t"
+ contig.getAlignmentStart()
+ "\t"
+ contig.getAlignmentEnd()
+ "\t"
+ contig.getReadName()
+ "."
+ contig.getReadGroup().getSample());
if (annotatedContigs.size() > 10 && numContigs % (annotatedContigs.size() / 10) == 0) {
log.info(" processed {}/{} contigs", numContigs, annotatedContigs.size());
}
numContigs++;
}
Map<String, String> stats = new LinkedHashMap<String, String>();
stats.put("contigName", contig.getReadName());
stats.put("sampleName", contig.getReadGroup().getSample());
for (Character c : kmerCodeNames.keySet()) {
stats.put(kmerCodeNames.get(c), String.valueOf(kmerStats.get(c)));
}
tw.addEntry(stats);
}
}
}
log.info("Writing kmer inheritance information...");
out.printf("%s\t%s\t%s\t%s\t%s\n", "Chromosome", "Start", "End", "Feature", "Parentage");
for (IGVEntry igvEntry : igvEntries) {
out.printf(
"%s\t%d\t%d\t%s\t%d\n",
igvEntry.chromosome,
igvEntry.start,
igvEntry.start + 1,
igvEntry.parentageName,
igvEntry.parentage);
}
sfw.close();
}
