@Override
protected Object doWork() {
IOUtil.assertFileIsReadable(INPUT);
IOUtil.assertFileIsReadable(REFERENCE_SEQUENCE);
IOUtil.assertFileIsReadable(CHAIN);
IOUtil.assertFileIsWritable(OUTPUT);
IOUtil.assertFileIsWritable(REJECT);
////////////////////////////////////////////////////////////////////////
// Setup the inputs
////////////////////////////////////////////////////////////////////////
final LiftOver liftOver = new LiftOver(CHAIN);
final VCFFileReader in = new VCFFileReader(INPUT, false);
logger.info("Loading up the target reference genome.");
final ReferenceSequenceFileWalker walker = new ReferenceSequenceFileWalker(REFERENCE_SEQUENCE);
final Map<String, byte[]> refSeqs = new HashMap<>();
for (final SAMSequenceRecord rec : walker.getSequenceDictionary().getSequences()) {
refSeqs.put(rec.getSequenceName(), walker.get(rec.getSequenceIndex()).getBases());
}
CloserUtil.close(walker);
////////////////////////////////////////////////////////////////////////
// Setup the outputs
////////////////////////////////////////////////////////////////////////
final VCFHeader inHeader = in.getFileHeader();
final VCFHeader outHeader = new VCFHeader(inHeader);
outHeader.setSequenceDictionary(walker.getSequenceDictionary());
final VariantContextWriter out =
new VariantContextWriterBuilder()
.setOption(Options.INDEX_ON_THE_FLY)
.setOutputFile(OUTPUT)
.setReferenceDictionary(walker.getSequenceDictionary())
.build();
out.writeHeader(outHeader);
final VariantContextWriter rejects =
new VariantContextWriterBuilder()
.setOutputFile(REJECT)
.unsetOption(Options.INDEX_ON_THE_FLY)
.build();
final VCFHeader rejectHeader = new VCFHeader(in.getFileHeader());
for (final VCFFilterHeaderLine line : FILTERS) rejectHeader.addMetaDataLine(line);
rejects.writeHeader(rejectHeader);
////////////////////////////////////////////////////////////////////////
// Read the input VCF, lift the records over and write to the sorting
// collection.
////////////////////////////////////////////////////////////////////////
long failedLiftover = 0, failedAlleleCheck = 0, total = 0;
logger.info("Lifting variants over and sorting.");
final SortingCollection<VariantContext> sorter =
SortingCollection.newInstance(
VariantContext.class,
new VCFRecordCodec(outHeader),
outHeader.getVCFRecordComparator(),
MAX_RECORDS_IN_RAM,
TMP_DIR);
ProgressLogger progress = new ProgressLogger(logger, 1000000, "read");
for (final VariantContext ctx : in) {
++total;
final Interval source =
new Interval(
ctx.getContig(),
ctx.getStart(),
ctx.getEnd(),
false,
ctx.getContig() + ":" + ctx.getStart() + "-" + ctx.getEnd());
final Interval target = liftOver.liftOver(source, 1.0);
if (target == null) {
rejects.add(new VariantContextBuilder(ctx).filter(FILTER_CANNOT_LIFTOVER).make());
failedLiftover++;
} else {
// Fix the alleles if we went from positive to negative strand
final List<Allele> alleles = new ArrayList<>();
for (final Allele oldAllele : ctx.getAlleles()) {
if (target.isPositiveStrand() || oldAllele.isSymbolic()) {
alleles.add(oldAllele);
} else {
alleles.add(
Allele.create(
SequenceUtil.reverseComplement(oldAllele.getBaseString()),
oldAllele.isReference()));
}
}
// Build the new variant context
final VariantContextBuilder builder =
new VariantContextBuilder(
ctx.getSource(), target.getContig(), target.getStart(), target.getEnd(), alleles);
builder.id(ctx.getID());
builder.attributes(ctx.getAttributes());
builder.genotypes(ctx.getGenotypes());
builder.filters(ctx.getFilters());
builder.log10PError(ctx.getLog10PError());
// Check that the reference allele still agrees with the reference sequence
boolean mismatchesReference = false;
for (final Allele allele : builder.getAlleles()) {
if (allele.isReference()) {
final byte[] ref = refSeqs.get(target.getContig());
final String refString =
StringUtil.bytesToString(ref, target.getStart() - 1, target.length());
if (!refString.equalsIgnoreCase(allele.getBaseString())) {
mismatchesReference = true;
}
break;
}
}
if (mismatchesReference) {
rejects.add(new VariantContextBuilder(ctx).filter(FILTER_MISMATCHING_REF_ALLELE).make());
failedAlleleCheck++;
} else {
sorter.add(builder.make());
}
}
progress.record(ctx.getContig(), ctx.getStart());
}
final NumberFormat pfmt = new DecimalFormat("0.0000%");
final String pct = pfmt.format((failedLiftover + failedAlleleCheck) / (double) total);
logger.info("Processed ", total, " variants.");
logger.info(Long.toString(failedLiftover), " variants failed to liftover.");
logger.info(
Long.toString(failedAlleleCheck),
" variants lifted over but had mismatching reference alleles after lift over.");
logger.info(pct, " of variants were not successfully lifted over and written to the output.");
rejects.close();
in.close();
////////////////////////////////////////////////////////////////////////
// Write the sorted outputs to the final output file
////////////////////////////////////////////////////////////////////////
sorter.doneAdding();
progress = new ProgressLogger(logger, 1000000, "written");
logger.info("Writing out sorted records to final VCF.");
for (final VariantContext ctx : sorter) {
out.add(ctx);
progress.record(ctx.getContig(), ctx.getStart());
}
out.close();
sorter.cleanup();
return null;
}
/**
* Main method for the program. Checks that all input files are present and readable and that the
* output file can be written to. Then iterates through all the records accumulating metrics.
* Finally writes metrics file
*/
protected int doWork() {
IOUtil.assertFileIsReadable(INPUT);
IOUtil.assertFileIsWritable(OUTPUT);
final SamReader reader =
SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).open(INPUT);
final Histogram<Integer> mismatchesHist = new Histogram<Integer>("Predicted", "Mismatches");
final Histogram<Integer> totalHist = new Histogram<Integer>("Predicted", "Total_Bases");
final Map<String, Histogram> mismatchesByTypeHist = new HashMap<String, Histogram>();
final Map<String, Histogram> totalByTypeHist = new HashMap<String, Histogram>();
// Set up the histograms
byte[] bases = {'A', 'C', 'G', 'T'};
for (final byte base : bases) {
final Histogram<Integer> h = new Histogram<Integer>("Predicted", (char) base + ">");
mismatchesByTypeHist.put((char) base + ">", h);
final Histogram<Integer> h2 = new Histogram<Integer>("Predicted", ">" + (char) base);
mismatchesByTypeHist.put(">" + (char) base, h2);
}
for (final byte base : bases) {
final Histogram<Integer> h = new Histogram<Integer>("Predicted", (char) base + ">");
totalByTypeHist.put((char) base + ">", h);
final Histogram<Integer> h2 = new Histogram<Integer>("Predicted", ">" + (char) base);
totalByTypeHist.put(">" + (char) base, h2);
}
for (final SAMRecord record : reader) {
// Ignore these as we don't know the truth
if (record.getReadUnmappedFlag() || record.isSecondaryOrSupplementary()) {
continue;
}
final byte[] readBases = record.getReadBases();
final byte[] readQualities = record.getBaseQualities();
final byte[] refBases = SequenceUtil.makeReferenceFromAlignment(record, false);
// We've seen stranger things
if (readQualities.length != readBases.length) {
throw new PicardException(
"Missing Qualities ("
+ readQualities.length
+ ","
+ readBases.length
+ ") : "
+ record.getSAMString());
}
if (refBases.length != readBases.length) {
throw new PicardException(
"The read length did not match the inferred reference length, please check your MD and CIGAR.");
}
int cycleIndex; // zero-based
if (record.getReadNegativeStrandFlag()) {
cycleIndex = readBases.length - 1 + CYCLE_OFFSET;
} else {
cycleIndex = CYCLE_OFFSET;
}
for (int i = 0; i < readBases.length; i++) {
if (-1 == CYCLE || cycleIndex == CYCLE) {
if ('-' != refBases[i] && '0' != refBases[i]) { // not insertion and not soft-clipped
if (!SequenceUtil.basesEqual(readBases[i], refBases[i])) { // mismatch
mismatchesHist.increment((int) readQualities[i]);
if (SequenceUtil.isValidBase(refBases[i])) {
mismatchesByTypeHist
.get((char) refBases[i] + ">")
.increment((int) readQualities[i]);
}
if (SequenceUtil.isValidBase(readBases[i])) {
mismatchesByTypeHist
.get(">" + (char) readBases[i])
.increment((int) readQualities[i]);
}
} else {
mismatchesHist.increment(
(int) readQualities[i], 0); // to make sure the bin will exist
}
totalHist.increment((int) readQualities[i]);
if (SequenceUtil.isValidBase(readBases[i])) {
totalByTypeHist.get(">" + (char) readBases[i]).increment((int) readQualities[i]);
}
if (SequenceUtil.isValidBase(refBases[i])) {
totalByTypeHist.get((char) refBases[i] + ">").increment((int) readQualities[i]);
}
}
}
cycleIndex += record.getReadNegativeStrandFlag() ? -1 : 1;
}
}
CloserUtil.close(reader);
final Histogram<Integer> hist = new Histogram<Integer>("Predicted", "Observed");
double sumOfSquaresError = 0.0;
// compute the aggregate phred values
for (final Integer key : mismatchesHist.keySet()) {
final double numMismatches = mismatchesHist.get(key).getValue();
final double numBases = totalHist.get(key).getValue();
final double phredErr = Math.log10(numMismatches / numBases) * -10.0;
sumOfSquaresError += (0 == numMismatches) ? 0.0 : (key - phredErr) * (key - phredErr);
hist.increment(key, phredErr);
// make sure the bin will exist
for (final byte base : bases) {
mismatchesByTypeHist.get(">" + (char) base).increment(key, 0.0);
mismatchesByTypeHist.get((char) base + ">").increment(key, 0.0);
totalByTypeHist.get(">" + (char) base).increment(key, 0.0);
totalByTypeHist.get((char) base + ">").increment(key, 0.0);
}
}
final QualityScoreAccuracyMetrics metrics = new QualityScoreAccuracyMetrics();
metrics.SUM_OF_SQUARE_ERROR = sumOfSquaresError;
final MetricsFile<QualityScoreAccuracyMetrics, Integer> out = getMetricsFile();
out.addMetric(metrics);
out.addHistogram(hist);
for (final byte base : bases) {
// >base : histograms for mismatches *to* the given base
Histogram<Integer> m = mismatchesByTypeHist.get(">" + (char) base);
Histogram<Integer> t = totalByTypeHist.get(">" + (char) base);
Histogram<Integer> h = new Histogram<Integer>(m.getBinLabel(), m.getValueLabel());
for (final Integer key : m.keySet()) {
final double numMismatches = m.get(key).getValue();
final double numBases = t.get(key).getValue();
final double phredErr = Math.log10(numMismatches / numBases) * -10.0;
h.increment(key, phredErr);
}
out.addHistogram(h);
// base> : histograms for mismatches *from* the given base
m = mismatchesByTypeHist.get((char) base + ">");
t = totalByTypeHist.get(">" + (char) base);
h = new Histogram<Integer>(m.getBinLabel(), m.getValueLabel());
for (final Integer key : m.keySet()) {
final double numMismatches = m.get(key).getValue();
final double numBases = t.get(key).getValue();
final double phredErr = Math.log10(numMismatches / numBases) * -10.0;
h.increment(key, phredErr);
}
out.addHistogram(h);
}
out.addHistogram(mismatchesHist);
out.addHistogram(totalHist);
out.write(OUTPUT);
return 0;
}