private void addItem(String activity, String time, String guid) {
if (map.get(guid) == null) map.put(guid, new Interval());
Interval interval = map.get(guid);
System.out.println(time);
Date date = new Date(time);
addTimeRange(date);
if (activity.equalsIgnoreCase(AWS_AUTHENTICATION_SUCCEEDED)) {
if (interval.getStart() == null) {
interval.setStart(date);
} else {
if (interval.getStart().before(date)) {
interval.setStart(date);
}
}
} else if (activity.equalsIgnoreCase(AWS_TRANSACTION_COMPLETED)) {
if (interval.getEnd() == null) {
interval.setEnd(date);
} else {
if (interval.getEnd().before(date)) {
interval.setEnd(date);
}
}
}
}
public void testConstructor_long_long_minMax() throws Throwable {
Interval test = new Interval(Long.MIN_VALUE, Long.MAX_VALUE);
assertEquals(Long.MIN_VALUE, test.getStartMillis());
assertEquals(Long.MAX_VALUE, test.getEndMillis());
assertEquals(new DateTime(Long.MIN_VALUE), test.getStart());
assertEquals(new DateTime(Long.MAX_VALUE), test.getEnd());
assertEquals(ISOChronology.getInstance(), test.getChronology());
assertEquals(test, test.toInterval());
assertEquals(
"-292275055-05-16T16:56:25.192+00:09:21/292278994-08-17T07:12:55.807Z", test.toString());
try {
test.toDuration();
fail();
} catch (ArithmeticException ex) {
}
try {
test.toDurationMillis();
fail();
} catch (ArithmeticException ex) {
}
try {
test.toPeriod();
fail();
} catch (RuntimeException ex) {
}
}
// the given list is sorted
// remove the duplicates of intervals with the same start
// the rule is like this: keep the longest length interval
private List<Interval> removeDuplicateStart() {
List<Interval> list = getIntervalListFromETask();
if (null == list) {
return null;
}
List<Interval> res = new ArrayList<Interval>();
int sz = list.size();
int i = 0;
while (i < sz) {
int j = i + 1;
Interval longestInterval = list.get(i);
int maxDuration = (int) (longestInterval.getEnd() - longestInterval.getStart());
while (j < sz && list.get(j).getStart() == list.get(i).getStart()) {
int curDuration = (int) (list.get(j).getEnd() - list.get(j).getStart());
if (curDuration > maxDuration) {
maxDuration = curDuration;
longestInterval = list.get(j);
}
j++;
}
res.add(longestInterval);
i = j;
}
return res;
}
/**
* Creates (if not already existing) and returns the partition in which the instant,value should
* be written. instant can be invalid (in case value only or no partitioning) value can be null
* (in case
*
* @return a Partition
*/
public synchronized Partition createAndGetPartition(long instant, Object value)
throws IOException {
Partition partition;
if (partitioningSpec.timeColumn != null) {
if ((pcache == null) || (pcache.start > instant) || (pcache.getEnd() <= instant)) {
Entry<Long, Interval> entry = intervals.floorEntry(instant);
if ((entry != null) && (instant < entry.getValue().getEnd())) {
pcache = entry.getValue();
} else { // no partition in this interval.
PartitionInfo pinfo = partitioningSpec.timePartitioningSchema.getPartitionInfo(instant);
pcache = new Interval(pinfo.partitionStart, pinfo.partitionEnd);
intervals.put(pcache.start, pcache);
}
}
}
partition = pcache.get(value);
if (partition == null) {
if (partitioningSpec.timeColumn != null) {
PartitionInfo pinfo = partitioningSpec.timePartitioningSchema.getPartitionInfo(instant);
partition = createPartition(pinfo, value);
} else {
partition = createPartition(value);
}
pcache.add(value, partition);
}
return partition;
}
@Override
public boolean hasNext() {
if (next != null) return true;
next = new ArrayList<Partition>();
while (it.hasNext()) {
Entry<Long, Interval> entry = it.next();
Interval intv = entry.getValue();
if (jumpToStart && (intv.getEnd() <= start)) {
continue;
} else {
jumpToStart = false;
}
if (partitioningSpec.type == _type.TIME) {
next.add(intv.partitions.values().iterator().next());
} else {
for (Partition p : intv.partitions.values()) {
if ((partitionValueFilter == null) || (partitionValueFilter.contains(p.getValue()))) {
next.add(p);
}
}
}
if (!next.isEmpty()) {
break;
}
}
if (next.isEmpty()) {
next = null;
return false;
} else {
return true;
}
}
public void testConstructor_long_long_max() throws Throwable {
Interval test = new Interval(Long.MAX_VALUE - 9, Long.MAX_VALUE);
assertEquals(Long.MAX_VALUE - 9, test.getStartMillis());
assertEquals(Long.MAX_VALUE, test.getEndMillis());
assertEquals(new DateTime(Long.MAX_VALUE - 9), test.getStart());
assertEquals(new DateTime(Long.MAX_VALUE), test.getEnd());
assertEquals(ISOChronology.getInstance(), test.getChronology());
assertEquals(test, test.toInterval());
assertEquals("292278994-08-17T07:12:55.798Z/292278994-08-17T07:12:55.807Z", test.toString());
assertEquals(9, test.toDurationMillis());
assertEquals(new Duration(9), test.toDuration());
}
public void testConstructor_long_long_min() throws Throwable {
Interval test = new Interval(Long.MIN_VALUE, Long.MIN_VALUE + 9);
assertEquals(Long.MIN_VALUE, test.getStartMillis());
assertEquals(Long.MIN_VALUE + 9, test.getEndMillis());
assertEquals(new DateTime(Long.MIN_VALUE), test.getStart());
assertEquals(new DateTime(Long.MIN_VALUE + 9), test.getEnd());
assertEquals(ISOChronology.getInstance(), test.getChronology());
assertEquals(test, test.toInterval());
assertEquals(
"-292275055-05-16T16:56:25.192+00:09:21/-292275055-05-16T16:56:25.201+00:09:21",
test.toString());
assertEquals(9, test.toDurationMillis());
assertEquals(new Duration(9), test.toDuration());
assertEquals(new Period(9), test.toPeriod());
}
public static <T> List<T> mergeListWithSortedMatchedPreAggregated(
List<? extends T> list,
List<? extends T> matched,
Function<T, Interval<Integer>> toIntervalFunc) {
List<T> merged = new ArrayList<T>(list.size()); // Approximate size
int last = 0;
for (T m : matched) {
Interval<Integer> interval = toIntervalFunc.apply(m);
int start = interval.getBegin();
int end = interval.getEnd();
if (start >= last) {
merged.addAll(list.subList(last, start));
merged.add(m);
last = end;
}
}
// Add rest of elements
if (last < list.size()) {
merged.addAll(list.subList(last, list.size()));
}
return merged;
}
public static <T> List<T> mergeListWithSortedMatched(
List<? extends T> list,
List<? extends HasInterval<Integer>> matched,
Function<List<? extends T>, T> aggregator) {
List<T> merged = new ArrayList<T>(list.size()); // Approximate size
int last = 0;
for (HasInterval<Integer> m : matched) {
Interval<Integer> interval = m.getInterval();
int start = interval.getBegin();
int end = interval.getEnd();
if (start >= last) {
merged.addAll(list.subList(last, start));
T t = aggregator.apply(list.subList(start, end));
merged.add(t);
last = end;
}
}
// Add rest of elements
if (last < list.size()) {
merged.addAll(list.subList(last, list.size()));
}
return merged;
}
@Override
protected Object doWork() {
IOUtil.assertFileIsReadable(INPUT);
IOUtil.assertFileIsReadable(REFERENCE_SEQUENCE);
IOUtil.assertFileIsReadable(CHAIN);
IOUtil.assertFileIsWritable(OUTPUT);
IOUtil.assertFileIsWritable(REJECT);
////////////////////////////////////////////////////////////////////////
// Setup the inputs
////////////////////////////////////////////////////////////////////////
final LiftOver liftOver = new LiftOver(CHAIN);
final VCFFileReader in = new VCFFileReader(INPUT, false);
logger.info("Loading up the target reference genome.");
final ReferenceSequenceFileWalker walker = new ReferenceSequenceFileWalker(REFERENCE_SEQUENCE);
final Map<String, byte[]> refSeqs = new HashMap<>();
for (final SAMSequenceRecord rec : walker.getSequenceDictionary().getSequences()) {
refSeqs.put(rec.getSequenceName(), walker.get(rec.getSequenceIndex()).getBases());
}
CloserUtil.close(walker);
////////////////////////////////////////////////////////////////////////
// Setup the outputs
////////////////////////////////////////////////////////////////////////
final VCFHeader inHeader = in.getFileHeader();
final VCFHeader outHeader = new VCFHeader(inHeader);
outHeader.setSequenceDictionary(walker.getSequenceDictionary());
final VariantContextWriter out =
new VariantContextWriterBuilder()
.setOption(Options.INDEX_ON_THE_FLY)
.setOutputFile(OUTPUT)
.setReferenceDictionary(walker.getSequenceDictionary())
.build();
out.writeHeader(outHeader);
final VariantContextWriter rejects =
new VariantContextWriterBuilder()
.setOutputFile(REJECT)
.unsetOption(Options.INDEX_ON_THE_FLY)
.build();
final VCFHeader rejectHeader = new VCFHeader(in.getFileHeader());
for (final VCFFilterHeaderLine line : FILTERS) rejectHeader.addMetaDataLine(line);
rejects.writeHeader(rejectHeader);
////////////////////////////////////////////////////////////////////////
// Read the input VCF, lift the records over and write to the sorting
// collection.
////////////////////////////////////////////////////////////////////////
long failedLiftover = 0, failedAlleleCheck = 0, total = 0;
logger.info("Lifting variants over and sorting.");
final SortingCollection<VariantContext> sorter =
SortingCollection.newInstance(
VariantContext.class,
new VCFRecordCodec(outHeader),
outHeader.getVCFRecordComparator(),
MAX_RECORDS_IN_RAM,
TMP_DIR);
ProgressLogger progress = new ProgressLogger(logger, 1000000, "read");
for (final VariantContext ctx : in) {
++total;
final Interval source =
new Interval(
ctx.getContig(),
ctx.getStart(),
ctx.getEnd(),
false,
ctx.getContig() + ":" + ctx.getStart() + "-" + ctx.getEnd());
final Interval target = liftOver.liftOver(source, 1.0);
if (target == null) {
rejects.add(new VariantContextBuilder(ctx).filter(FILTER_CANNOT_LIFTOVER).make());
failedLiftover++;
} else {
// Fix the alleles if we went from positive to negative strand
final List<Allele> alleles = new ArrayList<>();
for (final Allele oldAllele : ctx.getAlleles()) {
if (target.isPositiveStrand() || oldAllele.isSymbolic()) {
alleles.add(oldAllele);
} else {
alleles.add(
Allele.create(
SequenceUtil.reverseComplement(oldAllele.getBaseString()),
oldAllele.isReference()));
}
}
// Build the new variant context
final VariantContextBuilder builder =
new VariantContextBuilder(
ctx.getSource(), target.getContig(), target.getStart(), target.getEnd(), alleles);
builder.id(ctx.getID());
builder.attributes(ctx.getAttributes());
builder.genotypes(ctx.getGenotypes());
builder.filters(ctx.getFilters());
builder.log10PError(ctx.getLog10PError());
// Check that the reference allele still agrees with the reference sequence
boolean mismatchesReference = false;
for (final Allele allele : builder.getAlleles()) {
if (allele.isReference()) {
final byte[] ref = refSeqs.get(target.getContig());
final String refString =
StringUtil.bytesToString(ref, target.getStart() - 1, target.length());
if (!refString.equalsIgnoreCase(allele.getBaseString())) {
mismatchesReference = true;
}
break;
}
}
if (mismatchesReference) {
rejects.add(new VariantContextBuilder(ctx).filter(FILTER_MISMATCHING_REF_ALLELE).make());
failedAlleleCheck++;
} else {
sorter.add(builder.make());
}
}
progress.record(ctx.getContig(), ctx.getStart());
}
final NumberFormat pfmt = new DecimalFormat("0.0000%");
final String pct = pfmt.format((failedLiftover + failedAlleleCheck) / (double) total);
logger.info("Processed ", total, " variants.");
logger.info(Long.toString(failedLiftover), " variants failed to liftover.");
logger.info(
Long.toString(failedAlleleCheck),
" variants lifted over but had mismatching reference alleles after lift over.");
logger.info(pct, " of variants were not successfully lifted over and written to the output.");
rejects.close();
in.close();
////////////////////////////////////////////////////////////////////////
// Write the sorted outputs to the final output file
////////////////////////////////////////////////////////////////////////
sorter.doneAdding();
progress = new ProgressLogger(logger, 1000000, "written");
logger.info("Writing out sorted records to final VCF.");
for (final VariantContext ctx : sorter) {
out.add(ctx);
progress.record(ctx.getContig(), ctx.getStart());
}
out.close();
sorter.cleanup();
return null;
}
public boolean overlapsWith(Interval other) {
return this.start <= other.getEnd() && this.end >= other.getStart();
}