private void addItem(String activity, String time, String guid) {
    if (map.get(guid) == null) map.put(guid, new Interval());

    Interval interval = map.get(guid);

    System.out.println(time);
    Date date = new Date(time);
    addTimeRange(date);

    if (activity.equalsIgnoreCase(AWS_AUTHENTICATION_SUCCEEDED)) {
      if (interval.getStart() == null) {
        interval.setStart(date);
      } else {
        if (interval.getStart().before(date)) {
          interval.setStart(date);
        }
      }
    } else if (activity.equalsIgnoreCase(AWS_TRANSACTION_COMPLETED)) {
      if (interval.getEnd() == null) {
        interval.setEnd(date);
      } else {
        if (interval.getEnd().before(date)) {
          interval.setEnd(date);
        }
      }
    }
  }
 public void testConstructor_long_long_minMax() throws Throwable {
   Interval test = new Interval(Long.MIN_VALUE, Long.MAX_VALUE);
   assertEquals(Long.MIN_VALUE, test.getStartMillis());
   assertEquals(Long.MAX_VALUE, test.getEndMillis());
   assertEquals(new DateTime(Long.MIN_VALUE), test.getStart());
   assertEquals(new DateTime(Long.MAX_VALUE), test.getEnd());
   assertEquals(ISOChronology.getInstance(), test.getChronology());
   assertEquals(test, test.toInterval());
   assertEquals(
       "-292275055-05-16T16:56:25.192+00:09:21/292278994-08-17T07:12:55.807Z", test.toString());
   try {
     test.toDuration();
     fail();
   } catch (ArithmeticException ex) {
   }
   try {
     test.toDurationMillis();
     fail();
   } catch (ArithmeticException ex) {
   }
   try {
     test.toPeriod();
     fail();
   } catch (RuntimeException ex) {
   }
 }
Beispiel #3
0
 // the given list is sorted
 // remove the duplicates of intervals with the same start
 // the rule is like this: keep the longest length interval
 private List<Interval> removeDuplicateStart() {
   List<Interval> list = getIntervalListFromETask();
   if (null == list) {
     return null;
   }
   List<Interval> res = new ArrayList<Interval>();
   int sz = list.size();
   int i = 0;
   while (i < sz) {
     int j = i + 1;
     Interval longestInterval = list.get(i);
     int maxDuration = (int) (longestInterval.getEnd() - longestInterval.getStart());
     while (j < sz && list.get(j).getStart() == list.get(i).getStart()) {
       int curDuration = (int) (list.get(j).getEnd() - list.get(j).getStart());
       if (curDuration > maxDuration) {
         maxDuration = curDuration;
         longestInterval = list.get(j);
       }
       j++;
     }
     res.add(longestInterval);
     i = j;
   }
   return res;
 }
Beispiel #4
0
  /**
   * Creates (if not already existing) and returns the partition in which the instant,value should
   * be written. instant can be invalid (in case value only or no partitioning) value can be null
   * (in case
   *
   * @return a Partition
   */
  public synchronized Partition createAndGetPartition(long instant, Object value)
      throws IOException {
    Partition partition;
    if (partitioningSpec.timeColumn != null) {
      if ((pcache == null) || (pcache.start > instant) || (pcache.getEnd() <= instant)) {

        Entry<Long, Interval> entry = intervals.floorEntry(instant);
        if ((entry != null) && (instant < entry.getValue().getEnd())) {
          pcache = entry.getValue();
        } else { // no partition in this interval.
          PartitionInfo pinfo = partitioningSpec.timePartitioningSchema.getPartitionInfo(instant);
          pcache = new Interval(pinfo.partitionStart, pinfo.partitionEnd);
          intervals.put(pcache.start, pcache);
        }
      }
    }
    partition = pcache.get(value);
    if (partition == null) {
      if (partitioningSpec.timeColumn != null) {
        PartitionInfo pinfo = partitioningSpec.timePartitioningSchema.getPartitionInfo(instant);
        partition = createPartition(pinfo, value);
      } else {
        partition = createPartition(value);
      }
      pcache.add(value, partition);
    }
    return partition;
  }
Beispiel #5
0
    @Override
    public boolean hasNext() {
      if (next != null) return true;
      next = new ArrayList<Partition>();

      while (it.hasNext()) {
        Entry<Long, Interval> entry = it.next();
        Interval intv = entry.getValue();
        if (jumpToStart && (intv.getEnd() <= start)) {
          continue;
        } else {
          jumpToStart = false;
        }
        if (partitioningSpec.type == _type.TIME) {
          next.add(intv.partitions.values().iterator().next());
        } else {
          for (Partition p : intv.partitions.values()) {
            if ((partitionValueFilter == null) || (partitionValueFilter.contains(p.getValue()))) {
              next.add(p);
            }
          }
        }
        if (!next.isEmpty()) {
          break;
        }
      }
      if (next.isEmpty()) {
        next = null;
        return false;
      } else {
        return true;
      }
    }
 public void testConstructor_long_long_max() throws Throwable {
   Interval test = new Interval(Long.MAX_VALUE - 9, Long.MAX_VALUE);
   assertEquals(Long.MAX_VALUE - 9, test.getStartMillis());
   assertEquals(Long.MAX_VALUE, test.getEndMillis());
   assertEquals(new DateTime(Long.MAX_VALUE - 9), test.getStart());
   assertEquals(new DateTime(Long.MAX_VALUE), test.getEnd());
   assertEquals(ISOChronology.getInstance(), test.getChronology());
   assertEquals(test, test.toInterval());
   assertEquals("292278994-08-17T07:12:55.798Z/292278994-08-17T07:12:55.807Z", test.toString());
   assertEquals(9, test.toDurationMillis());
   assertEquals(new Duration(9), test.toDuration());
 }
 public void testConstructor_long_long_min() throws Throwable {
   Interval test = new Interval(Long.MIN_VALUE, Long.MIN_VALUE + 9);
   assertEquals(Long.MIN_VALUE, test.getStartMillis());
   assertEquals(Long.MIN_VALUE + 9, test.getEndMillis());
   assertEquals(new DateTime(Long.MIN_VALUE), test.getStart());
   assertEquals(new DateTime(Long.MIN_VALUE + 9), test.getEnd());
   assertEquals(ISOChronology.getInstance(), test.getChronology());
   assertEquals(test, test.toInterval());
   assertEquals(
       "-292275055-05-16T16:56:25.192+00:09:21/-292275055-05-16T16:56:25.201+00:09:21",
       test.toString());
   assertEquals(9, test.toDurationMillis());
   assertEquals(new Duration(9), test.toDuration());
   assertEquals(new Period(9), test.toPeriod());
 }
Beispiel #8
0
 public static <T> List<T> mergeListWithSortedMatchedPreAggregated(
     List<? extends T> list,
     List<? extends T> matched,
     Function<T, Interval<Integer>> toIntervalFunc) {
   List<T> merged = new ArrayList<T>(list.size()); // Approximate size
   int last = 0;
   for (T m : matched) {
     Interval<Integer> interval = toIntervalFunc.apply(m);
     int start = interval.getBegin();
     int end = interval.getEnd();
     if (start >= last) {
       merged.addAll(list.subList(last, start));
       merged.add(m);
       last = end;
     }
   }
   // Add rest of elements
   if (last < list.size()) {
     merged.addAll(list.subList(last, list.size()));
   }
   return merged;
 }
Beispiel #9
0
 public static <T> List<T> mergeListWithSortedMatched(
     List<? extends T> list,
     List<? extends HasInterval<Integer>> matched,
     Function<List<? extends T>, T> aggregator) {
   List<T> merged = new ArrayList<T>(list.size()); // Approximate size
   int last = 0;
   for (HasInterval<Integer> m : matched) {
     Interval<Integer> interval = m.getInterval();
     int start = interval.getBegin();
     int end = interval.getEnd();
     if (start >= last) {
       merged.addAll(list.subList(last, start));
       T t = aggregator.apply(list.subList(start, end));
       merged.add(t);
       last = end;
     }
   }
   // Add rest of elements
   if (last < list.size()) {
     merged.addAll(list.subList(last, list.size()));
   }
   return merged;
 }
Beispiel #10
0
  @Override
  protected Object doWork() {
    IOUtil.assertFileIsReadable(INPUT);
    IOUtil.assertFileIsReadable(REFERENCE_SEQUENCE);
    IOUtil.assertFileIsReadable(CHAIN);
    IOUtil.assertFileIsWritable(OUTPUT);
    IOUtil.assertFileIsWritable(REJECT);

    ////////////////////////////////////////////////////////////////////////
    // Setup the inputs
    ////////////////////////////////////////////////////////////////////////
    final LiftOver liftOver = new LiftOver(CHAIN);
    final VCFFileReader in = new VCFFileReader(INPUT, false);

    logger.info("Loading up the target reference genome.");
    final ReferenceSequenceFileWalker walker = new ReferenceSequenceFileWalker(REFERENCE_SEQUENCE);
    final Map<String, byte[]> refSeqs = new HashMap<>();
    for (final SAMSequenceRecord rec : walker.getSequenceDictionary().getSequences()) {
      refSeqs.put(rec.getSequenceName(), walker.get(rec.getSequenceIndex()).getBases());
    }
    CloserUtil.close(walker);

    ////////////////////////////////////////////////////////////////////////
    // Setup the outputs
    ////////////////////////////////////////////////////////////////////////
    final VCFHeader inHeader = in.getFileHeader();
    final VCFHeader outHeader = new VCFHeader(inHeader);
    outHeader.setSequenceDictionary(walker.getSequenceDictionary());
    final VariantContextWriter out =
        new VariantContextWriterBuilder()
            .setOption(Options.INDEX_ON_THE_FLY)
            .setOutputFile(OUTPUT)
            .setReferenceDictionary(walker.getSequenceDictionary())
            .build();
    out.writeHeader(outHeader);

    final VariantContextWriter rejects =
        new VariantContextWriterBuilder()
            .setOutputFile(REJECT)
            .unsetOption(Options.INDEX_ON_THE_FLY)
            .build();
    final VCFHeader rejectHeader = new VCFHeader(in.getFileHeader());
    for (final VCFFilterHeaderLine line : FILTERS) rejectHeader.addMetaDataLine(line);
    rejects.writeHeader(rejectHeader);

    ////////////////////////////////////////////////////////////////////////
    // Read the input VCF, lift the records over and write to the sorting
    // collection.
    ////////////////////////////////////////////////////////////////////////
    long failedLiftover = 0, failedAlleleCheck = 0, total = 0;
    logger.info("Lifting variants over and sorting.");

    final SortingCollection<VariantContext> sorter =
        SortingCollection.newInstance(
            VariantContext.class,
            new VCFRecordCodec(outHeader),
            outHeader.getVCFRecordComparator(),
            MAX_RECORDS_IN_RAM,
            TMP_DIR);

    ProgressLogger progress = new ProgressLogger(logger, 1000000, "read");

    for (final VariantContext ctx : in) {
      ++total;
      final Interval source =
          new Interval(
              ctx.getContig(),
              ctx.getStart(),
              ctx.getEnd(),
              false,
              ctx.getContig() + ":" + ctx.getStart() + "-" + ctx.getEnd());
      final Interval target = liftOver.liftOver(source, 1.0);

      if (target == null) {
        rejects.add(new VariantContextBuilder(ctx).filter(FILTER_CANNOT_LIFTOVER).make());
        failedLiftover++;
      } else {
        // Fix the alleles if we went from positive to negative strand
        final List<Allele> alleles = new ArrayList<>();
        for (final Allele oldAllele : ctx.getAlleles()) {
          if (target.isPositiveStrand() || oldAllele.isSymbolic()) {
            alleles.add(oldAllele);
          } else {
            alleles.add(
                Allele.create(
                    SequenceUtil.reverseComplement(oldAllele.getBaseString()),
                    oldAllele.isReference()));
          }
        }

        // Build the new variant context
        final VariantContextBuilder builder =
            new VariantContextBuilder(
                ctx.getSource(), target.getContig(), target.getStart(), target.getEnd(), alleles);

        builder.id(ctx.getID());
        builder.attributes(ctx.getAttributes());
        builder.genotypes(ctx.getGenotypes());
        builder.filters(ctx.getFilters());
        builder.log10PError(ctx.getLog10PError());

        // Check that the reference allele still agrees with the reference sequence
        boolean mismatchesReference = false;
        for (final Allele allele : builder.getAlleles()) {
          if (allele.isReference()) {
            final byte[] ref = refSeqs.get(target.getContig());
            final String refString =
                StringUtil.bytesToString(ref, target.getStart() - 1, target.length());

            if (!refString.equalsIgnoreCase(allele.getBaseString())) {
              mismatchesReference = true;
            }

            break;
          }
        }

        if (mismatchesReference) {
          rejects.add(new VariantContextBuilder(ctx).filter(FILTER_MISMATCHING_REF_ALLELE).make());
          failedAlleleCheck++;
        } else {
          sorter.add(builder.make());
        }
      }

      progress.record(ctx.getContig(), ctx.getStart());
    }

    final NumberFormat pfmt = new DecimalFormat("0.0000%");
    final String pct = pfmt.format((failedLiftover + failedAlleleCheck) / (double) total);
    logger.info("Processed ", total, " variants.");
    logger.info(Long.toString(failedLiftover), " variants failed to liftover.");
    logger.info(
        Long.toString(failedAlleleCheck),
        " variants lifted over but had mismatching reference alleles after lift over.");
    logger.info(pct, " of variants were not successfully lifted over and written to the output.");

    rejects.close();
    in.close();

    ////////////////////////////////////////////////////////////////////////
    // Write the sorted outputs to the final output file
    ////////////////////////////////////////////////////////////////////////
    sorter.doneAdding();
    progress = new ProgressLogger(logger, 1000000, "written");
    logger.info("Writing out sorted records to final VCF.");

    for (final VariantContext ctx : sorter) {
      out.add(ctx);
      progress.record(ctx.getContig(), ctx.getStart());
    }
    out.close();
    sorter.cleanup();

    return null;
  }
Beispiel #11
0
 public boolean overlapsWith(Interval other) {
   return this.start <= other.getEnd() && this.end >= other.getStart();
 }