Beispiel #1
0
  /**
   * Add an object to the end of the Journal.
   *
   * @param obj the object to add
   * @throws com.nfsdb.exceptions.JournalException if there is an error
   */
  public void append(T obj) throws JournalException {

    if (obj == null) {
      throw new JournalException("Cannot append NULL to %s", this);
    }

    if (!txActive) {
      beginTx();
    }

    if (checkOrder) {
      long timestamp = getTimestamp(obj);

      if (timestamp > appendTimestampHi) {
        switchAppendPartition(timestamp);
      }

      if (timestamp < appendTimestampLo) {
        throw new JournalException(
            "Cannot insert records out of order. maxHardTimestamp=%d (%s), timestamp=%d (%s): %s",
            appendTimestampLo,
            Dates.toString(appendTimestampLo),
            timestamp,
            Dates.toString(timestamp),
            this);
      }

      appendPartition.append(obj);
      appendTimestampLo = timestamp;
    } else {
      getAppendPartition().append(obj);
    }
  }
Beispiel #2
0
  private void splitAppend(Iterator<T> it, long hard, long soft, Partition<T> partition)
      throws JournalException {
    while (it.hasNext()) {
      T obj = it.next();
      if (doDiscard && getTimestamp(obj) < hard) {
        // discard
        continue;
      } else if (doDiscard) {
        doDiscard = false;
      }

      if (doJournal && getTimestamp(obj) < soft) {
        append(obj);
        continue;
      } else if (doJournal) {
        doJournal = false;
      }

      partition.append(obj);
    }
  }
Beispiel #3
0
  public void mergeAppend(PeekingIterator<T> data) throws JournalException {

    if (lagMillis == 0) {
      throw new JournalException("This journal is not configured to have lag partition");
    }

    beginTx();

    if (data == null || data.isEmpty()) {
      return;
    }

    long dataMaxTimestamp = getTimestamp(data.peekLast());
    long hard = getAppendTimestampLo();

    if (dataMaxTimestamp < hard) {
      return;
    }

    final Partition<T> lagPartition = openOrCreateLagPartition();
    this.doDiscard = true;
    this.doJournal = true;

    long dataMinTimestamp = getTimestamp(data.peekFirst());
    long lagMaxTimestamp = getMaxTimestamp();
    long lagMinTimestamp = lagPartition.size() == 0L ? 0 : getTimestamp(lagPartition.read(0));
    long soft = Math.max(dataMaxTimestamp, lagMaxTimestamp) - lagMillis;

    if (dataMinTimestamp > lagMaxTimestamp) {
      // this could be as simple as just appending data to lag
      // the only complication is that after adding records to lag it could swell beyond
      // the allocated "lagSwellTimestamp"
      // we should check if this is going to happen and optimise copying of data

      long lagSizeMillis;
      if (hard > 0L) {
        lagSizeMillis = dataMaxTimestamp - hard;
      } else if (lagMinTimestamp > 0L) {
        lagSizeMillis = dataMaxTimestamp - lagMinTimestamp;
      } else {
        lagSizeMillis = 0L;
      }

      if (lagSizeMillis > lagSwellMillis) {
        // data would  be too big and would stretch outside of swell timestamp
        // this is when lag partition should be split, but it is still a straight split without
        // re-order

        Partition<T> tempPartition = createTempPartition().open();
        splitAppend(lagPartition.bufferedIterator(), hard, soft, tempPartition);
        splitAppend(data, hard, soft, tempPartition);
        replaceIrregularPartition(tempPartition);
      } else {
        // simplest case, just append to lag
        lagPartition.append(data);
      }
    } else {

      Partition<T> tempPartition = createTempPartition().open();
      if (dataMinTimestamp > lagMinTimestamp && dataMaxTimestamp < lagMaxTimestamp) {
        //
        // overlap scenario 1: data is fully inside of lag
        //

        // calc boundaries of lag that intersects with data
        long lagMid1 = lagPartition.indexOf(dataMinTimestamp, BSearchType.OLDER_OR_SAME);
        long lagMid2 = lagPartition.indexOf(dataMaxTimestamp, BSearchType.NEWER_OR_SAME);

        // copy part of lag above data
        splitAppend(lagPartition.bufferedIterator(0, lagMid1), hard, soft, tempPartition);

        // merge lag with data and copy result to temp partition
        splitAppendMerge(
            data,
            lagPartition.bufferedIterator(lagMid1 + 1, lagMid2 - 1),
            hard,
            soft,
            tempPartition);

        // copy part of lag below data
        splitAppend(
            lagPartition.bufferedIterator(lagMid2, lagPartition.size() - 1),
            hard,
            soft,
            tempPartition);

      } else if (dataMaxTimestamp < lagMinTimestamp && dataMaxTimestamp <= lagMinTimestamp) {
        //
        // overlap scenario 2: data sits directly above lag
        //
        splitAppend(data, hard, soft, tempPartition);
        splitAppend(lagPartition.bufferedIterator(), hard, soft, tempPartition);
      } else if (dataMinTimestamp <= lagMinTimestamp && dataMaxTimestamp < lagMaxTimestamp) {
        //
        // overlap scenario 3: bottom part of data overlaps top part of lag
        //

        // calc overlap line
        long split = lagPartition.indexOf(dataMaxTimestamp, BSearchType.NEWER_OR_SAME);

        // merge lag with data and copy result to temp partition
        splitAppendMerge(
            data, lagPartition.bufferedIterator(0, split - 1), hard, soft, tempPartition);

        // copy part of lag below data
        splitAppend(
            lagPartition.bufferedIterator(split, lagPartition.size() - 1),
            hard,
            soft,
            tempPartition);
      } else if (dataMinTimestamp > lagMinTimestamp && dataMaxTimestamp >= lagMaxTimestamp) {
        //
        // overlap scenario 4: top part of data overlaps with bottom part of lag
        //
        long split = lagPartition.indexOf(dataMinTimestamp, BSearchType.OLDER_OR_SAME);

        // copy part of lag above overlap
        splitAppend(lagPartition.bufferedIterator(0, split), hard, soft, tempPartition);

        // merge lag with data and copy result to temp partition
        splitAppendMerge(
            data,
            lagPartition.bufferedIterator(split + 1, lagPartition.size() - 1),
            hard,
            soft,
            tempPartition);
      } else if (dataMinTimestamp <= lagMinTimestamp && dataMaxTimestamp >= lagMaxTimestamp) {
        //
        // overlap scenario 5: lag is fully inside of data
        //

        // merge lag with data and copy result to temp partition
        splitAppendMerge(data, lagPartition.bufferedIterator(), hard, soft, tempPartition);
      } else {
        throw new JournalRuntimeException(
            "Unsupported overlap type: lag min/max [%s/%s] data min/max: [%s/%s]",
            Dates.toString(lagMinTimestamp),
            Dates.toString(lagMaxTimestamp),
            Dates.toString(dataMinTimestamp),
            Dates.toString(dataMaxTimestamp));
      }

      replaceIrregularPartition(tempPartition);
    }
  }