Ejemplo n.º 1
0
    /**
     * Atomic update.
     *
     * @return <code>null</code>
     */
    @Override
    protected Void doTask() throws Exception {

      updateEvent.start();

      try {

        if (resourceManager.isOverflowAllowed()) throw new IllegalStateException();

        final SegmentMetadata segmentMetadata = buildResult.segmentMetadata;

        if (INFO) log.info("Begin: name=" + getOnlyResource() + ", newSegment=" + segmentMetadata);

        /*
         * Open the unisolated B+Tree on the live journal that is
         * absorbing writes. We are going to update its index metadata.
         *
         * Note: I am using AbstractTask#getIndex(String name) so that
         * the concurrency control logic will notice the changes to the
         * BTree and cause it to be checkpointed if this task succeeds
         * normally.
         */
        final ILocalBTreeView view = (ILocalBTreeView) getIndex(getOnlyResource());

        // make sure that this is the same scale-out index.
        assertSameIndex(indexUUID, view.getMutableBTree());

        if (view instanceof BTree) {

          /*
           * Note: there is an expectation that this is not a simple
           * BTree because this the build task is supposed to be
           * invoked after an overflow event, and that event should
           * have re-defined the view to include the BTree on the new
           * journal plus the historical view.
           *
           * One explanation for finding a simple view here is that
           * the view was a simple BTree on the old journal and the
           * data was copied from the old journal into the new journal
           * and then someone decided to do a build even through a
           * copy had already been done. However, this is not a very
           * good explanation since we try to avoid doing a build if
           * we have already done a copy!
           */

          throw new RuntimeException(
              "View is only a B+Tree: name="
                  + buildResult.name
                  + ", pmd="
                  + view.getIndexMetadata().getPartitionMetadata());
        }

        // The live B+Tree.
        final BTree btree = view.getMutableBTree();

        if (INFO)
          log.info(
              "src="
                  + getOnlyResource()
                  + ",counter="
                  + view.getCounter().get()
                  + ",checkpoint="
                  + btree.getCheckpoint());

        assert btree != null : "Expecting index: " + getOnlyResource();

        // clone the current metadata record for the live index.
        final IndexMetadata indexMetadata = btree.getIndexMetadata().clone();

        /*
         * This is the index partition definition on the live index -
         * the one that will be replaced with a new view as the result
         * of this atomic update.
         */
        final LocalPartitionMetadata currentpmd = indexMetadata.getPartitionMetadata();

        // Check pre-conditions.
        final IResourceMetadata[] currentResources = currentpmd.getResources();
        {
          if (currentpmd == null) {

            throw new IllegalStateException("Not an index partition: " + getOnlyResource());
          }

          if (!currentResources[0].getUUID().equals(getJournal().getRootBlockView().getUUID())) {

            throw new IllegalStateException(
                "Expecting live journal to be the first resource: " + currentResources);
          }

          /*
           * Note: I have commented out a bunch of pre-condition tests
           * that are not valid for histories such as:
           *
           * history=create() register(0) split(0)
           * copy(entryCount=314)
           *
           * This case arises when there are not enough index entries
           * written on the journal after a split to warrant a build
           * so the buffered writes are just copied to the new
           * journal. The resources in the view are:
           *
           * 1. journal 2. segment
           *
           * And this update will replace the segment.
           */

          // // the old journal's resource metadata.
          // final IResourceMetadata oldJournalMetadata =
          // oldResources[1];
          // assert oldJournalMetadata != null;
          // assert oldJournalMetadata instanceof JournalMetadata :
          // "name="
          // + getOnlyResource() + ", old pmd=" + oldpmd
          // + ", segmentMetadata=" + buildResult.segmentMetadata;
          //
          // // live journal must be newer.
          // assert journal.getRootBlockView().getCreateTime() >
          // oldJournalMetadata
          // .getCreateTime();
          // new index segment build from a view that did not include
          // data from the live journal.
          assert segmentMetadata.getCreateTime()
                  < getJournal().getRootBlockView().getFirstCommitTime()
              : "segment createTime LT journal 1st commit time"
                  + ": segmentMetadata="
                  + segmentMetadata
                  + ", journal: "
                  + getJournal().getRootBlockView();

          // if (oldResources.length == 3) {
          //
          // // the old index segment's resource metadata.
          // final IResourceMetadata oldSegmentMetadata =
          // oldResources[2];
          // assert oldSegmentMetadata != null;
          // assert oldSegmentMetadata instanceof SegmentMetadata;
          //
          // assert oldSegmentMetadata.getCreateTime() <=
          // oldJournalMetadata
          // .getCreateTime();
          //
          // }

        }

        // new view definition.
        final IResourceMetadata[] newResources =
            new IResourceMetadata[] {
              // the live journal.
              getJournal().getResourceMetadata(),
              // the newly built index segment.
              segmentMetadata
            };

        // describe the index partition.
        indexMetadata.setPartitionMetadata(
            new LocalPartitionMetadata( //
                currentpmd.getPartitionId(), //
                currentpmd.getSourcePartitionId(), //
                currentpmd.getLeftSeparatorKey(), //
                currentpmd.getRightSeparatorKey(), //
                newResources, //
                currentpmd.getIndexPartitionCause()
                //                        currentpmd.getHistory()
                //                                + OverflowActionEnum.Merge//
                //                                + "(lastCommitTime="
                //                                + segmentMetadata.getCreateTime()//
                //                                + ",btreeEntryCount="
                //                                + btree.getEntryCount()//
                //                                + ",segmentEntryCount="
                //                                + buildResult.builder.getCheckpoint().nentries//
                //                                + ",segment="
                //                                + segmentMetadata.getUUID()//
                //                                + ",counter="
                //                                + btree.getCounter().get()//
                //                                + ",oldResources="
                //                                + Arrays.toString(currentResources) + ") "
                ));

        // update the metadata associated with the btree
        btree.setIndexMetadata(indexMetadata);

        if (INFO)
          log.info(
              "Updated view: name="
                  + getOnlyResource()
                  + ", pmd="
                  + indexMetadata.getPartitionMetadata());

        /*
         * Verify that the btree recognizes that it needs to be
         * checkpointed.
         *
         * Note: The atomic commit point is when this task commits.
         */
        assert btree.needsCheckpoint();
        //            btree.writeCheckpoint();
        //            {
        //                final long id0 = btree.getCounter().get();
        //                final long pid = id0 >> 32;
        //                final long mask = 0xffffffffL;
        //                final int ctr = (int) (id0 & mask);
        //                log.warn("name="+getOnlyResource()+", counter="+id0+", pid="+pid+",
        // ctr="+ctr);
        //            }

        // notify successful index partition build.
        resourceManager.overflowCounters.indexPartitionMergeCounter.incrementAndGet();

        return null;

      } finally {

        updateEvent.end();
      }
    } // doTask()
Ejemplo n.º 2
0
  /**
   * Build an {@link IndexSegment} from the compacting merge of an index partition.
   *
   * @return The {@link BuildResult}.
   */
  protected BuildResult doTask() throws Exception {

    final Event e =
        new Event(
                resourceManager.getFederation(),
                new EventResource(vmd.indexMetadata),
                OverflowActionEnum.Merge,
                vmd.getParams())
            .start();

    BuildResult buildResult = null;
    try {

      try {

        if (resourceManager.isOverflowAllowed()) throw new IllegalStateException();

        /*
         * Build the index segment.
         *
         * Note: Since this is a compacting merge the view on the old
         * journal as of the last commit time will be fully captured by
         * the generated index segment. However, writes buffered by the
         * live journal WILL NOT be present in that index segment and
         * the post-condition view will include those writes.
         */

        // build the index segment.
        buildResult =
            resourceManager.buildIndexSegment(
                vmd.name,
                vmd.getView(),
                true /* compactingMerge */,
                vmd.commitTime,
                null /* fromKey */,
                null /* toKey */,
                e);

      } finally {

        /*
         * Release our hold on the source view - we only needed it when
         * we did the index segment build.
         */

        clearRefs();
      }

      if (buildResult.builder.getCheckpoint().length >= resourceManager.nominalShardSize) {

        /*
         * If sumSegBytes exceeds the threshold, then do a split here.
         */

        // FIXME reconcile return type and enable post-merge split.
        //                return new SplitCompactViewTask(vmd.name, buildResult);

      }

      /*
       * @todo error handling should be inside of the atomic update task
       * since it has more visibility into the state changes and when we
       * can no longer delete the new index segment.
       */
      try {

        // scale-out index UUID.
        final UUID indexUUID = vmd.indexMetadata.getIndexUUID();

        // submit task and wait for it to complete
        concurrencyManager
            .submit(
                new AtomicUpdateCompactingMergeTask(
                    resourceManager,
                    concurrencyManager,
                    vmd.name,
                    indexUUID,
                    buildResult,
                    e.newSubEvent(OverflowSubtaskEnum.AtomicUpdate, vmd.getParams())))
            .get();

        // /*
        // * Verify that the view was updated. If the atomic update task
        //                 * runs correctly then it will replace the IndexMetadata object
        //                 * on the mutable BTree with a new view containing only the live
        //                 * journal and the new index segment (for a compacting merge).
        //                 * We verify that right now to make sure that the state change
        //                 * to the BTree was noticed and resulted in a commit before
        //                 * returning control to us here.
        //                 *
        //                 * @todo comment this out or replicate for the index build task
        //                 * also?
        //                 */
        //                concurrencyManager
        //                        .submit(
        //                                new VerifyAtomicUpdateTask(resourceManager,
        //                                        concurrencyManager, vmd.name,
        //                                        indexUUID, result)).get();

      } catch (Throwable t) {

        // make it releasable.
        resourceManager.retentionSetRemove(buildResult.segmentMetadata.getUUID());

        // delete the generated index segment.
        resourceManager.deleteResource(
            buildResult.segmentMetadata.getUUID(), false /* isJournal */);

        // re-throw the exception
        throw new Exception(t);
      }

      if (resourceManager.compactingMergeWithAfterAction) {

        /*
         * Consider possible after-actions now that the view is compact.
         * If any is selected, then it will be executed in the current
         * thread.
         */
        final AbstractTask<?> afterActionTask = chooseAfterActionTask();

        if (afterActionTask != null) {

          afterActionTask.call();
        }
      }

      return buildResult;

    } finally {

      if (buildResult != null) {

        /*
         * At this point the index segment was either incorporated into
         * the new view in a restart safe manner or there was an error.
         * Either way, we now remove the index segment store's UUID from
         * the retentionSet so it will be subject to the release policy
         * of the StoreManager.
         */
        resourceManager.retentionSetRemove(buildResult.segmentMetadata.getUUID());
      }

      e.end();
    }
  }