コード例 #1
0
  /**
   * Removing the already merged segments from list.
   *
   * @param segments
   * @param loadsToMerge
   * @return
   */
  public static List<LoadMetadataDetails> filterOutNewlyAddedSegments(
      List<LoadMetadataDetails> segments, LoadMetadataDetails lastSeg) {

    // take complete list of segments.
    List<LoadMetadataDetails> list = new ArrayList<>(segments);

    List<LoadMetadataDetails> trimmedList =
        new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);

    // sort list
    CarbonDataMergerUtil.sortSegments(list);

    // first filter out newly added segments.
    trimmedList = list.subList(0, list.indexOf(lastSeg) + 1);

    return trimmedList;
  }
コード例 #2
0
  /**
   * To identify which all segments can be merged.
   *
   * @param storeLocation
   * @param carbonLoadModel
   * @param partitionCount
   * @param compactionSize
   * @return
   */
  public static List<LoadMetadataDetails> identifySegmentsToBeMerged(
      String storeLocation,
      CarbonLoadModel carbonLoadModel,
      int partitionCount,
      long compactionSize,
      List<LoadMetadataDetails> segments,
      CompactionType compactionType) {

    List sortedSegments = new ArrayList(segments);

    sortSegments(sortedSegments);

    // check preserve property and preserve the configured number of latest loads.

    List<LoadMetadataDetails> listOfSegmentsAfterPreserve =
        checkPreserveSegmentsPropertyReturnRemaining(sortedSegments);

    // filter the segments if the compaction based on days is configured.

    List<LoadMetadataDetails> listOfSegmentsLoadedInSameDateInterval =
        identifySegmentsToBeMergedBasedOnLoadedDate(listOfSegmentsAfterPreserve);
    List<LoadMetadataDetails> listOfSegmentsToBeMerged;
    // identify the segments to merge based on the Size of the segments across partition.
    if (compactionType.equals(CompactionType.MAJOR_COMPACTION)) {

      listOfSegmentsToBeMerged =
          identifySegmentsToBeMergedBasedOnSize(
              compactionSize,
              listOfSegmentsLoadedInSameDateInterval,
              carbonLoadModel,
              partitionCount,
              storeLocation);
    } else {

      listOfSegmentsToBeMerged =
          identifySegmentsToBeMergedBasedOnSegCount(listOfSegmentsLoadedInSameDateInterval);
    }

    return listOfSegmentsToBeMerged;
  }