/**
* Removing the already merged segments from list.
*
* @param segments
* @param loadsToMerge
* @return
*/
public static List<LoadMetadataDetails> filterOutNewlyAddedSegments(
List<LoadMetadataDetails> segments, LoadMetadataDetails lastSeg) {
// take complete list of segments.
List<LoadMetadataDetails> list = new ArrayList<>(segments);
List<LoadMetadataDetails> trimmedList =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
// sort list
CarbonDataMergerUtil.sortSegments(list);
// first filter out newly added segments.
trimmedList = list.subList(0, list.indexOf(lastSeg) + 1);
return trimmedList;
}
/**
* To identify which all segments can be merged.
*
* @param storeLocation
* @param carbonLoadModel
* @param partitionCount
* @param compactionSize
* @return
*/
public static List<LoadMetadataDetails> identifySegmentsToBeMerged(
String storeLocation,
CarbonLoadModel carbonLoadModel,
int partitionCount,
long compactionSize,
List<LoadMetadataDetails> segments,
CompactionType compactionType) {
List sortedSegments = new ArrayList(segments);
sortSegments(sortedSegments);
// check preserve property and preserve the configured number of latest loads.
List<LoadMetadataDetails> listOfSegmentsAfterPreserve =
checkPreserveSegmentsPropertyReturnRemaining(sortedSegments);
// filter the segments if the compaction based on days is configured.
List<LoadMetadataDetails> listOfSegmentsLoadedInSameDateInterval =
identifySegmentsToBeMergedBasedOnLoadedDate(listOfSegmentsAfterPreserve);
List<LoadMetadataDetails> listOfSegmentsToBeMerged;
// identify the segments to merge based on the Size of the segments across partition.
if (compactionType.equals(CompactionType.MAJOR_COMPACTION)) {
listOfSegmentsToBeMerged =
identifySegmentsToBeMergedBasedOnSize(
compactionSize,
listOfSegmentsLoadedInSameDateInterval,
carbonLoadModel,
partitionCount,
storeLocation);
} else {
listOfSegmentsToBeMerged =
identifySegmentsToBeMergedBasedOnSegCount(listOfSegmentsLoadedInSameDateInterval);
}
return listOfSegmentsToBeMerged;
}