/**
* Identify the segments to be merged based on the Size in case of Major compaction.
*
* @param compactionSize
* @param listOfSegmentsAfterPreserve
* @param carbonLoadModel
* @param partitionCount
* @param storeLocation
* @return
*/
private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnSize(
long compactionSize,
List<LoadMetadataDetails> listOfSegmentsAfterPreserve,
CarbonLoadModel carbonLoadModel,
int partitionCount,
String storeLocation) {
List<LoadMetadataDetails> segmentsToBeMerged =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
CarbonTableIdentifier tableIdentifier =
carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getCarbonTableIdentifier();
// total length
long totalLength = 0;
// check size of each segment , sum it up across partitions
for (LoadMetadataDetails segment : listOfSegmentsAfterPreserve) {
String segId = segment.getLoadName();
// variable to store one segment size across partition.
long sizeOfOneSegmentAcrossPartition =
getSizeOfOneSegmentAcrossPartition(partitionCount, storeLocation, tableIdentifier, segId);
// if size of a segment is greater than the Major compaction size. then ignore it.
if (sizeOfOneSegmentAcrossPartition > (compactionSize * 1024 * 1024)) {
// if already 2 segments have been found for merging then stop scan here and merge.
if (segmentsToBeMerged.size() > 1) {
break;
} else { // if only one segment is found then remove the earlier one in list.
// reset the total length to 0.
segmentsToBeMerged = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
totalLength = 0;
continue;
}
}
totalLength += sizeOfOneSegmentAcrossPartition;
// in case of major compaction the size doesnt matter. all the segments will be merged.
if (totalLength < (compactionSize * 1024 * 1024)) {
segmentsToBeMerged.add(segment);
} else { // if already 2 segments have been found for merging then stop scan here and merge.
if (segmentsToBeMerged.size() > 1) {
break;
} else { // if only one segment is found then remove the earlier one in list and put this.
// reset the total length to the current identified segment.
segmentsToBeMerged = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
segmentsToBeMerged.add(segment);
totalLength = sizeOfOneSegmentAcrossPartition;
}
}
}
return segmentsToBeMerged;
}
public static boolean updateLoadMetadataWithMergeStatus(
List<LoadMetadataDetails> loadsToMerge,
String metaDataFilepath,
String MergedLoadName,
CarbonLoadModel carbonLoadModel,
String mergeLoadStartTime,
CompactionType compactionType) {
boolean tableStatusUpdationStatus = false;
AbsoluteTableIdentifier absoluteTableIdentifier =
carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
try {
if (carbonLock.lockWithRetries()) {
LOGGER.info(
"Acquired lock for the table "
+ carbonLoadModel.getDatabaseName()
+ "."
+ carbonLoadModel.getTableName()
+ " for table status updation ");
CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath(
absoluteTableIdentifier.getStorePath(),
absoluteTableIdentifier.getCarbonTableIdentifier());
String statusFilePath = carbonTablePath.getTableStatusFilePath();
LoadMetadataDetails[] loadDetails = segmentStatusManager.readLoadMetadata(metaDataFilepath);
String mergedLoadNumber =
MergedLoadName.substring(
MergedLoadName.lastIndexOf(CarbonCommonConstants.LOAD_FOLDER)
+ CarbonCommonConstants.LOAD_FOLDER.length(),
MergedLoadName.length());
String modificationOrDeletionTimeStamp = CarbonLoaderUtil.readCurrentTime();
for (LoadMetadataDetails loadDetail : loadDetails) {
// check if this segment is merged.
if (loadsToMerge.contains(loadDetail)) {
// if the compacted load is deleted after the start of the compaction process,
// then need to discard the compaction process and treat it as failed compaction.
if (loadDetail
.getLoadStatus()
.equalsIgnoreCase(CarbonCommonConstants.MARKED_FOR_DELETE)) {
LOGGER.error(
"Compaction is aborted as the segment "
+ loadDetail.getLoadName()
+ " is deleted after the compaction is started.");
return tableStatusUpdationStatus;
}
loadDetail.setLoadStatus(CarbonCommonConstants.SEGMENT_COMPACTED);
loadDetail.setModificationOrdeletionTimesStamp(modificationOrDeletionTimeStamp);
loadDetail.setMergedLoadName(mergedLoadNumber);
}
}
// create entry for merged one.
LoadMetadataDetails loadMetadataDetails = new LoadMetadataDetails();
loadMetadataDetails.setPartitionCount(carbonLoadModel.getPartitionId());
loadMetadataDetails.setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS);
String loadEnddate = CarbonLoaderUtil.readCurrentTime();
loadMetadataDetails.setTimestamp(loadEnddate);
loadMetadataDetails.setLoadName(mergedLoadNumber);
loadMetadataDetails.setLoadStartTime(mergeLoadStartTime);
loadMetadataDetails.setPartitionCount("0");
// if this is a major compaction then set the segment as major compaction.
if (compactionType == CompactionType.MAJOR_COMPACTION) {
loadMetadataDetails.setMajorCompacted("true");
}
List<LoadMetadataDetails> updatedDetailsList = new ArrayList<>(Arrays.asList(loadDetails));
// put the merged folder entry
updatedDetailsList.add(loadMetadataDetails);
try {
segmentStatusManager.writeLoadDetailsIntoFile(
statusFilePath,
updatedDetailsList.toArray(new LoadMetadataDetails[updatedDetailsList.size()]));
tableStatusUpdationStatus = true;
} catch (IOException e) {
LOGGER.error("Error while writing metadata");
}
} else {
LOGGER.error(
"Could not able to obtain lock for table"
+ carbonLoadModel.getDatabaseName()
+ "."
+ carbonLoadModel.getTableName()
+ "for table status updation");
}
} finally {
if (carbonLock.unlock()) {
LOGGER.info(
"Table unlocked successfully after table status updation"
+ carbonLoadModel.getDatabaseName()
+ "."
+ carbonLoadModel.getTableName());
} else {
LOGGER.error(
"Unable to unlock Table lock for table"
+ carbonLoadModel.getDatabaseName()
+ "."
+ carbonLoadModel.getTableName()
+ " during table status updation");
}
}
return tableStatusUpdationStatus;
}