/** * Retain the number of segments which are to be preserved and return the remaining list of * segments. * * @param loadMetadataDetails * @param numberOfSegToBeRetained * @return */ private static List<LoadMetadataDetails> getValidLoadDetailsWithRetaining( List<LoadMetadataDetails> loadMetadataDetails, int numberOfSegToBeRetained) { List<LoadMetadataDetails> validList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); for (LoadMetadataDetails segment : loadMetadataDetails) { if (segment.getLoadStatus().equalsIgnoreCase(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS) || segment .getLoadStatus() .equalsIgnoreCase(CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS) || segment.getLoadStatus().equalsIgnoreCase(CarbonCommonConstants.MARKED_FOR_UPDATE)) { validList.add(segment); } } // check if valid list is big enough for removing the number of seg to be retained. // last element int removingIndex = validList.size() - 1; for (int i = validList.size(); i > 0; i--) { if (numberOfSegToBeRetained == 0) { break; } // remove last segment validList.remove(removingIndex--); numberOfSegToBeRetained--; } return validList; }
public static boolean updateLoadMetadataWithMergeStatus( List<LoadMetadataDetails> loadsToMerge, String metaDataFilepath, String MergedLoadName, CarbonLoadModel carbonLoadModel, String mergeLoadStartTime, CompactionType compactionType) { boolean tableStatusUpdationStatus = false; AbsoluteTableIdentifier absoluteTableIdentifier = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier(); SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier); ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock(); try { if (carbonLock.lockWithRetries()) { LOGGER.info( "Acquired lock for the table " + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName() + " for table status updation "); CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath( absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier()); String statusFilePath = carbonTablePath.getTableStatusFilePath(); LoadMetadataDetails[] loadDetails = segmentStatusManager.readLoadMetadata(metaDataFilepath); String mergedLoadNumber = MergedLoadName.substring( MergedLoadName.lastIndexOf(CarbonCommonConstants.LOAD_FOLDER) + CarbonCommonConstants.LOAD_FOLDER.length(), MergedLoadName.length()); String modificationOrDeletionTimeStamp = CarbonLoaderUtil.readCurrentTime(); for (LoadMetadataDetails loadDetail : loadDetails) { // check if this segment is merged. if (loadsToMerge.contains(loadDetail)) { // if the compacted load is deleted after the start of the compaction process, // then need to discard the compaction process and treat it as failed compaction. if (loadDetail .getLoadStatus() .equalsIgnoreCase(CarbonCommonConstants.MARKED_FOR_DELETE)) { LOGGER.error( "Compaction is aborted as the segment " + loadDetail.getLoadName() + " is deleted after the compaction is started."); return tableStatusUpdationStatus; } loadDetail.setLoadStatus(CarbonCommonConstants.SEGMENT_COMPACTED); loadDetail.setModificationOrdeletionTimesStamp(modificationOrDeletionTimeStamp); loadDetail.setMergedLoadName(mergedLoadNumber); } } // create entry for merged one. LoadMetadataDetails loadMetadataDetails = new LoadMetadataDetails(); loadMetadataDetails.setPartitionCount(carbonLoadModel.getPartitionId()); loadMetadataDetails.setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS); String loadEnddate = CarbonLoaderUtil.readCurrentTime(); loadMetadataDetails.setTimestamp(loadEnddate); loadMetadataDetails.setLoadName(mergedLoadNumber); loadMetadataDetails.setLoadStartTime(mergeLoadStartTime); loadMetadataDetails.setPartitionCount("0"); // if this is a major compaction then set the segment as major compaction. if (compactionType == CompactionType.MAJOR_COMPACTION) { loadMetadataDetails.setMajorCompacted("true"); } List<LoadMetadataDetails> updatedDetailsList = new ArrayList<>(Arrays.asList(loadDetails)); // put the merged folder entry updatedDetailsList.add(loadMetadataDetails); try { segmentStatusManager.writeLoadDetailsIntoFile( statusFilePath, updatedDetailsList.toArray(new LoadMetadataDetails[updatedDetailsList.size()])); tableStatusUpdationStatus = true; } catch (IOException e) { LOGGER.error("Error while writing metadata"); } } else { LOGGER.error( "Could not able to obtain lock for table" + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName() + "for table status updation"); } } finally { if (carbonLock.unlock()) { LOGGER.info( "Table unlocked successfully after table status updation" + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName()); } else { LOGGER.error( "Unable to unlock Table lock for table" + carbonLoadModel.getDatabaseName() + "." + carbonLoadModel.getTableName() + " during table status updation"); } } return tableStatusUpdationStatus; }