/**
* @param loadsOfSameDate
* @param segment
* @return
*/
private static Date initializeFirstSegment(
List<LoadMetadataDetails> loadsOfSameDate,
LoadMetadataDetails segment,
SimpleDateFormat sdf) {
String baselineLoadStartTime = segment.getLoadStartTime();
Date segDate1 = null;
try {
segDate1 = sdf.parse(baselineLoadStartTime);
} catch (ParseException e) {
LOGGER.error("Error while parsing segment start time" + e.getMessage());
}
loadsOfSameDate.add(segment);
return segDate1;
}
/**
* This method will return the list of loads which are loaded at the same interval. This property
* is configurable.
*
* @param listOfSegmentsBelowThresholdSize
* @return
*/
private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnLoadedDate(
List<LoadMetadataDetails> listOfSegmentsBelowThresholdSize) {
List<LoadMetadataDetails> loadsOfSameDate =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
long numberOfDaysAllowedToMerge = 0;
try {
numberOfDaysAllowedToMerge =
Long.parseLong(
CarbonProperties.getInstance()
.getProperty(
CarbonCommonConstants.DAYS_ALLOWED_TO_COMPACT,
CarbonCommonConstants.DEFAULT_DAYS_ALLOWED_TO_COMPACT));
if (numberOfDaysAllowedToMerge < 0 || numberOfDaysAllowedToMerge > 100) {
LOGGER.error(
"The specified value for property "
+ CarbonCommonConstants.DAYS_ALLOWED_TO_COMPACT
+ " is incorrect."
+ " Correct value should be in range of 0 -100. Taking the default value.");
numberOfDaysAllowedToMerge =
Long.parseLong(CarbonCommonConstants.DEFAULT_DAYS_ALLOWED_TO_COMPACT);
}
} catch (NumberFormatException e) {
numberOfDaysAllowedToMerge =
Long.parseLong(CarbonCommonConstants.DEFAULT_DAYS_ALLOWED_TO_COMPACT);
}
// if true then process loads according to the load date.
if (numberOfDaysAllowedToMerge > 0) {
// filter loads based on the loaded date
boolean first = true;
Date segDate1 = null;
SimpleDateFormat sdf = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
for (LoadMetadataDetails segment : listOfSegmentsBelowThresholdSize) {
if (first) {
segDate1 = initializeFirstSegment(loadsOfSameDate, segment, sdf);
first = false;
continue;
}
String segmentDate = segment.getLoadStartTime();
Date segDate2 = null;
try {
segDate2 = sdf.parse(segmentDate);
} catch (ParseException e) {
LOGGER.error("Error while parsing segment start time" + e.getMessage());
}
if (isTwoDatesPresentInRequiredRange(segDate1, segDate2, numberOfDaysAllowedToMerge)) {
loadsOfSameDate.add(segment);
}
// if the load is beyond merged date.
// then reset everything and continue search for loads.
else if (loadsOfSameDate.size() < 2) {
loadsOfSameDate.clear();
// need to add the next segment as first and to check further
segDate1 = initializeFirstSegment(loadsOfSameDate, segment, sdf);
} else { // case where a load is beyond merge date and there is at least 2 loads to merge.
break;
}
}
} else {
return listOfSegmentsBelowThresholdSize;
}
return loadsOfSameDate;
}
