コード例 #1
0
  /**
   * Retain the number of segments which are to be preserved and return the remaining list of
   * segments.
   *
   * @param loadMetadataDetails
   * @param numberOfSegToBeRetained
   * @return
   */
  private static List<LoadMetadataDetails> getValidLoadDetailsWithRetaining(
      List<LoadMetadataDetails> loadMetadataDetails, int numberOfSegToBeRetained) {

    List<LoadMetadataDetails> validList =
        new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    for (LoadMetadataDetails segment : loadMetadataDetails) {
      if (segment.getLoadStatus().equalsIgnoreCase(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS)
          || segment
              .getLoadStatus()
              .equalsIgnoreCase(CarbonCommonConstants.STORE_LOADSTATUS_PARTIAL_SUCCESS)
          || segment.getLoadStatus().equalsIgnoreCase(CarbonCommonConstants.MARKED_FOR_UPDATE)) {
        validList.add(segment);
      }
    }

    // check if valid list is big enough for removing the number of seg to be retained.
    // last element
    int removingIndex = validList.size() - 1;

    for (int i = validList.size(); i > 0; i--) {
      if (numberOfSegToBeRetained == 0) {
        break;
      }
      // remove last segment
      validList.remove(removingIndex--);
      numberOfSegToBeRetained--;
    }
    return validList;
  }
コード例 #2
0
  /**
   * Identify the segments to be merged based on the Size in case of Major compaction.
   *
   * @param compactionSize
   * @param listOfSegmentsAfterPreserve
   * @param carbonLoadModel
   * @param partitionCount
   * @param storeLocation
   * @return
   */
  private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnSize(
      long compactionSize,
      List<LoadMetadataDetails> listOfSegmentsAfterPreserve,
      CarbonLoadModel carbonLoadModel,
      int partitionCount,
      String storeLocation) {

    List<LoadMetadataDetails> segmentsToBeMerged =
        new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);

    CarbonTableIdentifier tableIdentifier =
        carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getCarbonTableIdentifier();

    // total length
    long totalLength = 0;

    // check size of each segment , sum it up across partitions
    for (LoadMetadataDetails segment : listOfSegmentsAfterPreserve) {

      String segId = segment.getLoadName();
      // variable to store one  segment size across partition.
      long sizeOfOneSegmentAcrossPartition =
          getSizeOfOneSegmentAcrossPartition(partitionCount, storeLocation, tableIdentifier, segId);

      // if size of a segment is greater than the Major compaction size. then ignore it.
      if (sizeOfOneSegmentAcrossPartition > (compactionSize * 1024 * 1024)) {
        // if already 2 segments have been found for merging then stop scan here and merge.
        if (segmentsToBeMerged.size() > 1) {
          break;
        } else { // if only one segment is found then remove the earlier one in list.
          // reset the total length to 0.
          segmentsToBeMerged = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
          totalLength = 0;
          continue;
        }
      }

      totalLength += sizeOfOneSegmentAcrossPartition;

      // in case of major compaction the size doesnt matter. all the segments will be merged.
      if (totalLength < (compactionSize * 1024 * 1024)) {
        segmentsToBeMerged.add(segment);
      } else { // if already 2 segments have been found for merging then stop scan here and merge.
        if (segmentsToBeMerged.size() > 1) {
          break;
        } else { // if only one segment is found then remove the earlier one in list and put this.
          // reset the total length to the current identified segment.
          segmentsToBeMerged = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
          segmentsToBeMerged.add(segment);
          totalLength = sizeOfOneSegmentAcrossPartition;
        }
      }
    }

    return segmentsToBeMerged;
  }
コード例 #3
0
 /**
  * For getting the comma separated valid segments for merging.
  *
  * @param loadMetadataDetails
  * @return
  */
 public static String getValidSegments(List<LoadMetadataDetails> loadMetadataDetails) {
   StringBuilder builder = new StringBuilder();
   for (LoadMetadataDetails segment : loadMetadataDetails) {
     // check if this load is an already merged load.
     if (null != segment.getMergedLoadName()) {
       builder.append(segment.getMergedLoadName() + ",");
     } else {
       builder.append(segment.getLoadName() + ",");
     }
   }
   builder.deleteCharAt(builder.length() - 1);
   return builder.toString();
 }
コード例 #4
0
 /**
  * @param loadsOfSameDate
  * @param segment
  * @return
  */
 private static Date initializeFirstSegment(
     List<LoadMetadataDetails> loadsOfSameDate,
     LoadMetadataDetails segment,
     SimpleDateFormat sdf) {
   String baselineLoadStartTime = segment.getLoadStartTime();
   Date segDate1 = null;
   try {
     segDate1 = sdf.parse(baselineLoadStartTime);
   } catch (ParseException e) {
     LOGGER.error("Error while parsing segment start time" + e.getMessage());
   }
   loadsOfSameDate.add(segment);
   return segDate1;
 }
コード例 #5
0
  /**
   * Identify the segments to be merged based on the segment count
   *
   * @param listOfSegmentsAfterPreserve
   * @return
   */
  private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnSegCount(
      List<LoadMetadataDetails> listOfSegmentsAfterPreserve) {

    List<LoadMetadataDetails> mergedSegments =
        new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    List<LoadMetadataDetails> unMergedSegments =
        new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);

    int[] noOfSegmentLevelsCount = CarbonProperties.getInstance().getCompactionSegmentLevelCount();

    int level1Size = 0;
    int level2Size = 0;
    boolean first = true;

    for (int levelCount : noOfSegmentLevelsCount) {
      if (first) {
        level1Size = levelCount;
        first = false;
      } else {
        level2Size = levelCount;
        break;
        // breaking as we are doing only 2 levels
      }
    }

    int unMergeCounter = 0;
    int mergeCounter = 0;

    // check size of each segment , sum it up across partitions
    for (LoadMetadataDetails segment : listOfSegmentsAfterPreserve) {

      String segName = segment.getLoadName();

      // if a segment is already merged 2 levels then it s name will become .2
      // need to exclude those segments from minor compaction.
      // if a segment is major compacted then should not be considered for minor.
      if (segName.endsWith(CarbonCommonConstants.LEVEL2_COMPACTION_INDEX)
          || (segment.isMajorCompacted() != null
              && segment.isMajorCompacted().equalsIgnoreCase("true"))) {
        continue;
      }

      // check if the segment is merged or not

      if (!isMergedSegment(segName)) {
        // if it is an unmerged segment then increment counter
        unMergeCounter++;
        unMergedSegments.add(segment);
        if (unMergeCounter == (level1Size)) {
          return unMergedSegments;
        }
      } else {
        mergeCounter++;
        mergedSegments.add(segment);
        if (mergeCounter == (level2Size)) {
          return mergedSegments;
        }
      }
    }
    return new ArrayList<>(0);
  }
コード例 #6
0
  /**
   * This method will return the list of loads which are loaded at the same interval. This property
   * is configurable.
   *
   * @param listOfSegmentsBelowThresholdSize
   * @return
   */
  private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnLoadedDate(
      List<LoadMetadataDetails> listOfSegmentsBelowThresholdSize) {

    List<LoadMetadataDetails> loadsOfSameDate =
        new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);

    long numberOfDaysAllowedToMerge = 0;
    try {
      numberOfDaysAllowedToMerge =
          Long.parseLong(
              CarbonProperties.getInstance()
                  .getProperty(
                      CarbonCommonConstants.DAYS_ALLOWED_TO_COMPACT,
                      CarbonCommonConstants.DEFAULT_DAYS_ALLOWED_TO_COMPACT));
      if (numberOfDaysAllowedToMerge < 0 || numberOfDaysAllowedToMerge > 100) {
        LOGGER.error(
            "The specified value for property "
                + CarbonCommonConstants.DAYS_ALLOWED_TO_COMPACT
                + " is incorrect."
                + " Correct value should be in range of 0 -100. Taking the default value.");
        numberOfDaysAllowedToMerge =
            Long.parseLong(CarbonCommonConstants.DEFAULT_DAYS_ALLOWED_TO_COMPACT);
      }

    } catch (NumberFormatException e) {
      numberOfDaysAllowedToMerge =
          Long.parseLong(CarbonCommonConstants.DEFAULT_DAYS_ALLOWED_TO_COMPACT);
    }
    // if true then process loads according to the load date.
    if (numberOfDaysAllowedToMerge > 0) {

      // filter loads based on the loaded date
      boolean first = true;
      Date segDate1 = null;
      SimpleDateFormat sdf = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
      for (LoadMetadataDetails segment : listOfSegmentsBelowThresholdSize) {

        if (first) {
          segDate1 = initializeFirstSegment(loadsOfSameDate, segment, sdf);
          first = false;
          continue;
        }
        String segmentDate = segment.getLoadStartTime();
        Date segDate2 = null;
        try {
          segDate2 = sdf.parse(segmentDate);
        } catch (ParseException e) {
          LOGGER.error("Error while parsing segment start time" + e.getMessage());
        }

        if (isTwoDatesPresentInRequiredRange(segDate1, segDate2, numberOfDaysAllowedToMerge)) {
          loadsOfSameDate.add(segment);
        }
        // if the load is beyond merged date.
        // then reset everything and continue search for loads.
        else if (loadsOfSameDate.size() < 2) {
          loadsOfSameDate.clear();
          // need to add the next segment as first and  to check further
          segDate1 = initializeFirstSegment(loadsOfSameDate, segment, sdf);
        } else { // case where a load is beyond merge date and there is at least 2 loads to merge.
          break;
        }
      }
    } else {
      return listOfSegmentsBelowThresholdSize;
    }

    return loadsOfSameDate;
  }
コード例 #7
0
  public static boolean updateLoadMetadataWithMergeStatus(
      List<LoadMetadataDetails> loadsToMerge,
      String metaDataFilepath,
      String MergedLoadName,
      CarbonLoadModel carbonLoadModel,
      String mergeLoadStartTime,
      CompactionType compactionType) {

    boolean tableStatusUpdationStatus = false;
    AbsoluteTableIdentifier absoluteTableIdentifier =
        carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();

    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);

    ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();

    try {
      if (carbonLock.lockWithRetries()) {
        LOGGER.info(
            "Acquired lock for the table "
                + carbonLoadModel.getDatabaseName()
                + "."
                + carbonLoadModel.getTableName()
                + " for table status updation ");

        CarbonTablePath carbonTablePath =
            CarbonStorePath.getCarbonTablePath(
                absoluteTableIdentifier.getStorePath(),
                absoluteTableIdentifier.getCarbonTableIdentifier());

        String statusFilePath = carbonTablePath.getTableStatusFilePath();

        LoadMetadataDetails[] loadDetails = segmentStatusManager.readLoadMetadata(metaDataFilepath);

        String mergedLoadNumber =
            MergedLoadName.substring(
                MergedLoadName.lastIndexOf(CarbonCommonConstants.LOAD_FOLDER)
                    + CarbonCommonConstants.LOAD_FOLDER.length(),
                MergedLoadName.length());

        String modificationOrDeletionTimeStamp = CarbonLoaderUtil.readCurrentTime();
        for (LoadMetadataDetails loadDetail : loadDetails) {
          // check if this segment is merged.
          if (loadsToMerge.contains(loadDetail)) {
            // if the compacted load is deleted after the start of the compaction process,
            // then need to discard the compaction process and treat it as failed compaction.
            if (loadDetail
                .getLoadStatus()
                .equalsIgnoreCase(CarbonCommonConstants.MARKED_FOR_DELETE)) {
              LOGGER.error(
                  "Compaction is aborted as the segment "
                      + loadDetail.getLoadName()
                      + " is deleted after the compaction is started.");
              return tableStatusUpdationStatus;
            }
            loadDetail.setLoadStatus(CarbonCommonConstants.SEGMENT_COMPACTED);
            loadDetail.setModificationOrdeletionTimesStamp(modificationOrDeletionTimeStamp);
            loadDetail.setMergedLoadName(mergedLoadNumber);
          }
        }

        // create entry for merged one.
        LoadMetadataDetails loadMetadataDetails = new LoadMetadataDetails();
        loadMetadataDetails.setPartitionCount(carbonLoadModel.getPartitionId());
        loadMetadataDetails.setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS);
        String loadEnddate = CarbonLoaderUtil.readCurrentTime();
        loadMetadataDetails.setTimestamp(loadEnddate);
        loadMetadataDetails.setLoadName(mergedLoadNumber);
        loadMetadataDetails.setLoadStartTime(mergeLoadStartTime);
        loadMetadataDetails.setPartitionCount("0");
        // if this is a major compaction then set the segment as major compaction.
        if (compactionType == CompactionType.MAJOR_COMPACTION) {
          loadMetadataDetails.setMajorCompacted("true");
        }

        List<LoadMetadataDetails> updatedDetailsList = new ArrayList<>(Arrays.asList(loadDetails));

        // put the merged folder entry
        updatedDetailsList.add(loadMetadataDetails);

        try {
          segmentStatusManager.writeLoadDetailsIntoFile(
              statusFilePath,
              updatedDetailsList.toArray(new LoadMetadataDetails[updatedDetailsList.size()]));
          tableStatusUpdationStatus = true;
        } catch (IOException e) {
          LOGGER.error("Error while writing metadata");
        }
      } else {
        LOGGER.error(
            "Could not able to obtain lock for table"
                + carbonLoadModel.getDatabaseName()
                + "."
                + carbonLoadModel.getTableName()
                + "for table status updation");
      }
    } finally {
      if (carbonLock.unlock()) {
        LOGGER.info(
            "Table unlocked successfully after table status updation"
                + carbonLoadModel.getDatabaseName()
                + "."
                + carbonLoadModel.getTableName());
      } else {
        LOGGER.error(
            "Unable to unlock Table lock for table"
                + carbonLoadModel.getDatabaseName()
                + "."
                + carbonLoadModel.getTableName()
                + " during table status updation");
      }
    }
    return tableStatusUpdationStatus;
  }