コード例 #1
0
  /**
   * For calculating the size of a segment across all partition.
   *
   * @param partitionCount
   * @param storeLocation
   * @param tableIdentifier
   * @param segId
   * @return
   */
  private static long getSizeOfOneSegmentAcrossPartition(
      int partitionCount,
      String storeLocation,
      CarbonTableIdentifier tableIdentifier,
      String segId) {
    long sizeOfOneSegmentAcrossPartition = 0;
    // calculate size across partitions
    for (int partition = 0; partition < partitionCount; partition++) {

      String loadPath =
          CarbonLoaderUtil.getStoreLocation(storeLocation, tableIdentifier, segId, partition + "");

      CarbonFile segmentFolder =
          FileFactory.getCarbonFile(loadPath, FileFactory.getFileType(loadPath));

      long sizeOfEachSegment = getSizeOfFactFileInLoad(segmentFolder);

      sizeOfOneSegmentAcrossPartition += sizeOfEachSegment;
    }
    return sizeOfOneSegmentAcrossPartition;
  }
コード例 #2
0
  public static boolean updateLoadMetadataWithMergeStatus(
      List<LoadMetadataDetails> loadsToMerge,
      String metaDataFilepath,
      String MergedLoadName,
      CarbonLoadModel carbonLoadModel,
      String mergeLoadStartTime,
      CompactionType compactionType) {

    boolean tableStatusUpdationStatus = false;
    AbsoluteTableIdentifier absoluteTableIdentifier =
        carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();

    SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);

    ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();

    try {
      if (carbonLock.lockWithRetries()) {
        LOGGER.info(
            "Acquired lock for the table "
                + carbonLoadModel.getDatabaseName()
                + "."
                + carbonLoadModel.getTableName()
                + " for table status updation ");

        CarbonTablePath carbonTablePath =
            CarbonStorePath.getCarbonTablePath(
                absoluteTableIdentifier.getStorePath(),
                absoluteTableIdentifier.getCarbonTableIdentifier());

        String statusFilePath = carbonTablePath.getTableStatusFilePath();

        LoadMetadataDetails[] loadDetails = segmentStatusManager.readLoadMetadata(metaDataFilepath);

        String mergedLoadNumber =
            MergedLoadName.substring(
                MergedLoadName.lastIndexOf(CarbonCommonConstants.LOAD_FOLDER)
                    + CarbonCommonConstants.LOAD_FOLDER.length(),
                MergedLoadName.length());

        String modificationOrDeletionTimeStamp = CarbonLoaderUtil.readCurrentTime();
        for (LoadMetadataDetails loadDetail : loadDetails) {
          // check if this segment is merged.
          if (loadsToMerge.contains(loadDetail)) {
            // if the compacted load is deleted after the start of the compaction process,
            // then need to discard the compaction process and treat it as failed compaction.
            if (loadDetail
                .getLoadStatus()
                .equalsIgnoreCase(CarbonCommonConstants.MARKED_FOR_DELETE)) {
              LOGGER.error(
                  "Compaction is aborted as the segment "
                      + loadDetail.getLoadName()
                      + " is deleted after the compaction is started.");
              return tableStatusUpdationStatus;
            }
            loadDetail.setLoadStatus(CarbonCommonConstants.SEGMENT_COMPACTED);
            loadDetail.setModificationOrdeletionTimesStamp(modificationOrDeletionTimeStamp);
            loadDetail.setMergedLoadName(mergedLoadNumber);
          }
        }

        // create entry for merged one.
        LoadMetadataDetails loadMetadataDetails = new LoadMetadataDetails();
        loadMetadataDetails.setPartitionCount(carbonLoadModel.getPartitionId());
        loadMetadataDetails.setLoadStatus(CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS);
        String loadEnddate = CarbonLoaderUtil.readCurrentTime();
        loadMetadataDetails.setTimestamp(loadEnddate);
        loadMetadataDetails.setLoadName(mergedLoadNumber);
        loadMetadataDetails.setLoadStartTime(mergeLoadStartTime);
        loadMetadataDetails.setPartitionCount("0");
        // if this is a major compaction then set the segment as major compaction.
        if (compactionType == CompactionType.MAJOR_COMPACTION) {
          loadMetadataDetails.setMajorCompacted("true");
        }

        List<LoadMetadataDetails> updatedDetailsList = new ArrayList<>(Arrays.asList(loadDetails));

        // put the merged folder entry
        updatedDetailsList.add(loadMetadataDetails);

        try {
          segmentStatusManager.writeLoadDetailsIntoFile(
              statusFilePath,
              updatedDetailsList.toArray(new LoadMetadataDetails[updatedDetailsList.size()]));
          tableStatusUpdationStatus = true;
        } catch (IOException e) {
          LOGGER.error("Error while writing metadata");
        }
      } else {
        LOGGER.error(
            "Could not able to obtain lock for table"
                + carbonLoadModel.getDatabaseName()
                + "."
                + carbonLoadModel.getTableName()
                + "for table status updation");
      }
    } finally {
      if (carbonLock.unlock()) {
        LOGGER.info(
            "Table unlocked successfully after table status updation"
                + carbonLoadModel.getDatabaseName()
                + "."
                + carbonLoadModel.getTableName());
      } else {
        LOGGER.error(
            "Unable to unlock Table lock for table"
                + carbonLoadModel.getDatabaseName()
                + "."
                + carbonLoadModel.getTableName()
                + " during table status updation");
      }
    }
    return tableStatusUpdationStatus;
  }