Esempio n. 1
0
 /**
  * make snapshots for the new segment by copying from one of the underlying merging segments. it's
  * guaranteed to be consistent(checked in CubeSegmentValidator)
  *
  * @param cube
  * @param newSeg
  */
 private void makeSnapshotForNewSegment(
     CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments) {
   CubeSegment lastSeg = mergingSegments.get(mergingSegments.size() - 1);
   for (Map.Entry<String, String> entry : lastSeg.getSnapshots().entrySet()) {
     newSeg.putSnapshotResPath(entry.getKey(), entry.getValue());
   }
 }
Esempio n. 2
0
  public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
    this.enableSharding = cubeSeg.isEnableSharding();
    this.cubeDesc = cubeSeg.getCubeDesc();
    this.colIO = new RowKeyColumnIO(new CubeDimEncMap(cubeSeg));

    this.splitBuffers = new SplittedBytes[splitLen];
    for (int i = 0; i < splitLen; i++) {
      this.splitBuffers[i] = new SplittedBytes(bytesLen);
    }
    this.bufferSize = 0;
  }
Esempio n. 3
0
  private DictionaryInfo mergeDictionaries(
      DictionaryManager dictMgr, CubeSegment cubeSeg, List<DictionaryInfo> dicts, TblColRef col)
      throws IOException {
    DictionaryInfo dictInfo = dictMgr.mergeDictionary(dicts);
    if (dictInfo != null) cubeSeg.putDictResPath(col, dictInfo.getResourcePath());

    return dictInfo;
  }
 @Override
 public int hashCode() {
   final int prime = 31;
   int result = 1;
   result = prime * result + ((cubeSeg == null) ? 0 : cubeSeg.hashCode());
   result = prime * result + ((cuboid == null) ? 0 : cuboid.hashCode());
   result = prime * result + ((fuzzyKeyString == null) ? 0 : fuzzyKeyString.hashCode());
   result = prime * result + ((startKeyString == null) ? 0 : startKeyString.hashCode());
   result = prime * result + ((stopKeyString == null) ? 0 : stopKeyString.hashCode());
   return result;
 }
Esempio n. 5
0
  /**
   * For the new segment, we need to create dictionaries for it, too. For those dictionaries on fact
   * table, create it by merging underlying dictionaries For those dictionaries on lookup table,
   * just copy it from any one of the merging segments, it's guaranteed to be consistent(checked in
   * CubeSegmentValidator)
   *
   * @param cube
   * @param newSeg
   * @throws IOException
   */
  private void makeDictForNewSegment(
      KylinConfig conf, CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments)
      throws IOException {
    HashSet<TblColRef> colsNeedMeringDict = new HashSet<TblColRef>();
    HashSet<TblColRef> colsNeedCopyDict = new HashSet<TblColRef>();
    DictionaryManager dictMgr = DictionaryManager.getInstance(conf);

    CubeDesc cubeDesc = cube.getDescriptor();

    for (TblColRef col : cubeDesc.getAllColumnsNeedDictionary()) {
      String dictTable = dictMgr.decideSourceData(cubeDesc.getModel(), true, col).getTable();
      if (cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) {
        colsNeedMeringDict.add(col);
      } else {
        colsNeedCopyDict.add(col);
      }
    }

    for (TblColRef col : colsNeedMeringDict) {
      logger.info("Merging fact table dictionary on : " + col);
      List<DictionaryInfo> dictInfos = new ArrayList<DictionaryInfo>();
      for (CubeSegment segment : mergingSegments) {
        logger.info("Including fact table dictionary of segment : " + segment);
        if (segment.getDictResPath(col) != null) {
          DictionaryInfo dictInfo = dictMgr.getDictionaryInfo(segment.getDictResPath(col));
          if (dictInfo != null) {
            dictInfos.add(dictInfo);
          } else {
            logger.warn("Failed to load DictionaryInfo from " + segment.getDictResPath(col));
          }
        }
      }
      mergeDictionaries(dictMgr, newSeg, dictInfos, col);
    }

    for (TblColRef col : colsNeedCopyDict) {
      String path = mergingSegments.get(0).getDictResPath(col);
      newSeg.putDictResPath(col, path);
    }
  }
Esempio n. 6
0
  private static void processSegment(
      KylinConfig config, CubeSegment cubeSeg, DistinctColumnValuesProvider factTableValueProvider)
      throws IOException {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    // dictionary
    for (TblColRef col : cubeSeg.getCubeDesc().getAllColumnsNeedDictionary()) {
      logger.info("Building dictionary for " + col);
      cubeMgr.buildDictionary(cubeSeg, col, factTableValueProvider);
    }

    for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) {
      // build snapshot
      if (dim.getTable() != null
          && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) {
        // CubeSegment seg = cube.getTheOnlySegment();
        logger.info("Building snapshot of " + dim.getTable());
        cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable());
        logger.info("Checking snapshot of " + dim.getTable());
        cubeMgr.getLookupTable(cubeSeg, dim); // load the table for sanity check
      }
    }
  }
 @Override
 public boolean equals(Object obj) {
   if (this == obj) return true;
   if (obj == null) return false;
   if (getClass() != obj.getClass()) return false;
   HBaseKeyRange other = (HBaseKeyRange) obj;
   if (cubeSeg == null) {
     if (other.cubeSeg != null) return false;
   } else if (!cubeSeg.equals(other.cubeSeg)) return false;
   if (cuboid == null) {
     if (other.cuboid != null) return false;
   } else if (!cuboid.equals(other.cuboid)) return false;
   if (fuzzyKeyString == null) {
     if (other.fuzzyKeyString != null) return false;
   } else if (!fuzzyKeyString.equals(other.fuzzyKeyString)) return false;
   if (startKeyString == null) {
     if (other.startKeyString != null) return false;
   } else if (!startKeyString.equals(other.startKeyString)) return false;
   if (stopKeyString == null) {
     if (other.stopKeyString != null) return false;
   } else if (!stopKeyString.equals(other.stopKeyString)) return false;
   return true;
 }
  private void init(Collection<ColumnValueRange> andDimensionRanges) {
    int size = andDimensionRanges.size();
    Map<TblColRef, String> startValues = Maps.newHashMapWithExpectedSize(size);
    Map<TblColRef, String> stopValues = Maps.newHashMapWithExpectedSize(size);
    Map<TblColRef, Set<String>> fuzzyValues = Maps.newHashMapWithExpectedSize(size);
    for (ColumnValueRange dimRange : andDimensionRanges) {
      TblColRef column = dimRange.getColumn();
      startValues.put(column, dimRange.getBeginValue());
      stopValues.put(column, dimRange.getEndValue());
      fuzzyValues.put(column, dimRange.getEqualValues());

      TblColRef partitionDateColumnRef =
          cubeSeg.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
      if (column.equals(partitionDateColumnRef)) {
        initPartitionRange(dimRange);
      }
    }

    AbstractRowKeyEncoder encoder = AbstractRowKeyEncoder.createInstance(cubeSeg, cuboid);

    encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE);

    this.startKey = encoder.encode(startValues);

    encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE);

    // In order to make stopRow inclusive add a trailing 0 byte. #See
    // Scan.setStopRow(byte [] stopRow)
    this.stopKey = Bytes.add(encoder.encode(stopValues), ZERO_TAIL_BYTES);

    // restore encoder defaults for later reuse (note
    // AbstractRowKeyEncoder.createInstance() caches instances)
    encoder.setBlankByte(AbstractRowKeyEncoder.DEFAULT_BLANK_BYTE);

    // always fuzzy match cuboid ID to lock on the selected cuboid
    this.fuzzyKeys = buildFuzzyKeys(fuzzyValues);
  }
 public boolean hitSegment() {
   return cubeSeg.getDateRangeStart() <= getPartitionColumnEndDate()
       && cubeSeg.getDateRangeEnd() >= getPartitionColumnStartDate();
 }
Esempio n. 10
0
 public void addHTableNamesForCube(CubeInstance cube, List<String> segFullNameList) {
   for (CubeSegment seg : cube.getSegments()) {
     String tableName = seg.getStorageLocationIdentifier();
     segFullNameList.add(tableName + "," + cube.getName());
   }
 }