/**
* make snapshots for the new segment by copying from one of the underlying merging segments. it's
* guaranteed to be consistent(checked in CubeSegmentValidator)
*
* @param cube
* @param newSeg
*/
private void makeSnapshotForNewSegment(
CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments) {
CubeSegment lastSeg = mergingSegments.get(mergingSegments.size() - 1);
for (Map.Entry<String, String> entry : lastSeg.getSnapshots().entrySet()) {
newSeg.putSnapshotResPath(entry.getKey(), entry.getValue());
}
}
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) {
this.enableSharding = cubeSeg.isEnableSharding();
this.cubeDesc = cubeSeg.getCubeDesc();
this.colIO = new RowKeyColumnIO(new CubeDimEncMap(cubeSeg));
this.splitBuffers = new SplittedBytes[splitLen];
for (int i = 0; i < splitLen; i++) {
this.splitBuffers[i] = new SplittedBytes(bytesLen);
}
this.bufferSize = 0;
}
private DictionaryInfo mergeDictionaries(
DictionaryManager dictMgr, CubeSegment cubeSeg, List<DictionaryInfo> dicts, TblColRef col)
throws IOException {
DictionaryInfo dictInfo = dictMgr.mergeDictionary(dicts);
if (dictInfo != null) cubeSeg.putDictResPath(col, dictInfo.getResourcePath());
return dictInfo;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((cubeSeg == null) ? 0 : cubeSeg.hashCode());
result = prime * result + ((cuboid == null) ? 0 : cuboid.hashCode());
result = prime * result + ((fuzzyKeyString == null) ? 0 : fuzzyKeyString.hashCode());
result = prime * result + ((startKeyString == null) ? 0 : startKeyString.hashCode());
result = prime * result + ((stopKeyString == null) ? 0 : stopKeyString.hashCode());
return result;
}
/**
* For the new segment, we need to create dictionaries for it, too. For those dictionaries on fact
* table, create it by merging underlying dictionaries For those dictionaries on lookup table,
* just copy it from any one of the merging segments, it's guaranteed to be consistent(checked in
* CubeSegmentValidator)
*
* @param cube
* @param newSeg
* @throws IOException
*/
private void makeDictForNewSegment(
KylinConfig conf, CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments)
throws IOException {
HashSet<TblColRef> colsNeedMeringDict = new HashSet<TblColRef>();
HashSet<TblColRef> colsNeedCopyDict = new HashSet<TblColRef>();
DictionaryManager dictMgr = DictionaryManager.getInstance(conf);
CubeDesc cubeDesc = cube.getDescriptor();
for (TblColRef col : cubeDesc.getAllColumnsNeedDictionary()) {
String dictTable = dictMgr.decideSourceData(cubeDesc.getModel(), true, col).getTable();
if (cubeDesc.getFactTable().equalsIgnoreCase(dictTable)) {
colsNeedMeringDict.add(col);
} else {
colsNeedCopyDict.add(col);
}
}
for (TblColRef col : colsNeedMeringDict) {
logger.info("Merging fact table dictionary on : " + col);
List<DictionaryInfo> dictInfos = new ArrayList<DictionaryInfo>();
for (CubeSegment segment : mergingSegments) {
logger.info("Including fact table dictionary of segment : " + segment);
if (segment.getDictResPath(col) != null) {
DictionaryInfo dictInfo = dictMgr.getDictionaryInfo(segment.getDictResPath(col));
if (dictInfo != null) {
dictInfos.add(dictInfo);
} else {
logger.warn("Failed to load DictionaryInfo from " + segment.getDictResPath(col));
}
}
}
mergeDictionaries(dictMgr, newSeg, dictInfos, col);
}
for (TblColRef col : colsNeedCopyDict) {
String path = mergingSegments.get(0).getDictResPath(col);
newSeg.putDictResPath(col, path);
}
}
private static void processSegment(
KylinConfig config, CubeSegment cubeSeg, DistinctColumnValuesProvider factTableValueProvider)
throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(config);
// dictionary
for (TblColRef col : cubeSeg.getCubeDesc().getAllColumnsNeedDictionary()) {
logger.info("Building dictionary for " + col);
cubeMgr.buildDictionary(cubeSeg, col, factTableValueProvider);
}
for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) {
// build snapshot
if (dim.getTable() != null
&& !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) {
// CubeSegment seg = cube.getTheOnlySegment();
logger.info("Building snapshot of " + dim.getTable());
cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable());
logger.info("Checking snapshot of " + dim.getTable());
cubeMgr.getLookupTable(cubeSeg, dim); // load the table for sanity check
}
}
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
HBaseKeyRange other = (HBaseKeyRange) obj;
if (cubeSeg == null) {
if (other.cubeSeg != null) return false;
} else if (!cubeSeg.equals(other.cubeSeg)) return false;
if (cuboid == null) {
if (other.cuboid != null) return false;
} else if (!cuboid.equals(other.cuboid)) return false;
if (fuzzyKeyString == null) {
if (other.fuzzyKeyString != null) return false;
} else if (!fuzzyKeyString.equals(other.fuzzyKeyString)) return false;
if (startKeyString == null) {
if (other.startKeyString != null) return false;
} else if (!startKeyString.equals(other.startKeyString)) return false;
if (stopKeyString == null) {
if (other.stopKeyString != null) return false;
} else if (!stopKeyString.equals(other.stopKeyString)) return false;
return true;
}
private void init(Collection<ColumnValueRange> andDimensionRanges) {
int size = andDimensionRanges.size();
Map<TblColRef, String> startValues = Maps.newHashMapWithExpectedSize(size);
Map<TblColRef, String> stopValues = Maps.newHashMapWithExpectedSize(size);
Map<TblColRef, Set<String>> fuzzyValues = Maps.newHashMapWithExpectedSize(size);
for (ColumnValueRange dimRange : andDimensionRanges) {
TblColRef column = dimRange.getColumn();
startValues.put(column, dimRange.getBeginValue());
stopValues.put(column, dimRange.getEndValue());
fuzzyValues.put(column, dimRange.getEqualValues());
TblColRef partitionDateColumnRef =
cubeSeg.getCubeDesc().getModel().getPartitionDesc().getPartitionDateColumnRef();
if (column.equals(partitionDateColumnRef)) {
initPartitionRange(dimRange);
}
}
AbstractRowKeyEncoder encoder = AbstractRowKeyEncoder.createInstance(cubeSeg, cuboid);
encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE);
this.startKey = encoder.encode(startValues);
encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE);
// In order to make stopRow inclusive add a trailing 0 byte. #See
// Scan.setStopRow(byte [] stopRow)
this.stopKey = Bytes.add(encoder.encode(stopValues), ZERO_TAIL_BYTES);
// restore encoder defaults for later reuse (note
// AbstractRowKeyEncoder.createInstance() caches instances)
encoder.setBlankByte(AbstractRowKeyEncoder.DEFAULT_BLANK_BYTE);
// always fuzzy match cuboid ID to lock on the selected cuboid
this.fuzzyKeys = buildFuzzyKeys(fuzzyValues);
}
public boolean hitSegment() {
return cubeSeg.getDateRangeStart() <= getPartitionColumnEndDate()
&& cubeSeg.getDateRangeEnd() >= getPartitionColumnStartDate();
}
public void addHTableNamesForCube(CubeInstance cube, List<String> segFullNameList) {
for (CubeSegment seg : cube.getSegments()) {
String tableName = seg.getStorageLocationIdentifier();
segFullNameList.add(tableName + "," + cube.getName());
}
}