@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
KylinConfig conf = context.getConfig();
final CubeManager mgr = CubeManager.getInstance(conf);
final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
final CubeSegment newSegment =
cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
final List<CubeSegment> mergingSegments = getMergingSegments(cube);
Collections.sort(mergingSegments);
try {
checkLookupSnapshotsMustIncremental(mergingSegments);
makeDictForNewSegment(conf, cube, newSegment, mergingSegments);
makeSnapshotForNewSegment(cube, newSegment, mergingSegments);
CubeUpdate cubeBuilder = new CubeUpdate(cube);
cubeBuilder.setToUpdateSegs(newSegment);
mgr.updateCube(cubeBuilder);
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
} catch (IOException e) {
logger.error("fail to merge dictionary or lookup snapshots", e);
return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
}
}
private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
CubingJob job = cubingJobBuilder.buildJob(segment);
jobService.addJob(job);
waitForJob(job.getId());
return job.getId();
}
public void checkAll() {
List<String> segFullNameList = Lists.newArrayList();
CubeManager cubeMgr = CubeManager.getInstance(dstCfg);
for (CubeInstance cube : cubeMgr.listAllCubes()) {
addHTableNamesForCube(cube, segFullNameList);
}
check(segFullNameList);
}
@Test
public void testExistingProject() throws Exception {
ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());
int originalProjectCount = prjMgr.listAllProjects().size();
int originalCubeCount = cubeMgr.listAllCubes().size();
ResourceStore store = getStore();
// clean legacy in case last run failed
store.deleteResource("/cube/new_cube_in_default.json");
CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
CubeInstance createdCube =
cubeMgr.createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null);
assertTrue(createdCube == cubeMgr.getCube("new_cube_in_default"));
System.out.println(JsonUtil.writeValueAsIndentString(createdCube));
assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);
CubeInstance droppedCube = cubeMgr.dropCube("new_cube_in_default", true);
assertTrue(createdCube == droppedCube);
assertNull(cubeMgr.getCube("new_cube_in_default"));
assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);
}
@Test
public void testNewProject() throws Exception {
ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());
int originalProjectCount = prjMgr.listAllProjects().size();
int originalCubeCount = cubeMgr.listAllCubes().size();
int originalCubeCountInDefault = prjMgr.listAllRealizations("default").size();
ResourceStore store = getStore();
// clean legacy in case last run failed
store.deleteResource("/cube/cube_in_alien_project.json");
CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
CubeInstance createdCube = cubeMgr.createCube("cube_in_alien_project", "alien", desc, null);
assertTrue(createdCube == cubeMgr.getCube("cube_in_alien_project"));
ProjectManager proMgr = ProjectManager.getInstance(getTestConfig());
Set<IRealization> realizations = proMgr.listAllRealizations("alien");
assertTrue(realizations.contains(createdCube));
System.out.println(JsonUtil.writeValueAsIndentString(createdCube));
assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1);
assertTrue(
prjMgr
.listAllRealizations("ALIEN")
.iterator()
.next()
.getName()
.equalsIgnoreCase("CUBE_IN_ALIEN_PROJECT"));
assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);
prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "default", null);
assertTrue(prjMgr.listAllRealizations("ALIEN").size() == 0);
assertTrue(prjMgr.listAllRealizations("default").size() == originalCubeCountInDefault + 1);
assertTrue(
ProjectManager.getInstance(getTestConfig())
.listAllRealizations("default")
.contains(createdCube));
prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "alien", null);
assertTrue(prjMgr.listAllRealizations("ALIEN").size() == 1);
assertTrue(prjMgr.listAllRealizations("default").size() == originalCubeCountInDefault);
assertTrue(
ProjectManager.getInstance(getTestConfig())
.listAllRealizations("alien")
.contains(createdCube));
CubeInstance droppedCube = cubeMgr.dropCube("cube_in_alien_project", true);
assertTrue(createdCube == droppedCube);
assertNull(cubeMgr.getCube("cube_in_alien_project"));
assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1);
assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);
prjMgr.dropProject("alien");
assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
}
@Before
public void setUp() throws Exception {
this.createTestMetadata();
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
cube = cubeMgr.getCube("test_kylin_cube_without_slr_left_join_empty");
Assert.assertNotNull(cube);
storageEngine = StorageFactory.createQuery(cube);
String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
context = new StorageContext();
context.setConnUrl(url);
mockup = new StorageMockUtils(cube.getModel());
}
@Test
public void testProjectsDrop() throws IOException {
ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty");
assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
assertTrue(prjMgr.listAllRealizations("default").contains(cube));
cubeMgr.dropCube(cube.getName(), false);
assertTrue(!prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
assertTrue(!prjMgr.listAllRealizations("default").contains(cube));
}
@Override
protected void setup(Context context) throws IOException {
super.bindCurrentConfiguration(context.getConfiguration());
cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();
// only used in Build job, not in Merge job
cuboidLevel = context.getConfiguration().getInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, 0);
KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();
cubeDesc = CubeManager.getInstance(config).getCube(cubeName).getDescriptor();
measuresDescs = cubeDesc.getMeasures();
codec = new BufferedMeasureEncoder(measuresDescs);
aggs = new MeasureAggregators(measuresDescs);
input = new Object[measuresDescs.size()];
result = new Object[measuresDescs.size()];
needAggr = new boolean[measuresDescs.size()];
if (cuboidLevel > 0) {
for (int i = 0; i < measuresDescs.size(); i++) {
needAggr[i] = !measuresDescs.get(i).getFunction().getMeasureType().onlyAggrInBaseCuboid();
}
}
}
@Test
public void testProjectsLoadAfterProjectChange() throws IOException {
ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty");
assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
prjMgr.removeRealizationsFromProjects(RealizationType.CUBE, cube.getName());
assertTrue(!prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
prjMgr.moveRealizationToProject(RealizationType.CUBE, cube.getName(), "default", "tester");
assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
}
@Test
public void testDecode() throws Exception {
CubeDesc cubeDesc =
CubeManager.getInstance(getTestConfig())
.getCube("test_kylin_cube_with_slr_ready")
.getDescriptor();
HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];
MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
BigDecimal sum = new BigDecimal("333.1234567");
BigDecimal min = new BigDecimal("333.1111111");
BigDecimal max = new BigDecimal("333.1999999");
LongMutable count = new LongMutable(2);
LongMutable item_count = new LongMutable(100);
ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
codec.encode(new Object[] {sum, min, max, count, item_count}, buf);
buf.flip();
byte[] valueBytes = new byte[buf.limit()];
System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());
RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
for (MeasureDesc measure : cubeDesc.getMeasures()) {
FunctionDesc aggrFunc = measure.getFunction();
int index = hbaseCol.findMeasure(aggrFunc);
rowValueDecoder.setProjectIndex(index);
}
rowValueDecoder.decodeAndConvertJavaObj(valueBytes);
Object[] measureValues = rowValueDecoder.getValues();
// BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer
assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues));
}
@SuppressWarnings("unused")
// called by reflection
private List<String> testLeftJoinCube2() throws Exception {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
List<String> result = Lists.newArrayList();
final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
// this cube's start date is 0, end date is 20120601000000
long dateStart =
cubeManager
.getCube(cubeName)
.getDescriptor()
.getModel()
.getPartitionDesc()
.getPartitionDateStart();
long dateEnd = f.parse("2012-06-01").getTime();
clearSegment(cubeName);
result.add(buildSegment(cubeName, dateStart, dateEnd));
// then submit an append job, start date is 20120601000000, end
// date is 20220101000000
dateStart = f.parse("2012-06-01").getTime();
dateEnd = f.parse("2022-01-01").getTime();
result.add(buildSegment(cubeName, dateStart, dateEnd));
return result;
}
public void checkCube(String cubeName) {
List<String> segFullNameList = Lists.newArrayList();
CubeInstance cube = CubeManager.getInstance(dstCfg).getCube(cubeName);
addHTableNamesForCube(cube, segFullNameList);
check(segFullNameList);
}
@Override
public int run(String[] args) throws Exception {
Options options = new Options();
try {
options.addOption(OPTION_INPUT_PATH);
options.addOption(OPTION_HTABLE_NAME);
options.addOption(OPTION_CUBE_NAME);
parseOptions(options, args);
String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
// e.g
// /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/
// end with "/"
String input = getOptionValue(OPTION_INPUT_PATH);
Configuration conf = HBaseConfiguration.create(getConf());
FileSystem fs = FileSystem.get(conf);
String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
KylinConfig config = KylinConfig.getInstanceFromEnv();
CubeManager cubeMgr = CubeManager.getInstance(config);
CubeInstance cube = cubeMgr.getCube(cubeName);
CubeDesc cubeDesc = cube.getDescriptor();
FsPermission permission = new FsPermission((short) 0777);
for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) {
String cfName = cf.getName();
fs.setPermission(new Path(input + cfName), permission);
}
String[] newArgs = new String[2];
newArgs[0] = input;
newArgs[1] = tableName;
log.debug("Start to run LoadIncrementalHFiles");
int ret = ToolRunner.run(new LoadIncrementalHFiles(conf), newArgs);
log.debug("End to run LoadIncrementalHFiles");
return ret;
} catch (Exception e) {
printUsage(options);
throw e;
}
}
public static void processSegment(
KylinConfig config,
String cubeName,
String segmentName,
DistinctColumnValuesProvider factTableValueProvider)
throws IOException {
CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
CubeSegment segment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);
processSegment(config, segment, factTableValueProvider);
}
private static void processSegment(
KylinConfig config, CubeSegment cubeSeg, DistinctColumnValuesProvider factTableValueProvider)
throws IOException {
CubeManager cubeMgr = CubeManager.getInstance(config);
// dictionary
for (TblColRef col : cubeSeg.getCubeDesc().getAllColumnsNeedDictionary()) {
logger.info("Building dictionary for " + col);
cubeMgr.buildDictionary(cubeSeg, col, factTableValueProvider);
}
for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) {
// build snapshot
if (dim.getTable() != null
&& !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) {
// CubeSegment seg = cube.getTheOnlySegment();
logger.info("Building snapshot of " + dim.getTable());
cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable());
logger.info("Checking snapshot of " + dim.getTable());
cubeMgr.getLookupTable(cubeSeg, dim); // load the table for sanity check
}
}
}
@Test(expected = IllegalArgumentException.class)
public void testError() throws Exception {
CubeDesc cubeDesc =
CubeManager.getInstance(getTestConfig())
.getCube("test_kylin_cube_with_slr_ready")
.getDescriptor();
HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];
MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
BigDecimal sum = new BigDecimal("11111111111111111111333.1234567");
BigDecimal min = new BigDecimal("333.1111111");
BigDecimal max = new BigDecimal("333.1999999");
LongWritable count = new LongWritable(2);
LongMutable item_count = new LongMutable(100);
ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
codec.encode(new Object[] {sum, min, max, count, item_count}, buf);
}
@SuppressWarnings("unused")
// called by reflection
private List<String> testLeftJoinCube() throws Exception {
String cubeName = "test_kylin_cube_with_slr_left_join_empty";
clearSegment(cubeName);
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
long dateStart =
cubeManager
.getCube(cubeName)
.getDescriptor()
.getModel()
.getPartitionDesc()
.getPartitionDateStart();
long dateEnd = f.parse("2050-11-12").getTime();
// this cube's start date is 0, end date is 20501112000000
List<String> result = Lists.newArrayList();
result.add(buildSegment(cubeName, dateStart, dateEnd));
return result;
}
@Before
public void before() throws Exception {
HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);
DeployUtil.initCliWorkDir();
DeployUtil.deployMetadata();
DeployUtil.overrideJobJarLocations();
final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
jobService = ExecutableManager.getInstance(kylinConfig);
scheduler = DefaultScheduler.getInstance();
scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
if (!scheduler.hasStarted()) {
throw new RuntimeException("scheduler has not been started");
}
cubeManager = CubeManager.getInstance(kylinConfig);
jobEngineConfig = new JobEngineConfig(kylinConfig);
for (String jobId : jobService.getAllJobIds()) {
if (jobService.getJob(jobId) instanceof CubingJob) {
jobService.deleteJob(jobId);
}
}
}
private void clearSegment(String cubeName) throws Exception {
CubeInstance cube = cubeManager.getCube(cubeName);
cube.getSegments().clear();
cubeManager.updateCube(cube);
}