@Test
public void testExistingProject() throws Exception {
ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());
int originalProjectCount = prjMgr.listAllProjects().size();
int originalCubeCount = cubeMgr.listAllCubes().size();
ResourceStore store = getStore();
// clean legacy in case last run failed
store.deleteResource("/cube/new_cube_in_default.json");
CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
CubeInstance createdCube =
cubeMgr.createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null);
assertTrue(createdCube == cubeMgr.getCube("new_cube_in_default"));
System.out.println(JsonUtil.writeValueAsIndentString(createdCube));
assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);
CubeInstance droppedCube = cubeMgr.dropCube("new_cube_in_default", true);
assertTrue(createdCube == droppedCube);
assertNull(cubeMgr.getCube("new_cube_in_default"));
assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);
}
@Test
public void testNewProject() throws Exception {
ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());
int originalProjectCount = prjMgr.listAllProjects().size();
int originalCubeCount = cubeMgr.listAllCubes().size();
int originalCubeCountInDefault = prjMgr.listAllRealizations("default").size();
ResourceStore store = getStore();
// clean legacy in case last run failed
store.deleteResource("/cube/cube_in_alien_project.json");
CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
CubeInstance createdCube = cubeMgr.createCube("cube_in_alien_project", "alien", desc, null);
assertTrue(createdCube == cubeMgr.getCube("cube_in_alien_project"));
ProjectManager proMgr = ProjectManager.getInstance(getTestConfig());
Set<IRealization> realizations = proMgr.listAllRealizations("alien");
assertTrue(realizations.contains(createdCube));
System.out.println(JsonUtil.writeValueAsIndentString(createdCube));
assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1);
assertTrue(
prjMgr
.listAllRealizations("ALIEN")
.iterator()
.next()
.getName()
.equalsIgnoreCase("CUBE_IN_ALIEN_PROJECT"));
assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);
prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "default", null);
assertTrue(prjMgr.listAllRealizations("ALIEN").size() == 0);
assertTrue(prjMgr.listAllRealizations("default").size() == originalCubeCountInDefault + 1);
assertTrue(
ProjectManager.getInstance(getTestConfig())
.listAllRealizations("default")
.contains(createdCube));
prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "alien", null);
assertTrue(prjMgr.listAllRealizations("ALIEN").size() == 1);
assertTrue(prjMgr.listAllRealizations("default").size() == originalCubeCountInDefault);
assertTrue(
ProjectManager.getInstance(getTestConfig())
.listAllRealizations("alien")
.contains(createdCube));
CubeInstance droppedCube = cubeMgr.dropCube("cube_in_alien_project", true);
assertTrue(createdCube == droppedCube);
assertNull(cubeMgr.getCube("cube_in_alien_project"));
assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1);
assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);
prjMgr.dropProject("alien");
assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
}
@SuppressWarnings("unused")
// called by reflection
private List<String> testLeftJoinCube2() throws Exception {
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
List<String> result = Lists.newArrayList();
final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
// this cube's start date is 0, end date is 20120601000000
long dateStart =
cubeManager
.getCube(cubeName)
.getDescriptor()
.getModel()
.getPartitionDesc()
.getPartitionDateStart();
long dateEnd = f.parse("2012-06-01").getTime();
clearSegment(cubeName);
result.add(buildSegment(cubeName, dateStart, dateEnd));
// then submit an append job, start date is 20120601000000, end
// date is 20220101000000
dateStart = f.parse("2012-06-01").getTime();
dateEnd = f.parse("2022-01-01").getTime();
result.add(buildSegment(cubeName, dateStart, dateEnd));
return result;
}
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
KylinConfig conf = context.getConfig();
final CubeManager mgr = CubeManager.getInstance(conf);
final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
final CubeSegment newSegment =
cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
final List<CubeSegment> mergingSegments = getMergingSegments(cube);
Collections.sort(mergingSegments);
try {
checkLookupSnapshotsMustIncremental(mergingSegments);
makeDictForNewSegment(conf, cube, newSegment, mergingSegments);
makeSnapshotForNewSegment(cube, newSegment, mergingSegments);
CubeUpdate cubeBuilder = new CubeUpdate(cube);
cubeBuilder.setToUpdateSegs(newSegment);
mgr.updateCube(cubeBuilder);
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
} catch (IOException e) {
logger.error("fail to merge dictionary or lookup snapshots", e);
return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
}
}
private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
CubingJob job = cubingJobBuilder.buildJob(segment);
jobService.addJob(job);
waitForJob(job.getId());
return job.getId();
}
@Before
public void setUp() throws Exception {
this.createTestMetadata();
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
cube = cubeMgr.getCube("test_kylin_cube_without_slr_left_join_empty");
Assert.assertNotNull(cube);
storageEngine = StorageFactory.createQuery(cube);
String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
context = new StorageContext();
context.setConnUrl(url);
mockup = new StorageMockUtils(cube.getModel());
}
@Test
public void testProjectsDrop() throws IOException {
ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty");
assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
assertTrue(prjMgr.listAllRealizations("default").contains(cube));
cubeMgr.dropCube(cube.getName(), false);
assertTrue(!prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
assertTrue(!prjMgr.listAllRealizations("default").contains(cube));
}
@Test
public void testProjectsLoadAfterProjectChange() throws IOException {
ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty");
assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
prjMgr.removeRealizationsFromProjects(RealizationType.CUBE, cube.getName());
assertTrue(!prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
prjMgr.moveRealizationToProject(RealizationType.CUBE, cube.getName(), "default", "tester");
assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
}
@Override
public int run(String[] args) throws Exception {
Options options = new Options();
try {
options.addOption(OPTION_INPUT_PATH);
options.addOption(OPTION_HTABLE_NAME);
options.addOption(OPTION_CUBE_NAME);
parseOptions(options, args);
String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
// e.g
// /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/
// end with "/"
String input = getOptionValue(OPTION_INPUT_PATH);
Configuration conf = HBaseConfiguration.create(getConf());
FileSystem fs = FileSystem.get(conf);
String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
KylinConfig config = KylinConfig.getInstanceFromEnv();
CubeManager cubeMgr = CubeManager.getInstance(config);
CubeInstance cube = cubeMgr.getCube(cubeName);
CubeDesc cubeDesc = cube.getDescriptor();
FsPermission permission = new FsPermission((short) 0777);
for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) {
String cfName = cf.getName();
fs.setPermission(new Path(input + cfName), permission);
}
String[] newArgs = new String[2];
newArgs[0] = input;
newArgs[1] = tableName;
log.debug("Start to run LoadIncrementalHFiles");
int ret = ToolRunner.run(new LoadIncrementalHFiles(conf), newArgs);
log.debug("End to run LoadIncrementalHFiles");
return ret;
} catch (Exception e) {
printUsage(options);
throw e;
}
}
@SuppressWarnings("unused")
// called by reflection
private List<String> testLeftJoinCube() throws Exception {
String cubeName = "test_kylin_cube_with_slr_left_join_empty";
clearSegment(cubeName);
SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
f.setTimeZone(TimeZone.getTimeZone("GMT"));
long dateStart =
cubeManager
.getCube(cubeName)
.getDescriptor()
.getModel()
.getPartitionDesc()
.getPartitionDateStart();
long dateEnd = f.parse("2050-11-12").getTime();
// this cube's start date is 0, end date is 20501112000000
List<String> result = Lists.newArrayList();
result.add(buildSegment(cubeName, dateStart, dateEnd));
return result;
}
private void clearSegment(String cubeName) throws Exception {
CubeInstance cube = cubeManager.getCube(cubeName);
cube.getSegments().clear();
cubeManager.updateCube(cube);
}
