コード例 #1
0
ファイル: ProjectManagerTest.java プロジェクト: haoch/kylin
  @Test
  public void testExistingProject() throws Exception {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());

    int originalProjectCount = prjMgr.listAllProjects().size();
    int originalCubeCount = cubeMgr.listAllCubes().size();
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/new_cube_in_default.json");

    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube =
        cubeMgr.createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null);
    assertTrue(createdCube == cubeMgr.getCube("new_cube_in_default"));

    System.out.println(JsonUtil.writeValueAsIndentString(createdCube));

    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);

    CubeInstance droppedCube = cubeMgr.dropCube("new_cube_in_default", true);

    assertTrue(createdCube == droppedCube);
    assertNull(cubeMgr.getCube("new_cube_in_default"));
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);
  }
コード例 #2
0
ファイル: ProjectManagerTest.java プロジェクト: haoch/kylin
  @Test
  public void testNewProject() throws Exception {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());

    int originalProjectCount = prjMgr.listAllProjects().size();
    int originalCubeCount = cubeMgr.listAllCubes().size();
    int originalCubeCountInDefault = prjMgr.listAllRealizations("default").size();
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/cube_in_alien_project.json");

    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube = cubeMgr.createCube("cube_in_alien_project", "alien", desc, null);
    assertTrue(createdCube == cubeMgr.getCube("cube_in_alien_project"));
    ProjectManager proMgr = ProjectManager.getInstance(getTestConfig());
    Set<IRealization> realizations = proMgr.listAllRealizations("alien");
    assertTrue(realizations.contains(createdCube));

    System.out.println(JsonUtil.writeValueAsIndentString(createdCube));

    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1);
    assertTrue(
        prjMgr
            .listAllRealizations("ALIEN")
            .iterator()
            .next()
            .getName()
            .equalsIgnoreCase("CUBE_IN_ALIEN_PROJECT"));
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);

    prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "default", null);
    assertTrue(prjMgr.listAllRealizations("ALIEN").size() == 0);
    assertTrue(prjMgr.listAllRealizations("default").size() == originalCubeCountInDefault + 1);
    assertTrue(
        ProjectManager.getInstance(getTestConfig())
            .listAllRealizations("default")
            .contains(createdCube));

    prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "alien", null);
    assertTrue(prjMgr.listAllRealizations("ALIEN").size() == 1);
    assertTrue(prjMgr.listAllRealizations("default").size() == originalCubeCountInDefault);
    assertTrue(
        ProjectManager.getInstance(getTestConfig())
            .listAllRealizations("alien")
            .contains(createdCube));

    CubeInstance droppedCube = cubeMgr.dropCube("cube_in_alien_project", true);

    assertTrue(createdCube == droppedCube);
    assertNull(cubeMgr.getCube("cube_in_alien_project"));
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);

    prjMgr.dropProject("alien");
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
  }
コード例 #3
0
  @SuppressWarnings("unused")
  // called by reflection
  private List<String> testLeftJoinCube2() throws Exception {
    SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
    f.setTimeZone(TimeZone.getTimeZone("GMT"));
    List<String> result = Lists.newArrayList();
    final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
    // this cube's start date is 0, end date is 20120601000000
    long dateStart =
        cubeManager
            .getCube(cubeName)
            .getDescriptor()
            .getModel()
            .getPartitionDesc()
            .getPartitionDateStart();
    long dateEnd = f.parse("2012-06-01").getTime();

    clearSegment(cubeName);
    result.add(buildSegment(cubeName, dateStart, dateEnd));

    // then submit an append job, start date is 20120601000000, end
    // date is 20220101000000
    dateStart = f.parse("2012-06-01").getTime();
    dateEnd = f.parse("2022-01-01").getTime();
    result.add(buildSegment(cubeName, dateStart, dateEnd));
    return result;
  }
コード例 #4
0
ファイル: MergeDictionaryStep.java プロジェクト: haoch/kylin
  @Override
  protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig conf = context.getConfig();
    final CubeManager mgr = CubeManager.getInstance(conf);
    final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final CubeSegment newSegment =
        cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
    final List<CubeSegment> mergingSegments = getMergingSegments(cube);

    Collections.sort(mergingSegments);

    try {
      checkLookupSnapshotsMustIncremental(mergingSegments);

      makeDictForNewSegment(conf, cube, newSegment, mergingSegments);
      makeSnapshotForNewSegment(cube, newSegment, mergingSegments);

      CubeUpdate cubeBuilder = new CubeUpdate(cube);
      cubeBuilder.setToUpdateSegs(newSegment);
      mgr.updateCube(cubeBuilder);
      return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
    } catch (IOException e) {
      logger.error("fail to merge dictionary or lookup snapshots", e);
      return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
    }
  }
コード例 #5
0
 private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
   CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
   CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
   CubingJob job = cubingJobBuilder.buildJob(segment);
   jobService.addJob(job);
   waitForJob(job.getId());
   return job.getId();
 }
コード例 #6
0
ファイル: ITStorageTest.java プロジェクト: yiming187/kylin
  @Before
  public void setUp() throws Exception {
    this.createTestMetadata();

    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    cube = cubeMgr.getCube("test_kylin_cube_without_slr_left_join_empty");
    Assert.assertNotNull(cube);
    storageEngine = StorageFactory.createQuery(cube);
    String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
    context = new StorageContext();
    context.setConnUrl(url);
    mockup = new StorageMockUtils(cube.getModel());
  }
コード例 #7
0
ファイル: ProjectManagerTest.java プロジェクト: haoch/kylin
  @Test
  public void testProjectsDrop() throws IOException {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());

    CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty");
    assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
    assertTrue(prjMgr.listAllRealizations("default").contains(cube));

    cubeMgr.dropCube(cube.getName(), false);

    assertTrue(!prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
    assertTrue(!prjMgr.listAllRealizations("default").contains(cube));
  }
コード例 #8
0
ファイル: ProjectManagerTest.java プロジェクト: haoch/kylin
  @Test
  public void testProjectsLoadAfterProjectChange() throws IOException {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());

    CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty");
    assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));

    prjMgr.removeRealizationsFromProjects(RealizationType.CUBE, cube.getName());

    assertTrue(!prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));

    prjMgr.moveRealizationToProject(RealizationType.CUBE, cube.getName(), "default", "tester");

    assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
  }
コード例 #9
0
ファイル: BulkLoadJob.java プロジェクト: Tiroshan/Kylin
  @Override
  public int run(String[] args) throws Exception {
    Options options = new Options();

    try {
      options.addOption(OPTION_INPUT_PATH);
      options.addOption(OPTION_HTABLE_NAME);
      options.addOption(OPTION_CUBE_NAME);
      parseOptions(options, args);

      String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
      // e.g
      // /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/
      // end with "/"
      String input = getOptionValue(OPTION_INPUT_PATH);

      Configuration conf = HBaseConfiguration.create(getConf());
      FileSystem fs = FileSystem.get(conf);

      String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
      KylinConfig config = KylinConfig.getInstanceFromEnv();
      CubeManager cubeMgr = CubeManager.getInstance(config);
      CubeInstance cube = cubeMgr.getCube(cubeName);
      CubeDesc cubeDesc = cube.getDescriptor();
      FsPermission permission = new FsPermission((short) 0777);
      for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) {
        String cfName = cf.getName();
        fs.setPermission(new Path(input + cfName), permission);
      }

      String[] newArgs = new String[2];
      newArgs[0] = input;
      newArgs[1] = tableName;

      log.debug("Start to run LoadIncrementalHFiles");
      int ret = ToolRunner.run(new LoadIncrementalHFiles(conf), newArgs);
      log.debug("End to run LoadIncrementalHFiles");
      return ret;
    } catch (Exception e) {
      printUsage(options);
      throw e;
    }
  }
コード例 #10
0
  @SuppressWarnings("unused")
  // called by reflection
  private List<String> testLeftJoinCube() throws Exception {
    String cubeName = "test_kylin_cube_with_slr_left_join_empty";
    clearSegment(cubeName);

    SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
    f.setTimeZone(TimeZone.getTimeZone("GMT"));
    long dateStart =
        cubeManager
            .getCube(cubeName)
            .getDescriptor()
            .getModel()
            .getPartitionDesc()
            .getPartitionDateStart();
    long dateEnd = f.parse("2050-11-12").getTime();

    // this cube's start date is 0, end date is 20501112000000
    List<String> result = Lists.newArrayList();
    result.add(buildSegment(cubeName, dateStart, dateEnd));
    return result;
  }
コード例 #11
0
 private void clearSegment(String cubeName) throws Exception {
   CubeInstance cube = cubeManager.getCube(cubeName);
   cube.getSegments().clear();
   cubeManager.updateCube(cube);
 }