Beispiel #1
0
  @Override
  protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig conf = context.getConfig();
    final CubeManager mgr = CubeManager.getInstance(conf);
    final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final CubeSegment newSegment =
        cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
    final List<CubeSegment> mergingSegments = getMergingSegments(cube);

    Collections.sort(mergingSegments);

    try {
      checkLookupSnapshotsMustIncremental(mergingSegments);

      makeDictForNewSegment(conf, cube, newSegment, mergingSegments);
      makeSnapshotForNewSegment(cube, newSegment, mergingSegments);

      CubeUpdate cubeBuilder = new CubeUpdate(cube);
      cubeBuilder.setToUpdateSegs(newSegment);
      mgr.updateCube(cubeBuilder);
      return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
    } catch (IOException e) {
      logger.error("fail to merge dictionary or lookup snapshots", e);
      return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
    }
  }
 private String buildSegment(String cubeName, long startDate, long endDate) throws Exception {
   CubeSegment segment = cubeManager.appendSegments(cubeManager.getCube(cubeName), endDate);
   CubingJobBuilder cubingJobBuilder = new CubingJobBuilder(jobEngineConfig);
   CubingJob job = cubingJobBuilder.buildJob(segment);
   jobService.addJob(job);
   waitForJob(job.getId());
   return job.getId();
 }
  public void checkAll() {
    List<String> segFullNameList = Lists.newArrayList();

    CubeManager cubeMgr = CubeManager.getInstance(dstCfg);
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
      addHTableNamesForCube(cube, segFullNameList);
    }

    check(segFullNameList);
  }
Beispiel #4
0
  @Test
  public void testExistingProject() throws Exception {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());

    int originalProjectCount = prjMgr.listAllProjects().size();
    int originalCubeCount = cubeMgr.listAllCubes().size();
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/new_cube_in_default.json");

    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube =
        cubeMgr.createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null);
    assertTrue(createdCube == cubeMgr.getCube("new_cube_in_default"));

    System.out.println(JsonUtil.writeValueAsIndentString(createdCube));

    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);

    CubeInstance droppedCube = cubeMgr.dropCube("new_cube_in_default", true);

    assertTrue(createdCube == droppedCube);
    assertNull(cubeMgr.getCube("new_cube_in_default"));
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);
  }
Beispiel #5
0
  @Test
  public void testNewProject() throws Exception {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());

    int originalProjectCount = prjMgr.listAllProjects().size();
    int originalCubeCount = cubeMgr.listAllCubes().size();
    int originalCubeCountInDefault = prjMgr.listAllRealizations("default").size();
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/cube_in_alien_project.json");

    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube = cubeMgr.createCube("cube_in_alien_project", "alien", desc, null);
    assertTrue(createdCube == cubeMgr.getCube("cube_in_alien_project"));
    ProjectManager proMgr = ProjectManager.getInstance(getTestConfig());
    Set<IRealization> realizations = proMgr.listAllRealizations("alien");
    assertTrue(realizations.contains(createdCube));

    System.out.println(JsonUtil.writeValueAsIndentString(createdCube));

    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1);
    assertTrue(
        prjMgr
            .listAllRealizations("ALIEN")
            .iterator()
            .next()
            .getName()
            .equalsIgnoreCase("CUBE_IN_ALIEN_PROJECT"));
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);

    prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "default", null);
    assertTrue(prjMgr.listAllRealizations("ALIEN").size() == 0);
    assertTrue(prjMgr.listAllRealizations("default").size() == originalCubeCountInDefault + 1);
    assertTrue(
        ProjectManager.getInstance(getTestConfig())
            .listAllRealizations("default")
            .contains(createdCube));

    prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "alien", null);
    assertTrue(prjMgr.listAllRealizations("ALIEN").size() == 1);
    assertTrue(prjMgr.listAllRealizations("default").size() == originalCubeCountInDefault);
    assertTrue(
        ProjectManager.getInstance(getTestConfig())
            .listAllRealizations("alien")
            .contains(createdCube));

    CubeInstance droppedCube = cubeMgr.dropCube("cube_in_alien_project", true);

    assertTrue(createdCube == droppedCube);
    assertNull(cubeMgr.getCube("cube_in_alien_project"));
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);

    prjMgr.dropProject("alien");
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
  }
Beispiel #6
0
  @Before
  public void setUp() throws Exception {
    this.createTestMetadata();

    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    cube = cubeMgr.getCube("test_kylin_cube_without_slr_left_join_empty");
    Assert.assertNotNull(cube);
    storageEngine = StorageFactory.createQuery(cube);
    String url = KylinConfig.getInstanceFromEnv().getStorageUrl();
    context = new StorageContext();
    context.setConnUrl(url);
    mockup = new StorageMockUtils(cube.getModel());
  }
Beispiel #7
0
  @Test
  public void testProjectsDrop() throws IOException {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());

    CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty");
    assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
    assertTrue(prjMgr.listAllRealizations("default").contains(cube));

    cubeMgr.dropCube(cube.getName(), false);

    assertTrue(!prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
    assertTrue(!prjMgr.listAllRealizations("default").contains(cube));
  }
Beispiel #8
0
  @Override
  protected void setup(Context context) throws IOException {
    super.bindCurrentConfiguration(context.getConfiguration());
    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME).toUpperCase();

    // only used in Build job, not in Merge job
    cuboidLevel = context.getConfiguration().getInt(BatchConstants.CFG_CUBE_CUBOID_LEVEL, 0);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    cubeDesc = CubeManager.getInstance(config).getCube(cubeName).getDescriptor();
    measuresDescs = cubeDesc.getMeasures();

    codec = new BufferedMeasureEncoder(measuresDescs);
    aggs = new MeasureAggregators(measuresDescs);

    input = new Object[measuresDescs.size()];
    result = new Object[measuresDescs.size()];
    needAggr = new boolean[measuresDescs.size()];

    if (cuboidLevel > 0) {
      for (int i = 0; i < measuresDescs.size(); i++) {
        needAggr[i] = !measuresDescs.get(i).getFunction().getMeasureType().onlyAggrInBaseCuboid();
      }
    }
  }
Beispiel #9
0
  @Test
  public void testProjectsLoadAfterProjectChange() throws IOException {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());

    CubeInstance cube = cubeMgr.getCube("test_kylin_cube_with_slr_empty");
    assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));

    prjMgr.removeRealizationsFromProjects(RealizationType.CUBE, cube.getName());

    assertTrue(!prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));

    prjMgr.moveRealizationToProject(RealizationType.CUBE, cube.getName(), "default", "tester");

    assertTrue(prjMgr.getRealizationsByTable("default", "default.test_kylin_fact").contains(cube));
  }
Beispiel #10
0
  @Test
  public void testDecode() throws Exception {
    CubeDesc cubeDesc =
        CubeManager.getInstance(getTestConfig())
            .getCube("test_kylin_cube_with_slr_ready")
            .getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongMutable count = new LongMutable(2);
    LongMutable item_count = new LongMutable(100);
    ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
    codec.encode(new Object[] {sum, min, max, count, item_count}, buf);

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
      FunctionDesc aggrFunc = measure.getFunction();
      int index = hbaseCol.findMeasure(aggrFunc);
      rowValueDecoder.setProjectIndex(index);
    }

    rowValueDecoder.decodeAndConvertJavaObj(valueBytes);
    Object[] measureValues = rowValueDecoder.getValues();
    // BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer
    assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues));
  }
  @SuppressWarnings("unused")
  // called by reflection
  private List<String> testLeftJoinCube2() throws Exception {
    SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
    f.setTimeZone(TimeZone.getTimeZone("GMT"));
    List<String> result = Lists.newArrayList();
    final String cubeName = "test_kylin_cube_without_slr_left_join_empty";
    // this cube's start date is 0, end date is 20120601000000
    long dateStart =
        cubeManager
            .getCube(cubeName)
            .getDescriptor()
            .getModel()
            .getPartitionDesc()
            .getPartitionDateStart();
    long dateEnd = f.parse("2012-06-01").getTime();

    clearSegment(cubeName);
    result.add(buildSegment(cubeName, dateStart, dateEnd));

    // then submit an append job, start date is 20120601000000, end
    // date is 20220101000000
    dateStart = f.parse("2012-06-01").getTime();
    dateEnd = f.parse("2022-01-01").getTime();
    result.add(buildSegment(cubeName, dateStart, dateEnd));
    return result;
  }
  public void checkCube(String cubeName) {
    List<String> segFullNameList = Lists.newArrayList();

    CubeInstance cube = CubeManager.getInstance(dstCfg).getCube(cubeName);
    addHTableNamesForCube(cube, segFullNameList);

    check(segFullNameList);
  }
Beispiel #13
0
  @Override
  public int run(String[] args) throws Exception {
    Options options = new Options();

    try {
      options.addOption(OPTION_INPUT_PATH);
      options.addOption(OPTION_HTABLE_NAME);
      options.addOption(OPTION_CUBE_NAME);
      parseOptions(options, args);

      String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
      // e.g
      // /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/
      // end with "/"
      String input = getOptionValue(OPTION_INPUT_PATH);

      Configuration conf = HBaseConfiguration.create(getConf());
      FileSystem fs = FileSystem.get(conf);

      String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
      KylinConfig config = KylinConfig.getInstanceFromEnv();
      CubeManager cubeMgr = CubeManager.getInstance(config);
      CubeInstance cube = cubeMgr.getCube(cubeName);
      CubeDesc cubeDesc = cube.getDescriptor();
      FsPermission permission = new FsPermission((short) 0777);
      for (HBaseColumnFamilyDesc cf : cubeDesc.getHBaseMapping().getColumnFamily()) {
        String cfName = cf.getName();
        fs.setPermission(new Path(input + cfName), permission);
      }

      String[] newArgs = new String[2];
      newArgs[0] = input;
      newArgs[1] = tableName;

      log.debug("Start to run LoadIncrementalHFiles");
      int ret = ToolRunner.run(new LoadIncrementalHFiles(conf), newArgs);
      log.debug("End to run LoadIncrementalHFiles");
      return ret;
    } catch (Exception e) {
      printUsage(options);
      throw e;
    }
  }
Beispiel #14
0
  public static void processSegment(
      KylinConfig config,
      String cubeName,
      String segmentName,
      DistinctColumnValuesProvider factTableValueProvider)
      throws IOException {
    CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
    CubeSegment segment = cube.getSegment(segmentName, SegmentStatusEnum.NEW);

    processSegment(config, segment, factTableValueProvider);
  }
Beispiel #15
0
  private static void processSegment(
      KylinConfig config, CubeSegment cubeSeg, DistinctColumnValuesProvider factTableValueProvider)
      throws IOException {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    // dictionary
    for (TblColRef col : cubeSeg.getCubeDesc().getAllColumnsNeedDictionary()) {
      logger.info("Building dictionary for " + col);
      cubeMgr.buildDictionary(cubeSeg, col, factTableValueProvider);
    }

    for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) {
      // build snapshot
      if (dim.getTable() != null
          && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) {
        // CubeSegment seg = cube.getTheOnlySegment();
        logger.info("Building snapshot of " + dim.getTable());
        cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable());
        logger.info("Checking snapshot of " + dim.getTable());
        cubeMgr.getLookupTable(cubeSeg, dim); // load the table for sanity check
      }
    }
  }
Beispiel #16
0
  @Test(expected = IllegalArgumentException.class)
  public void testError() throws Exception {
    CubeDesc cubeDesc =
        CubeManager.getInstance(getTestConfig())
            .getCube("test_kylin_cube_with_slr_ready")
            .getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("11111111111111111111333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    LongMutable item_count = new LongMutable(100);
    ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
    codec.encode(new Object[] {sum, min, max, count, item_count}, buf);
  }
  @SuppressWarnings("unused")
  // called by reflection
  private List<String> testLeftJoinCube() throws Exception {
    String cubeName = "test_kylin_cube_with_slr_left_join_empty";
    clearSegment(cubeName);

    SimpleDateFormat f = new SimpleDateFormat("yyyy-MM-dd");
    f.setTimeZone(TimeZone.getTimeZone("GMT"));
    long dateStart =
        cubeManager
            .getCube(cubeName)
            .getDescriptor()
            .getModel()
            .getPartitionDesc()
            .getPartitionDateStart();
    long dateEnd = f.parse("2050-11-12").getTime();

    // this cube's start date is 0, end date is 20501112000000
    List<String> result = Lists.newArrayList();
    result.add(buildSegment(cubeName, dateStart, dateEnd));
    return result;
  }
  @Before
  public void before() throws Exception {
    HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);

    DeployUtil.initCliWorkDir();
    DeployUtil.deployMetadata();
    DeployUtil.overrideJobJarLocations();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
      throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);
    jobEngineConfig = new JobEngineConfig(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
      if (jobService.getJob(jobId) instanceof CubingJob) {
        jobService.deleteJob(jobId);
      }
    }
  }
 private void clearSegment(String cubeName) throws Exception {
   CubeInstance cube = cubeManager.getCube(cubeName);
   cube.getSegments().clear();
   cubeManager.updateCube(cube);
 }