Exemplo n.º 1
0
  public List<Record> runNewPartitionRecord() throws Exception {
    HiveMetastoreTarget hiveTarget = new HiveMetastoreTargetBuilder().build();

    TargetRunner runner =
        new TargetRunner.Builder(HiveMetastoreTarget.class, hiveTarget)
            .setOnRecordError(OnRecordError.TO_ERROR)
            .build();
    runner.runInit();

    Assert.assertEquals("There should be no error records", 0, runner.getErrorRecords().size());
    LinkedHashMap<String, String> partitionVals = new LinkedHashMap<String, String>();
    partitionVals.put("dt", "2016");

    Field newPartitionField =
        HiveMetastoreUtil.newPartitionMetadataFieldBuilder(
            "default", "tbl", partitionVals, "/user/hive/warehouse/tbl/dt=2016");
    Record record = RecordCreator.create();
    record.set(newPartitionField);
    runner.runWrite(ImmutableList.of(record));

    try {
      return runner.getEventRecords();
    } finally {
      runner.runDestroy();
    }
  }
Exemplo n.º 2
0
  @SuppressWarnings("unchecked")
  public List<Record> runNewTableRecord() throws Exception {
    HiveMetastoreTarget hiveTarget = new HiveMetastoreTargetBuilder().build();

    TargetRunner runner =
        new TargetRunner.Builder(HiveMetastoreTarget.class, hiveTarget)
            .setOnRecordError(OnRecordError.STOP_PIPELINE)
            .build();
    runner.runInit();

    LinkedHashMap<String, HiveTypeInfo> columns = new LinkedHashMap<>();
    columns.put("name", HiveType.STRING.getSupport().generateHiveTypeInfoFromResultSet("STRING"));
    columns.put(
        "surname", HiveType.STRING.getSupport().generateHiveTypeInfoFromResultSet("STRING"));

    LinkedHashMap<String, HiveTypeInfo> partitions = new LinkedHashMap<>();
    partitions.put("dt", HiveType.STRING.getSupport().generateHiveTypeInfoFromResultSet("STRING"));

    Field newTableField =
        HiveMetastoreUtil.newSchemaMetadataFieldBuilder(
            "default",
            "tbl",
            columns,
            partitions,
            true,
            BaseHiveIT.getDefaultWareHouseDir(),
            HiveMetastoreUtil.generateAvroSchema(columns, "tbl"));

    Record record = RecordCreator.create();
    record.set(newTableField);
    Assert.assertTrue(HiveMetastoreUtil.isSchemaChangeRecord(record));

    runner.runWrite(ImmutableList.of(record));

    assertTableStructure(
        "default.tbl",
        ImmutablePair.of("tbl.name", Types.VARCHAR),
        ImmutablePair.of("tbl.surname", Types.VARCHAR),
        ImmutablePair.of("tbl.dt", Types.VARCHAR));

    try {
      return runner.getEventRecords();
    } finally {
      runner.runDestroy();
    }
  }