public Deployment load(File deploymentFile) throws IOException {
String json = Files.toString(deploymentFile, UTF_8);
DeploymentRepresentation data = jsonCodec.fromJson(json);
File deploymentDir = new File(baseDir, "installation");
if (!deploymentDir.isDirectory()) {
deploymentDir = new File(baseDir, "deployment");
}
Deployment deployment = data.toDeployment(deploymentDir, getDataDir(), location);
return deployment;
}
private void testRoundTrip(HiveColumnHandle expected) {
String json = codec.toJson(expected);
HiveColumnHandle actual = codec.fromJson(json);
assertEquals(actual.getClientId(), expected.getClientId());
assertEquals(actual.getName(), expected.getName());
assertEquals(actual.getHiveType(), expected.getHiveType());
assertEquals(actual.getHiveColumnIndex(), expected.getHiveColumnIndex());
assertEquals(actual.isPartitionKey(), expected.isPartitionKey());
}
@Test
public void testBackwardsCompatible() {
ClientTypeSignature signature =
new ClientTypeSignature(
StandardTypes.ARRAY,
ImmutableList.of(
new ClientTypeSignatureParameter(
TypeSignatureParameter.of(BIGINT.getTypeSignature()))));
ClientTypeSignature legacy =
CLIENT_TYPE_SIGNATURE_CODEC.fromJson(
"{\"rawType\":\"array\",\"literalArguments\":[],\"typeArguments\":[{\"rawType\":\"bigint\",\"literalArguments\":[],\"typeArguments\":[]}]}");
assertEquals(legacy, signature);
}
@Test
public void testJsonRoundTrip() {
JsonCodec<LocalFileSplit> codec = jsonCodec(LocalFileSplit.class);
String json = codec.toJson(split);
LocalFileSplit copy = codec.fromJson(json);
assertEquals(copy.getAddress(), split.getAddress());
assertEquals(copy.getTableName(), split.getTableName());
assertEquals(copy.getEffectivePredicate(), split.getEffectivePredicate());
assertEquals(copy.getAddresses(), ImmutableList.of(address));
assertEquals(copy.isRemotelyAccessible(), false);
}
@Test
public void testJsonRoundTrip() {
Properties schema = new Properties();
schema.setProperty("foo", "bar");
schema.setProperty("bar", "baz");
ImmutableList<HivePartitionKey> partitionKeys =
ImmutableList.of(
new HivePartitionKey("a", HIVE_STRING, "apple"),
new HivePartitionKey("b", HiveType.HIVE_LONG, "42"));
ImmutableList<HostAddress> addresses =
ImmutableList.of(
HostAddress.fromParts("127.0.0.1", 44), HostAddress.fromParts("127.0.0.1", 45));
HiveSplit expected =
new HiveSplit(
"clientId",
"db",
"table",
"partitionId",
"path",
42,
88,
schema,
partitionKeys,
addresses,
OptionalInt.empty(),
true,
TupleDomain.<HiveColumnHandle>all(),
ImmutableMap.of(1, HIVE_STRING));
String json = codec.toJson(expected);
HiveSplit actual = codec.fromJson(json);
assertEquals(actual.getClientId(), expected.getClientId());
assertEquals(actual.getDatabase(), expected.getDatabase());
assertEquals(actual.getTable(), expected.getTable());
assertEquals(actual.getPartitionName(), expected.getPartitionName());
assertEquals(actual.getPath(), expected.getPath());
assertEquals(actual.getStart(), expected.getStart());
assertEquals(actual.getLength(), expected.getLength());
assertEquals(actual.getSchema(), expected.getSchema());
assertEquals(actual.getPartitionKeys(), expected.getPartitionKeys());
assertEquals(actual.getAddresses(), expected.getAddresses());
assertEquals(actual.getColumnCoercions(), expected.getColumnCoercions());
assertEquals(actual.isForceLocalScheduling(), expected.isForceLocalScheduling());
}
@Test
public void testRoundTrip() {
ObjectMapperProvider objectMapperProvider = new ObjectMapperProvider();
objectMapperProvider.setJsonDeserializers(
ImmutableMap.<Class<?>, JsonDeserializer<?>>of(
Type.class, new TypeDeserializer(new TypeRegistry())));
JsonCodec<Signature> codec =
new JsonCodecFactory(objectMapperProvider, true).jsonCodec(Signature.class);
Signature expected =
new Signature("function", BIGINT, ImmutableList.of(BOOLEAN, DOUBLE, VARCHAR));
String json = codec.toJson(expected);
Signature actual = codec.fromJson(json);
assertEquals(actual.getName(), expected.getName());
assertEquals(actual.getReturnType(), expected.getReturnType());
assertEquals(actual.getArgumentTypes(), expected.getArgumentTypes());
}
public static Map.Entry<SchemaTableName, KafkaTopicDescription> loadTpchTopicDescription(
JsonCodec<KafkaTopicDescription> topicDescriptionJsonCodec,
String topicName,
SchemaTableName schemaTableName)
throws IOException {
KafkaTopicDescription tpchTemplate =
topicDescriptionJsonCodec.fromJson(
ByteStreams.toByteArray(
TestUtils.class.getResourceAsStream(
format("/tpch/%s.json", schemaTableName.getTableName()))));
return new AbstractMap.SimpleImmutableEntry<>(
schemaTableName,
new KafkaTopicDescription(
schemaTableName.getTableName(),
schemaTableName.getSchemaName(),
topicName,
tpchTemplate.getKey(),
tpchTemplate.getMessage()));
}
private static void assertJsonRoundTrip(ClientTypeSignature signature) {
String json = CLIENT_TYPE_SIGNATURE_CODEC.toJson(signature);
ClientTypeSignature copy = CLIENT_TYPE_SIGNATURE_CODEC.fromJson(json);
assertEquals(copy, signature);
}
@Test
public void testWriter() throws Exception {
List<Long> columnIds = ImmutableList.of(1L, 2L, 4L, 6L, 7L, 8L, 9L, 10L);
ArrayType arrayType = new ArrayType(BIGINT);
ArrayType arrayOfArrayType = new ArrayType(arrayType);
MapType mapType = new MapType(createVarcharType(10), BOOLEAN);
List<Type> columnTypes =
ImmutableList.of(
BIGINT,
createVarcharType(10),
VARBINARY,
DOUBLE,
BOOLEAN,
arrayType,
mapType,
arrayOfArrayType);
File file = new File(directory, System.nanoTime() + ".orc");
byte[] bytes1 = octets(0x00, 0xFE, 0xFF);
byte[] bytes3 = octets(0x01, 0x02, 0x19, 0x80);
RowPagesBuilder rowPagesBuilder =
RowPagesBuilder.rowPagesBuilder(columnTypes)
.row(
123L,
"hello",
wrappedBuffer(bytes1),
123.456,
true,
arrayBlockOf(BIGINT, 1, 2),
mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true),
arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5)))
.row(
null,
"world",
null,
Double.POSITIVE_INFINITY,
null,
arrayBlockOf(BIGINT, 3, null),
mapBlockOf(createVarcharType(5), BOOLEAN, "k2", null),
arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7)))
.row(
456L,
"bye \u2603",
wrappedBuffer(bytes3),
Double.NaN,
false,
arrayBlockOf(BIGINT),
mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false),
arrayBlockOf(arrayType, arrayBlockOf(BIGINT)));
try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(new EmptyClassLoader());
OrcFileWriter writer = new OrcFileWriter(columnIds, columnTypes, file)) {
writer.appendPages(rowPagesBuilder.build());
}
try (OrcDataSource dataSource = fileOrcDataSource(file)) {
OrcRecordReader reader = createReader(dataSource, columnIds, columnTypes);
assertEquals(reader.getReaderRowCount(), 3);
assertEquals(reader.getReaderPosition(), 0);
assertEquals(reader.getFileRowCount(), reader.getReaderRowCount());
assertEquals(reader.getFilePosition(), reader.getFilePosition());
assertEquals(reader.nextBatch(), 3);
assertEquals(reader.getReaderPosition(), 0);
assertEquals(reader.getFilePosition(), reader.getFilePosition());
Block column0 = reader.readBlock(BIGINT, 0);
assertEquals(column0.isNull(0), false);
assertEquals(column0.isNull(1), true);
assertEquals(column0.isNull(2), false);
assertEquals(BIGINT.getLong(column0, 0), 123L);
assertEquals(BIGINT.getLong(column0, 2), 456L);
Block column1 = reader.readBlock(createVarcharType(10), 1);
assertEquals(createVarcharType(10).getSlice(column1, 0), utf8Slice("hello"));
assertEquals(createVarcharType(10).getSlice(column1, 1), utf8Slice("world"));
assertEquals(createVarcharType(10).getSlice(column1, 2), utf8Slice("bye \u2603"));
Block column2 = reader.readBlock(VARBINARY, 2);
assertEquals(VARBINARY.getSlice(column2, 0), wrappedBuffer(bytes1));
assertEquals(column2.isNull(1), true);
assertEquals(VARBINARY.getSlice(column2, 2), wrappedBuffer(bytes3));
Block column3 = reader.readBlock(DOUBLE, 3);
assertEquals(column3.isNull(0), false);
assertEquals(column3.isNull(1), false);
assertEquals(column3.isNull(2), false);
assertEquals(DOUBLE.getDouble(column3, 0), 123.456);
assertEquals(DOUBLE.getDouble(column3, 1), Double.POSITIVE_INFINITY);
assertEquals(DOUBLE.getDouble(column3, 2), Double.NaN);
Block column4 = reader.readBlock(BOOLEAN, 4);
assertEquals(column4.isNull(0), false);
assertEquals(column4.isNull(1), true);
assertEquals(column4.isNull(2), false);
assertEquals(BOOLEAN.getBoolean(column4, 0), true);
assertEquals(BOOLEAN.getBoolean(column4, 2), false);
Block column5 = reader.readBlock(arrayType, 5);
assertEquals(column5.getPositionCount(), 3);
assertTrue(
arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 0), arrayBlockOf(BIGINT, 1, 2)));
assertTrue(
arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 1), arrayBlockOf(BIGINT, 3, null)));
assertTrue(arrayBlocksEqual(BIGINT, arrayType.getObject(column5, 2), arrayBlockOf(BIGINT)));
Block column6 = reader.readBlock(mapType, 6);
assertEquals(column6.getPositionCount(), 3);
assertTrue(
mapBlocksEqual(
createVarcharType(5),
BOOLEAN,
arrayType.getObject(column6, 0),
mapBlockOf(createVarcharType(5), BOOLEAN, "k1", true)));
assertTrue(
mapBlocksEqual(
createVarcharType(5),
BOOLEAN,
arrayType.getObject(column6, 1),
mapBlockOf(createVarcharType(5), BOOLEAN, "k2", null)));
assertTrue(
mapBlocksEqual(
createVarcharType(5),
BOOLEAN,
arrayType.getObject(column6, 2),
mapBlockOf(createVarcharType(5), BOOLEAN, "k3", false)));
Block column7 = reader.readBlock(arrayOfArrayType, 7);
assertEquals(column7.getPositionCount(), 3);
assertTrue(
arrayBlocksEqual(
arrayType,
arrayOfArrayType.getObject(column7, 0),
arrayBlockOf(arrayType, arrayBlockOf(BIGINT, 5))));
assertTrue(
arrayBlocksEqual(
arrayType,
arrayOfArrayType.getObject(column7, 1),
arrayBlockOf(arrayType, null, arrayBlockOf(BIGINT, 6, 7))));
assertTrue(
arrayBlocksEqual(
arrayType,
arrayOfArrayType.getObject(column7, 2),
arrayBlockOf(arrayType, arrayBlockOf(BIGINT))));
assertEquals(reader.nextBatch(), -1);
assertEquals(reader.getReaderPosition(), 3);
assertEquals(reader.getFilePosition(), reader.getFilePosition());
OrcFileMetadata orcFileMetadata =
METADATA_CODEC.fromJson(reader.getUserMetadata().get(OrcFileMetadata.KEY).getBytes());
assertEquals(
orcFileMetadata,
new OrcFileMetadata(
ImmutableMap.<Long, TypeSignature>builder()
.put(1L, BIGINT.getTypeSignature())
.put(2L, createVarcharType(10).getTypeSignature())
.put(4L, VARBINARY.getTypeSignature())
.put(6L, DOUBLE.getTypeSignature())
.put(7L, BOOLEAN.getTypeSignature())
.put(8L, arrayType.getTypeSignature())
.put(9L, mapType.getTypeSignature())
.put(10L, arrayOfArrayType.getTypeSignature())
.build()));
}
File crcFile = new File(file.getParentFile(), "." + file.getName() + ".crc");
assertFalse(crcFile.exists());
}
