/** Tests the case where we try to convert a string to a long incorrectly. */ @Test( expected = org.apache.nifi.processors.kite.AvroRecordConverter.AvroConversionException.class) public void testIllegalConversion() throws Exception { // We will convert s1 from string to long (or leave it null), ignore s2, // convert l1 from long to string, and leave l2 the same. Schema input = SchemaBuilder.record("Input") .namespace("com.cloudera.edh") .fields() .nullableString("s1", "") .requiredString("s2") .optionalLong("l1") .requiredLong("l2") .endRecord(); Schema output = SchemaBuilder.record("Output") .namespace("com.cloudera.edh") .fields() .optionalLong("s1") .optionalString("l1") .requiredLong("l2") .endRecord(); AvroRecordConverter converter = new AvroRecordConverter(input, output, EMPTY_MAPPING); Record inputRecord = new Record(input); inputRecord.put("s1", "blah"); inputRecord.put("s2", "blah"); inputRecord.put("l1", null); inputRecord.put("l2", 5L); converter.convert(inputRecord); }
/** Tests the case where we don't use a mapping file and just map records by name. */ @Test public void testDefaultConversion() throws Exception { // We will convert s1 from string to long (or leave it null), ignore s2, // convert s3 to from string to double, convert l1 from long to string, // and leave l2 the same. Schema input = SchemaBuilder.record("Input") .namespace("com.cloudera.edh") .fields() .nullableString("s1", "") .requiredString("s2") .requiredString("s3") .optionalLong("l1") .requiredLong("l2") .endRecord(); Schema output = SchemaBuilder.record("Output") .namespace("com.cloudera.edh") .fields() .optionalLong("s1") .optionalString("l1") .requiredLong("l2") .requiredDouble("s3") .endRecord(); AvroRecordConverter converter = new AvroRecordConverter(input, output, EMPTY_MAPPING, LocaleUtils.toLocale("en_US")); Record inputRecord = new Record(input); inputRecord.put("s1", null); inputRecord.put("s2", "blah"); inputRecord.put("s3", "5.5"); inputRecord.put("l1", null); inputRecord.put("l2", 5L); Record outputRecord = converter.convert(inputRecord); assertNull(outputRecord.get("s1")); assertNull(outputRecord.get("l1")); assertEquals(5L, outputRecord.get("l2")); assertEquals(5.5, outputRecord.get("s3")); inputRecord.put("s1", "500"); inputRecord.put("s2", "blah"); inputRecord.put("s3", "5.5e-5"); inputRecord.put("l1", 100L); inputRecord.put("l2", 2L); outputRecord = converter.convert(inputRecord); assertEquals(500L, outputRecord.get("s1")); assertEquals("100", outputRecord.get("l1")); assertEquals(2L, outputRecord.get("l2")); assertEquals(5.5e-5, outputRecord.get("s3")); }
@Override public ValidationResult validate(String subject, String uri, ValidationContext context) { Configuration conf = getConfiguration(context.getProperty(CONF_XML_FILES).getValue()); String inputUri = context.getProperty(INPUT_SCHEMA).getValue(); String error = null; final boolean elPresent = context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(uri); if (!elPresent) { try { Schema outputSchema = getSchema(uri, conf); Schema inputSchema = getSchema(inputUri, conf); // Get the explicitly mapped fields. This is identical to // logic in onTrigger, but ValidationContext and // ProcessContext share no ancestor, so we cannot generalize // the code. Map<String, String> fieldMapping = new HashMap<>(); for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) { if (entry.getKey().isDynamic()) { fieldMapping.put(entry.getKey().getName(), entry.getValue()); } } AvroRecordConverter converter = new AvroRecordConverter(inputSchema, outputSchema, fieldMapping); Collection<String> unmappedFields = converter.getUnmappedFields(); if (unmappedFields.size() > 0) { error = "The following fields are unmapped: " + unmappedFields; } } catch (SchemaNotFoundException e) { error = e.getMessage(); } } return new ValidationResult.Builder() .subject(subject) .input(uri) .explanation(error) .valid(error == null) .build(); }
/** Tests the case where we want to default map one field and explicitly map another. */ @Test public void testExplicitMapping() throws Exception { // We will convert s1 from string to long (or leave it null), ignore s2, // convert l1 from long to string, and leave l2 the same. Schema input = NESTED_RECORD_SCHEMA; Schema parent = NESTED_PARENT_SCHEMA; Schema output = UNNESTED_OUTPUT_SCHEMA; Map<String, String> mapping = ImmutableMap.of("parent.id", "parentId"); AvroRecordConverter converter = new AvroRecordConverter(input, output, mapping); Record inputRecord = new Record(input); inputRecord.put("l1", 5L); inputRecord.put("s1", "1000"); Record parentRecord = new Record(parent); parentRecord.put("id", 200L); parentRecord.put("name", "parent"); inputRecord.put("parent", parentRecord); Record outputRecord = converter.convert(inputRecord); assertEquals(5L, outputRecord.get("l1")); assertEquals(1000L, outputRecord.get("s1")); assertEquals(200L, outputRecord.get("parentId")); }
@Test public void testGetUnmappedFields() throws Exception { Schema input = SchemaBuilder.record("Input") .namespace("com.cloudera.edh") .fields() .nullableString("s1", "") .requiredString("s2") .optionalLong("l1") .requiredLong("l2") .endRecord(); Schema output = SchemaBuilder.record("Output") .namespace("com.cloudera.edh") .fields() .optionalLong("field") .endRecord(); // Test the case where the field isn't mapped at all. AvroRecordConverter converter = new AvroRecordConverter(input, output, EMPTY_MAPPING); assertEquals(ImmutableList.of("field"), converter.getUnmappedFields()); // Test the case where we tried to map from a non-existent field. converter = new AvroRecordConverter(input, output, ImmutableMap.of("nonExistentField", "field")); assertEquals(ImmutableList.of("field"), converter.getUnmappedFields()); // Test the case where we tried to map from a non-existent record. converter = new AvroRecordConverter(input, output, ImmutableMap.of("parent.nonExistentField", "field")); assertEquals(ImmutableList.of("field"), converter.getUnmappedFields()); // Test a valid case converter = new AvroRecordConverter(input, output, ImmutableMap.of("l2", "field")); assertEquals(Collections.EMPTY_LIST, converter.getUnmappedFields()); }
public class TestAvroRecordConverter { static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); static final Map<String, String> EMPTY_MAPPING = ImmutableMap.of(); static final String NESTED_RECORD_SCHEMA_STRING = "{\n" + " \"type\": \"record\",\n" + " \"name\": \"NestedInput\",\n" + " \"namespace\": \"org.apache.example\",\n" + " \"fields\": [\n" + " {\n" + " \"name\": \"l1\",\n" + " \"type\": \"long\"\n" + " },\n" + " {\n" + " \"name\": \"s1\",\n" + " \"type\": \"string\"\n" + " },\n" + " {\n" + " \"name\": \"parent\",\n" + " \"type\": [\"null\", {\n" + " \"type\": \"record\",\n" + " \"name\": \"parent\",\n" + " \"fields\": [\n" + " { \"name\": \"id\", \"type\": \"long\" },\n" + " { \"name\": \"name\", \"type\": \"string\" }\n" + " ]" + " } ]" + " }" + " ] }"; static final Schema NESTED_RECORD_SCHEMA = new Schema.Parser().parse(NESTED_RECORD_SCHEMA_STRING); static final Schema NESTED_PARENT_SCHEMA = AvroRecordConverter.getNonNullSchema(NESTED_RECORD_SCHEMA.getField("parent").schema()); static final Schema UNNESTED_OUTPUT_SCHEMA = SchemaBuilder.record("Output") .namespace("org.apache.example") .fields() .requiredLong("l1") .requiredLong("s1") .optionalLong("parentId") .endRecord(); /** Tests the case where we don't use a mapping file and just map records by name. */ @Test public void testDefaultConversion() throws Exception { // We will convert s1 from string to long (or leave it null), ignore s2, // convert s3 to from string to double, convert l1 from long to string, // and leave l2 the same. Schema input = SchemaBuilder.record("Input") .namespace("com.cloudera.edh") .fields() .nullableString("s1", "") .requiredString("s2") .requiredString("s3") .optionalLong("l1") .requiredLong("l2") .endRecord(); Schema output = SchemaBuilder.record("Output") .namespace("com.cloudera.edh") .fields() .optionalLong("s1") .optionalString("l1") .requiredLong("l2") .requiredDouble("s3") .endRecord(); AvroRecordConverter converter = new AvroRecordConverter(input, output, EMPTY_MAPPING, LocaleUtils.toLocale("en_US")); Record inputRecord = new Record(input); inputRecord.put("s1", null); inputRecord.put("s2", "blah"); inputRecord.put("s3", "5.5"); inputRecord.put("l1", null); inputRecord.put("l2", 5L); Record outputRecord = converter.convert(inputRecord); assertNull(outputRecord.get("s1")); assertNull(outputRecord.get("l1")); assertEquals(5L, outputRecord.get("l2")); assertEquals(5.5, outputRecord.get("s3")); inputRecord.put("s1", "500"); inputRecord.put("s2", "blah"); inputRecord.put("s3", "5.5e-5"); inputRecord.put("l1", 100L); inputRecord.put("l2", 2L); outputRecord = converter.convert(inputRecord); assertEquals(500L, outputRecord.get("s1")); assertEquals("100", outputRecord.get("l1")); assertEquals(2L, outputRecord.get("l2")); assertEquals(5.5e-5, outputRecord.get("s3")); } /** Tests the case where we want to default map one field and explicitly map another. */ @Test public void testExplicitMapping() throws Exception { // We will convert s1 from string to long (or leave it null), ignore s2, // convert l1 from long to string, and leave l2 the same. Schema input = NESTED_RECORD_SCHEMA; Schema parent = NESTED_PARENT_SCHEMA; Schema output = UNNESTED_OUTPUT_SCHEMA; Map<String, String> mapping = ImmutableMap.of("parent.id", "parentId"); AvroRecordConverter converter = new AvroRecordConverter(input, output, mapping); Record inputRecord = new Record(input); inputRecord.put("l1", 5L); inputRecord.put("s1", "1000"); Record parentRecord = new Record(parent); parentRecord.put("id", 200L); parentRecord.put("name", "parent"); inputRecord.put("parent", parentRecord); Record outputRecord = converter.convert(inputRecord); assertEquals(5L, outputRecord.get("l1")); assertEquals(1000L, outputRecord.get("s1")); assertEquals(200L, outputRecord.get("parentId")); } /** Tests the case where we try to convert a string to a long incorrectly. */ @Test( expected = org.apache.nifi.processors.kite.AvroRecordConverter.AvroConversionException.class) public void testIllegalConversion() throws Exception { // We will convert s1 from string to long (or leave it null), ignore s2, // convert l1 from long to string, and leave l2 the same. Schema input = SchemaBuilder.record("Input") .namespace("com.cloudera.edh") .fields() .nullableString("s1", "") .requiredString("s2") .optionalLong("l1") .requiredLong("l2") .endRecord(); Schema output = SchemaBuilder.record("Output") .namespace("com.cloudera.edh") .fields() .optionalLong("s1") .optionalString("l1") .requiredLong("l2") .endRecord(); AvroRecordConverter converter = new AvroRecordConverter(input, output, EMPTY_MAPPING); Record inputRecord = new Record(input); inputRecord.put("s1", "blah"); inputRecord.put("s2", "blah"); inputRecord.put("l1", null); inputRecord.put("l2", 5L); converter.convert(inputRecord); } @Test public void testGetUnmappedFields() throws Exception { Schema input = SchemaBuilder.record("Input") .namespace("com.cloudera.edh") .fields() .nullableString("s1", "") .requiredString("s2") .optionalLong("l1") .requiredLong("l2") .endRecord(); Schema output = SchemaBuilder.record("Output") .namespace("com.cloudera.edh") .fields() .optionalLong("field") .endRecord(); // Test the case where the field isn't mapped at all. AvroRecordConverter converter = new AvroRecordConverter(input, output, EMPTY_MAPPING); assertEquals(ImmutableList.of("field"), converter.getUnmappedFields()); // Test the case where we tried to map from a non-existent field. converter = new AvroRecordConverter(input, output, ImmutableMap.of("nonExistentField", "field")); assertEquals(ImmutableList.of("field"), converter.getUnmappedFields()); // Test the case where we tried to map from a non-existent record. converter = new AvroRecordConverter(input, output, ImmutableMap.of("parent.nonExistentField", "field")); assertEquals(ImmutableList.of("field"), converter.getUnmappedFields()); // Test a valid case converter = new AvroRecordConverter(input, output, ImmutableMap.of("l2", "field")); assertEquals(Collections.EMPTY_LIST, converter.getUnmappedFields()); } }