@Test
public void testNestedKeep5() throws StageException {
Record record = createNestedRecord();
/*
* keep all entries of a list by specifying path just upto the list
*/
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration("fields", ImmutableList.of("/USA[0]/SanFrancisco/folsom/streets[0]"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/folsom/streets[0]"));
// Its a list and it is empty. use wild card [*] to preserve the contents of the list
Assert.assertEquals(
2, resultRecord.get("/USA[0]/SanFrancisco/folsom/streets[0]").getValueAsList().size());
} finally {
runner.runDestroy();
}
}
public List<Record> runNewPartitionRecord() throws Exception {
HiveMetastoreTarget hiveTarget = new HiveMetastoreTargetBuilder().build();
TargetRunner runner =
new TargetRunner.Builder(HiveMetastoreTarget.class, hiveTarget)
.setOnRecordError(OnRecordError.TO_ERROR)
.build();
runner.runInit();
Assert.assertEquals("There should be no error records", 0, runner.getErrorRecords().size());
LinkedHashMap<String, String> partitionVals = new LinkedHashMap<String, String>();
partitionVals.put("dt", "2016");
Field newPartitionField =
HiveMetastoreUtil.newPartitionMetadataFieldBuilder(
"default", "tbl", partitionVals, "/user/hive/warehouse/tbl/dt=2016");
Record record = RecordCreator.create();
record.set(newPartitionField);
runner.runWrite(ImmutableList.of(record));
try {
return runner.getEventRecords();
} finally {
runner.runDestroy();
}
}
@Test
public void testGetFieldNull() throws Exception {
// initial data in record
Record record = RecordCreator.create();
Map<String, Field> map = new HashMap<>();
map.put("null_int", Field.create(Field.Type.INTEGER, null));
map.put("null_string", Field.create(Field.Type.STRING, null));
map.put("null_boolean", Field.create(Field.Type.BOOLEAN, null));
map.put("null_list", Field.create(Field.Type.LIST, null));
map.put("null_map", Field.create(Field.Type.MAP, null));
// original record has value in the field, so getFieldNull should return the value
map.put("null_datetime", Field.createDatetime(new Date()));
record.set(Field.create(map));
Processor processor =
new JythonProcessor(
ProcessingMode.RECORD,
"for record in records:\n"
+ " if sdcFunctions.getFieldNull(record, '/null_int') == NULL_INTEGER:\n"
+ " record.value['null_int'] = 123 \n"
+ " if sdcFunctions.getFieldNull(record, '/null_string') == NULL_STRING:\n"
+ " record.value['null_string'] = 'test' \n"
+ " if sdcFunctions.getFieldNull(record, '/null_boolean') == NULL_BOOLEAN:\n"
+ " record.value['null_boolean'] = True \n"
+ " if sdcFunctions.getFieldNull(record, '/null_list') is NULL_LIST:\n"
+ " record.value['null_list'] = ['elem1', 'elem2'] \n"
+ " if sdcFunctions.getFieldNull(record, '/null_map') == NULL_MAP:\n"
+ " record.value['null_map'] = {'x': 'X', 'y': 'Y'} \n"
+ " if sdcFunctions.getFieldNull(record, '/null_datetime') == NULL_DATETIME:\n"
+ // this should be false
" record.value['null_datetime'] = NULL_DATETIME \n"
+ " output.write(record);\n");
ScriptingProcessorTestUtil.verifyNullField(JythonProcessor.class, processor, record);
}
@Test
public void testWildCardRemove1() throws StageException {
Record record = createNestedRecord();
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration(
"fields", ImmutableList.of("/USA[*]/SanFrancisco/*/streets[*][*]/name"))
.addConfiguration("filterOperation", FilterOperation.REMOVE)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[1][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[1][1]/name"));
Assert.assertEquals(
resultRecord.get("/USA[1]/SantaMonica/cole/streets[0][0]/name").getValueAsString(), "e");
Assert.assertEquals(
resultRecord.get("/USA[1]/SantaMonica/cole/streets[0][1]/name").getValueAsString(), "f");
} finally {
runner.runDestroy();
}
}
@Test
public void testNewFieldWithTypedNull() throws Exception {
// initial data in record is empty
Record record = RecordCreator.create();
Map<String, Field> map = new HashMap<>();
record.set(Field.create(map));
Processor processor =
new JythonProcessor(
ProcessingMode.RECORD,
"for record in records:\n"
+ " record.value['null_int'] = NULL_INTEGER\n"
+ " record.value['null_long'] = NULL_LONG\n"
+ " record.value['null_float'] = NULL_FLOAT\n"
+ " record.value['null_double'] = NULL_DOUBLE\n"
+ " record.value['null_date'] = NULL_DATE\n"
+ " record.value['null_datetime'] = NULL_DATETIME\n"
+ " record.value['null_boolean'] = NULL_BOOLEAN\n"
+ " record.value['null_decimal'] = NULL_DECIMAL\n"
+ " record.value['null_byteArray'] = NULL_BYTE_ARRAY\n"
+ " record.value['null_string'] = NULL_STRING\n"
+ " record.value['null_list'] = NULL_LIST\n"
+ " record.value['null_map'] = NULL_MAP\n"
+ " record.value['null_time'] = NULL_TIME\n"
+ " output.write(record)\n");
ScriptingProcessorTestUtil.verifyTypedFieldWithNullValue(
JythonDProcessor.class, processor, record);
}
@Test
public void testKeepNonExistingFiled() throws StageException {
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration("fields", ImmutableList.of("/city"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
Map<String, Field> map = new LinkedHashMap<>();
map.put("name", Field.create("a"));
map.put("age", Field.create("b"));
map.put("streetAddress", Field.create("c"));
Record record = RecordCreator.create("s", "s:1");
record.set(Field.create(map));
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Field field = output.getRecords().get("a").get(0).get();
Assert.assertTrue(field.getValue() instanceof Map);
Map<String, Field> result = field.getValueAsMap();
Assert.assertTrue(result.size() == 0);
} finally {
runner.runDestroy();
}
}
@Test
public void testMultipleRegexMatchingSameField() throws StageException {
FieldRenamerConfig renameConfig1 = new FieldRenamerConfig();
renameConfig1.fromFieldExpression = "/sql(.*)";
renameConfig1.toFieldExpression = "/sqlRename$1";
FieldRenamerConfig renamerConfig2 = new FieldRenamerConfig();
renamerConfig2.fromFieldExpression = "/s(.*)";
renamerConfig2.toFieldExpression = "/sRename$1";
Map<String, Field> map = new LinkedHashMap<>();
map.put("sqlField", Field.create(Field.Type.STRING, "foo"));
Record record = RecordCreator.create("s", "s:1");
record.set(Field.create(Field.Type.MAP, map));
try {
FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.CONTINUE;
errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
errorHandler.existingToFieldHandling = ExistingToFieldHandling.TO_ERROR;
FieldRenamerProcessor processor =
new FieldRenamerProcessor(ImmutableList.of(renameConfig1, renamerConfig2), errorHandler);
// Test non-existent source with existing target field
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
.setOnRecordError(OnRecordError.STOP_PIPELINE)
.addOutputLane("a")
.build();
runner.runInit();
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.fail("Should throw error if multiple regex match the same field");
} catch (OnRecordErrorException e) {
Assert.assertEquals(Errors.FIELD_RENAMER_03, e.getErrorCode());
}
FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.CONTINUE;
errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.CONTINUE;
errorHandler.existingToFieldHandling = ExistingToFieldHandling.TO_ERROR;
FieldRenamerProcessor processor =
new FieldRenamerProcessor(ImmutableList.of(renameConfig1, renamerConfig2), errorHandler);
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
.setOnRecordError(OnRecordError.TO_ERROR)
.addOutputLane("a")
.build();
runner.runInit();
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record r = output.getRecords().get("a").get(0);
Assert.assertTrue(r.has("/sqlField"));
}
@Test
public void testRenameMultipleListElementsWithConstIdxExpr() throws StageException {
FieldRenamerConfig renameConfig1 = new FieldRenamerConfig();
renameConfig1.fromFieldExpression = "/listOfInts[0]";
renameConfig1.toFieldExpression = "/nonExisting0";
FieldRenamerConfig renameConfig2 = new FieldRenamerConfig();
renameConfig2.fromFieldExpression = "/listOfInts[1]";
renameConfig2.toFieldExpression = "/nonExisting1";
FieldRenamerConfig renameConfig3 = new FieldRenamerConfig();
renameConfig3.fromFieldExpression = "/listOfInts[2]";
renameConfig3.toFieldExpression = "/nonExisting2";
FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.TO_ERROR;
errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
errorHandler.existingToFieldHandling = ExistingToFieldHandling.REPLACE;
// Reverse order in configuration so as to preserve array indices
FieldRenamerProcessor processor =
new FieldRenamerProcessor(
ImmutableList.of(renameConfig3, renameConfig2, renameConfig1), errorHandler);
// Test non-existent source with existing target field
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
.addOutputLane("a")
.build();
runner.runInit();
try {
Map<String, Field> map = new LinkedHashMap<>();
map.put(
"listOfInts",
Field.create(
ImmutableList.of(
Field.create(Field.Type.INTEGER, 1),
Field.create(Field.Type.INTEGER, 2),
Field.create(Field.Type.INTEGER, 3))));
Record record = RecordCreator.create("s", "s:1");
record.set(Field.create(map));
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Map<String, Field> result = output.getRecords().get("a").get(0).get().getValueAsMap();
Assert.assertTrue(result.containsKey("listOfInts"));
Assert.assertTrue(result.get("listOfInts").getValueAsList().isEmpty());
Assert.assertTrue(result.containsKey("nonExisting0"));
Assert.assertTrue(result.containsKey("nonExisting1"));
Assert.assertTrue(result.containsKey("nonExisting2"));
Assert.assertEquals(1, result.get("nonExisting0").getValueAsInteger());
Assert.assertEquals(2, result.get("nonExisting1").getValueAsInteger());
Assert.assertEquals(3, result.get("nonExisting2").getValueAsInteger());
} finally {
runner.runDestroy();
}
}
public static List<Record> createStringRecords() {
List<Record> records = new ArrayList<>(9);
for (int i = 0; i < 9; i++) {
Record r = RecordCreator.create("s", "s:1", (TEST_STRING + i).getBytes(), MIME);
r.set(Field.create((TEST_STRING + i)));
records.add(r);
}
return records;
}
@Test
public void testLookup() throws Exception {
String ip = "128.101.101.101";
List<GeolocationFieldConfig> configs = new ArrayList<>();
GeolocationFieldConfig config;
config = new GeolocationFieldConfig();
config.inputFieldName = "/ipAsInt";
config.outputFieldName = "/intIpCountry";
config.targetType = GeolocationField.COUNTRY_NAME;
configs.add(config);
config = new GeolocationFieldConfig();
config.inputFieldName = "/ipAsIntString";
config.outputFieldName = "/intStringIpCountry";
config.targetType = GeolocationField.COUNTRY_NAME;
configs.add(config);
config = new GeolocationFieldConfig();
config.inputFieldName = "/ipAsString";
config.outputFieldName = "/stringIpCountry";
config.targetType = GeolocationField.COUNTRY_NAME;
configs.add(config);
ProcessorRunner runner =
new ProcessorRunner.Builder(GeolocationDProcessor.class)
.addConfiguration("fieldTypeConverterConfigs", configs)
.addConfiguration("geoIP2DBFile", databaseFile.getAbsolutePath())
.addOutputLane("a")
.build();
runner.runInit();
try {
Map<String, Field> map = new LinkedHashMap<>();
map.put("ipAsInt", Field.create(GeolocationProcessor.ipAsStringToInt(ip)));
map.put(
"ipAsIntString", Field.create(String.valueOf(GeolocationProcessor.ipAsStringToInt(ip))));
map.put("ipAsString", Field.create(ip));
Record record = RecordCreator.create("s", "s:1");
record.set(Field.create(map));
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(0, runner.getErrorRecords().size());
Assert.assertEquals(1, output.getRecords().get("a").size());
Field field = output.getRecords().get("a").get(0).get();
Assert.assertTrue(field.getValue() instanceof Map);
Map<String, Field> result = field.getValueAsMap();
Assert.assertEquals(String.valueOf(result), 6, result.size());
Assert.assertEquals(
"United States",
Utils.checkNotNull(result.get("intStringIpCountry"), "intStringIpCountry").getValue());
Assert.assertEquals(
"United States",
Utils.checkNotNull(result.get("intIpCountry"), "intIpCountry").getValue());
Assert.assertEquals(
"United States",
Utils.checkNotNull(result.get("stringIpCountry"), "stringIpCountry").getValue());
} finally {
runner.runDestroy();
}
}
static void checkRecords(List<Record> expectedRecords, List<Record> actualRecords)
throws Exception {
Assert.assertEquals(
"Record Size Does not match.", expectedRecords.size(), actualRecords.size());
for (int i = 0; i < actualRecords.size(); i++) {
Record actualRecord = actualRecords.get(i);
Record expectedRecord = expectedRecords.get(i);
checkField("", expectedRecord.get(), actualRecord.get());
}
}
public static List<Record> produce20Records() throws IOException {
List<Record> list = new ArrayList<>();
for (int i = 0; i < 20; i++) {
Record record = RecordCreator.create();
Map<String, Field> map = new HashMap<>();
map.put("name", Field.create("NAME" + i));
map.put("lastStatusChange", Field.create(i));
record.set(Field.create(map));
list.add(record);
}
return list;
}
@Test
public void testNewFieldWithTypedNull() throws Exception {
// initial data in record is empty
Record record = RecordCreator.create();
Map<String, Field> map = new HashMap<>();
record.set(Field.create(map));
final String script =
Resources.toString(Resources.getResource("AssignTypedNullField.groovy"), Charsets.UTF_8);
Processor processor = new GroovyProcessor(ProcessingMode.BATCH, script);
ScriptingProcessorTestUtil.verifyTypedFieldWithNullValue(
GroovyProcessor.class, processor, record);
}
@Test
public void testRegexInComplexListType() throws StageException {
FieldRenamerConfig renameConfig = new FieldRenamerConfig();
// Any field containing a non-word character should be in single quotes
renameConfig.fromFieldExpression = "/(*)[(*)]/'SQL(#)(.*)'";
renameConfig.toFieldExpression = "/$1[$2]/SQL$4";
FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.TO_ERROR;
errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
errorHandler.existingToFieldHandling = ExistingToFieldHandling.TO_ERROR;
FieldRenamerProcessor processor =
new FieldRenamerProcessor(ImmutableList.of(renameConfig), errorHandler);
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
.setOnRecordError(OnRecordError.TO_ERROR)
.addOutputLane("a")
.build();
runner.runInit();
try {
Map<String, Field> innerMap1 = new LinkedHashMap<>();
innerMap1.put("SQL#1", Field.create(Field.Type.STRING, "foo1"));
Map<String, Field> innerMap2 = new LinkedHashMap<>();
innerMap2.put("SQL#2", Field.create(Field.Type.STRING, "foo2"));
List<Field> list = new LinkedList<>();
list.add(Field.create(innerMap1));
list.add(Field.create(innerMap2));
Map<String, Field> map = new HashMap<>();
map.put("list", Field.create(list));
Record record = RecordCreator.create("s", "s:1");
record.set(Field.create(map));
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record r = output.getRecords().get("a").get(0);
Assert.assertFalse(r.getEscapedFieldPaths().contains("/list[0]/'SQL#1'"));
Assert.assertFalse(r.getEscapedFieldPaths().contains("/list[1]/'SQL#2'"));
Assert.assertTrue(r.getEscapedFieldPaths().contains("/list[0]/SQL1"));
Assert.assertTrue(r.getEscapedFieldPaths().contains("/list[1]/SQL2"));
} finally {
runner.runDestroy();
}
}
protected static void insertRows(String insertTemplate, List<Record> records)
throws SQLException {
try (Statement st = connection.createStatement()) {
for (Record record : records) {
List<String> values = new ArrayList<>();
for (String fieldPath : record.getEscapedFieldPaths()) {
// Skip root field
if (!fieldPath.equals("")) {
values.add(getStringRepOfFieldValueForInsert(record.get(fieldPath)));
}
}
st.addBatch(String.format(insertTemplate, values.toArray()));
}
st.executeBatch();
}
}
private static void handleNonRepeatedField(
Record record,
Map<String, Field> valueAsMap,
String fieldPath,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Map<String, Object> defaultValueMap,
Descriptors.Descriptor desc,
Descriptors.FieldDescriptor f,
DynamicMessage.Builder builder)
throws DataGeneratorException {
Object val;
String keyName = f.getName();
if (valueAsMap.containsKey(keyName)) {
val =
getValue(
f,
valueAsMap.get(keyName),
record,
fieldPath + FORWARD_SLASH + f.getName(),
messageTypeToExtensionMap,
defaultValueMap);
} else {
// record does not contain field, look up default value
String key = desc.getFullName() + "." + f.getName();
if (!defaultValueMap.containsKey(key) && !f.isOptional()) {
throw new DataGeneratorException(Errors.PROTOBUF_04, record.getHeader().getSourceId(), key);
}
val = defaultValueMap.get(key);
}
if (val != null) {
builder.setField(f, val);
}
}
@SuppressWarnings("unchecked")
public List<Record> runNewTableRecord() throws Exception {
HiveMetastoreTarget hiveTarget = new HiveMetastoreTargetBuilder().build();
TargetRunner runner =
new TargetRunner.Builder(HiveMetastoreTarget.class, hiveTarget)
.setOnRecordError(OnRecordError.STOP_PIPELINE)
.build();
runner.runInit();
LinkedHashMap<String, HiveTypeInfo> columns = new LinkedHashMap<>();
columns.put("name", HiveType.STRING.getSupport().generateHiveTypeInfoFromResultSet("STRING"));
columns.put(
"surname", HiveType.STRING.getSupport().generateHiveTypeInfoFromResultSet("STRING"));
LinkedHashMap<String, HiveTypeInfo> partitions = new LinkedHashMap<>();
partitions.put("dt", HiveType.STRING.getSupport().generateHiveTypeInfoFromResultSet("STRING"));
Field newTableField =
HiveMetastoreUtil.newSchemaMetadataFieldBuilder(
"default",
"tbl",
columns,
partitions,
true,
BaseHiveIT.getDefaultWareHouseDir(),
HiveMetastoreUtil.generateAvroSchema(columns, "tbl"));
Record record = RecordCreator.create();
record.set(newTableField);
Assert.assertTrue(HiveMetastoreUtil.isSchemaChangeRecord(record));
runner.runWrite(ImmutableList.of(record));
assertTableStructure(
"default.tbl",
ImmutablePair.of("tbl.name", Types.VARCHAR),
ImmutablePair.of("tbl.surname", Types.VARCHAR),
ImmutablePair.of("tbl.dt", Types.VARCHAR));
try {
return runner.getEventRecords();
} finally {
runner.runDestroy();
}
}
public static String csvRecordToString(Record r, CSVFormat csvFormat) throws IOException {
StringWriter stringWriter = new StringWriter();
CSVPrinter csvPrinter = new CSVPrinter(stringWriter, csvFormat);
csvPrinter.printRecord(CsvUtil.fieldToCsv(r.get()));
csvPrinter.flush();
csvPrinter.close();
return stringWriter.toString();
}
@Test
public void testNestedKeep6() throws StageException {
Record record = createNestedRecord();
/*
* keep all entries of a map by specifying path just upto the map
*/
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration("fields", ImmutableList.of("/USA[0]/SanFrancisco/noe"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe"));
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[0][0]/name").getValueAsString(), "a");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[0][1]/name").getValueAsString(), "b");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[1][0]/name").getValueAsString(), "c");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[1][1]/name").getValueAsString(), "d");
} finally {
runner.runDestroy();
}
}
/**
* Serializes a record to a protobuf message using the specified descriptor.
*
* @param record Record to serialize
* @param desc Protobuf descriptor
* @param messageTypeToExtensionMap Protobuf extension map
* @param defaultValueMap Protobuf default field values
* @return serialized message
* @throws DataGeneratorException
*/
public static DynamicMessage sdcFieldToProtobufMsg(
Record record,
Descriptors.Descriptor desc,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Map<String, Object> defaultValueMap)
throws DataGeneratorException {
return sdcFieldToProtobufMsg(
record, record.get(), "", desc, messageTypeToExtensionMap, defaultValueMap);
}
@Test
public void testGetFieldNull() throws Exception {
// initial data in record
Record record = RecordCreator.create();
Map<String, Field> map = new HashMap<>();
map.put("null_int", Field.create(Field.Type.INTEGER, null));
map.put("null_string", Field.create(Field.Type.STRING, null));
map.put("null_boolean", Field.create(Field.Type.BOOLEAN, null));
map.put("null_list", Field.create(Field.Type.LIST, null));
map.put("null_map", Field.create(Field.Type.MAP, null));
// original record has value in the field, so getFieldNull should return the value
map.put("null_datetime", Field.createDatetime(new Date()));
record.set(Field.create(map));
final String script =
Resources.toString(Resources.getResource("GetFieldNullScript.groovy"), Charsets.UTF_8);
Processor processor = new GroovyProcessor(ProcessingMode.BATCH, script);
ScriptingProcessorTestUtil.verifyNullField(GroovyProcessor.class, processor, record);
}
@Test
public void testTargetFieldExistsReplace() throws StageException {
// Standard overwrite condition. Source and target fields exist
FieldRenamerConfig renameConfig = new FieldRenamerConfig();
renameConfig.fromFieldExpression = "/existing";
renameConfig.toFieldExpression = "/overwrite";
FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.CONTINUE;
errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
errorHandler.existingToFieldHandling = ExistingToFieldHandling.REPLACE;
FieldRenamerProcessor processor =
new FieldRenamerProcessor(ImmutableList.of(renameConfig), errorHandler);
// Test non-existent source with existing target field
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
.addOutputLane("a")
.build();
runner.runInit();
try {
Map<String, Field> map = new LinkedHashMap<>();
map.put("existing", Field.create(Field.Type.STRING, "foo"));
map.put("overwrite", Field.create(Field.Type.STRING, "bar"));
Record record = RecordCreator.create("s", "s:1");
record.set(Field.create(map));
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Field field = output.getRecords().get("a").get(0).get();
Assert.assertTrue(field.getValue() instanceof Map);
Map<String, Field> result = field.getValueAsMap();
Assert.assertEquals(String.valueOf(result), 1, result.size());
Assert.assertTrue(result.containsKey("overwrite"));
Assert.assertTrue(!result.containsKey("existing"));
Assert.assertEquals("foo", result.get("overwrite").getValue());
} finally {
runner.runDestroy();
}
}
@Test
public void testSourceWithQuotedSubstring() throws StageException {
// source should be processed as quoted string.
FieldRenamerConfig renameConfig1 = new FieldRenamerConfig();
renameConfig1.fromFieldExpression = "/'attr|OrderNum'";
renameConfig1.toFieldExpression = "/theOrderNum";
FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.CONTINUE;
errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
errorHandler.existingToFieldHandling = ExistingToFieldHandling.REPLACE;
FieldRenamerProcessor processor =
new FieldRenamerProcessor(ImmutableList.of(renameConfig1), errorHandler);
// Test non-existent source with existing target field
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
.addOutputLane("a")
.build();
runner.runInit();
try {
Map<String, Field> map = new LinkedHashMap<>();
map.put("attr|OrderNum", Field.create(Field.Type.STRING, "foo"));
Record record = RecordCreator.create("s", "s:1");
record.set(Field.create(map));
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Field field = output.getRecords().get("a").get(0).get();
Assert.assertTrue(field.getValue() instanceof Map);
Map<String, Field> result = field.getValueAsMap();
Assert.assertEquals(String.valueOf(result), 1, result.size());
Assert.assertFalse(result.containsKey("/'attr|OrderNum'"));
Assert.assertFalse(result.containsKey("'attr|OrderNum'"));
Assert.assertFalse(result.containsKey("attr|OrderNum"));
Assert.assertTrue(result.containsKey("theOrderNum"));
} finally {
runner.runDestroy();
}
}
@Test
public void testDifferentMatchesRegex() throws StageException {
FieldRenamerConfig renameConfig = new FieldRenamerConfig();
// Any field containing a non-word character should be in single quotes
renameConfig.fromFieldExpression = "/'(.*)(#)(.*)'";
renameConfig.toFieldExpression = "/$1hash$3";
FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.TO_ERROR;
errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
errorHandler.existingToFieldHandling = ExistingToFieldHandling.TO_ERROR;
FieldRenamerProcessor processor =
new FieldRenamerProcessor(ImmutableList.of(renameConfig), errorHandler);
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
.setOnRecordError(OnRecordError.STOP_PIPELINE)
.addOutputLane("a")
.build();
runner.runInit();
Map<String, Field> map = new LinkedHashMap<>();
map.put("#abcd", Field.create("hashabcd"));
Record record1 = RecordCreator.create("s", "s:1");
record1.set(Field.create(Field.Type.MAP, map));
map = new LinkedHashMap<>();
map.put("ab#cd", Field.create("abhashcd"));
Record record2 = RecordCreator.create("s", "s:2");
record2.set(Field.create(Field.Type.MAP, map));
map = new LinkedHashMap<>();
map.put("abcd#", Field.create("abcdhash"));
Record record3 = RecordCreator.create("s", "s:3");
record3.set(Field.create(Field.Type.MAP, map));
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record1, record2, record3));
Assert.assertEquals(3, output.getRecords().get("a").size());
for (Record record : output.getRecords().get("a")) {
Map<String, Field> fieldMap = record.get().getValueAsMap();
for (Map.Entry<String, Field> fieldEntry : fieldMap.entrySet()) {
Assert.assertEquals(fieldEntry.getKey(), fieldEntry.getValue().getValueAsString());
}
}
} finally {
runner.runDestroy();
}
}
public static List<Record> createCsvRecords() throws IOException {
List<Record> records = new ArrayList<>();
String line;
BufferedReader bufferedReader =
new BufferedReader(
new FileReader(
FlumeTestUtil.class.getClassLoader().getResource("testFlumeTarget.csv").getFile()));
while ((line = bufferedReader.readLine()) != null) {
String columns[] = line.split(",");
List<Field> list = new ArrayList<>();
for (String column : columns) {
Map<String, Field> map = new LinkedHashMap<>();
map.put("value", Field.create(column));
list.add(Field.create(map));
}
Record record = RecordCreator.create("s", "s:1", null, null);
record.set(Field.create(list));
records.add(record);
}
return records;
}
@Test
public void testNestedKeep3() throws StageException {
/*
* try to retain the second element from the root array list.
* Make sure that this turns out to be the first element in the resulting record
*/
Record record = createNestedRecord();
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration(
"fields", ImmutableList.of("/USA[1]/SantaMonica/cole/streets[0][1]/name"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertTrue(resultRecord.has("/USA[0]/SantaMonica/cole/streets[0][0]/name"));
Assert.assertEquals(
"f", resultRecord.get("/USA[0]/SantaMonica/cole/streets[0][0]/name").getValueAsString());
Assert.assertFalse(resultRecord.has("/USA[0]/SantaMonica/cole/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco"));
Assert.assertFalse(resultRecord.has("/USA[1]"));
} finally {
runner.runDestroy();
}
}
@Test
public void testNonExistingSourceAndTargetFields() throws StageException {
// If neither the source or target fields exist, then field renaming is a noop, and should
// succeed
FieldRenamerConfig renameConfig = new FieldRenamerConfig();
renameConfig.fromFieldExpression = "/nonExisting";
renameConfig.toFieldExpression = "/alsoNonExisting";
FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.CONTINUE;
errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
errorHandler.existingToFieldHandling = ExistingToFieldHandling.TO_ERROR;
FieldRenamerProcessor processor =
new FieldRenamerProcessor(ImmutableList.of(renameConfig), errorHandler);
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
.addOutputLane("a")
.build();
runner.runInit();
try {
Map<String, Field> map = new LinkedHashMap<>();
map.put("name", Field.create(Field.Type.STRING, null));
Record record = RecordCreator.create("s", "s:1");
record.set(Field.create(map));
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Field field = output.getRecords().get("a").get(0).get();
Assert.assertTrue(field.getValue() instanceof Map);
Map<String, Field> result = field.getValueAsMap();
Assert.assertEquals(String.valueOf(result), 1, result.size());
Assert.assertTrue(result.containsKey("name"));
Assert.assertEquals(null, result.get("name").getValue());
} finally {
runner.runDestroy();
}
}
@Test
public void testNestedKeep2() throws StageException {
/*
* In a deep nested record try to retain arbitrary paths
*/
Record record = createNestedRecord();
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration(
"fields",
ImmutableList.of(
"/USA[0]/SanFrancisco/noe/streets[1][1]/name",
"/USA[1]/SantaMonica/cole/streets[0][1]/name"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][0]/name"));
Assert.assertEquals(
"d", resultRecord.get("/USA[0]/SanFrancisco/noe/streets[0][0]/name").getValueAsString());
Assert.assertTrue(resultRecord.has("/USA[1]/SantaMonica/cole/streets[0][0]/name"));
Assert.assertEquals(
"f", resultRecord.get("/USA[1]/SantaMonica/cole/streets[0][0]/name").getValueAsString());
Assert.assertFalse(resultRecord.has("/USA[1]/SantaMonica/cole/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/folsom/streets[0][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/folsom/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[1][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[1][1]/name"));
} finally {
runner.runDestroy();
}
}
@Test
public void testTargetFieldExistsError() throws StageException {
// If overwrite is set to false, overwriting should result in an error
FieldRenamerConfig renameConfig = new FieldRenamerConfig();
renameConfig.fromFieldExpression = "/existing";
renameConfig.toFieldExpression = "/overwrite";
FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.CONTINUE;
errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
errorHandler.existingToFieldHandling = ExistingToFieldHandling.TO_ERROR;
FieldRenamerProcessor processor =
new FieldRenamerProcessor(ImmutableList.of(renameConfig), errorHandler);
// Test non-existent source with existing target field
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
.setOnRecordError(OnRecordError.TO_ERROR)
.addOutputLane("a")
.build();
runner.runInit();
try {
Map<String, Field> map = new LinkedHashMap<>();
map.put("existing", Field.create(Field.Type.STRING, "foo"));
map.put("overwrite", Field.create(Field.Type.STRING, "bar"));
Record record = RecordCreator.create("s", "s:1");
record.set(Field.create(map));
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(0, output.getRecords().get("a").size());
Assert.assertEquals(1, runner.getErrorRecords().size());
} finally {
runner.runDestroy();
}
}
@Test
public void testRegexInNonComplexType() throws StageException {
FieldRenamerConfig renameConfig = new FieldRenamerConfig();
// Any field containing a non-word character should be in single quotes
renameConfig.fromFieldExpression = "/'sql(#)(.*)'";
renameConfig.toFieldExpression = "/sql$2";
FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.CONTINUE;
errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
errorHandler.existingToFieldHandling = ExistingToFieldHandling.TO_ERROR;
FieldRenamerProcessor processor =
new FieldRenamerProcessor(ImmutableList.of(renameConfig), errorHandler);
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
.setOnRecordError(OnRecordError.TO_ERROR)
.addOutputLane("a")
.build();
runner.runInit();
try {
Map<String, Field> map = new LinkedHashMap<>();
map.put("sql#1", Field.create(Field.Type.STRING, "foo"));
Record record = RecordCreator.create("s", "s:1");
record.set(Field.create(Field.Type.MAP, map));
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record r = output.getRecords().get("a").get(0);
Assert.assertFalse(r.has("/'sql#1'"));
Assert.assertTrue(r.has("/sql1"));
} finally {
runner.runDestroy();
}
}