@Test
public void testNestedKeep6() throws StageException {
Record record = createNestedRecord();
/*
* keep all entries of a map by specifying path just upto the map
*/
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration("fields", ImmutableList.of("/USA[0]/SanFrancisco/noe"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe"));
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[0][0]/name").getValueAsString(), "a");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[0][1]/name").getValueAsString(), "b");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[1][0]/name").getValueAsString(), "c");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[1][1]/name").getValueAsString(), "d");
} finally {
runner.runDestroy();
}
}
@Test
public void testCategoryRegexInNonComplexType() throws StageException {
FieldRenamerConfig renameConfig = new FieldRenamerConfig();
// Any field containing a non-word character should be in single quotes
renameConfig.fromFieldExpression = "/'(.*)[#&@|](.*)'";
renameConfig.toFieldExpression = "/$1_$2";
FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.CONTINUE;
errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
errorHandler.existingToFieldHandling = ExistingToFieldHandling.APPEND_NUMBERS;
FieldRenamerProcessor processor =
new FieldRenamerProcessor(ImmutableList.of(renameConfig), errorHandler);
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
.setOnRecordError(OnRecordError.TO_ERROR)
.addOutputLane("a")
.build();
runner.runInit();
try {
Map<String, Field> map = new LinkedHashMap<>();
map.put("a#b", Field.create(Field.Type.STRING, "foo1"));
map.put("a_b", Field.create(Field.Type.STRING, "foo2"));
map.put("a&b", Field.create(Field.Type.STRING, "foo3"));
map.put("a|b", Field.create(Field.Type.STRING, "foo4"));
map.put("a@b", Field.create(Field.Type.STRING, "foo5"));
Record record = RecordCreator.create("s", "s:1");
record.set(Field.create(Field.Type.MAP, map));
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record r = output.getRecords().get("a").get(0);
Assert.assertFalse(r.has("/'a#b'"));
Assert.assertFalse(r.has("/'a&b'"));
Assert.assertFalse(r.has("/'a|b'"));
Assert.assertFalse(r.has("/'a&b'"));
Assert.assertTrue(r.has("/a_b"));
Assert.assertEquals("foo2", r.get("/a_b").getValueAsString());
Assert.assertTrue(r.has("/a_b1"));
Assert.assertEquals("foo1", r.get("/a_b1").getValueAsString());
Assert.assertTrue(r.has("/a_b2"));
Assert.assertEquals("foo3", r.get("/a_b2").getValueAsString());
Assert.assertTrue(r.has("/a_b3"));
Assert.assertEquals("foo4", r.get("/a_b3").getValueAsString());
Assert.assertTrue(r.has("/a_b4"));
Assert.assertEquals("foo5", r.get("/a_b4").getValueAsString());
} finally {
runner.runDestroy();
}
}
@Test
public void testWildCardRemove1() throws StageException {
Record record = createNestedRecord();
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration(
"fields", ImmutableList.of("/USA[*]/SanFrancisco/*/streets[*][*]/name"))
.addConfiguration("filterOperation", FilterOperation.REMOVE)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[1][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[1][1]/name"));
Assert.assertEquals(
resultRecord.get("/USA[1]/SantaMonica/cole/streets[0][0]/name").getValueAsString(), "e");
Assert.assertEquals(
resultRecord.get("/USA[1]/SantaMonica/cole/streets[0][1]/name").getValueAsString(), "f");
} finally {
runner.runDestroy();
}
}
static void checkRecords(List<Record> expectedRecords, List<Record> actualRecords)
throws Exception {
Assert.assertEquals(
"Record Size Does not match.", expectedRecords.size(), actualRecords.size());
for (int i = 0; i < actualRecords.size(); i++) {
Record actualRecord = actualRecords.get(i);
Record expectedRecord = expectedRecords.get(i);
checkField("", expectedRecord.get(), actualRecord.get());
}
}
@Test
public void testNestedKeep5() throws StageException {
Record record = createNestedRecord();
/*
* keep all entries of a list by specifying path just upto the list
*/
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration("fields", ImmutableList.of("/USA[0]/SanFrancisco/folsom/streets[0]"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/folsom/streets[0]"));
// Its a list and it is empty. use wild card [*] to preserve the contents of the list
Assert.assertEquals(
2, resultRecord.get("/USA[0]/SanFrancisco/folsom/streets[0]").getValueAsList().size());
} finally {
runner.runDestroy();
}
}
@Test
public void testNestedKeep3() throws StageException {
/*
* try to retain the second element from the root array list.
* Make sure that this turns out to be the first element in the resulting record
*/
Record record = createNestedRecord();
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration(
"fields", ImmutableList.of("/USA[1]/SantaMonica/cole/streets[0][1]/name"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertTrue(resultRecord.has("/USA[0]/SantaMonica/cole/streets[0][0]/name"));
Assert.assertEquals(
"f", resultRecord.get("/USA[0]/SantaMonica/cole/streets[0][0]/name").getValueAsString());
Assert.assertFalse(resultRecord.has("/USA[0]/SantaMonica/cole/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco"));
Assert.assertFalse(resultRecord.has("/USA[1]"));
} finally {
runner.runDestroy();
}
}
public static String csvRecordToString(Record r, CSVFormat csvFormat) throws IOException {
StringWriter stringWriter = new StringWriter();
CSVPrinter csvPrinter = new CSVPrinter(stringWriter, csvFormat);
csvPrinter.printRecord(CsvUtil.fieldToCsv(r.get()));
csvPrinter.flush();
csvPrinter.close();
return stringWriter.toString();
}
/**
* Serializes a record to a protobuf message using the specified descriptor.
*
* @param record Record to serialize
* @param desc Protobuf descriptor
* @param messageTypeToExtensionMap Protobuf extension map
* @param defaultValueMap Protobuf default field values
* @return serialized message
* @throws DataGeneratorException
*/
public static DynamicMessage sdcFieldToProtobufMsg(
Record record,
Descriptors.Descriptor desc,
Map<String, Set<Descriptors.FieldDescriptor>> messageTypeToExtensionMap,
Map<String, Object> defaultValueMap)
throws DataGeneratorException {
return sdcFieldToProtobufMsg(
record, record.get(), "", desc, messageTypeToExtensionMap, defaultValueMap);
}
@Test
public void testTargetFieldExistsAppendNumbers() throws StageException {
FieldRenamerConfig renameConfig = new FieldRenamerConfig();
// Any field containing a non-word character should be in single quotes
renameConfig.fromFieldExpression = "/field";
renameConfig.toFieldExpression = "/col";
FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.CONTINUE;
errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
errorHandler.existingToFieldHandling = ExistingToFieldHandling.APPEND_NUMBERS;
FieldRenamerProcessor processor =
new FieldRenamerProcessor(ImmutableList.of(renameConfig), errorHandler);
// Test non-existent source with existing target field
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
.addOutputLane("a")
.build();
runner.runInit();
try {
Map<String, Field> map = new LinkedHashMap<>();
map.put("field", Field.create(Field.Type.STRING, "field"));
map.put("col", Field.create(Field.Type.STRING, "col"));
Record record = RecordCreator.create("s", "s:1");
record.set(Field.create(Field.Type.MAP, map));
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record r = output.getRecords().get("a").get(0);
Assert.assertFalse(r.has("/field"));
Assert.assertTrue(r.has("/col"));
Assert.assertTrue(r.has("/col1"));
Assert.assertEquals("col", r.get("/col").getValueAsString());
Assert.assertEquals("field", r.get("/col1").getValueAsString());
} finally {
runner.runDestroy();
}
}
@Test
public void testNestedKeep2() throws StageException {
/*
* In a deep nested record try to retain arbitrary paths
*/
Record record = createNestedRecord();
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration(
"fields",
ImmutableList.of(
"/USA[0]/SanFrancisco/noe/streets[1][1]/name",
"/USA[1]/SantaMonica/cole/streets[0][1]/name"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][0]/name"));
Assert.assertEquals(
"d", resultRecord.get("/USA[0]/SanFrancisco/noe/streets[0][0]/name").getValueAsString());
Assert.assertTrue(resultRecord.has("/USA[1]/SantaMonica/cole/streets[0][0]/name"));
Assert.assertEquals(
"f", resultRecord.get("/USA[1]/SantaMonica/cole/streets[0][0]/name").getValueAsString());
Assert.assertFalse(resultRecord.has("/USA[1]/SantaMonica/cole/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/folsom/streets[0][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/folsom/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[1][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[1][1]/name"));
} finally {
runner.runDestroy();
}
}
@Test
public void testWildCardKeep2() throws StageException {
/*
* Use wil card in array within array
*/
Record record = createNestedRecord();
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration("fields", ImmutableList.of("/USA[0]/SanFrancisco/noe/streets[0][*]"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0]"));
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[0][0]/name").getValueAsString(), "a");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[0][1]/name").getValueAsString(), "b");
} finally {
runner.runDestroy();
}
/*
* Use wil card in array
*/
record = createNestedRecord();
runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration("fields", ImmutableList.of("/USA[0]/SanFrancisco/noe/streets[*]"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe/streets"));
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[0][0]/name").getValueAsString(), "a");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[0][1]/name").getValueAsString(), "b");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[1][0]/name").getValueAsString(), "c");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[1][1]/name").getValueAsString(), "d");
} finally {
runner.runDestroy();
}
}
@Test
public void testDifferentMatchesRegex() throws StageException {
FieldRenamerConfig renameConfig = new FieldRenamerConfig();
// Any field containing a non-word character should be in single quotes
renameConfig.fromFieldExpression = "/'(.*)(#)(.*)'";
renameConfig.toFieldExpression = "/$1hash$3";
FieldRenamerProcessorErrorHandler errorHandler = new FieldRenamerProcessorErrorHandler();
errorHandler.nonExistingFromFieldHandling = OnStagePreConditionFailure.TO_ERROR;
errorHandler.multipleFromFieldsMatching = OnStagePreConditionFailure.TO_ERROR;
errorHandler.existingToFieldHandling = ExistingToFieldHandling.TO_ERROR;
FieldRenamerProcessor processor =
new FieldRenamerProcessor(ImmutableList.of(renameConfig), errorHandler);
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldRenamerDProcessor.class, processor)
.setOnRecordError(OnRecordError.STOP_PIPELINE)
.addOutputLane("a")
.build();
runner.runInit();
Map<String, Field> map = new LinkedHashMap<>();
map.put("#abcd", Field.create("hashabcd"));
Record record1 = RecordCreator.create("s", "s:1");
record1.set(Field.create(Field.Type.MAP, map));
map = new LinkedHashMap<>();
map.put("ab#cd", Field.create("abhashcd"));
Record record2 = RecordCreator.create("s", "s:2");
record2.set(Field.create(Field.Type.MAP, map));
map = new LinkedHashMap<>();
map.put("abcd#", Field.create("abcdhash"));
Record record3 = RecordCreator.create("s", "s:3");
record3.set(Field.create(Field.Type.MAP, map));
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record1, record2, record3));
Assert.assertEquals(3, output.getRecords().get("a").size());
for (Record record : output.getRecords().get("a")) {
Map<String, Field> fieldMap = record.get().getValueAsMap();
for (Map.Entry<String, Field> fieldEntry : fieldMap.entrySet()) {
Assert.assertEquals(fieldEntry.getKey(), fieldEntry.getValue().getValueAsString());
}
}
} finally {
runner.runDestroy();
}
}
@Test
public void testSingleRecordMap() throws Exception {
List<JdbcFieldColumnMapping> columnMappings =
ImmutableList.of(new JdbcFieldColumnMapping("P_ID", "/p_id"));
JdbcLookupDProcessor processor = new JdbcLookupDProcessor();
processor.hikariConfigBean = createConfigBean(h2ConnectionString, username, password);
ProcessorRunner processorRunner =
new ProcessorRunner.Builder(JdbcLookupDProcessor.class, processor)
.addConfiguration("query", mapQuery)
.addConfiguration("columnMappings", columnMappings)
.addConfiguration("maxClobSize", 1000)
.addConfiguration("maxBlobSize", 1000)
.addOutputLane("lane")
.build();
Record record = RecordCreator.create();
LinkedHashMap<String, Field> fields = new LinkedHashMap<>();
fields.put("first_name", Field.create("Adam"));
fields.put("last_name", Field.create("Kunicki"));
record.set(Field.create(fields));
List<Record> singleRecord = ImmutableList.of(record);
processorRunner.runInit();
try {
StageRunner.Output output = processorRunner.runProcess(singleRecord);
Assert.assertEquals(1, output.getRecords().get("lane").size());
record = output.getRecords().get("lane").get(0);
Assert.assertNotEquals(null, record.get("/p_id"));
Assert.assertEquals(1, record.get("/p_id").getValueAsInteger());
} finally {
processorRunner.runDestroy();
}
}
protected static void insertRows(String insertTemplate, List<Record> records)
throws SQLException {
try (Statement st = connection.createStatement()) {
for (Record record : records) {
List<String> values = new ArrayList<>();
for (String fieldPath : record.getEscapedFieldPaths()) {
// Skip root field
if (!fieldPath.equals("")) {
values.add(getStringRepOfFieldValueForInsert(record.get(fieldPath)));
}
}
st.addBatch(String.format(insertTemplate, values.toArray()));
}
st.executeBatch();
}
}
@Override
protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
Field field = record.get(fieldPath);
String[] splits = null;
ErrorCode error = null;
if (field == null || field.getValue() == null) {
error = Errors.SPLITTER_01;
} else {
String str;
try {
str = field.getValueAsString();
} catch (IllegalArgumentException e) {
throw new OnRecordErrorException(Errors.SPLITTER_04, fieldPath, field.getType().name());
}
splits = str.split(separatorStr, fieldPaths.length);
if (splits.length < fieldPaths.length) {
error = Errors.SPLITTER_02;
}
}
if (error == null || onStagePreConditionFailure == OnStagePreConditionFailure.CONTINUE) {
for (int i = 0; i < fieldPaths.length; i++) {
try {
if (splits != null && splits.length > i) {
record.set(fieldPaths[i], Field.create(splits[i]));
} else {
record.set(fieldPaths[i], Field.create(Field.Type.STRING, null));
}
} catch (IllegalArgumentException e) {
throw new OnRecordErrorException(
Errors.SPLITTER_05, fieldPath, record.getHeader().getSourceId(), e.toString());
}
}
if (removeUnsplitValue) {
record.delete(fieldPath);
}
batchMaker.addRecord(record);
} else {
throw new OnRecordErrorException(error, record.getHeader().getSourceId(), fieldPath);
}
}
protected List<Record> processKafkaMessage(String messageId, byte[] payload)
throws StageException {
List<Record> records = new ArrayList<>();
try (DataParser parser = parserFactory.getParser(messageId, payload)) {
Record record = parser.parse();
while (record != null) {
records.add(record);
record = parser.parse();
}
} catch (IOException | DataParserException ex) {
handleException(messageId, ex);
}
if (produceSingleRecordPerMessage) {
List<Field> list = new ArrayList<>();
for (Record record : records) {
list.add(record.get());
}
Record record = records.get(0);
record.set(Field.create(list));
records.clear();
records.add(record);
}
return records;
}
protected List<Record> processMessage(String messageId, Object message) throws StageException {
List<Record> records = new ArrayList<>();
if (dataFormat == DataFormat.AVRO) {
try (DataParser parser = parserFactory.getParser(messageId, (byte[]) message)) {
Record record = parser.parse();
if (record != null) {
records.add(record);
}
} catch (IOException | DataParserException ex) {
LOG.debug("Got exception: '{}'", ex, ex);
handleException(messageId, ex);
}
} else {
try (DataParser parser = parserFactory.getParser(messageId, String.valueOf(message))) {
Record record = parser.parse();
while (record != null) {
records.add(record);
record = parser.parse();
}
} catch (IOException | DataParserException ex) {
LOG.debug("Got exception: '{}'", ex, ex);
handleException(messageId, ex);
}
}
if (produceSingleRecordPerMessage) {
List<Field> list = new ArrayList<>();
for (Record record : records) {
list.add(record.get());
}
Record record = records.get(0);
record.set(Field.create(list));
records.clear();
records.add(record);
}
return records;
}
@Test
public void testNestedKeep4() throws StageException {
Record record = createNestedRecord();
/*
* Keep non existing nested path "/USA[0]/SanFrancisco/cole".
* Only objects upto /USA[0]/SanFrancisco should exist
*/
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration(
"fields",
ImmutableList.of(
"/USA[0]/SanFrancisco/cole/streets[0][0]/name",
"/USA[0]/SanFrancisco/cole/streets[1][0]/name"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco"));
Assert.assertEquals(0, resultRecord.get("/USA[0]/SanFrancisco").getValueAsMap().size());
Assert.assertFalse(resultRecord.has("/USA[1]/SantaMonica/cole/streets[0][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[1]/SantaMonica/cole/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/folsom/streets[0][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/folsom/streets[0][1]/name"));
} finally {
runner.runDestroy();
}
}
@Test
public void testNestedKeep1() throws StageException {
/*
* In a deep nested field path try to retain the second elements of an array within an array
*/
Record record = createNestedRecord();
// Keep only second elements of array within array "/USA[0]/SanFrancisco/noe/streets"
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration(
"fields",
ImmutableList.of(
"/USA[0]/SanFrancisco/noe/streets[0][1]/name",
"/USA[0]/SanFrancisco/noe/streets[1][1]/name"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
// Note that since the first element was removed, the second element is the new first element
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][0]/name"));
Assert.assertEquals(
"b", resultRecord.get("/USA[0]/SanFrancisco/noe/streets[0][0]/name").getValueAsString());
// Note that since the first element was removed, the second element is the new first element
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[1][0]/name"));
Assert.assertEquals(
"d", resultRecord.get("/USA[0]/SanFrancisco/noe/streets[1][0]/name").getValueAsString());
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[1][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[1]/SantaMonica/cole/streets[0][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[1]/SantaMonica/cole/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/folsom/streets[0][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/folsom/streets[0][1]/name"));
} finally {
runner.runDestroy();
}
/*
* In a deep nested field path try to retain the first elements of an array within an array
*/
record = createNestedRecord();
// Keep only first elements of array within array "/USA[0]/SanFrancisco/noe/streets"
runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration(
"fields",
ImmutableList.of(
"/USA[0]/SanFrancisco/noe/streets[0][0]/name",
"/USA[0]/SanFrancisco/noe/streets[1][0]/name"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][0]/name"));
Assert.assertEquals(
"a", resultRecord.get("/USA[0]/SanFrancisco/noe/streets[0][0]/name").getValueAsString());
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[1][0]/name"));
Assert.assertEquals(
"c", resultRecord.get("/USA[0]/SanFrancisco/noe/streets[1][0]/name").getValueAsString());
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[1][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[1]/SantaMonica/cole/streets[0][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[1]/SantaMonica/cole/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/folsom/streets[0][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/folsom/streets[0][1]/name"));
} finally {
runner.runDestroy();
}
}
private Record createNestedRecord() {
Field name1 = Field.create("a");
Field name2 = Field.create("b");
Map<String, Field> nameMap1 = new HashMap<>();
nameMap1.put("name", name1);
Map<String, Field> nameMap2 = new HashMap<>();
nameMap2.put("name", name2);
Field name3 = Field.create("c");
Field name4 = Field.create("d");
Map<String, Field> nameMap3 = new HashMap<>();
nameMap3.put("name", name3);
Map<String, Field> nameMap4 = new HashMap<>();
nameMap4.put("name", name4);
Field name5 = Field.create("e");
Field name6 = Field.create("f");
Map<String, Field> nameMap5 = new HashMap<>();
nameMap5.put("name", name5);
Map<String, Field> nameMap6 = new HashMap<>();
nameMap6.put("name", name6);
Field name7 = Field.create("g");
Field name8 = Field.create("h");
Map<String, Field> nameMap7 = new HashMap<>();
nameMap7.put("name", name7);
Map<String, Field> nameMap8 = new HashMap<>();
nameMap8.put("name", name8);
Field first =
Field.create(
Field.Type.LIST, ImmutableList.of(Field.create(nameMap1), Field.create(nameMap2)));
Field second =
Field.create(
Field.Type.LIST, ImmutableList.of(Field.create(nameMap3), Field.create(nameMap4)));
Field third =
Field.create(
Field.Type.LIST, ImmutableList.of(Field.create(nameMap5), Field.create(nameMap6)));
Field fourth =
Field.create(
Field.Type.LIST, ImmutableList.of(Field.create(nameMap7), Field.create(nameMap8)));
Map<String, Field> noe = new HashMap<>();
noe.put("streets", Field.create(ImmutableList.of(first, second)));
Map<String, Field> folsom = new HashMap<>();
folsom.put("streets", Field.create(ImmutableList.of(fourth)));
Map<String, Field> cole = new HashMap<>();
cole.put("streets", Field.create(ImmutableList.of(third)));
Map<String, Field> sfArea = new HashMap<>();
sfArea.put("noe", Field.create(noe));
sfArea.put("folsom", Field.create(folsom));
Map<String, Field> utahArea = new HashMap<>();
utahArea.put("cole", Field.create(cole));
Map<String, Field> california = new HashMap<>();
california.put("SanFrancisco", Field.create(sfArea));
Map<String, Field> utah = new HashMap<>();
utah.put("SantaMonica", Field.create(utahArea));
Map<String, Field> map = new LinkedHashMap<>();
map.put(
"USA",
Field.create(
Field.Type.LIST, ImmutableList.of(Field.create(california), Field.create(utah))));
Record record = RecordCreator.create("s", "s:1");
record.set(Field.create(map));
// Nested record looks like this:
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[0][0]/name").getValueAsString(), "a");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[0][1]/name").getValueAsString(), "b");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[1][0]/name").getValueAsString(), "c");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[1][1]/name").getValueAsString(), "d");
Assert.assertEquals(
record.get("/USA[1]/SantaMonica/cole/streets[0][0]/name").getValueAsString(), "e");
Assert.assertEquals(
record.get("/USA[1]/SantaMonica/cole/streets[0][1]/name").getValueAsString(), "f");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/folsom/streets[0][0]/name").getValueAsString(), "g");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/folsom/streets[0][1]/name").getValueAsString(), "h");
return record;
}
@Test
public void testWildCardKeep3() throws StageException {
/*
* Use wil card in map and array
*/
Record record = createNestedRecord();
ProcessorRunner runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration(
"fields", ImmutableList.of("/USA[0]/SanFrancisco/*/streets[*][1]/name"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][0]/name"));
Assert.assertEquals(
"b", resultRecord.get("/USA[0]/SanFrancisco/noe/streets[0][0]/name").getValueAsString());
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[0][1]/name"));
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[1][0]/name"));
Assert.assertEquals(
"d", resultRecord.get("/USA[0]/SanFrancisco/noe/streets[1][0]/name").getValueAsString());
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/noe/streets[1][1]/name"));
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/folsom/streets[0][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[0]/SanFrancisco/folsom/streets[0][1]/name"));
Assert.assertFalse(resultRecord.has("/USA[1]/SantaMonica/cole/streets[0][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[1]/SantaMonica/cole/streets[0][1]/name"));
} finally {
runner.runDestroy();
}
/*
* Use wil card in map. Make sure the entire tree of elements is preserved
*/
record = createNestedRecord();
runner =
new ProcessorRunner.Builder(FieldFilterDProcessor.class)
.addConfiguration("fields", ImmutableList.of("/USA[0]/SanFrancisco/*"))
.addConfiguration("filterOperation", FilterOperation.KEEP)
.addOutputLane("a")
.build();
runner.runInit();
try {
StageRunner.Output output = runner.runProcess(ImmutableList.of(record));
Assert.assertEquals(1, output.getRecords().get("a").size());
Record resultRecord = output.getRecords().get("a").get(0);
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/noe"));
Assert.assertTrue(resultRecord.has("/USA[0]/SanFrancisco/folsom"));
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[0][0]/name").getValueAsString(), "a");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[0][1]/name").getValueAsString(), "b");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[1][0]/name").getValueAsString(), "c");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/noe/streets[1][1]/name").getValueAsString(), "d");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/folsom/streets[0][0]/name").getValueAsString(), "g");
Assert.assertEquals(
record.get("/USA[0]/SanFrancisco/folsom/streets[0][1]/name").getValueAsString(), "h");
Assert.assertFalse(resultRecord.has("/USA[1]/SantaMonica/cole/streets[0][0]/name"));
Assert.assertFalse(resultRecord.has("/USA[1]/SantaMonica/cole/streets[0][1]/name"));
} finally {
runner.runDestroy();
}
}
@Test(timeout = 30000)
public void testProduceDelimitedWithHeader() throws Exception {
SourceRunner sourceRunner =
new SourceRunner.Builder(ClusterHdfsDSource.class)
.addOutputLane("lane")
.setExecutionMode(ExecutionMode.CLUSTER_BATCH)
.addConfiguration("hdfsUri", miniDFS.getURI().toString())
.addConfiguration("hdfsDirLocations", Arrays.asList(dir.toUri().getPath()))
.addConfiguration("recursive", false)
.addConfiguration("hdfsConfigs", new HashMap<String, String>())
.addConfiguration("dataFormat", DataFormat.DELIMITED)
.addConfiguration("csvFileFormat", CsvMode.CSV)
.addConfiguration("csvHeader", CsvHeader.WITH_HEADER)
.addConfiguration("csvMaxObjectLen", 4096)
.addConfiguration("csvRecordType", CsvRecordType.LIST)
.addConfiguration("textMaxLineLen", 1024)
.addConfiguration("produceSingleRecordPerMessage", false)
.addConfiguration("regex", null)
.addConfiguration("grokPatternDefinition", null)
.addConfiguration("enableLog4jCustomLogFormat", false)
.addConfiguration("customLogFormat", null)
.addConfiguration("fieldPathsToGroupName", null)
.addConfiguration("log4jCustomLogFormat", null)
.addConfiguration("grokPattern", null)
.addConfiguration("hdfsKerberos", false)
.addConfiguration("hdfsConfDir", hadoopConfDir)
.setResourcesDir(resourcesDir)
.build();
sourceRunner.runInit();
List<Map.Entry> list = new ArrayList<>();
list.add(new Pair("HEADER_COL_1,HEADER_COL_2", null));
list.add(new Pair("path::" + "1", new String("a,b\nC,D\nc,d")));
Thread th = createThreadForAddingBatch(sourceRunner, list);
try {
StageRunner.Output output = sourceRunner.runProduce(null, 5);
String newOffset = output.getNewOffset();
Assert.assertEquals("path::" + "1", newOffset);
List<Record> records = output.getRecords().get("lane");
Assert.assertEquals(3, records.size());
Record record = records.get(0);
Assert.assertEquals(
"a",
record.get().getValueAsList().get(0).getValueAsMap().get("value").getValueAsString());
Assert.assertEquals(
"HEADER_COL_1",
record.get().getValueAsList().get(0).getValueAsMap().get("header").getValueAsString());
Assert.assertEquals(
"b",
record.get().getValueAsList().get(1).getValueAsMap().get("value").getValueAsString());
Assert.assertEquals(
"HEADER_COL_2",
record.get().getValueAsList().get(1).getValueAsMap().get("header").getValueAsString());
record = records.get(1);
Assert.assertEquals(
"C",
record.get().getValueAsList().get(0).getValueAsMap().get("value").getValueAsString());
Assert.assertEquals(
"HEADER_COL_1",
record.get().getValueAsList().get(0).getValueAsMap().get("header").getValueAsString());
Assert.assertEquals(
"D",
record.get().getValueAsList().get(1).getValueAsMap().get("value").getValueAsString());
Assert.assertEquals(
"HEADER_COL_2",
record.get().getValueAsList().get(1).getValueAsMap().get("header").getValueAsString());
record = records.get(2);
Assert.assertEquals(
"c",
record.get().getValueAsList().get(0).getValueAsMap().get("value").getValueAsString());
Assert.assertEquals(
"HEADER_COL_1",
record.get().getValueAsList().get(0).getValueAsMap().get("header").getValueAsString());
Assert.assertEquals(
"d",
record.get().getValueAsList().get(1).getValueAsMap().get("value").getValueAsString());
Assert.assertEquals(
"HEADER_COL_2",
record.get().getValueAsList().get(1).getValueAsMap().get("header").getValueAsString());
if (sourceRunner != null) {
sourceRunner.runDestroy();
}
} finally {
th.interrupt();
}
}
@Test(timeout = 30000)
public void testProduceAvroData() throws Exception {
SourceRunner sourceRunner =
new SourceRunner.Builder(ClusterHdfsDSource.class)
.addOutputLane("lane")
.setExecutionMode(ExecutionMode.CLUSTER_BATCH)
.addConfiguration("hdfsUri", miniDFS.getURI().toString())
.addConfiguration("hdfsDirLocations", Arrays.asList(dir.toUri().getPath()))
.addConfiguration("recursive", false)
.addConfiguration("hdfsConfigs", new HashMap<String, String>())
.addConfiguration("dataFormat", DataFormat.AVRO)
.addConfiguration("csvFileFormat", CsvMode.CSV)
.addConfiguration("csvHeader", CsvHeader.WITH_HEADER)
.addConfiguration("csvMaxObjectLen", 4096)
.addConfiguration("textMaxLineLen", 1024)
.addConfiguration("produceSingleRecordPerMessage", false)
.addConfiguration("regex", null)
.addConfiguration("grokPatternDefinition", null)
.addConfiguration("enableLog4jCustomLogFormat", false)
.addConfiguration("customLogFormat", null)
.addConfiguration("fieldPathsToGroupName", null)
.addConfiguration("log4jCustomLogFormat", null)
.addConfiguration("grokPattern", null)
.addConfiguration("hdfsKerberos", false)
.addConfiguration("hdfsConfDir", hadoopConfDir)
.setResourcesDir(resourcesDir)
.build();
sourceRunner.runInit();
List<Map.Entry> list = new ArrayList<>();
list.add(
new Pair(
"path::" + "1" + "::1",
createAvroData("a", 30, ImmutableList.of("*****@*****.**", "*****@*****.**"))));
list.add(
new Pair(
"path::" + "1" + "::2",
createAvroData("b", 40, ImmutableList.of("*****@*****.**", "*****@*****.**"))));
Thread th = createThreadForAddingBatch(sourceRunner, list);
try {
StageRunner.Output output = sourceRunner.runProduce(null, 5);
String newOffset = output.getNewOffset();
Assert.assertEquals("path::" + "1::2", newOffset);
List<Record> records = output.getRecords().get("lane");
Assert.assertEquals(2, records.size());
Record record = records.get(0);
Assert.assertTrue(record.has("/name"));
Assert.assertEquals("a", record.get("/name").getValueAsString());
Assert.assertTrue(record.has("/age"));
Assert.assertEquals(30, record.get("/age").getValueAsInteger());
Assert.assertTrue(record.has("/emails"));
Assert.assertTrue(record.get("/emails").getValueAsList() instanceof List);
List<Field> emails = record.get("/emails").getValueAsList();
Assert.assertEquals(2, emails.size());
Assert.assertEquals("*****@*****.**", emails.get(0).getValueAsString());
Assert.assertEquals("*****@*****.**", emails.get(1).getValueAsString());
record = records.get(1);
Assert.assertTrue(record.has("/name"));
Assert.assertEquals("b", record.get("/name").getValueAsString());
Assert.assertTrue(record.has("/age"));
Assert.assertEquals(40, record.get("/age").getValueAsInteger());
Assert.assertTrue(record.has("/emails"));
Assert.assertTrue(record.get("/emails").getValueAsList() instanceof List);
emails = record.get("/emails").getValueAsList();
Assert.assertEquals(2, emails.size());
Assert.assertEquals("*****@*****.**", emails.get(0).getValueAsString());
Assert.assertEquals("*****@*****.**", emails.get(1).getValueAsString());
} finally {
th.interrupt();
}
}