@Test
public void testBasicConversion() throws IOException {
TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
runner.assertNotValid();
runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
runner.assertValid();
runner.enqueue(streamFor(CSV_CONTENT));
runner.run();
long converted = runner.getCounterValue("Converted records");
long errors = runner.getCounterValue("Conversion errors");
Assert.assertEquals("Should convert 2 rows", 2, converted);
Assert.assertEquals("Should reject 1 row", 1, errors);
runner.assertTransferCount("success", 1);
runner.assertTransferCount("failure", 0);
runner.assertTransferCount("incompatible", 1);
MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
String failureContent =
new String(runner.getContentAsByteArray(incompatible), StandardCharsets.UTF_8);
Assert.assertEquals("Should reject an invalid string and double", CSV_CONTENT, failureContent);
Assert.assertEquals(
"Should accumulate error messages", FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
@Test
public void testSingleRegisteredCallbackRemoveProtocol() throws MalformedURLException {
controller.registerUCSClientCallback(new URL("http://localhost:8080/App/listener"));
testRunner.setProperty(UCSGetUCSClientCallbacks.CALLBACK_ATTRIBUTE_NAME, "callback.url");
testRunner.setProperty(UCSGetUCSClientCallbacks.REMOVE_PROTOCOL_FROM_URL, "true");
testRunner.enqueue(new byte[] {});
testRunner.run();
testRunner.assertAllFlowFilesTransferred(UCSGetUCSClientCallbacks.REL_SUCCESS, 1);
MockFlowFile flowFile =
testRunner.getFlowFilesForRelationship(UCSGetUCSClientCallbacks.REL_SUCCESS).get(0);
assertThat(flowFile.getAttribute("callback.url"), is("localhost:8080/App/listener"));
}
@Test
public void testEmptyContent() throws IOException {
TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
runner.assertNotValid();
runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
runner.assertValid();
runner.enqueue(streamFor(""));
runner.run();
long converted = runner.getCounterValue("Converted records");
long errors = runner.getCounterValue("Conversion errors");
Assert.assertEquals("Should convert 0 rows", 0, converted);
Assert.assertEquals("Should reject 0 row", 0, errors);
runner.assertTransferCount("success", 0);
runner.assertTransferCount("failure", 1);
runner.assertTransferCount("incompatible", 0);
MockFlowFile incompatible = runner.getFlowFilesForRelationship("failure").get(0);
Assert.assertEquals(
"Should set an error message", "No incoming records", incompatible.getAttribute("errors"));
}
@Test
public void testAlternateCharset() throws IOException {
TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
runner.setProperty(ConvertCSVToAvro.CHARSET, "utf16");
runner.assertValid();
runner.enqueue(streamFor(CSV_CONTENT, Charset.forName("UTF-16")));
runner.run();
long converted = runner.getCounterValue("Converted records");
long errors = runner.getCounterValue("Conversion errors");
Assert.assertEquals("Should convert 2 rows", 2, converted);
Assert.assertEquals("Should reject 1 row", 1, errors);
runner.assertTransferCount("success", 1);
runner.assertTransferCount("failure", 0);
runner.assertTransferCount("incompatible", 1);
MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
Assert.assertEquals(
"Should accumulate error messages", FAILURE_SUMMARY, incompatible.getAttribute("errors"));
}
@Test
public void testNumberHashNumberRangePutGetDeleteGetSuccess() {
final TestRunner putRunner = TestRunners.newTestRunner(PutDynamoDB.class);
putRunner.setProperty(AbstractWriteDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
putRunner.setProperty(AbstractWriteDynamoDBProcessor.REGION, REGION);
putRunner.setProperty(AbstractWriteDynamoDBProcessor.TABLE, numberHashNumberRangeTableName);
putRunner.setProperty(AbstractWriteDynamoDBProcessor.HASH_KEY_NAME, "hashN");
putRunner.setProperty(AbstractWriteDynamoDBProcessor.RANGE_KEY_NAME, "rangeN");
putRunner.setProperty(
AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
putRunner.setProperty(
AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
putRunner.setProperty(AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE, "40");
putRunner.setProperty(AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE, "50");
putRunner.setProperty(AbstractWriteDynamoDBProcessor.JSON_DOCUMENT, "document");
String document = "{\"40\":\"50\"}";
putRunner.enqueue(document.getBytes());
putRunner.run(1);
putRunner.assertAllFlowFilesTransferred(AbstractWriteDynamoDBProcessor.REL_SUCCESS, 1);
List<MockFlowFile> flowFiles =
putRunner.getFlowFilesForRelationship(AbstractWriteDynamoDBProcessor.REL_SUCCESS);
for (MockFlowFile flowFile : flowFiles) {
assertEquals(document, new String(flowFile.toByteArray()));
}
final TestRunner getRunner = TestRunners.newTestRunner(GetDynamoDB.class);
getRunner.setProperty(AbstractDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, numberHashNumberRangeTableName);
getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashN");
getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeN");
getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "40");
getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "50");
getRunner.setProperty(
AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
getRunner.setProperty(
AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "document");
getRunner.enqueue(new byte[] {});
getRunner.run(1);
getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_SUCCESS, 1);
flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_SUCCESS);
for (MockFlowFile flowFile : flowFiles) {
assertEquals(document, new String(flowFile.toByteArray()));
}
final TestRunner deleteRunner = TestRunners.newTestRunner(DeleteDynamoDB.class);
deleteRunner.setProperty(DeleteDynamoDB.CREDENTIALS_FILE, CREDENTIALS_FILE);
deleteRunner.setProperty(DeleteDynamoDB.REGION, REGION);
deleteRunner.setProperty(DeleteDynamoDB.TABLE, numberHashNumberRangeTableName);
deleteRunner.setProperty(DeleteDynamoDB.HASH_KEY_NAME, "hashN");
deleteRunner.setProperty(DeleteDynamoDB.RANGE_KEY_NAME, "rangeN");
deleteRunner.setProperty(DeleteDynamoDB.HASH_KEY_VALUE, "40");
deleteRunner.setProperty(DeleteDynamoDB.RANGE_KEY_VALUE, "50");
deleteRunner.setProperty(
AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
deleteRunner.setProperty(
AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
deleteRunner.enqueue(new byte[] {});
deleteRunner.run(1);
deleteRunner.assertAllFlowFilesTransferred(DeleteDynamoDB.REL_SUCCESS, 1);
flowFiles = deleteRunner.getFlowFilesForRelationship(DeleteDynamoDB.REL_SUCCESS);
for (MockFlowFile flowFile : flowFiles) {
System.out.println(flowFile.getAttributes());
assertEquals("", new String(flowFile.toByteArray()));
}
// Final check after delete
final TestRunner getRunnerAfterDelete = TestRunners.newTestRunner(GetDynamoDB.class);
getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
getRunnerAfterDelete.setProperty(
AbstractDynamoDBProcessor.TABLE, numberHashNumberRangeTableName);
getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashN");
getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeN");
getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "40");
getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "50");
getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "document");
getRunnerAfterDelete.setProperty(
AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
getRunnerAfterDelete.setProperty(
AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
getRunnerAfterDelete.enqueue(new byte[] {});
getRunnerAfterDelete.run(1);
getRunnerAfterDelete.assertAllFlowFilesTransferred(GetDynamoDB.REL_NOT_FOUND, 1);
flowFiles = getRunnerAfterDelete.getFlowFilesForRelationship(GetDynamoDB.REL_NOT_FOUND);
for (MockFlowFile flowFile : flowFiles) {
String error = flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_KEY_ERROR_NOT_FOUND);
assertTrue(error.startsWith(AbstractDynamoDBProcessor.DYNAMODB_KEY_ERROR_NOT_FOUND_MESSAGE));
}
}