示例#1
0
  @Test
  public void testBasicConversion() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor(CSV_CONTENT));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 1 row", 1, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 1);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
    String failureContent =
        new String(runner.getContentAsByteArray(incompatible), StandardCharsets.UTF_8);
    Assert.assertEquals("Should reject an invalid string and double", CSV_CONTENT, failureContent);
    Assert.assertEquals(
        "Should accumulate error messages", FAILURE_SUMMARY, incompatible.getAttribute("errors"));
  }
  @Test
  public void testSingleRegisteredCallbackRemoveProtocol() throws MalformedURLException {

    controller.registerUCSClientCallback(new URL("http://localhost:8080/App/listener"));

    testRunner.setProperty(UCSGetUCSClientCallbacks.CALLBACK_ATTRIBUTE_NAME, "callback.url");
    testRunner.setProperty(UCSGetUCSClientCallbacks.REMOVE_PROTOCOL_FROM_URL, "true");

    testRunner.enqueue(new byte[] {});
    testRunner.run();

    testRunner.assertAllFlowFilesTransferred(UCSGetUCSClientCallbacks.REL_SUCCESS, 1);
    MockFlowFile flowFile =
        testRunner.getFlowFilesForRelationship(UCSGetUCSClientCallbacks.REL_SUCCESS).get(0);

    assertThat(flowFile.getAttribute("callback.url"), is("localhost:8080/App/listener"));
  }
示例#3
0
  @Test
  public void testEmptyContent() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor(""));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 0 rows", 0, converted);
    Assert.assertEquals("Should reject 0 row", 0, errors);

    runner.assertTransferCount("success", 0);
    runner.assertTransferCount("failure", 1);
    runner.assertTransferCount("incompatible", 0);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("failure").get(0);
    Assert.assertEquals(
        "Should set an error message", "No incoming records", incompatible.getAttribute("errors"));
  }
示例#4
0
  @Test
  public void testAlternateCharset() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.setProperty(ConvertCSVToAvro.CHARSET, "utf16");
    runner.assertValid();

    runner.enqueue(streamFor(CSV_CONTENT, Charset.forName("UTF-16")));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 1 row", 1, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 1);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
    Assert.assertEquals(
        "Should accumulate error messages", FAILURE_SUMMARY, incompatible.getAttribute("errors"));
  }
  @Test
  public void testNumberHashNumberRangePutGetDeleteGetSuccess() {
    final TestRunner putRunner = TestRunners.newTestRunner(PutDynamoDB.class);

    putRunner.setProperty(AbstractWriteDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.REGION, REGION);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.TABLE, numberHashNumberRangeTableName);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.HASH_KEY_NAME, "hashN");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.RANGE_KEY_NAME, "rangeN");
    putRunner.setProperty(
        AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    putRunner.setProperty(
        AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE, "40");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE, "50");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.JSON_DOCUMENT, "document");
    String document = "{\"40\":\"50\"}";
    putRunner.enqueue(document.getBytes());

    putRunner.run(1);

    putRunner.assertAllFlowFilesTransferred(AbstractWriteDynamoDBProcessor.REL_SUCCESS, 1);

    List<MockFlowFile> flowFiles =
        putRunner.getFlowFilesForRelationship(AbstractWriteDynamoDBProcessor.REL_SUCCESS);
    for (MockFlowFile flowFile : flowFiles) {
      assertEquals(document, new String(flowFile.toByteArray()));
    }

    final TestRunner getRunner = TestRunners.newTestRunner(GetDynamoDB.class);

    getRunner.setProperty(AbstractDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, numberHashNumberRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashN");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeN");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "40");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "50");
    getRunner.setProperty(
        AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    getRunner.setProperty(
        AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "document");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_SUCCESS, 1);

    flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_SUCCESS);
    for (MockFlowFile flowFile : flowFiles) {
      assertEquals(document, new String(flowFile.toByteArray()));
    }

    final TestRunner deleteRunner = TestRunners.newTestRunner(DeleteDynamoDB.class);

    deleteRunner.setProperty(DeleteDynamoDB.CREDENTIALS_FILE, CREDENTIALS_FILE);
    deleteRunner.setProperty(DeleteDynamoDB.REGION, REGION);
    deleteRunner.setProperty(DeleteDynamoDB.TABLE, numberHashNumberRangeTableName);
    deleteRunner.setProperty(DeleteDynamoDB.HASH_KEY_NAME, "hashN");
    deleteRunner.setProperty(DeleteDynamoDB.RANGE_KEY_NAME, "rangeN");
    deleteRunner.setProperty(DeleteDynamoDB.HASH_KEY_VALUE, "40");
    deleteRunner.setProperty(DeleteDynamoDB.RANGE_KEY_VALUE, "50");
    deleteRunner.setProperty(
        AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    deleteRunner.setProperty(
        AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    deleteRunner.enqueue(new byte[] {});

    deleteRunner.run(1);

    deleteRunner.assertAllFlowFilesTransferred(DeleteDynamoDB.REL_SUCCESS, 1);

    flowFiles = deleteRunner.getFlowFilesForRelationship(DeleteDynamoDB.REL_SUCCESS);
    for (MockFlowFile flowFile : flowFiles) {
      System.out.println(flowFile.getAttributes());
      assertEquals("", new String(flowFile.toByteArray()));
    }

    // Final check after delete
    final TestRunner getRunnerAfterDelete = TestRunners.newTestRunner(GetDynamoDB.class);

    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunnerAfterDelete.setProperty(
        AbstractDynamoDBProcessor.TABLE, numberHashNumberRangeTableName);
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashN");
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeN");
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "40");
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "50");
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "document");
    getRunnerAfterDelete.setProperty(
        AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    getRunnerAfterDelete.setProperty(
        AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    getRunnerAfterDelete.enqueue(new byte[] {});

    getRunnerAfterDelete.run(1);
    getRunnerAfterDelete.assertAllFlowFilesTransferred(GetDynamoDB.REL_NOT_FOUND, 1);

    flowFiles = getRunnerAfterDelete.getFlowFilesForRelationship(GetDynamoDB.REL_NOT_FOUND);
    for (MockFlowFile flowFile : flowFiles) {
      String error = flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_KEY_ERROR_NOT_FOUND);
      assertTrue(error.startsWith(AbstractDynamoDBProcessor.DYNAMODB_KEY_ERROR_NOT_FOUND_MESSAGE));
    }
  }