Ejemplo n.º 1
0
  @Test
  public void testSimpleListUsingCredentialsProviderService() throws Throwable {
    putTestFile("a", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    putTestFile("b/c", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    putTestFile("d/e", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));

    final TestRunner runner = TestRunners.newTestRunner(new ListS3());

    final AWSCredentialsProviderControllerService serviceImpl =
        new AWSCredentialsProviderControllerService();

    runner.addControllerService("awsCredentialsProvider", serviceImpl);

    runner.setProperty(
        serviceImpl,
        AbstractAWSProcessor.CREDENTIALS_FILE,
        System.getProperty("user.home") + "/aws-credentials.properties");
    runner.enableControllerService(serviceImpl);
    runner.assertValid(serviceImpl);

    runner.setProperty(ListS3.AWS_CREDENTIALS_PROVIDER_SERVICE, "awsCredentialsProvider");
    runner.setProperty(ListS3.REGION, REGION);
    runner.setProperty(ListS3.BUCKET, BUCKET_NAME);

    runner.run();

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    flowFiles.get(0).assertAttributeEquals("filename", "a");
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
  }
Ejemplo n.º 2
0
  @Test
  public void testCSVProperties() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    ConvertCSVToAvro processor = new ConvertCSVToAvro();
    ProcessContext context = runner.getProcessContext();

    // check defaults
    processor.createCSVProperties(context);
    Assert.assertEquals("Charset should match", "utf8", processor.props.charset);
    Assert.assertEquals("Delimiter should match", ",", processor.props.delimiter);
    Assert.assertEquals("Quote should match", "\"", processor.props.quote);
    Assert.assertEquals("Escape should match", "\\", processor.props.escape);
    Assert.assertEquals("Header flag should match", false, processor.props.useHeader);
    Assert.assertEquals("Lines to skip should match", 0, processor.props.linesToSkip);

    runner.setProperty(ConvertCSVToAvro.CHARSET, "utf16");
    runner.setProperty(ConvertCSVToAvro.DELIMITER, "|");
    runner.setProperty(ConvertCSVToAvro.QUOTE, "'");
    runner.setProperty(ConvertCSVToAvro.ESCAPE, "\u2603");
    runner.setProperty(ConvertCSVToAvro.HAS_HEADER, "true");
    runner.setProperty(ConvertCSVToAvro.LINES_TO_SKIP, "2");

    // check updates
    processor.createCSVProperties(context);
    Assert.assertEquals("Charset should match", "utf16", processor.props.charset);
    Assert.assertEquals("Delimiter should match", "|", processor.props.delimiter);
    Assert.assertEquals("Quote should match", "'", processor.props.quote);
    Assert.assertEquals("Escape should match", "\u2603", processor.props.escape);
    Assert.assertEquals("Header flag should match", true, processor.props.useHeader);
    Assert.assertEquals("Lines to skip should match", 2, processor.props.linesToSkip);
  }
Ejemplo n.º 3
0
  @Test
  public void testBasicConversion() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor(CSV_CONTENT));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 1 row", 1, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 1);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
    String failureContent =
        new String(runner.getContentAsByteArray(incompatible), StandardCharsets.UTF_8);
    Assert.assertEquals("Should reject an invalid string and double", CSV_CONTENT, failureContent);
    Assert.assertEquals(
        "Should accumulate error messages", FAILURE_SUMMARY, incompatible.getAttribute("errors"));
  }
Ejemplo n.º 4
0
  @Test
  public void testSimplePutUsingCredentailsProviderService() throws Throwable {
    final TestRunner runner = TestRunners.newTestRunner(new PutSQS());

    runner.setProperty(PutSQS.TIMEOUT, "30 secs");
    String queueUrl = "Add queue url here";
    runner.setProperty(PutSQS.QUEUE_URL, queueUrl);
    runner.setValidateExpressionUsage(false);
    final AWSCredentialsProviderControllerService serviceImpl =
        new AWSCredentialsProviderControllerService();

    runner.addControllerService("awsCredentialsProvider", serviceImpl);

    runner.setProperty(
        serviceImpl,
        AbstractAWSProcessor.CREDENTIALS_FILE,
        System.getProperty("user.home") + "/aws-credentials.properties");
    runner.enableControllerService(serviceImpl);

    runner.assertValid(serviceImpl);

    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "1.txt");
    runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
    runner.setProperty(PutSQS.AWS_CREDENTIALS_PROVIDER_SERVICE, "awsCredentialsProvider");
    runner.run(1);

    final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(PutSQS.REL_SUCCESS);
    for (final MockFlowFile mff : flowFiles) {
      System.out.println(mff.getAttributes());
      System.out.println(new String(mff.toByteArray()));
    }
  }
Ejemplo n.º 5
0
  @Test
  public void testContentsOfFileRetrieved() throws IOException {
    String key = "folder/1.txt";
    putTestFile(key, getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));

    final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object());

    runner.setProperty(FetchS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(FetchS3Object.REGION, REGION);
    runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME);

    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", key);
    runner.enqueue(new byte[0], attrs);

    runner.run(1);

    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);

    final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
    final MockFlowFile out = ffs.iterator().next();

    final byte[] expectedBytes = Files.readAllBytes(getResourcePath(SAMPLE_FILE_RESOURCE_NAME));
    out.assertContentEquals(new String(expectedBytes));

    for (final Map.Entry<String, String> entry : out.getAttributes().entrySet()) {
      System.out.println(entry.getKey() + " : " + entry.getValue());
    }
  }
Ejemplo n.º 6
0
  @Test
  public void validateSuccessfulConsumeAndTransferToSuccess() throws Exception {
    JmsTemplate jmsTemplate = CommonTest.buildJmsTemplateForDestination("cooQueue", false);
    JMSPublisher sender = new JMSPublisher(jmsTemplate, mock(ComponentLog.class));
    sender.publish("Hey dude!".getBytes());
    TestRunner runner = TestRunners.newTestRunner(new ConsumeJMS());
    JMSConnectionFactoryProviderDefinition cs = mock(JMSConnectionFactoryProviderDefinition.class);
    when(cs.getIdentifier()).thenReturn("cfProvider");
    when(cs.getConnectionFactory()).thenReturn(jmsTemplate.getConnectionFactory());
    runner.addControllerService("cfProvider", cs);
    runner.enableControllerService(cs);

    runner.setProperty(PublishJMS.CF_SERVICE, "cfProvider");
    runner.setProperty(ConsumeJMS.DESTINATION, "cooQueue");
    runner.setProperty(ConsumeJMS.DESTINATION_TYPE, ConsumeJMS.QUEUE);
    runner.run(1, false);
    //
    final MockFlowFile successFF =
        runner.getFlowFilesForRelationship(PublishJMS.REL_SUCCESS).get(0);
    assertNotNull(successFF);
    assertEquals("cooQueue", successFF.getAttributes().get(JmsHeaders.DESTINATION));
    successFF.assertContentEquals("Hey dude!".getBytes());

    ((CachingConnectionFactory) jmsTemplate.getConnectionFactory()).destroy();
  }
  @Test
  public void testStringHashStringRangePutGetWithHashOnlyKeyFailure() {
    final TestRunner putRunner = TestRunners.newTestRunner(PutDynamoDB.class);

    putRunner.setProperty(AbstractWriteDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.REGION, REGION);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.JSON_DOCUMENT, "document");
    String document = "{\"name\":\"john\"}";
    putRunner.enqueue(document.getBytes());

    putRunner.run(1);

    putRunner.assertAllFlowFilesTransferred(AbstractWriteDynamoDBProcessor.REL_SUCCESS, 1);

    List<MockFlowFile> flowFiles =
        putRunner.getFlowFilesForRelationship(AbstractWriteDynamoDBProcessor.REL_SUCCESS);
    for (MockFlowFile flowFile : flowFiles) {
      System.out.println(flowFile.getAttributes());
      assertEquals(document, new String(flowFile.toByteArray()));
    }

    final TestRunner getRunner = TestRunners.newTestRunner(GetDynamoDB.class);

    getRunner.setProperty(AbstractDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "document");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);

    flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
      validateServiceExceptionAttribute(flowFile);
      assertEquals("", new String(flowFile.toByteArray()));
    }
  }
Ejemplo n.º 8
0
  @Before
  public void init() {
    ProcessorLog logger = Mockito.mock(ProcessorLog.class);
    sender = new CapturingChannelSender("localhost", 12345, 0, logger);
    proc = new TestablePutSplunk(sender);

    runner = TestRunners.newTestRunner(proc);
    runner.setProperty(PutSplunk.PORT, "12345");
  }
Ejemplo n.º 9
0
  @Test
  public void testUnableToCreateConnectionShouldRouteToFailure() {
    PutSplunk proc = new UnableToConnectPutSplunk();
    runner = TestRunners.newTestRunner(proc);
    runner.setProperty(PutSplunk.PORT, "12345");

    final String message = "This is one message, should send the whole FlowFile";

    runner.enqueue(message);
    runner.run();
    runner.assertAllFlowFilesTransferred(PutSplunk.REL_FAILURE, 1);
  }
Ejemplo n.º 10
0
  @Test
  public void testSendStreamToQueue() throws Exception {
    PutJMS putJms = new PutJMS();
    TestRunner putRunner = TestRunners.newTestRunner(putJms);
    putRunner.setProperty(JmsProperties.JMS_PROVIDER, JmsProperties.ACTIVEMQ_PROVIDER);
    putRunner.setProperty(JmsProperties.URL, "vm://localhost?broker.persistent=false");
    putRunner.setProperty(JmsProperties.DESTINATION_TYPE, JmsProperties.DESTINATION_TYPE_QUEUE);
    putRunner.setProperty(JmsProperties.DESTINATION_NAME, "queue.testing");
    putRunner.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, JmsProperties.ACK_MODE_AUTO);
    WrappedMessageProducer wrappedProducer =
        JmsFactory.createMessageProducer(putRunner.getProcessContext(), true);
    final Session jmsSession = wrappedProducer.getSession();
    final MessageProducer producer = wrappedProducer.getProducer();

    final StreamMessage message = jmsSession.createStreamMessage();
    message.writeBytes("Hello Stream".getBytes());

    producer.send(message);
    jmsSession.commit();

    GetJMSQueue getJmsQueue = new GetJMSQueue();
    TestRunner runner = TestRunners.newTestRunner(getJmsQueue);
    runner.setProperty(JmsProperties.JMS_PROVIDER, JmsProperties.ACTIVEMQ_PROVIDER);
    runner.setProperty(JmsProperties.URL, "vm://localhost?broker.persistent=false");
    runner.setProperty(JmsProperties.DESTINATION_NAME, "queue.testing");
    runner.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, JmsProperties.ACK_MODE_AUTO);

    runner.run();

    List<MockFlowFile> flowFiles =
        runner.getFlowFilesForRelationship(new Relationship.Builder().name("success").build());

    assertTrue(flowFiles.size() == 1);
    MockFlowFile successFlowFile = flowFiles.get(0);
    successFlowFile.assertContentEquals("Hello Stream");
    successFlowFile.assertAttributeEquals("jms.JMSDestination", "queue.testing");

    producer.close();
    jmsSession.close();
  }
Ejemplo n.º 11
0
  @Test
  public void testSimplePut() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new PutSQS());
    runner.setProperty(PutSNS.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(PutSQS.TIMEOUT, "30 secs");
    runner.setProperty(
        PutSQS.QUEUE_URL, "https://sqs.us-west-2.amazonaws.com/100515378163/test-queue-000000000");
    Assert.assertTrue(runner.setProperty("x-custom-prop", "hello").isValid());

    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "1.txt");
    runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
    runner.run(1);

    runner.assertAllFlowFilesTransferred(PutSQS.REL_SUCCESS, 1);
  }
Ejemplo n.º 12
0
  @Test
  public void testTryToFetchNotExistingFile() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object());

    runner.setProperty(FetchS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(FetchS3Object.REGION, REGION);
    runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME);

    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "no-such-a-file");
    runner.enqueue(new byte[0], attrs);

    runner.run(1);

    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_FAILURE, 1);
  }
Ejemplo n.º 13
0
  @Test
  public void testNonXmlContent() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
    runner.setProperty(TransformXml.XSLT_FILE_NAME, "src/test/resources/TestTransformXml/math.xsl");

    final Map<String, String> attributes = new HashMap<>();
    runner.enqueue("not xml".getBytes(), attributes);

    runner.run();

    runner.assertAllFlowFilesTransferred(TransformXml.REL_FAILURE);
    final MockFlowFile original =
        runner.getFlowFilesForRelationship(TransformXml.REL_FAILURE).get(0);
    final String originalContent = new String(original.toByteArray(), StandardCharsets.UTF_8);

    original.assertContentEquals("not xml");
  }
Ejemplo n.º 14
0
  @Ignore("this test fails")
  @Test
  public void testTransformMath() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
    runner.setProperty("header", "Test for mod");
    runner.setProperty(TransformXml.XSLT_FILE_NAME, "src/test/resources/TestTransformXml/math.xsl");

    final Map<String, String> attributes = new HashMap<>();
    runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes);
    runner.run();

    runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
    final MockFlowFile transformed =
        runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
    final String transformedContent = new String(transformed.toByteArray(), StandardCharsets.UTF_8);

    transformed.assertContentEquals(Paths.get("src/test/resources/TestTransformXml/math.html"));
  }
Ejemplo n.º 15
0
  @Test
  public void testSimpleGet() throws IOException {
    putTestFile("test-file", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));

    final TestRunner runner = TestRunners.newTestRunner(new FetchS3Object());

    runner.setProperty(FetchS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(FetchS3Object.REGION, REGION);
    runner.setProperty(FetchS3Object.BUCKET, BUCKET_NAME);

    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "test-file");
    runner.enqueue(new byte[0], attrs);

    runner.run(1);

    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
  }
Ejemplo n.º 16
0
  @Test
  public void testSimpleListWithDelimiter() throws Throwable {
    putTestFile("a", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    putTestFile("b/c", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));
    putTestFile("d/e", getFileFromResourceName(SAMPLE_FILE_RESOURCE_NAME));

    final TestRunner runner = TestRunners.newTestRunner(new ListS3());

    runner.setProperty(ListS3.CREDENTIALS_FILE, CREDENTIALS_FILE);
    runner.setProperty(ListS3.REGION, REGION);
    runner.setProperty(ListS3.BUCKET, BUCKET_NAME);
    runner.setProperty(ListS3.DELIMITER, "/");

    runner.run();

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 1);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    flowFiles.get(0).assertAttributeEquals("filename", "a");
  }
Ejemplo n.º 17
0
  @Test
  public void testBasicConversionNoErrors() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor("1,green\n2,blue,\n3,grey,12.95"));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 3 rows", 3, converted);
    Assert.assertEquals("Should reject 0 row", 0, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 0);
  }
Ejemplo n.º 18
0
  @org.junit.Ignore
  public void testSendObjectToQueue() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(GetJMSQueue.class);
    runner.setProperty(JmsProperties.JMS_PROVIDER, JmsProperties.ACTIVEMQ_PROVIDER);
    runner.setProperty(JmsProperties.URL, "tcp://localhost:61616");
    runner.setProperty(JmsProperties.DESTINATION_TYPE, JmsProperties.DESTINATION_TYPE_QUEUE);
    runner.setProperty(JmsProperties.DESTINATION_NAME, "queue.testing");
    runner.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, JmsProperties.ACK_MODE_AUTO);
    WrappedMessageProducer wrappedProducer =
        JmsFactory.createMessageProducer(runner.getProcessContext(), true);
    final Session jmsSession = wrappedProducer.getSession();
    final MessageProducer producer = wrappedProducer.getProducer();

    // Revision class is used because test just needs any Serializable class in core NiFi
    final ObjectMessage message = jmsSession.createObjectMessage(new Revision(1L, "ID"));

    producer.send(message);
    jmsSession.commit();
    producer.close();
    jmsSession.close();
  }
Ejemplo n.º 19
0
  @org.junit.Ignore
  public void testSendMapToQueue() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(GetJMSQueue.class);
    runner.setProperty(JmsProperties.JMS_PROVIDER, JmsProperties.ACTIVEMQ_PROVIDER);
    runner.setProperty(JmsProperties.URL, "tcp://localhost:61616");
    runner.setProperty(JmsProperties.DESTINATION_TYPE, JmsProperties.DESTINATION_TYPE_QUEUE);
    runner.setProperty(JmsProperties.DESTINATION_NAME, "queue.testing");
    runner.setProperty(JmsProperties.ACKNOWLEDGEMENT_MODE, JmsProperties.ACK_MODE_AUTO);
    WrappedMessageProducer wrappedProducer =
        JmsFactory.createMessageProducer(runner.getProcessContext(), true);
    final Session jmsSession = wrappedProducer.getSession();
    final MessageProducer producer = wrappedProducer.getProducer();

    final MapMessage message = jmsSession.createMapMessage();
    message.setString("foo!", "bar");
    message.setString("bacon", "meat");

    producer.send(message);
    jmsSession.commit();
    producer.close();
    jmsSession.close();
  }
Ejemplo n.º 20
0
  @Test
  public void testEmptyContent() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.assertNotValid();
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.assertValid();

    runner.enqueue(streamFor(""));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 0 rows", 0, converted);
    Assert.assertEquals("Should reject 0 row", 0, errors);

    runner.assertTransferCount("success", 0);
    runner.assertTransferCount("failure", 1);
    runner.assertTransferCount("incompatible", 0);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("failure").get(0);
    Assert.assertEquals(
        "Should set an error message", "No incoming records", incompatible.getAttribute("errors"));
  }
Ejemplo n.º 21
0
  @Test
  public void testAlternateCharset() throws IOException {
    TestRunner runner = TestRunners.newTestRunner(ConvertCSVToAvro.class);
    runner.setProperty(ConvertCSVToAvro.SCHEMA, SCHEMA.toString());
    runner.setProperty(ConvertCSVToAvro.CHARSET, "utf16");
    runner.assertValid();

    runner.enqueue(streamFor(CSV_CONTENT, Charset.forName("UTF-16")));
    runner.run();

    long converted = runner.getCounterValue("Converted records");
    long errors = runner.getCounterValue("Conversion errors");
    Assert.assertEquals("Should convert 2 rows", 2, converted);
    Assert.assertEquals("Should reject 1 row", 1, errors);

    runner.assertTransferCount("success", 1);
    runner.assertTransferCount("failure", 0);
    runner.assertTransferCount("incompatible", 1);

    MockFlowFile incompatible = runner.getFlowFilesForRelationship("incompatible").get(0);
    Assert.assertEquals(
        "Should accumulate error messages", FAILURE_SUMMARY, incompatible.getAttribute("errors"));
  }
Ejemplo n.º 22
0
  @Ignore("this test fails")
  @Test
  public void testTransformCsv() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
    runner.setProperty(
        TransformXml.XSLT_FILE_NAME, "src/test/resources/TestTransformXml/tokens.xsl");
    runner.setProperty("uuid_0", "${uuid_0}");
    runner.setProperty("uuid_1", "${uuid_1}");

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("uuid_0", "uuid_0");
    attributes.put("uuid_1", "uuid_1");

    StringBuilder builder = new StringBuilder();
    builder.append("<data>\n");

    InputStream in =
        new FileInputStream(new File("src/test/resources/TestTransformXml/tokens.csv"));
    BufferedReader reader = new BufferedReader(new InputStreamReader(in));

    String line = null;
    while ((line = reader.readLine()) != null) {
      builder.append(line).append("\n");
    }
    builder.append("</data>");
    String data = builder.toString();
    runner.enqueue(data.getBytes(), attributes);
    runner.run();

    runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
    final MockFlowFile transformed =
        runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
    final String transformedContent =
        new String(transformed.toByteArray(), StandardCharsets.ISO_8859_1);

    transformed.assertContentEquals(Paths.get("src/test/resources/TestTransformXml/tokens.xml"));
  }
 @Override
 protected TestRunner createTestRunner() {
   return TestRunners.newTestRunner(new UCSGetUCSClientCallbacks());
 }
 @Before
 public void init() throws Exception {
   testRunner = TestRunners.newTestRunner(new UCSRouteMessageOnContent());
 }
Ejemplo n.º 25
0
 @Test
 public void testStylesheetNotFound() throws IOException {
   final TestRunner controller = TestRunners.newTestRunner(TransformXml.class);
   controller.setProperty(TransformXml.XSLT_FILE_NAME, "/no/path/to/math.xsl");
   controller.assertNotValid();
 }
  @Test
  public void testNumberHashNumberRangePutGetDeleteGetSuccess() {
    final TestRunner putRunner = TestRunners.newTestRunner(PutDynamoDB.class);

    putRunner.setProperty(AbstractWriteDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.REGION, REGION);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.TABLE, numberHashNumberRangeTableName);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.HASH_KEY_NAME, "hashN");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.RANGE_KEY_NAME, "rangeN");
    putRunner.setProperty(
        AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    putRunner.setProperty(
        AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE, "40");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE, "50");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.JSON_DOCUMENT, "document");
    String document = "{\"40\":\"50\"}";
    putRunner.enqueue(document.getBytes());

    putRunner.run(1);

    putRunner.assertAllFlowFilesTransferred(AbstractWriteDynamoDBProcessor.REL_SUCCESS, 1);

    List<MockFlowFile> flowFiles =
        putRunner.getFlowFilesForRelationship(AbstractWriteDynamoDBProcessor.REL_SUCCESS);
    for (MockFlowFile flowFile : flowFiles) {
      assertEquals(document, new String(flowFile.toByteArray()));
    }

    final TestRunner getRunner = TestRunners.newTestRunner(GetDynamoDB.class);

    getRunner.setProperty(AbstractDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, numberHashNumberRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashN");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeN");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "40");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "50");
    getRunner.setProperty(
        AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    getRunner.setProperty(
        AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "document");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_SUCCESS, 1);

    flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_SUCCESS);
    for (MockFlowFile flowFile : flowFiles) {
      assertEquals(document, new String(flowFile.toByteArray()));
    }

    final TestRunner deleteRunner = TestRunners.newTestRunner(DeleteDynamoDB.class);

    deleteRunner.setProperty(DeleteDynamoDB.CREDENTIALS_FILE, CREDENTIALS_FILE);
    deleteRunner.setProperty(DeleteDynamoDB.REGION, REGION);
    deleteRunner.setProperty(DeleteDynamoDB.TABLE, numberHashNumberRangeTableName);
    deleteRunner.setProperty(DeleteDynamoDB.HASH_KEY_NAME, "hashN");
    deleteRunner.setProperty(DeleteDynamoDB.RANGE_KEY_NAME, "rangeN");
    deleteRunner.setProperty(DeleteDynamoDB.HASH_KEY_VALUE, "40");
    deleteRunner.setProperty(DeleteDynamoDB.RANGE_KEY_VALUE, "50");
    deleteRunner.setProperty(
        AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    deleteRunner.setProperty(
        AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    deleteRunner.enqueue(new byte[] {});

    deleteRunner.run(1);

    deleteRunner.assertAllFlowFilesTransferred(DeleteDynamoDB.REL_SUCCESS, 1);

    flowFiles = deleteRunner.getFlowFilesForRelationship(DeleteDynamoDB.REL_SUCCESS);
    for (MockFlowFile flowFile : flowFiles) {
      System.out.println(flowFile.getAttributes());
      assertEquals("", new String(flowFile.toByteArray()));
    }

    // Final check after delete
    final TestRunner getRunnerAfterDelete = TestRunners.newTestRunner(GetDynamoDB.class);

    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunnerAfterDelete.setProperty(
        AbstractDynamoDBProcessor.TABLE, numberHashNumberRangeTableName);
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashN");
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeN");
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "40");
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "50");
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "document");
    getRunnerAfterDelete.setProperty(
        AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    getRunnerAfterDelete.setProperty(
        AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    getRunnerAfterDelete.enqueue(new byte[] {});

    getRunnerAfterDelete.run(1);
    getRunnerAfterDelete.assertAllFlowFilesTransferred(GetDynamoDB.REL_NOT_FOUND, 1);

    flowFiles = getRunnerAfterDelete.getFlowFilesForRelationship(GetDynamoDB.REL_NOT_FOUND);
    for (MockFlowFile flowFile : flowFiles) {
      String error = flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_KEY_ERROR_NOT_FOUND);
      assertTrue(error.startsWith(AbstractDynamoDBProcessor.DYNAMODB_KEY_ERROR_NOT_FOUND_MESSAGE));
    }
  }