Beispiel #1
0
  @Test
  public void testSimplePutUsingCredentailsProviderService() throws Throwable {
    final TestRunner runner = TestRunners.newTestRunner(new PutSQS());

    runner.setProperty(PutSQS.TIMEOUT, "30 secs");
    String queueUrl = "Add queue url here";
    runner.setProperty(PutSQS.QUEUE_URL, queueUrl);
    runner.setValidateExpressionUsage(false);
    final AWSCredentialsProviderControllerService serviceImpl =
        new AWSCredentialsProviderControllerService();

    runner.addControllerService("awsCredentialsProvider", serviceImpl);

    runner.setProperty(
        serviceImpl,
        AbstractAWSProcessor.CREDENTIALS_FILE,
        System.getProperty("user.home") + "/aws-credentials.properties");
    runner.enableControllerService(serviceImpl);

    runner.assertValid(serviceImpl);

    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "1.txt");
    runner.enqueue(Paths.get("src/test/resources/hello.txt"), attrs);
    runner.setProperty(PutSQS.AWS_CREDENTIALS_PROVIDER_SERVICE, "awsCredentialsProvider");
    runner.run(1);

    final List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(PutSQS.REL_SUCCESS);
    for (final MockFlowFile mff : flowFiles) {
      System.out.println(mff.getAttributes());
      System.out.println(new String(mff.toByteArray()));
    }
  }
  @Test
  public void testStringHashStringRangePutGetWithHashOnlyKeyFailure() {
    final TestRunner putRunner = TestRunners.newTestRunner(PutDynamoDB.class);

    putRunner.setProperty(AbstractWriteDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.REGION, REGION);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.JSON_DOCUMENT, "document");
    String document = "{\"name\":\"john\"}";
    putRunner.enqueue(document.getBytes());

    putRunner.run(1);

    putRunner.assertAllFlowFilesTransferred(AbstractWriteDynamoDBProcessor.REL_SUCCESS, 1);

    List<MockFlowFile> flowFiles =
        putRunner.getFlowFilesForRelationship(AbstractWriteDynamoDBProcessor.REL_SUCCESS);
    for (MockFlowFile flowFile : flowFiles) {
      System.out.println(flowFile.getAttributes());
      assertEquals(document, new String(flowFile.toByteArray()));
    }

    final TestRunner getRunner = TestRunners.newTestRunner(GetDynamoDB.class);

    getRunner.setProperty(AbstractDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "document");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);

    flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
      validateServiceExceptionAttribute(flowFile);
      assertEquals("", new String(flowFile.toByteArray()));
    }
  }
  @Test
  public void testNonXmlContent() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
    runner.setProperty(TransformXml.XSLT_FILE_NAME, "src/test/resources/TestTransformXml/math.xsl");

    final Map<String, String> attributes = new HashMap<>();
    runner.enqueue("not xml".getBytes(), attributes);

    runner.run();

    runner.assertAllFlowFilesTransferred(TransformXml.REL_FAILURE);
    final MockFlowFile original =
        runner.getFlowFilesForRelationship(TransformXml.REL_FAILURE).get(0);
    final String originalContent = new String(original.toByteArray(), StandardCharsets.UTF_8);

    original.assertContentEquals("not xml");
  }
  @Ignore("this test fails")
  @Test
  public void testTransformMath() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
    runner.setProperty("header", "Test for mod");
    runner.setProperty(TransformXml.XSLT_FILE_NAME, "src/test/resources/TestTransformXml/math.xsl");

    final Map<String, String> attributes = new HashMap<>();
    runner.enqueue(Paths.get("src/test/resources/TestTransformXml/math.xml"), attributes);
    runner.run();

    runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
    final MockFlowFile transformed =
        runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
    final String transformedContent = new String(transformed.toByteArray(), StandardCharsets.UTF_8);

    transformed.assertContentEquals(Paths.get("src/test/resources/TestTransformXml/math.html"));
  }
  @Ignore("this test fails")
  @Test
  public void testTransformCsv() throws IOException {
    final TestRunner runner = TestRunners.newTestRunner(new TransformXml());
    runner.setProperty(
        TransformXml.XSLT_FILE_NAME, "src/test/resources/TestTransformXml/tokens.xsl");
    runner.setProperty("uuid_0", "${uuid_0}");
    runner.setProperty("uuid_1", "${uuid_1}");

    final Map<String, String> attributes = new HashMap<>();
    attributes.put("uuid_0", "uuid_0");
    attributes.put("uuid_1", "uuid_1");

    StringBuilder builder = new StringBuilder();
    builder.append("<data>\n");

    InputStream in =
        new FileInputStream(new File("src/test/resources/TestTransformXml/tokens.csv"));
    BufferedReader reader = new BufferedReader(new InputStreamReader(in));

    String line = null;
    while ((line = reader.readLine()) != null) {
      builder.append(line).append("\n");
    }
    builder.append("</data>");
    String data = builder.toString();
    runner.enqueue(data.getBytes(), attributes);
    runner.run();

    runner.assertAllFlowFilesTransferred(TransformXml.REL_SUCCESS);
    final MockFlowFile transformed =
        runner.getFlowFilesForRelationship(TransformXml.REL_SUCCESS).get(0);
    final String transformedContent =
        new String(transformed.toByteArray(), StandardCharsets.ISO_8859_1);

    transformed.assertContentEquals(Paths.get("src/test/resources/TestTransformXml/tokens.xml"));
  }
  @Test
  public void testNumberHashNumberRangePutGetDeleteGetSuccess() {
    final TestRunner putRunner = TestRunners.newTestRunner(PutDynamoDB.class);

    putRunner.setProperty(AbstractWriteDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.REGION, REGION);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.TABLE, numberHashNumberRangeTableName);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.HASH_KEY_NAME, "hashN");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.RANGE_KEY_NAME, "rangeN");
    putRunner.setProperty(
        AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    putRunner.setProperty(
        AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE, "40");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE, "50");
    putRunner.setProperty(AbstractWriteDynamoDBProcessor.JSON_DOCUMENT, "document");
    String document = "{\"40\":\"50\"}";
    putRunner.enqueue(document.getBytes());

    putRunner.run(1);

    putRunner.assertAllFlowFilesTransferred(AbstractWriteDynamoDBProcessor.REL_SUCCESS, 1);

    List<MockFlowFile> flowFiles =
        putRunner.getFlowFilesForRelationship(AbstractWriteDynamoDBProcessor.REL_SUCCESS);
    for (MockFlowFile flowFile : flowFiles) {
      assertEquals(document, new String(flowFile.toByteArray()));
    }

    final TestRunner getRunner = TestRunners.newTestRunner(GetDynamoDB.class);

    getRunner.setProperty(AbstractDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, numberHashNumberRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashN");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeN");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "40");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "50");
    getRunner.setProperty(
        AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    getRunner.setProperty(
        AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "document");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_SUCCESS, 1);

    flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_SUCCESS);
    for (MockFlowFile flowFile : flowFiles) {
      assertEquals(document, new String(flowFile.toByteArray()));
    }

    final TestRunner deleteRunner = TestRunners.newTestRunner(DeleteDynamoDB.class);

    deleteRunner.setProperty(DeleteDynamoDB.CREDENTIALS_FILE, CREDENTIALS_FILE);
    deleteRunner.setProperty(DeleteDynamoDB.REGION, REGION);
    deleteRunner.setProperty(DeleteDynamoDB.TABLE, numberHashNumberRangeTableName);
    deleteRunner.setProperty(DeleteDynamoDB.HASH_KEY_NAME, "hashN");
    deleteRunner.setProperty(DeleteDynamoDB.RANGE_KEY_NAME, "rangeN");
    deleteRunner.setProperty(DeleteDynamoDB.HASH_KEY_VALUE, "40");
    deleteRunner.setProperty(DeleteDynamoDB.RANGE_KEY_VALUE, "50");
    deleteRunner.setProperty(
        AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    deleteRunner.setProperty(
        AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    deleteRunner.enqueue(new byte[] {});

    deleteRunner.run(1);

    deleteRunner.assertAllFlowFilesTransferred(DeleteDynamoDB.REL_SUCCESS, 1);

    flowFiles = deleteRunner.getFlowFilesForRelationship(DeleteDynamoDB.REL_SUCCESS);
    for (MockFlowFile flowFile : flowFiles) {
      System.out.println(flowFile.getAttributes());
      assertEquals("", new String(flowFile.toByteArray()));
    }

    // Final check after delete
    final TestRunner getRunnerAfterDelete = TestRunners.newTestRunner(GetDynamoDB.class);

    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.CREDENTIALS_FILE, CREDENTIALS_FILE);
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunnerAfterDelete.setProperty(
        AbstractDynamoDBProcessor.TABLE, numberHashNumberRangeTableName);
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashN");
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeN");
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "40");
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "50");
    getRunnerAfterDelete.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "document");
    getRunnerAfterDelete.setProperty(
        AbstractWriteDynamoDBProcessor.HASH_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    getRunnerAfterDelete.setProperty(
        AbstractWriteDynamoDBProcessor.RANGE_KEY_VALUE_TYPE,
        AbstractWriteDynamoDBProcessor.ALLOWABLE_VALUE_NUMBER);
    getRunnerAfterDelete.enqueue(new byte[] {});

    getRunnerAfterDelete.run(1);
    getRunnerAfterDelete.assertAllFlowFilesTransferred(GetDynamoDB.REL_NOT_FOUND, 1);

    flowFiles = getRunnerAfterDelete.getFlowFilesForRelationship(GetDynamoDB.REL_NOT_FOUND);
    for (MockFlowFile flowFile : flowFiles) {
      String error = flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_KEY_ERROR_NOT_FOUND);
      assertTrue(error.startsWith(AbstractDynamoDBProcessor.DYNAMODB_KEY_ERROR_NOT_FOUND_MESSAGE));
    }
  }