private AmazonS3Source createSource() {
S3ConfigBean s3ConfigBean = new S3ConfigBean();
s3ConfigBean.basicConfig = new BasicConfig();
s3ConfigBean.basicConfig.maxWaitTime = 1000;
s3ConfigBean.basicConfig.maxBatchSize = 60000;
s3ConfigBean.dataFormatConfig = new DataFormatConfig();
s3ConfigBean.dataFormat = DataFormat.TEXT;
s3ConfigBean.dataFormatConfig.charset = "UTF-8";
s3ConfigBean.dataFormatConfig.textMaxLineLen = 1024;
s3ConfigBean.dataFormatConfig.compression = Compression.NONE;
s3ConfigBean.dataFormatConfig.filePatternInArchive = "*";
s3ConfigBean.errorConfig = new S3ErrorConfig();
s3ConfigBean.errorConfig.errorHandlingOption = PostProcessingOptions.NONE;
s3ConfigBean.errorConfig.errorFolder = ERROR_FOLDER;
s3ConfigBean.errorConfig.errorBucket = ERROR_BUCKET;
s3ConfigBean.postProcessingConfig = new S3PostProcessingConfig();
s3ConfigBean.postProcessingConfig.archivingOption = S3ArchivingOption.MOVE_TO_BUCKET;
s3ConfigBean.postProcessingConfig.postProcessing = PostProcessingOptions.ARCHIVE;
s3ConfigBean.postProcessingConfig.postProcessBucket = POSTPROCESS_BUCKET;
s3ConfigBean.postProcessingConfig.postProcessFolder = POSTPROCESS_FOLDER;
s3ConfigBean.s3FileConfig = new S3FileConfig();
s3ConfigBean.s3FileConfig.overrunLimit = 65;
s3ConfigBean.s3FileConfig.filePattern = "*/*/*.log";
s3ConfigBean.s3Config = new S3Config();
s3ConfigBean.s3Config.setEndPointForTest("http://localhost:" + port);
s3ConfigBean.s3Config.bucket = BUCKET_NAME;
s3ConfigBean.s3Config.accessKeyId = "foo";
s3ConfigBean.s3Config.secretAccessKey = "bar";
s3ConfigBean.s3Config.folder = "";
s3ConfigBean.s3Config.delimiter = "/";
s3ConfigBean.advancedConfig = new S3AdvancedConfig();
s3ConfigBean.advancedConfig.useProxy = false;
return new AmazonS3Source(s3ConfigBean);
}
private AmazonS3Source createTarGzipAvroSource() {
S3ConfigBean s3ConfigBean = new S3ConfigBean();
s3ConfigBean.basicConfig = new BasicConfig();
s3ConfigBean.basicConfig.maxWaitTime = 1000;
s3ConfigBean.basicConfig.maxBatchSize = 60000;
s3ConfigBean.dataFormatConfig = new DataFormatConfig();
s3ConfigBean.dataFormat = DataFormat.AVRO;
s3ConfigBean.dataFormatConfig.charset = "UTF-8";
s3ConfigBean.dataFormatConfig.textMaxLineLen = 102400;
s3ConfigBean.errorConfig = new S3ErrorConfig();
s3ConfigBean.errorConfig.errorHandlingOption = PostProcessingOptions.NONE;
s3ConfigBean.postProcessingConfig = new S3PostProcessingConfig();
s3ConfigBean.postProcessingConfig.postProcessing = PostProcessingOptions.NONE;
s3ConfigBean.s3FileConfig = new S3FileConfig();
s3ConfigBean.s3FileConfig.overrunLimit = 65;
s3ConfigBean.s3FileConfig.filePattern = "*/testAvro*.tar.gz";
s3ConfigBean.dataFormatConfig.compression = Compression.COMPRESSED_ARCHIVE;
s3ConfigBean.dataFormatConfig.filePatternInArchive = "[!.]*.avro";
s3ConfigBean.s3Config = new S3Config();
s3ConfigBean.s3Config.setEndPointForTest("http://localhost:" + port);
s3ConfigBean.s3Config.bucket = BUCKET_NAME;
s3ConfigBean.s3Config.accessKeyId = "foo";
s3ConfigBean.s3Config.secretAccessKey = "bar";
s3ConfigBean.s3Config.folder = "";
s3ConfigBean.s3Config.delimiter = "/";
s3ConfigBean.advancedConfig = new S3AdvancedConfig();
s3ConfigBean.advancedConfig.useProxy = false;
return new AmazonS3Source(s3ConfigBean);
}
