private AmazonS3Source createSource() { S3ConfigBean s3ConfigBean = new S3ConfigBean(); s3ConfigBean.basicConfig = new BasicConfig(); s3ConfigBean.basicConfig.maxWaitTime = 1000; s3ConfigBean.basicConfig.maxBatchSize = 60000; s3ConfigBean.dataFormatConfig = new DataFormatConfig(); s3ConfigBean.dataFormat = DataFormat.TEXT; s3ConfigBean.dataFormatConfig.charset = "UTF-8"; s3ConfigBean.dataFormatConfig.textMaxLineLen = 1024; s3ConfigBean.dataFormatConfig.compression = Compression.NONE; s3ConfigBean.dataFormatConfig.filePatternInArchive = "*"; s3ConfigBean.errorConfig = new S3ErrorConfig(); s3ConfigBean.errorConfig.errorHandlingOption = PostProcessingOptions.NONE; s3ConfigBean.errorConfig.errorFolder = ERROR_FOLDER; s3ConfigBean.errorConfig.errorBucket = ERROR_BUCKET; s3ConfigBean.postProcessingConfig = new S3PostProcessingConfig(); s3ConfigBean.postProcessingConfig.archivingOption = S3ArchivingOption.MOVE_TO_BUCKET; s3ConfigBean.postProcessingConfig.postProcessing = PostProcessingOptions.ARCHIVE; s3ConfigBean.postProcessingConfig.postProcessBucket = POSTPROCESS_BUCKET; s3ConfigBean.postProcessingConfig.postProcessFolder = POSTPROCESS_FOLDER; s3ConfigBean.s3FileConfig = new S3FileConfig(); s3ConfigBean.s3FileConfig.overrunLimit = 65; s3ConfigBean.s3FileConfig.filePattern = "*/*/*.log"; s3ConfigBean.s3Config = new S3Config(); s3ConfigBean.s3Config.setEndPointForTest("http://localhost:" + port); s3ConfigBean.s3Config.bucket = BUCKET_NAME; s3ConfigBean.s3Config.accessKeyId = "foo"; s3ConfigBean.s3Config.secretAccessKey = "bar"; s3ConfigBean.s3Config.folder = ""; s3ConfigBean.s3Config.delimiter = "/"; s3ConfigBean.advancedConfig = new S3AdvancedConfig(); s3ConfigBean.advancedConfig.useProxy = false; return new AmazonS3Source(s3ConfigBean); }
private AmazonS3Source createTarGzipAvroSource() { S3ConfigBean s3ConfigBean = new S3ConfigBean(); s3ConfigBean.basicConfig = new BasicConfig(); s3ConfigBean.basicConfig.maxWaitTime = 1000; s3ConfigBean.basicConfig.maxBatchSize = 60000; s3ConfigBean.dataFormatConfig = new DataFormatConfig(); s3ConfigBean.dataFormat = DataFormat.AVRO; s3ConfigBean.dataFormatConfig.charset = "UTF-8"; s3ConfigBean.dataFormatConfig.textMaxLineLen = 102400; s3ConfigBean.errorConfig = new S3ErrorConfig(); s3ConfigBean.errorConfig.errorHandlingOption = PostProcessingOptions.NONE; s3ConfigBean.postProcessingConfig = new S3PostProcessingConfig(); s3ConfigBean.postProcessingConfig.postProcessing = PostProcessingOptions.NONE; s3ConfigBean.s3FileConfig = new S3FileConfig(); s3ConfigBean.s3FileConfig.overrunLimit = 65; s3ConfigBean.s3FileConfig.filePattern = "*/testAvro*.tar.gz"; s3ConfigBean.dataFormatConfig.compression = Compression.COMPRESSED_ARCHIVE; s3ConfigBean.dataFormatConfig.filePatternInArchive = "[!.]*.avro"; s3ConfigBean.s3Config = new S3Config(); s3ConfigBean.s3Config.setEndPointForTest("http://localhost:" + port); s3ConfigBean.s3Config.bucket = BUCKET_NAME; s3ConfigBean.s3Config.accessKeyId = "foo"; s3ConfigBean.s3Config.secretAccessKey = "bar"; s3ConfigBean.s3Config.folder = ""; s3ConfigBean.s3Config.delimiter = "/"; s3ConfigBean.advancedConfig = new S3AdvancedConfig(); s3ConfigBean.advancedConfig.useProxy = false; return new AmazonS3Source(s3ConfigBean); }