@Test
public void testFailsOnVerifyFile() throws Exception {
File invalidJsonFile = new File(tempStageDir, "invalidjson2.snappy");
Files.copy(new File(Resources.getResource("invalidjson2.snappy").toURI()), invalidJsonFile);
assertTrue(invalidJsonFile.exists());
uploader =
new S3Uploader(
storageSystem,
serverConfig,
new EventPartitioner(),
executor,
executor,
s3UploaderStats);
uploader.start();
assertFalse(invalidJsonFile.exists());
assertTrue(new File(tempStageDir.getPath() + "/failed", invalidJsonFile.getName()).exists());
assertFalse(storageSystem.hasReceivedFile(invalidJsonFile));
S3UploaderStats s3UploaderStatsArgumentVerifier =
testingReportCollectionFactory.getArgumentVerifier(S3UploaderStats.class);
verify(s3UploaderStatsArgumentVerifier).processedFiles(UNKNOWN_EVENT_TYPE, CORRUPT);
verifyNoMoreInteractions(s3UploaderStatsArgumentVerifier);
CounterStat processedFilesCounterStat =
testingReportCollectionFactory
.getReportCollection(S3UploaderStats.class)
.processedFiles(UNKNOWN_EVENT_TYPE, CORRUPT);
verify(processedFilesCounterStat).add(1);
verifyNoMoreInteractions(processedFilesCounterStat);
}
@Test
public void testPulseMetricsForUploadAttempts() throws Exception {
storageSystem.succeedOnAttempt(2);
File pendingFile = new File(tempStageDir, "pending.json.snappy");
Files.copy(new File(Resources.getResource("pending.json.snappy").toURI()), pendingFile);
// attempt 1 to upload from staging directory fails and file is moved to retry directory
uploader =
new S3Uploader(
storageSystem,
serverConfig,
new EventPartitioner(),
executor,
executor,
s3UploaderStats);
uploader.start();
S3UploaderStats s3UploaderStatsArgumentVerifier =
testingReportCollectionFactory.getArgumentVerifier(S3UploaderStats.class);
verify(s3UploaderStatsArgumentVerifier).uploadAttempts(ARBITRARY_EVENT_TYPE, FAILURE);
verify(s3UploaderStatsArgumentVerifier).processedTime(ARBITRARY_EVENT_TYPE);
verifyNoMoreInteractions(s3UploaderStatsArgumentVerifier);
S3UploaderStats s3UploaderStatsReportCollection =
testingReportCollectionFactory.getReportCollection(S3UploaderStats.class);
CounterStat uploadAttemptsCounterStat =
s3UploaderStatsReportCollection.uploadAttempts(ARBITRARY_EVENT_TYPE, FAILURE);
TimeStat processedTimeTimerStat =
s3UploaderStatsReportCollection.processedTime(ARBITRARY_EVENT_TYPE);
verify(uploadAttemptsCounterStat).add(1);
verify(processedTimeTimerStat).time();
verifyNoMoreInteractions(uploadAttemptsCounterStat);
verifyNoMoreInteractions(processedTimeTimerStat);
}
@Test
public void testUploadPendingFiles() throws Exception {
File pendingFile = new File(tempStageDir, "pending.json.snappy");
Files.copy(new File(Resources.getResource("pending.json.snappy").toURI()), pendingFile);
assertTrue(pendingFile.exists());
uploader =
new S3Uploader(
storageSystem,
serverConfig,
new EventPartitioner(),
executor,
executor,
s3UploaderStats);
uploader.start();
assertFalse(pendingFile.exists());
assertTrue(storageSystem.hasReceivedFile(pendingFile));
S3UploaderStats s3UploaderStatsArgumentVerifier =
testingReportCollectionFactory.getArgumentVerifier(S3UploaderStats.class);
verify(s3UploaderStatsArgumentVerifier).processedFiles(ARBITRARY_EVENT_TYPE, UPLOADED);
verify(s3UploaderStatsArgumentVerifier).uploadAttempts(ARBITRARY_EVENT_TYPE, SUCCESS);
verify(s3UploaderStatsArgumentVerifier).processedTime(ARBITRARY_EVENT_TYPE);
verifyNoMoreInteractions(s3UploaderStatsArgumentVerifier);
S3UploaderStats s3UploaderStatsReportCollection =
testingReportCollectionFactory.getReportCollection(S3UploaderStats.class);
CounterStat processedFilesCounterStat =
s3UploaderStatsReportCollection.processedFiles(ARBITRARY_EVENT_TYPE, UPLOADED);
CounterStat uploadAttemptsCounterStat =
s3UploaderStatsReportCollection.uploadAttempts(ARBITRARY_EVENT_TYPE, SUCCESS);
TimeStat processedTimeTimerStat =
s3UploaderStatsReportCollection.processedTime(ARBITRARY_EVENT_TYPE);
verify(processedFilesCounterStat).add(1);
verify(uploadAttemptsCounterStat).add(1);
verify(processedTimeTimerStat).time();
verifyNoMoreInteractions(processedFilesCounterStat);
verifyNoMoreInteractions(uploadAttemptsCounterStat);
verifyNoMoreInteractions(processedTimeTimerStat);
}
@BeforeMethod
public void setup() {
storageSystem = new DummyStorageSystem();
tempStageDir = Files.createTempDir();
tempStageDir.deleteOnExit();
serverConfig =
new ServerConfig()
.setLocalStagingDirectory(tempStageDir)
.setS3StagingLocation("s3://fake-location")
.setRetryPeriod(new Duration(3, TimeUnit.MINUTES))
.setRetryDelay(new Duration(1, TimeUnit.MINUTES));
executor = new SerialScheduledExecutorService();
testingReportCollectionFactory = new TestingReportCollectionFactory();
s3UploaderStats = testingReportCollectionFactory.createReportCollection(S3UploaderStats.class);
}