Exemplo n.º 1
0
  @Test
  public void testFailsOnVerifyFile() throws Exception {
    File invalidJsonFile = new File(tempStageDir, "invalidjson2.snappy");
    Files.copy(new File(Resources.getResource("invalidjson2.snappy").toURI()), invalidJsonFile);

    assertTrue(invalidJsonFile.exists());
    uploader =
        new S3Uploader(
            storageSystem,
            serverConfig,
            new EventPartitioner(),
            executor,
            executor,
            s3UploaderStats);
    uploader.start();
    assertFalse(invalidJsonFile.exists());
    assertTrue(new File(tempStageDir.getPath() + "/failed", invalidJsonFile.getName()).exists());
    assertFalse(storageSystem.hasReceivedFile(invalidJsonFile));

    S3UploaderStats s3UploaderStatsArgumentVerifier =
        testingReportCollectionFactory.getArgumentVerifier(S3UploaderStats.class);
    verify(s3UploaderStatsArgumentVerifier).processedFiles(UNKNOWN_EVENT_TYPE, CORRUPT);
    verifyNoMoreInteractions(s3UploaderStatsArgumentVerifier);

    CounterStat processedFilesCounterStat =
        testingReportCollectionFactory
            .getReportCollection(S3UploaderStats.class)
            .processedFiles(UNKNOWN_EVENT_TYPE, CORRUPT);
    verify(processedFilesCounterStat).add(1);
    verifyNoMoreInteractions(processedFilesCounterStat);
  }
Exemplo n.º 2
0
  @Test
  public void testPulseMetricsForUploadAttempts() throws Exception {
    storageSystem.succeedOnAttempt(2);
    File pendingFile = new File(tempStageDir, "pending.json.snappy");
    Files.copy(new File(Resources.getResource("pending.json.snappy").toURI()), pendingFile);

    // attempt 1 to upload from staging directory fails and file is moved to retry directory
    uploader =
        new S3Uploader(
            storageSystem,
            serverConfig,
            new EventPartitioner(),
            executor,
            executor,
            s3UploaderStats);
    uploader.start();

    S3UploaderStats s3UploaderStatsArgumentVerifier =
        testingReportCollectionFactory.getArgumentVerifier(S3UploaderStats.class);
    verify(s3UploaderStatsArgumentVerifier).uploadAttempts(ARBITRARY_EVENT_TYPE, FAILURE);
    verify(s3UploaderStatsArgumentVerifier).processedTime(ARBITRARY_EVENT_TYPE);
    verifyNoMoreInteractions(s3UploaderStatsArgumentVerifier);

    S3UploaderStats s3UploaderStatsReportCollection =
        testingReportCollectionFactory.getReportCollection(S3UploaderStats.class);
    CounterStat uploadAttemptsCounterStat =
        s3UploaderStatsReportCollection.uploadAttempts(ARBITRARY_EVENT_TYPE, FAILURE);
    TimeStat processedTimeTimerStat =
        s3UploaderStatsReportCollection.processedTime(ARBITRARY_EVENT_TYPE);
    verify(uploadAttemptsCounterStat).add(1);
    verify(processedTimeTimerStat).time();
    verifyNoMoreInteractions(uploadAttemptsCounterStat);
    verifyNoMoreInteractions(processedTimeTimerStat);
  }
Exemplo n.º 3
0
  @Test
  public void testUploadPendingFiles() throws Exception {
    File pendingFile = new File(tempStageDir, "pending.json.snappy");
    Files.copy(new File(Resources.getResource("pending.json.snappy").toURI()), pendingFile);

    assertTrue(pendingFile.exists());
    uploader =
        new S3Uploader(
            storageSystem,
            serverConfig,
            new EventPartitioner(),
            executor,
            executor,
            s3UploaderStats);
    uploader.start();
    assertFalse(pendingFile.exists());
    assertTrue(storageSystem.hasReceivedFile(pendingFile));

    S3UploaderStats s3UploaderStatsArgumentVerifier =
        testingReportCollectionFactory.getArgumentVerifier(S3UploaderStats.class);
    verify(s3UploaderStatsArgumentVerifier).processedFiles(ARBITRARY_EVENT_TYPE, UPLOADED);
    verify(s3UploaderStatsArgumentVerifier).uploadAttempts(ARBITRARY_EVENT_TYPE, SUCCESS);
    verify(s3UploaderStatsArgumentVerifier).processedTime(ARBITRARY_EVENT_TYPE);
    verifyNoMoreInteractions(s3UploaderStatsArgumentVerifier);

    S3UploaderStats s3UploaderStatsReportCollection =
        testingReportCollectionFactory.getReportCollection(S3UploaderStats.class);
    CounterStat processedFilesCounterStat =
        s3UploaderStatsReportCollection.processedFiles(ARBITRARY_EVENT_TYPE, UPLOADED);
    CounterStat uploadAttemptsCounterStat =
        s3UploaderStatsReportCollection.uploadAttempts(ARBITRARY_EVENT_TYPE, SUCCESS);
    TimeStat processedTimeTimerStat =
        s3UploaderStatsReportCollection.processedTime(ARBITRARY_EVENT_TYPE);
    verify(processedFilesCounterStat).add(1);
    verify(uploadAttemptsCounterStat).add(1);
    verify(processedTimeTimerStat).time();
    verifyNoMoreInteractions(processedFilesCounterStat);
    verifyNoMoreInteractions(uploadAttemptsCounterStat);
    verifyNoMoreInteractions(processedTimeTimerStat);
  }
Exemplo n.º 4
0
  @BeforeMethod
  public void setup() {
    storageSystem = new DummyStorageSystem();
    tempStageDir = Files.createTempDir();
    tempStageDir.deleteOnExit();

    serverConfig =
        new ServerConfig()
            .setLocalStagingDirectory(tempStageDir)
            .setS3StagingLocation("s3://fake-location")
            .setRetryPeriod(new Duration(3, TimeUnit.MINUTES))
            .setRetryDelay(new Duration(1, TimeUnit.MINUTES));

    executor = new SerialScheduledExecutorService();

    testingReportCollectionFactory = new TestingReportCollectionFactory();
    s3UploaderStats = testingReportCollectionFactory.createReportCollection(S3UploaderStats.class);
  }