예제 #1
0
  @Test
  public void testRetryUntilSuccess() throws Exception {
    storageSystem.succeedOnAttempt(3);
    File pendingFile = new File(tempStageDir, "pending.json.snappy");
    Files.copy(new File(Resources.getResource("pending.json.snappy").toURI()), pendingFile);

    String retryDir = tempStageDir.getPath() + "/retry";

    assertEquals(storageSystem.getAttempts(pendingFile), 0);
    // attempt 1 to upload from staging directory fails and file is moved to retry directory
    assertTrue(pendingFile.exists());
    uploader =
        new S3Uploader(
            storageSystem,
            serverConfig,
            new EventPartitioner(),
            executor,
            executor,
            s3UploaderStats);
    uploader.start();
    assertFalse(pendingFile.exists());
    assertTrue(new File(retryDir, pendingFile.getName()).exists());
    assertFalse(storageSystem.hasReceivedFile(pendingFile));
    assertEquals(storageSystem.getAttempts(pendingFile), 1);

    // attempt 2: file is moved to staging from retry directory, and fails and hence gets back to
    // retry directory
    executor.elapseTime(1, TimeUnit.MINUTES);
    assertFalse(pendingFile.exists());
    assertTrue(new File(retryDir, pendingFile.getName()).exists());
    assertFalse(storageSystem.hasReceivedFile(pendingFile));
    assertEquals(storageSystem.getAttempts(pendingFile), 2);

    // retryExecutor hasn't run again
    executor.elapseTime(1, TimeUnit.MINUTES);
    assertFalse(pendingFile.exists());
    assertTrue(new File(retryDir, pendingFile.getName()).exists());
    assertFalse(storageSystem.hasReceivedFile(pendingFile));
    assertEquals(storageSystem.getAttempts(pendingFile), 2);

    // attempt 3: file is moved to staging from retry directory, succeeds and hence is deleted from
    // local directories
    executor.elapseTime(2, TimeUnit.MINUTES);
    assertFalse(pendingFile.exists());
    assertFalse(new File(retryDir, pendingFile.getName()).exists());
    assertTrue(storageSystem.hasReceivedFile(pendingFile));
    assertEquals(storageSystem.getAttempts(pendingFile), 3);
  }
예제 #2
0
  @Test
  public void testInvalidFilesInRetryDirectory() throws Exception {
    uploader =
        new S3Uploader(
            storageSystem,
            serverConfig,
            new EventPartitioner(),
            executor,
            executor,
            s3UploaderStats);
    uploader.start();

    executor.elapseTime(1, TimeUnit.MINUTES);
    String retryDir = tempStageDir.getPath() + "/retry";
    String failedDir = tempStageDir.getPath() + "/failed";

    File invalidJsonFile = new File(retryDir, "invalidjson.snappy");
    Files.copy(new File(Resources.getResource("invalidjson.snappy").toURI()), invalidJsonFile);

    File invalidFile = new File(retryDir, "invalidFile.snappy");
    Files.copy(new File(Resources.getResource("invalidjson2.snappy").toURI()), invalidFile);

    File directory = new File(retryDir, "subdir");
    directory.mkdir();

    assertTrue(invalidJsonFile.exists());
    assertTrue(invalidFile.exists());
    assertTrue(directory.exists());

    executor.elapseTime(3, TimeUnit.MINUTES);

    assertTrue(new File(failedDir, invalidJsonFile.getName()).exists());
    assertFalse(new File(retryDir, invalidJsonFile.getName()).exists());
    assertFalse(new File(tempStageDir.getPath(), invalidJsonFile.getName()).exists());

    assertTrue(new File(failedDir, invalidFile.getName()).exists());
    assertFalse(new File(retryDir, invalidFile.getName()).exists());
    assertFalse(new File(tempStageDir.getPath(), invalidFile.getName()).exists());

    assertTrue(directory.exists());
  }