@Test(
dataProvider = "evictorTestInvalidDataProvider",
expectedExceptions = URISyntaxException.class)
public void testFeedEvictorForInvalidTableStorage(
String retentionLimit, String dateSeparator, boolean isExternal) throws Exception {
final String tableName = isExternal ? EXTERNAL_TABLE_NAME : TABLE_NAME;
final String timeZone = "UTC";
final String dateMask = "yyyy" + dateSeparator + "MM" + dateSeparator + "dd";
List<String> candidatePartitions = getCandidatePartitions("days(10)", dateMask, timeZone, 3);
addPartitions(tableName, candidatePartitions, isExternal);
try {
stream.clear();
final String tableUri =
DATABASE_NAME
+ "/"
+ tableName
+ "/ds=${YEAR}"
+ dateSeparator
+ "${MONTH}"
+ dateSeparator
+ "${DAY};region=us";
String feedBasePath = METASTORE_URL + tableUri;
String logFile = STORAGE_URL + "/falcon/staging/feed/instancePaths-2013-09-13-01-00.csv";
FeedEvictor.main(
new String[] {
"-feedBasePath", feedBasePath,
"-retentionType", "instance",
"-retentionLimit", retentionLimit,
"-timeZone", timeZone,
"-frequency", "daily",
"-logFile", logFile,
"-falconFeedStorageType", Storage.TYPE.TABLE.name(),
});
Assert.fail("Exception must have been thrown");
} finally {
dropPartitions(tableName, candidatePartitions);
}
}
@Test(dataProvider = "multiColDatedEvictorTestDataProvider")
public void testFeedEvictorForMultiColDatedTableStorage(String retentionLimit, boolean isExternal)
throws Exception {
final String tableName =
isExternal ? MULTI_COL_DATED_EXTERNAL_TABLE_NAME : MULTI_COL_DATED_TABLE_NAME;
final String timeZone = "UTC";
List<Map<String, String>> candidatePartitions =
getMultiColDatedCandidatePartitions("days(10)", timeZone, 3);
addMultiColDatedPartitions(tableName, candidatePartitions, isExternal);
List<HCatPartition> partitions = client.getPartitions(DATABASE_NAME, tableName);
Assert.assertEquals(partitions.size(), candidatePartitions.size());
Pair<Date, Date> range = getDateRange(retentionLimit);
List<HCatPartition> filteredPartitions =
getMultiColDatedFilteredPartitions(tableName, timeZone, range);
try {
stream.clear();
final String tableUri =
DATABASE_NAME + "/" + tableName + "/year=${YEAR};month=${MONTH};day=${DAY};region=us";
String feedBasePath = METASTORE_URL + tableUri;
String logFile = STORAGE_URL + "/falcon/staging/feed/instancePaths-2013-09-13-01-00.csv";
FeedEvictor.main(
new String[] {
"-feedBasePath", feedBasePath,
"-retentionType", "instance",
"-retentionLimit", retentionLimit,
"-timeZone", timeZone,
"-frequency", "daily",
"-logFile", logFile,
"-falconFeedStorageType", Storage.TYPE.TABLE.name(),
});
StringBuilder expectedInstancePaths = new StringBuilder();
List<Map<String, String>> expectedInstancesEvicted =
getMultiColDatedExpectedEvictedInstances(
candidatePartitions, range.first, timeZone, expectedInstancePaths);
int expectedSurvivorSize = candidatePartitions.size() - expectedInstancesEvicted.size();
List<HCatPartition> survivingPartitions = client.getPartitions(DATABASE_NAME, tableName);
Assert.assertEquals(
survivingPartitions.size(),
expectedSurvivorSize,
"Unexpected number of surviving partitions");
Assert.assertEquals(
expectedInstancesEvicted.size(),
filteredPartitions.size(),
"Unexpected number of evicted partitions");
final String actualInstancesEvicted = readLogFile(new Path(logFile));
validateInstancePaths(actualInstancesEvicted, expectedInstancePaths.toString());
if (isExternal) {
verifyMultiColDatedFSPartitionsAreDeleted(candidatePartitions, range.first, timeZone);
}
} finally {
dropMultiColDatedPartitions(tableName, candidatePartitions);
Assert.assertEquals(client.getPartitions(DATABASE_NAME, tableName).size(), 0);
}
}