@Test(enabled = false)
public void testEvictionWithEmptyDirs() throws Exception {
try {
Configuration conf = cluster.getConf();
FileSystem fs = FileSystem.get(conf);
fs.delete(new Path("/"), true);
stream.clear();
Pair<List<String>, List<String>> pair =
generateInstances(
fs, "feed1", "yyyy/MM/dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data", false);
final String storageUrl = cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY);
String dataPath =
LocationType.DATA.name()
+ "="
+ storageUrl
+ "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}/?{MONTH}/?{DAY}/more/?{YEAR}";
String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
long beforeDelCount =
fs.getContentSummary(new Path(("/data/YYYY/feed1/mmHH/dd/MM/"))).getDirectoryCount();
FeedEvictor.main(
new String[] {
"-feedBasePath", dataPath,
"-retentionType", "instance",
"-retentionLimit", "days(10)",
"-timeZone", "UTC",
"-frequency", "daily",
"-logFile", logFile,
"-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
});
compare(map.get("feed1"), stream.getBuffer());
String expectedInstancePaths = getExpectedInstancePaths(dataPath.replaceAll(storageUrl, ""));
Assert.assertEquals(readLogFile(new Path(logFile)), expectedInstancePaths);
String deletedPath = expectedInstancePaths.split(",")[0].split("=")[1];
Assert.assertFalse(fs.exists(new Path(deletedPath)));
// empty parents
Assert.assertFalse(fs.exists(new Path(deletedPath).getParent()));
Assert.assertFalse(fs.exists(new Path(deletedPath).getParent().getParent()));
// base path not deleted
Assert.assertTrue(fs.exists(new Path("/data/YYYY/feed1/mmHH/dd/MM/")));
// non-eligible empty dirs
long afterDelCount =
fs.getContentSummary(new Path(("/data/YYYY/feed1/mmHH/dd/MM/"))).getDirectoryCount();
Assert.assertEquals((beforeDelCount - afterDelCount), 19);
for (String path : pair.second) {
Assert.assertTrue(fs.exists(new Path(path)));
}
} catch (Exception e) {
Assert.fail("Unknown exception", e);
}
}
@Test
public void testEviction4() throws Exception {
try {
Configuration conf = cluster.getConf();
FileSystem fs = FileSystem.get(conf);
fs.delete(new Path("/"), true);
stream.clear();
Pair<List<String>, List<String>> pair = createTestData("/data");
FeedEvictor.main(
new String[] {
"-feedBasePath",
LocationType.DATA.name()
+ "="
+ cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY)
+ "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}",
"-retentionType", "instance",
"-retentionLimit", "months(5)",
"-timeZone", "UTC",
"-frequency", "hourly",
"-logFile",
conf.get(HadoopClientFactory.FS_DEFAULT_NAME_KEY)
+ "/falcon/staging/feed/2012-01-01-04-00",
"-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
});
Assert.assertEquals("instances=NULL", stream.getBuffer());
stream.clear();
String dataPath = "/data/YYYY/feed4/dd/MM/02/more/hello";
String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
FeedEvictor.main(
new String[] {
"-feedBasePath",
LocationType.DATA.name()
+ "="
+ cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY)
+ dataPath,
"-retentionType", "instance",
"-retentionLimit", "hours(5)",
"-timeZone", "UTC",
"-frequency", "hourly",
"-logFile", logFile,
"-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
});
Assert.assertEquals("instances=NULL", stream.getBuffer());
Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
assertFailures(fs, pair);
} catch (Exception e) {
Assert.fail("Unknown exception", e);
}
}
@Test
public void testEviction2() throws Exception {
try {
Configuration conf = cluster.getConf();
FileSystem fs = FileSystem.get(conf);
fs.delete(new Path("/"), true);
stream.clear();
Pair<List<String>, List<String>> pair =
createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
final String storageUrl = cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY);
String dataPath =
LocationType.DATA.name()
+ "="
+ storageUrl
+ "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
FeedEvictor.main(
new String[] {
"-feedBasePath", dataPath,
"-retentionType", "instance",
"-retentionLimit", "days(10)",
"-timeZone", "UTC",
"-frequency", "daily",
"-logFile", logFile,
"-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
});
assertFailures(fs, pair);
compare(map.get("feed1"), stream.getBuffer());
String expectedInstancePaths = getExpectedInstancePaths(dataPath);
Assert.assertEquals(readLogFile(new Path(logFile)), expectedInstancePaths);
String deletedPath = expectedInstancePaths.split(",")[0].split("=")[1];
Assert.assertFalse(fs.exists(new Path(deletedPath)));
// empty parents
Assert.assertFalse(fs.exists(new Path(deletedPath).getParent()));
Assert.assertFalse(fs.exists(new Path(deletedPath).getParent().getParent()));
// base path not deleted
Assert.assertTrue(fs.exists(new Path("/data/YYYY/feed1/mmHH/dd/MM/")));
} catch (Exception e) {
Assert.fail("Unknown exception", e);
}
}
@BeforeClass
public void setup() throws Exception {
StartupProperties.get()
.setProperty("falcon.state.store.impl", "org.apache.falcon.state.store.InMemoryStateStore");
super.setup();
this.dfsCluster = EmbeddedCluster.newCluster("testCluster");
this.conf = dfsCluster.getConf();
}
private String readLogFile(Path logFile) throws IOException {
Configuration conf = cluster.getConf();
FileSystem fs = FileSystem.get(conf);
ByteArrayOutputStream writer = new ByteArrayOutputStream();
InputStream date = fs.open(logFile);
IOUtils.copyBytes(date, writer, 4096, true);
return writer.toString();
}
@Test
public void testEviction6() throws Exception {
try {
Configuration conf = cluster.getConf();
FileSystem fs = FileSystem.get(conf);
fs.delete(new Path("/"), true);
stream.clear();
Pair<List<String>, List<String>> pair =
createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/stats");
createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/meta");
final String storageUrl = cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY);
String dataPath =
"DATA="
+ storageUrl
+ "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"
+ "#STATS="
+ storageUrl
+ "/stats/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"
+ "#META="
+ storageUrl
+ "/meta/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
FeedEvictor.main(
new String[] {
"-feedBasePath", dataPath,
"-retentionType", "instance",
"-retentionLimit", "days(10)",
"-timeZone", "UTC",
"-frequency", "daily",
"-logFile", logFile,
"-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
});
assertFailures(fs, pair);
Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
} catch (Exception e) {
Assert.fail("Unknown exception", e);
}
}
@BeforeClass
public void setup() throws Exception {
super.setup();
createDB(DB_SQL_FILE);
falconJPAService.init();
this.dfsCluster = EmbeddedCluster.newCluster("testCluster");
this.conf = dfsCluster.getConf();
registerServices();
}
protected Entity storeEntity(EntityType type, String name, String resource, String writeEndpoint)
throws Exception {
Unmarshaller unmarshaller = type.getUnmarshaller();
ConfigurationStore store = ConfigurationStore.get();
switch (type) {
case CLUSTER:
Cluster cluster = (Cluster) unmarshaller.unmarshal(this.getClass().getResource(resource));
if (name != null) {
store.remove(type, name);
cluster.setName(name);
}
store.publish(type, cluster);
if (writeEndpoint != null) {
ClusterHelper.getInterface(cluster, Interfacetype.WRITE).setEndpoint(writeEndpoint);
FileSystem fs = new Path(writeEndpoint).getFileSystem(EmbeddedCluster.newConfiguration());
fs.create(
new Path(
ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath(),
"libext/FEED/retention/ext.jar"))
.close();
fs.create(
new Path(
ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath(),
"libext/FEED/replication/ext.jar"))
.close();
}
return cluster;
case FEED:
Feed feed = (Feed) unmarshaller.unmarshal(this.getClass().getResource(resource));
if (name != null) {
store.remove(type, name);
feed.setName(name);
}
store.publish(type, feed);
return feed;
case PROCESS:
Process process = (Process) unmarshaller.unmarshal(this.getClass().getResource(resource));
if (name != null) {
store.remove(type, name);
process.setName(name);
}
store.publish(type, process);
return process;
default:
}
throw new IllegalArgumentException("Unhandled type: " + type);
}
private Pair<List<String>, List<String>> createTestData(
String feed, String mask, int period, TimeUnit timeUnit, String locationType)
throws Exception {
Configuration conf = cluster.getConf();
FileSystem fs = FileSystem.get(conf);
List<String> outOfRange = new ArrayList<String>();
List<String> inRange = new ArrayList<String>();
Pair<List<String>, List<String>> pair = createTestData(locationType);
outOfRange.addAll(pair.second);
inRange.addAll(pair.first);
pair = generateInstances(fs, feed, mask, period, timeUnit, locationType, true);
outOfRange.addAll(pair.second);
inRange.addAll(pair.first);
return Pair.of(inRange, outOfRange);
}
private Pair<List<String>, List<String>> createTestData(String locationType) throws Exception {
Configuration conf = cluster.getConf();
FileSystem fs = FileSystem.get(conf);
List<String> outOfRange = new ArrayList<String>();
List<String> inRange = new ArrayList<String>();
touch(fs, locationType + "/YYYY/feed3/dd/MM/02/more/hello", true);
touch(fs, locationType + "/YYYY/feed4/dd/MM/02/more/hello", true);
touch(fs, locationType + "/YYYY/feed1/mmHH/dd/MM/bad-va-lue/more/hello", true);
touch(fs, locationType + "/somedir/feed1/mmHH/dd/MM/bad-va-lue/more/hello", true);
outOfRange.add(locationType + "/YYYY/feed3/dd/MM/02/more/hello");
outOfRange.add(locationType + "/YYYY/feed4/dd/MM/02/more/hello");
outOfRange.add(locationType + "/YYYY/feed1/mmHH/dd/MM/bad-va-lue/more/hello");
outOfRange.add(locationType + "/somedir/feed1/mmHH/dd/MM/bad-va-lue/more/hello");
return Pair.of(inRange, outOfRange);
}
@AfterClass
public void close() throws Exception {
cluster.shutdown();
}
@BeforeClass
public void start() throws Exception {
cluster = EmbeddedCluster.newCluster("test");
hdfsUrl = cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY);
FeedEvictor.OUT.set(stream);
}
