コード例 #1
0
  @Test
  public void testEviction4() throws Exception {
    try {
      Configuration conf = cluster.getConf();
      FileSystem fs = FileSystem.get(conf);
      fs.delete(new Path("/"), true);
      stream.clear();

      Pair<List<String>, List<String>> pair = createTestData("/data");
      FeedEvictor.main(
          new String[] {
            "-feedBasePath",
                LocationType.DATA.name()
                    + "="
                    + cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY)
                    + "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}",
            "-retentionType", "instance",
            "-retentionLimit", "months(5)",
            "-timeZone", "UTC",
            "-frequency", "hourly",
            "-logFile",
                conf.get(HadoopClientFactory.FS_DEFAULT_NAME_KEY)
                    + "/falcon/staging/feed/2012-01-01-04-00",
            "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
          });
      Assert.assertEquals("instances=NULL", stream.getBuffer());

      stream.clear();
      String dataPath = "/data/YYYY/feed4/dd/MM/02/more/hello";
      String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
      FeedEvictor.main(
          new String[] {
            "-feedBasePath",
                LocationType.DATA.name()
                    + "="
                    + cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY)
                    + dataPath,
            "-retentionType", "instance",
            "-retentionLimit", "hours(5)",
            "-timeZone", "UTC",
            "-frequency", "hourly",
            "-logFile", logFile,
            "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
          });
      Assert.assertEquals("instances=NULL", stream.getBuffer());

      Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));

      assertFailures(fs, pair);
    } catch (Exception e) {
      Assert.fail("Unknown exception", e);
    }
  }
コード例 #2
0
  @Test(enabled = false)
  public void testEvictionWithEmptyDirs() throws Exception {
    try {
      Configuration conf = cluster.getConf();
      FileSystem fs = FileSystem.get(conf);
      fs.delete(new Path("/"), true);
      stream.clear();

      Pair<List<String>, List<String>> pair =
          generateInstances(
              fs, "feed1", "yyyy/MM/dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data", false);
      final String storageUrl = cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY);
      String dataPath =
          LocationType.DATA.name()
              + "="
              + storageUrl
              + "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}/?{MONTH}/?{DAY}/more/?{YEAR}";
      String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
      long beforeDelCount =
          fs.getContentSummary(new Path(("/data/YYYY/feed1/mmHH/dd/MM/"))).getDirectoryCount();

      FeedEvictor.main(
          new String[] {
            "-feedBasePath", dataPath,
            "-retentionType", "instance",
            "-retentionLimit", "days(10)",
            "-timeZone", "UTC",
            "-frequency", "daily",
            "-logFile", logFile,
            "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
          });

      compare(map.get("feed1"), stream.getBuffer());

      String expectedInstancePaths = getExpectedInstancePaths(dataPath.replaceAll(storageUrl, ""));
      Assert.assertEquals(readLogFile(new Path(logFile)), expectedInstancePaths);

      String deletedPath = expectedInstancePaths.split(",")[0].split("=")[1];
      Assert.assertFalse(fs.exists(new Path(deletedPath)));
      // empty parents
      Assert.assertFalse(fs.exists(new Path(deletedPath).getParent()));
      Assert.assertFalse(fs.exists(new Path(deletedPath).getParent().getParent()));
      // base path not deleted
      Assert.assertTrue(fs.exists(new Path("/data/YYYY/feed1/mmHH/dd/MM/")));
      // non-eligible empty dirs
      long afterDelCount =
          fs.getContentSummary(new Path(("/data/YYYY/feed1/mmHH/dd/MM/"))).getDirectoryCount();
      Assert.assertEquals((beforeDelCount - afterDelCount), 19);
      for (String path : pair.second) {
        Assert.assertTrue(fs.exists(new Path(path)));
      }

    } catch (Exception e) {
      Assert.fail("Unknown exception", e);
    }
  }
コード例 #3
0
  @Test
  public void testEviction2() throws Exception {
    try {
      Configuration conf = cluster.getConf();
      FileSystem fs = FileSystem.get(conf);
      fs.delete(new Path("/"), true);
      stream.clear();

      Pair<List<String>, List<String>> pair =
          createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS, "/data");
      final String storageUrl = cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY);
      String dataPath =
          LocationType.DATA.name()
              + "="
              + storageUrl
              + "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
      String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";

      FeedEvictor.main(
          new String[] {
            "-feedBasePath", dataPath,
            "-retentionType", "instance",
            "-retentionLimit", "days(10)",
            "-timeZone", "UTC",
            "-frequency", "daily",
            "-logFile", logFile,
            "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
          });

      assertFailures(fs, pair);
      compare(map.get("feed1"), stream.getBuffer());

      String expectedInstancePaths = getExpectedInstancePaths(dataPath);
      Assert.assertEquals(readLogFile(new Path(logFile)), expectedInstancePaths);

      String deletedPath = expectedInstancePaths.split(",")[0].split("=")[1];
      Assert.assertFalse(fs.exists(new Path(deletedPath)));
      // empty parents
      Assert.assertFalse(fs.exists(new Path(deletedPath).getParent()));
      Assert.assertFalse(fs.exists(new Path(deletedPath).getParent().getParent()));
      // base path not deleted
      Assert.assertTrue(fs.exists(new Path("/data/YYYY/feed1/mmHH/dd/MM/")));

    } catch (Exception e) {
      Assert.fail("Unknown exception", e);
    }
  }
コード例 #4
0
  @Test
  public void testEviction3() throws Exception {
    try {
      Configuration conf = cluster.getConf();
      FileSystem fs = FileSystem.get(conf);
      fs.delete(new Path("/"), true);
      stream.clear();

      Pair<List<String>, List<String>> pair =
          createTestData("feed2", "yyyyMMddHH/'more'/yyyy", 5, TimeUnit.HOURS, "/data");
      final String storageUrl = cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY);
      String dataPath =
          LocationType.DATA.name()
              + "="
              + storageUrl
              + "/data/YYYY/feed2/mmHH/dd/MM/?{YEAR}?{MONTH}?{DAY}?{HOUR}/more/?{YEAR}";
      String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
      FeedEvictor.main(
          new String[] {
            "-feedBasePath", dataPath,
            "-retentionType", "instance",
            "-retentionLimit", "hours(5)",
            "-timeZone", "UTC",
            "-frequency", "hourly",
            "-logFile", logFile,
            "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
          });
      assertFailures(fs, pair);

      compare(map.get("feed2"), stream.getBuffer());

      Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));

    } catch (Exception e) {
      Assert.fail("Unknown exception", e);
    }
  }