Beispiel #1
0
 private boolean exists(URI uri, HCatClient client, boolean closeClient)
     throws HCatAccessorException {
   try {
     HCatURI hcatURI = new HCatURI(uri.toString());
     List<HCatPartition> partitions =
         client.getPartitions(hcatURI.getDb(), hcatURI.getTable(), hcatURI.getPartitionMap());
     return (partitions != null && !partitions.isEmpty());
   } catch (ConnectionFailureException e) {
     throw new HCatAccessorException(ErrorCode.E1501, e);
   } catch (HCatException e) {
     throw new HCatAccessorException(ErrorCode.E0902, e);
   } catch (URISyntaxException e) {
     throw new HCatAccessorException(ErrorCode.E0902, e);
   } finally {
     closeQuietly(client, closeClient);
   }
 }
  @Test(dataProvider = "multiColDatedEvictorTestDataProvider")
  public void testFeedEvictorForMultiColDatedTableStorage(String retentionLimit, boolean isExternal)
      throws Exception {
    final String tableName =
        isExternal ? MULTI_COL_DATED_EXTERNAL_TABLE_NAME : MULTI_COL_DATED_TABLE_NAME;
    final String timeZone = "UTC";

    List<Map<String, String>> candidatePartitions =
        getMultiColDatedCandidatePartitions("days(10)", timeZone, 3);
    addMultiColDatedPartitions(tableName, candidatePartitions, isExternal);

    List<HCatPartition> partitions = client.getPartitions(DATABASE_NAME, tableName);
    Assert.assertEquals(partitions.size(), candidatePartitions.size());
    Pair<Date, Date> range = getDateRange(retentionLimit);
    List<HCatPartition> filteredPartitions =
        getMultiColDatedFilteredPartitions(tableName, timeZone, range);

    try {
      stream.clear();

      final String tableUri =
          DATABASE_NAME + "/" + tableName + "/year=${YEAR};month=${MONTH};day=${DAY};region=us";
      String feedBasePath = METASTORE_URL + tableUri;
      String logFile = STORAGE_URL + "/falcon/staging/feed/instancePaths-2013-09-13-01-00.csv";

      FeedEvictor.main(
          new String[] {
            "-feedBasePath", feedBasePath,
            "-retentionType", "instance",
            "-retentionLimit", retentionLimit,
            "-timeZone", timeZone,
            "-frequency", "daily",
            "-logFile", logFile,
            "-falconFeedStorageType", Storage.TYPE.TABLE.name(),
          });

      StringBuilder expectedInstancePaths = new StringBuilder();
      List<Map<String, String>> expectedInstancesEvicted =
          getMultiColDatedExpectedEvictedInstances(
              candidatePartitions, range.first, timeZone, expectedInstancePaths);
      int expectedSurvivorSize = candidatePartitions.size() - expectedInstancesEvicted.size();

      List<HCatPartition> survivingPartitions = client.getPartitions(DATABASE_NAME, tableName);
      Assert.assertEquals(
          survivingPartitions.size(),
          expectedSurvivorSize,
          "Unexpected number of surviving partitions");

      Assert.assertEquals(
          expectedInstancesEvicted.size(),
          filteredPartitions.size(),
          "Unexpected number of evicted partitions");

      final String actualInstancesEvicted = readLogFile(new Path(logFile));
      validateInstancePaths(actualInstancesEvicted, expectedInstancePaths.toString());

      if (isExternal) {
        verifyMultiColDatedFSPartitionsAreDeleted(candidatePartitions, range.first, timeZone);
      }
    } finally {
      dropMultiColDatedPartitions(tableName, candidatePartitions);
      Assert.assertEquals(client.getPartitions(DATABASE_NAME, tableName).size(), 0);
    }
  }