private void finalizePartitionsUpTo(String topic, Calendar calendar)
throws IOException, ParseException, InterruptedException {
NavigableSet<Calendar> partitionDates =
getPartitions(topic).headSet(calendar, true).descendingSet();
final String s3Prefix = "s3n://" + mConfig.getS3Bucket() + "/" + mConfig.getS3Path();
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
format.setTimeZone(TimeZone.getTimeZone("UTC"));
for (Calendar partition : partitionDates) {
String partitionStr = format.format(partition.getTime());
String[] partitions = {"dt=" + partitionStr};
LogFilePath logFilePath =
new LogFilePath(
s3Prefix, topic, partitions, mConfig.getGeneration(), 0, 0, mFileExtension);
String logFileDir = logFilePath.getLogFileDir();
assert FileUtil.exists(logFileDir) : "FileUtil.exists(" + logFileDir + ")";
String successFilePath = logFileDir + "/_SUCCESS";
if (FileUtil.exists(successFilePath)) {
return;
}
try {
mQuboleClient.addPartition(mConfig.getHivePrefix() + topic, "dt='" + partitionStr + "'");
} catch (Exception e) {
LOG.error("failed to finalize topic " + topic + " partition dt=" + partitionStr, e);
continue;
}
LOG.info("touching file " + successFilePath);
FileUtil.touch(successFilePath);
}
}
