private void finalizePartitionsUpTo(String topic, Calendar calendar) throws IOException, ParseException, InterruptedException { NavigableSet<Calendar> partitionDates = getPartitions(topic).headSet(calendar, true).descendingSet(); final String s3Prefix = "s3n://" + mConfig.getS3Bucket() + "/" + mConfig.getS3Path(); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); format.setTimeZone(TimeZone.getTimeZone("UTC")); for (Calendar partition : partitionDates) { String partitionStr = format.format(partition.getTime()); String[] partitions = {"dt=" + partitionStr}; LogFilePath logFilePath = new LogFilePath( s3Prefix, topic, partitions, mConfig.getGeneration(), 0, 0, mFileExtension); String logFileDir = logFilePath.getLogFileDir(); assert FileUtil.exists(logFileDir) : "FileUtil.exists(" + logFileDir + ")"; String successFilePath = logFileDir + "/_SUCCESS"; if (FileUtil.exists(successFilePath)) { return; } try { mQuboleClient.addPartition(mConfig.getHivePrefix() + topic, "dt='" + partitionStr + "'"); } catch (Exception e) { LOG.error("failed to finalize topic " + topic + " partition dt=" + partitionStr, e); continue; } LOG.info("touching file " + successFilePath); FileUtil.touch(successFilePath); } }
private NavigableSet<Calendar> getPartitions(String topic) throws IOException, ParseException { final String s3Prefix = "s3n://" + mConfig.getS3Bucket() + "/" + mConfig.getS3Path(); String[] partitions = {"dt="}; LogFilePath logFilePath = new LogFilePath(s3Prefix, topic, partitions, mConfig.getGeneration(), 0, 0, mFileExtension); String parentDir = logFilePath.getLogFileParentDir(); String[] partitionDirs = FileUtil.list(parentDir); Pattern pattern = Pattern.compile(".*/dt=(\\d\\d\\d\\d-\\d\\d-\\d\\d)$"); TreeSet<Calendar> result = new TreeSet<Calendar>(); for (String partitionDir : partitionDirs) { Matcher matcher = pattern.matcher(partitionDir); if (matcher.find()) { String date = matcher.group(1); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd"); format.setTimeZone(TimeZone.getTimeZone("UTC")); Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); calendar.setTime(format.parse(date)); result.add(calendar); } } return result; }
public void finalizePartitions() throws Exception { List<String> topics = mZookeeperConnector.getCommittedOffsetTopics(); for (String topic : topics) { if (!topic.matches(mConfig.getKafkaTopicFilter())) { LOG.info("skipping topic " + topic); } else { LOG.info("finalizing topic " + topic); long finalizedTimestampMillis = getFinalizedTimestampMillis(topic); LOG.info("finalized timestamp for topic " + topic + " is " + finalizedTimestampMillis); if (finalizedTimestampMillis != -1) { Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); calendar.setTimeInMillis(finalizedTimestampMillis); // Introduce a lag of one day and one hour. calendar.add(Calendar.HOUR, -1); calendar.add(Calendar.DAY_OF_MONTH, -1); finalizePartitionsUpTo(topic, calendar); } } } }