Exemplo n.º 1
0
 private void finalizePartitionsUpTo(String topic, Calendar calendar)
     throws IOException, ParseException, InterruptedException {
   NavigableSet<Calendar> partitionDates =
       getPartitions(topic).headSet(calendar, true).descendingSet();
   final String s3Prefix = "s3n://" + mConfig.getS3Bucket() + "/" + mConfig.getS3Path();
   SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
   format.setTimeZone(TimeZone.getTimeZone("UTC"));
   for (Calendar partition : partitionDates) {
     String partitionStr = format.format(partition.getTime());
     String[] partitions = {"dt=" + partitionStr};
     LogFilePath logFilePath =
         new LogFilePath(
             s3Prefix, topic, partitions, mConfig.getGeneration(), 0, 0, mFileExtension);
     String logFileDir = logFilePath.getLogFileDir();
     assert FileUtil.exists(logFileDir) : "FileUtil.exists(" + logFileDir + ")";
     String successFilePath = logFileDir + "/_SUCCESS";
     if (FileUtil.exists(successFilePath)) {
       return;
     }
     try {
       mQuboleClient.addPartition(mConfig.getHivePrefix() + topic, "dt='" + partitionStr + "'");
     } catch (Exception e) {
       LOG.error("failed to finalize topic " + topic + " partition dt=" + partitionStr, e);
       continue;
     }
     LOG.info("touching file " + successFilePath);
     FileUtil.touch(successFilePath);
   }
 }
Exemplo n.º 2
0
 private NavigableSet<Calendar> getPartitions(String topic) throws IOException, ParseException {
   final String s3Prefix = "s3n://" + mConfig.getS3Bucket() + "/" + mConfig.getS3Path();
   String[] partitions = {"dt="};
   LogFilePath logFilePath =
       new LogFilePath(s3Prefix, topic, partitions, mConfig.getGeneration(), 0, 0, mFileExtension);
   String parentDir = logFilePath.getLogFileParentDir();
   String[] partitionDirs = FileUtil.list(parentDir);
   Pattern pattern = Pattern.compile(".*/dt=(\\d\\d\\d\\d-\\d\\d-\\d\\d)$");
   TreeSet<Calendar> result = new TreeSet<Calendar>();
   for (String partitionDir : partitionDirs) {
     Matcher matcher = pattern.matcher(partitionDir);
     if (matcher.find()) {
       String date = matcher.group(1);
       SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
       format.setTimeZone(TimeZone.getTimeZone("UTC"));
       Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
       calendar.setTime(format.parse(date));
       result.add(calendar);
     }
   }
   return result;
 }
Exemplo n.º 3
0
 public void finalizePartitions() throws Exception {
   List<String> topics = mZookeeperConnector.getCommittedOffsetTopics();
   for (String topic : topics) {
     if (!topic.matches(mConfig.getKafkaTopicFilter())) {
       LOG.info("skipping topic " + topic);
     } else {
       LOG.info("finalizing topic " + topic);
       long finalizedTimestampMillis = getFinalizedTimestampMillis(topic);
       LOG.info("finalized timestamp for topic " + topic + " is " + finalizedTimestampMillis);
       if (finalizedTimestampMillis != -1) {
         Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
         calendar.setTimeInMillis(finalizedTimestampMillis);
         // Introduce a lag of one day and one hour.
         calendar.add(Calendar.HOUR, -1);
         calendar.add(Calendar.DAY_OF_MONTH, -1);
         finalizePartitionsUpTo(topic, calendar);
       }
     }
   }
 }