Пример #1
0
 private void finalizePartitionsUpTo(String topic, Calendar calendar)
     throws IOException, ParseException, InterruptedException {
   NavigableSet<Calendar> partitionDates =
       getPartitions(topic).headSet(calendar, true).descendingSet();
   final String s3Prefix = "s3n://" + mConfig.getS3Bucket() + "/" + mConfig.getS3Path();
   SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
   format.setTimeZone(TimeZone.getTimeZone("UTC"));
   for (Calendar partition : partitionDates) {
     String partitionStr = format.format(partition.getTime());
     String[] partitions = {"dt=" + partitionStr};
     LogFilePath logFilePath =
         new LogFilePath(
             s3Prefix, topic, partitions, mConfig.getGeneration(), 0, 0, mFileExtension);
     String logFileDir = logFilePath.getLogFileDir();
     assert FileUtil.exists(logFileDir) : "FileUtil.exists(" + logFileDir + ")";
     String successFilePath = logFileDir + "/_SUCCESS";
     if (FileUtil.exists(successFilePath)) {
       return;
     }
     try {
       mQuboleClient.addPartition(mConfig.getHivePrefix() + topic, "dt='" + partitionStr + "'");
     } catch (Exception e) {
       LOG.error("failed to finalize topic " + topic + " partition dt=" + partitionStr, e);
       continue;
     }
     LOG.info("touching file " + successFilePath);
     FileUtil.touch(successFilePath);
   }
 }
Пример #2
0
 public static FileSystem getFileSystem(String path) throws IOException {
   Configuration conf = new Configuration();
   if (mConfig != null) {
     conf.set("fs.s3n.awsAccessKeyId", mConfig.getAwsAccessKey());
     conf.set("fs.s3n.awsSecretAccessKey", mConfig.getAwsSecretKey());
   }
   return FileSystem.get(URI.create(path), conf);
 }
Пример #3
0
 public PartitionFinalizer(SecorConfig config) throws Exception {
   mConfig = config;
   mKafkaClient = new KafkaClient(mConfig);
   mZookeeperConnector = new ZookeeperConnector(mConfig);
   mMessageParser =
       (TimestampedMessageParser)
           ReflectionUtil.createMessageParser(mConfig.getMessageParserClass(), mConfig);
   mQuboleClient = new QuboleClient(mConfig);
   if (mConfig.getCompressionCodec() != null && !mConfig.getCompressionCodec().isEmpty()) {
     CompressionCodec codec =
         CompressionUtil.createCompressionCodec(mConfig.getCompressionCodec());
     mFileExtension = codec.getDefaultExtension();
   } else {
     mFileExtension = "";
   }
 }
Пример #4
0
  @Override
  public void setUp() throws Exception {
    SecorConfig mConfig = Mockito.mock(SecorConfig.class);
    Mockito.when(mConfig.getMessageTimestampName()).thenReturn("ts");
    Mockito.when(mConfig.getTimeZone()).thenReturn(TimeZone.getTimeZone("UTC"));
    mMessagePackParser = new MessagePackParser(mConfig);
    mObjectMapper = new ObjectMapper(new MessagePackFactory());

    HashMap<String, Object> mapWithSecondTimestamp = new HashMap<String, Object>();
    mapWithSecondTimestamp.put("ts", 1405970352);
    mMessageWithSecondsTimestamp =
        new Message("test", 0, 0, null, mObjectMapper.writeValueAsBytes(mapWithSecondTimestamp));

    HashMap<String, Object> mapWithMillisTimestamp = new HashMap<String, Object>();
    mapWithMillisTimestamp.put("ts", 1405970352123l);
    mapWithMillisTimestamp.put("isActive", true);
    mapWithMillisTimestamp.put("email", "*****@*****.**");
    mapWithMillisTimestamp.put("age", 27);
    mMessageWithMillisTimestamp =
        new Message("test", 0, 0, null, mObjectMapper.writeValueAsBytes(mapWithMillisTimestamp));

    HashMap<String, Object> mapWithMillisFloatTimestamp = new HashMap<String, Object>();
    mapWithMillisFloatTimestamp.put("ts", 1405970352123.0);
    mapWithMillisFloatTimestamp.put("isActive", false);
    mapWithMillisFloatTimestamp.put("email", "*****@*****.**");
    mapWithMillisFloatTimestamp.put("age", 35);
    mMessageWithMillisFloatTimestamp =
        new Message(
            "test", 0, 0, null, mObjectMapper.writeValueAsBytes(mapWithMillisFloatTimestamp));

    HashMap<String, Object> mapWithMillisStringTimestamp = new HashMap<String, Object>();
    mapWithMillisStringTimestamp.put("ts", "1405970352123");
    mapWithMillisStringTimestamp.put("isActive", null);
    mapWithMillisStringTimestamp.put("email", "*****@*****.**");
    mapWithMillisStringTimestamp.put("age", 67);
    mMessageWithMillisStringTimestamp =
        new Message(
            "test", 0, 0, null, mObjectMapper.writeValueAsBytes(mapWithMillisStringTimestamp));
  }
Пример #5
0
 private NavigableSet<Calendar> getPartitions(String topic) throws IOException, ParseException {
   final String s3Prefix = "s3n://" + mConfig.getS3Bucket() + "/" + mConfig.getS3Path();
   String[] partitions = {"dt="};
   LogFilePath logFilePath =
       new LogFilePath(s3Prefix, topic, partitions, mConfig.getGeneration(), 0, 0, mFileExtension);
   String parentDir = logFilePath.getLogFileParentDir();
   String[] partitionDirs = FileUtil.list(parentDir);
   Pattern pattern = Pattern.compile(".*/dt=(\\d\\d\\d\\d-\\d\\d-\\d\\d)$");
   TreeSet<Calendar> result = new TreeSet<Calendar>();
   for (String partitionDir : partitionDirs) {
     Matcher matcher = pattern.matcher(partitionDir);
     if (matcher.find()) {
       String date = matcher.group(1);
       SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
       format.setTimeZone(TimeZone.getTimeZone("UTC"));
       Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
       calendar.setTime(format.parse(date));
       result.add(calendar);
     }
   }
   return result;
 }
Пример #6
0
 public void finalizePartitions() throws Exception {
   List<String> topics = mZookeeperConnector.getCommittedOffsetTopics();
   for (String topic : topics) {
     if (!topic.matches(mConfig.getKafkaTopicFilter())) {
       LOG.info("skipping topic " + topic);
     } else {
       LOG.info("finalizing topic " + topic);
       long finalizedTimestampMillis = getFinalizedTimestampMillis(topic);
       LOG.info("finalized timestamp for topic " + topic + " is " + finalizedTimestampMillis);
       if (finalizedTimestampMillis != -1) {
         Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
         calendar.setTimeInMillis(finalizedTimestampMillis);
         // Introduce a lag of one day and one hour.
         calendar.add(Calendar.HOUR, -1);
         calendar.add(Calendar.DAY_OF_MONTH, -1);
         finalizePartitionsUpTo(topic, calendar);
       }
     }
   }
 }
Пример #7
0
 public QuboleClient(SecorConfig config) {
   mApiToken = config.getQuboleApiToken();
 }
Пример #8
0
 private String getPrefix() {
   return "s3n://" + mConfig.getS3Bucket() + "/" + mConfig.getS3Path();
 }