@Override
public Job addInputPaths(HadoopDruidIndexerConfig config, Job job) throws IOException {
final Set<Interval> intervals = Sets.newTreeSet(Comparators.intervals());
Optional<Set<Interval>> optionalIntervals = config.getSegmentGranularIntervals();
if (optionalIntervals.isPresent()) {
for (Interval segmentInterval : optionalIntervals.get()) {
for (Interval dataInterval : dataGranularity.getIterable(segmentInterval)) {
intervals.add(dataInterval);
}
}
}
Path betaInput = new Path(inputPath);
FileSystem fs = betaInput.getFileSystem(job.getConfiguration());
Set<String> paths = Sets.newTreeSet();
Pattern fileMatcher = Pattern.compile(filePattern);
DateTimeFormatter customFormatter = null;
if (pathFormat != null) {
customFormatter = DateTimeFormat.forPattern(pathFormat);
}
for (Interval interval : intervals) {
DateTime t = interval.getStart();
String intervalPath = null;
if (customFormatter != null) {
intervalPath = customFormatter.print(t);
} else {
intervalPath = dataGranularity.toPath(t);
}
Path granularPath = new Path(betaInput, intervalPath);
log.info("Checking path[%s]", granularPath);
for (FileStatus status : FSSpideringIterator.spiderIterable(fs, granularPath)) {
final Path filePath = status.getPath();
if (fileMatcher.matcher(filePath.toString()).matches()) {
paths.add(filePath.toString());
}
}
}
for (String path : paths) {
log.info("Appending path[%s]", path);
FileInputFormat.addInputPath(job, new Path(path));
}
return job;
}
@Override
public SegmentIdentifier allocate(final DateTime timestamp, final String previousSegmentId)
throws IOException {
synchronized (counters) {
final long timestampTruncated = granularity.truncate(timestamp).getMillis();
if (!counters.containsKey(timestampTruncated)) {
counters.put(timestampTruncated, new AtomicInteger());
}
final int partitionNum = counters.get(timestampTruncated).getAndIncrement();
return new SegmentIdentifier(
dataSource,
granularity.bucket(new DateTime(timestampTruncated)),
VERSION,
new NumberedShardSpec(partitionNum, 0));
}
}