public TableLevelWatermarker(State state) {
this.tableWatermarks = Maps.newHashMap();
// Load previous watermarks in case of sourceState
if (state instanceof SourceState) {
SourceState sourceState = (SourceState) state;
for (Map.Entry<String, Iterable<WorkUnitState>> datasetWorkUnitStates :
sourceState.getPreviousWorkUnitStatesByDatasetUrns().entrySet()) {
// Use the minimum of all previous watermarks for this dataset
List<LongWatermark> previousWatermarks =
FluentIterable.from(datasetWorkUnitStates.getValue())
.filter(Predicates.not(PartitionLevelWatermarker.WATERMARK_WORKUNIT_PREDICATE))
.transform(
new Function<WorkUnitState, LongWatermark>() {
@Override
public LongWatermark apply(WorkUnitState w) {
return w.getActualHighWatermark(LongWatermark.class);
}
})
.toList();
if (!previousWatermarks.isEmpty()) {
this.tableWatermarks.put(
datasetWorkUnitStates.getKey(), Collections.min(previousWatermarks));
}
}
log.info("Loaded table watermarks from previous state " + this.tableWatermarks);
}
}
