@Override
public List<WorkUnit> getWorkunits(SourceState state) {
initLogger(state);
List<WorkUnit> workUnits = Lists.newArrayList();
String nameSpaceName = state.getProp(ConfigurationKeys.EXTRACT_NAMESPACE_NAME_KEY);
String entityName = state.getProp(ConfigurationKeys.SOURCE_ENTITY);
String extractTableName = state.getProp(ConfigurationKeys.EXTRACT_TABLE_NAME_KEY);
// If extract table name is not found then use the entity name
if (StringUtils.isBlank(extractTableName)) {
extractTableName =
Utils.escapeSpecialCharacters(
entityName, ConfigurationKeys.ESCAPE_CHARS_IN_TABLE_NAME, "_");
}
TableType tableType =
TableType.valueOf(state.getProp(ConfigurationKeys.EXTRACT_TABLE_TYPE_KEY).toUpperCase());
long previousWatermark = this.getLatestWatermarkFromMetadata(state);
Map<Long, Long> sortedPartitions = Maps.newTreeMap();
sortedPartitions.putAll(new Partitioner(state).getPartitions(previousWatermark));
// Use extract table name to create extract
SourceState partitionState = new SourceState();
partitionState.addAll(state);
Extract extract = partitionState.createExtract(tableType, nameSpaceName, extractTableName);
// Setting current time for the full extract
if (Boolean.valueOf(state.getProp(ConfigurationKeys.EXTRACT_IS_FULL_KEY))) {
extract.setFullTrue(System.currentTimeMillis());
}
for (Entry<Long, Long> entry : sortedPartitions.entrySet()) {
partitionState.setProp(ConfigurationKeys.WORK_UNIT_LOW_WATER_MARK_KEY, entry.getKey());
partitionState.setProp(ConfigurationKeys.WORK_UNIT_HIGH_WATER_MARK_KEY, entry.getValue());
workUnits.add(partitionState.createWorkUnit(extract));
}
LOG.info("Total number of work units for the current run: " + workUnits.size());
List<WorkUnit> previousWorkUnits = this.getPreviousWorkUnitsForRetry(state);
LOG.info("Total number of incomplete tasks from the previous run: " + previousWorkUnits.size());
workUnits.addAll(previousWorkUnits);
return workUnits;
}
