@Override
public DynamicSplitResult requestDynamicSplit(DynamicSplitRequest splitRequest) {
checkNotNull(splitRequest);
ApproximateSplitRequest splitProgress = splitRequestToApproximateSplitRequest(splitRequest);
com.google.api.services.dataflow.model.Position cloudPosition = splitProgress.getPosition();
if (cloudPosition == null) {
LOG.warn("Concat only supports split at a Position. Requested: {}", splitRequest);
return null;
}
ConcatPosition concatPosition = cloudPosition.getConcatPosition();
if (concatPosition == null) {
LOG.warn(
"ConcatReader only supports split at a ConcatPosition. Requested: {}", cloudPosition);
return null;
}
if (rangeTracker.trySplitAtPosition(concatPosition.getIndex())) {
com.google.api.services.dataflow.model.Position positionToSplit =
new com.google.api.services.dataflow.model.Position();
positionToSplit.setConcatPosition(
new ConcatPosition().setIndex((int) rangeTracker.getStopPosition().longValue()));
return new DynamicSplitResultWithPosition(cloudPositionToReaderPosition(positionToSplit));
} else {
LOG.debug("Could not perform the dynamic split request " + splitRequest);
return null;
}
}
@Override
public Progress getProgress() {
if (currentIteratorIndex < 0) {
// Reading has not been started yet.
return null;
}
ConcatPosition concatPosition = new ConcatPosition();
concatPosition.setIndex(currentIteratorIndex);
Progress progressOfCurrentIterator = currentIterator.getProgress();
if (!(progressOfCurrentIterator instanceof DataflowReaderProgress)) {
throw new IllegalArgumentException(
"Cannot process progress "
+ progressOfCurrentIterator
+ " since ConcatReader can only handle readers that generate a progress of type "
+ "DataflowReaderProgress");
}
com.google.api.services.dataflow.model.Position positionOfCurrentIterator =
((DataflowReaderProgress) progressOfCurrentIterator).cloudProgress.getPosition();
if (positionOfCurrentIterator != null) {
concatPosition.setPosition(positionOfCurrentIterator);
}
ApproximateReportedProgress progress = new ApproximateReportedProgress();
com.google.api.services.dataflow.model.Position currentPosition =
new com.google.api.services.dataflow.model.Position();
currentPosition.setConcatPosition(concatPosition);
progress.setPosition(currentPosition);
return cloudProgressToReaderProgress(progress);
}
@Override
public Progress getProgress() {
// Currently we assume that only a record index position is reported as
// current progress. An implementer can override this method to update
// other metrics, e.g. completion percentage or remaining time.
com.google.api.services.dataflow.model.Position currentPosition =
new com.google.api.services.dataflow.model.Position();
currentPosition.setRecordIndex((long) nextIndex);
ApproximateReportedProgress progress = new ApproximateReportedProgress();
progress.setPosition(currentPosition);
return cloudProgressToReaderProgress(progress);
}
@Override
public DynamicSplitResult requestDynamicSplit(DynamicSplitRequest splitRequest) {
checkNotNull(splitRequest);
com.google.api.services.dataflow.model.Position splitPosition =
splitRequestToApproximateSplitRequest(splitRequest).getPosition();
if (splitPosition == null) {
LOG.warn("InMemoryReader only supports split at a Position. Requested: {}", splitRequest);
return null;
}
Long splitIndex = splitPosition.getRecordIndex();
if (splitIndex == null) {
LOG.warn(
"InMemoryReader only supports split at a record index. Requested: {}", splitPosition);
return null;
}
if (!tracker.trySplitAtPosition(splitIndex)) {
return null;
}
return new DynamicSplitResultWithPosition(cloudPositionToReaderPosition(splitPosition));
}