@Override public DynamicSplitResult requestDynamicSplit(DynamicSplitRequest splitRequest) { checkNotNull(splitRequest); ApproximateSplitRequest splitProgress = splitRequestToApproximateSplitRequest(splitRequest); com.google.api.services.dataflow.model.Position cloudPosition = splitProgress.getPosition(); if (cloudPosition == null) { LOG.warn("Concat only supports split at a Position. Requested: {}", splitRequest); return null; } ConcatPosition concatPosition = cloudPosition.getConcatPosition(); if (concatPosition == null) { LOG.warn( "ConcatReader only supports split at a ConcatPosition. Requested: {}", cloudPosition); return null; } if (rangeTracker.trySplitAtPosition(concatPosition.getIndex())) { com.google.api.services.dataflow.model.Position positionToSplit = new com.google.api.services.dataflow.model.Position(); positionToSplit.setConcatPosition( new ConcatPosition().setIndex((int) rangeTracker.getStopPosition().longValue())); return new DynamicSplitResultWithPosition(cloudPositionToReaderPosition(positionToSplit)); } else { LOG.debug("Could not perform the dynamic split request " + splitRequest); return null; } }
@Override public Progress getProgress() { if (currentIteratorIndex < 0) { // Reading has not been started yet. return null; } ConcatPosition concatPosition = new ConcatPosition(); concatPosition.setIndex(currentIteratorIndex); Progress progressOfCurrentIterator = currentIterator.getProgress(); if (!(progressOfCurrentIterator instanceof DataflowReaderProgress)) { throw new IllegalArgumentException( "Cannot process progress " + progressOfCurrentIterator + " since ConcatReader can only handle readers that generate a progress of type " + "DataflowReaderProgress"); } com.google.api.services.dataflow.model.Position positionOfCurrentIterator = ((DataflowReaderProgress) progressOfCurrentIterator).cloudProgress.getPosition(); if (positionOfCurrentIterator != null) { concatPosition.setPosition(positionOfCurrentIterator); } ApproximateReportedProgress progress = new ApproximateReportedProgress(); com.google.api.services.dataflow.model.Position currentPosition = new com.google.api.services.dataflow.model.Position(); currentPosition.setConcatPosition(concatPosition); progress.setPosition(currentPosition); return cloudProgressToReaderProgress(progress); }
@Override public Progress getProgress() { // Currently we assume that only a record index position is reported as // current progress. An implementer can override this method to update // other metrics, e.g. completion percentage or remaining time. com.google.api.services.dataflow.model.Position currentPosition = new com.google.api.services.dataflow.model.Position(); currentPosition.setRecordIndex((long) nextIndex); ApproximateReportedProgress progress = new ApproximateReportedProgress(); progress.setPosition(currentPosition); return cloudProgressToReaderProgress(progress); }
@Override public DynamicSplitResult requestDynamicSplit(DynamicSplitRequest splitRequest) { checkNotNull(splitRequest); com.google.api.services.dataflow.model.Position splitPosition = splitRequestToApproximateSplitRequest(splitRequest).getPosition(); if (splitPosition == null) { LOG.warn("InMemoryReader only supports split at a Position. Requested: {}", splitRequest); return null; } Long splitIndex = splitPosition.getRecordIndex(); if (splitIndex == null) { LOG.warn( "InMemoryReader only supports split at a record index. Requested: {}", splitPosition); return null; } if (!tracker.trySplitAtPosition(splitIndex)) { return null; } return new DynamicSplitResultWithPosition(cloudPositionToReaderPosition(splitPosition)); }