/**
* Push all new objects from the specified {@link Ref} to the remote.
*
* @param ref the local ref that points to new commit data
* @param refspec the remote branch to push to
*/
@Override
public void pushNewData(Ref ref, String refspec, ProgressListener progress)
throws SynchronizationException {
Optional<Ref> remoteRef = HttpUtils.getRemoteRef(repositoryURL, refspec);
checkPush(ref, remoteRef);
beginPush();
progress.setDescription("Uploading objects to " + refspec);
progress.setProgress(0);
CommitTraverser traverser = getPushTraverser(remoteRef);
traverser.traverse(ref.getObjectId());
List<ObjectId> toSend = new LinkedList<ObjectId>(traverser.commits);
Collections.reverse(toSend);
Set<ObjectId> have = new HashSet<ObjectId>(traverser.have);
Deduplicator deduplicator = deduplicationService.createDeduplicator();
try {
sendPackedObjects(toSend, have, deduplicator, progress);
} finally {
deduplicator.release();
}
ObjectId originalRemoteRefValue = ObjectId.NULL;
if (remoteRef.isPresent()) {
originalRemoteRefValue = remoteRef.get().getObjectId();
}
String nameToSet =
remoteRef.isPresent() ? remoteRef.get().getName() : Ref.HEADS_PREFIX + refspec;
endPush(nameToSet, ref.getObjectId(), originalRemoteRefValue.toString());
}
/**
* Fetch all new objects from the specified {@link Ref} from the remote.
*
* @param ref the remote ref that points to new commit data
* @param fetchLimit the maximum depth to fetch
*/
@Override
public void fetchNewData(Ref ref, Optional<Integer> fetchLimit, ProgressListener progress) {
CommitTraverser traverser = getFetchTraverser(fetchLimit);
try {
progress.setDescription("Fetching objects from " + ref.getName());
traverser.traverse(ref.getObjectId());
List<ObjectId> want = new LinkedList<ObjectId>();
want.addAll(traverser.commits);
Collections.reverse(want);
Set<ObjectId> have = new HashSet<ObjectId>();
have.addAll(traverser.have);
while (!want.isEmpty()) {
progress.setProgress(0);
fetchMoreData(want, have, progress);
}
} catch (Exception e) {
Throwables.propagate(e);
}
}
/**
* Executes the export operation using the parameters that have been specified.
*
* @return a FeatureCollection with the specified features
*/
@Override
protected SimpleFeatureStore _call() {
final ObjectDatabase database = objectDatabase();
if (filterFeatureTypeId != null) {
RevObject filterType = database.getIfPresent(filterFeatureTypeId);
checkArgument(
filterType instanceof RevFeatureType, "Provided filter feature type is does not exist");
}
final SimpleFeatureStore targetStore = getTargetStore();
final String refspec = resolveRefSpec();
final String treePath = refspec.substring(refspec.indexOf(':') + 1);
final RevTree rootTree = resolveRootTree(refspec);
final NodeRef typeTreeRef = resolTypeTreeRef(refspec, treePath, rootTree);
final ObjectId defaultMetadataId = typeTreeRef.getMetadataId();
final RevTree typeTree = database.getTree(typeTreeRef.getObjectId());
final ProgressListener progressListener = getProgressListener();
progressListener.started();
progressListener.setDescription(
"Exporting from " + path + " to " + targetStore.getName().getLocalPart() + "... ");
final Iterator<SimpleFeature> filtered;
{
final Iterator<SimpleFeature> plainFeatures =
getFeatures(typeTree, database, defaultMetadataId, progressListener);
Iterator<SimpleFeature> adaptedFeatures = adaptToArguments(plainFeatures, defaultMetadataId);
Iterator<Optional<Feature>> transformed =
Iterators.transform(adaptedFeatures, ExportOp.this.function);
Iterator<SimpleFeature> result =
Iterators.filter(
Iterators.transform(
transformed,
new Function<Optional<Feature>, SimpleFeature>() {
@Override
public SimpleFeature apply(Optional<Feature> input) {
return (SimpleFeature) input.orNull();
}
}),
Predicates.notNull());
// check the resulting schema has something to contribute
PeekingIterator<SimpleFeature> peekingIt = Iterators.peekingIterator(result);
if (peekingIt.hasNext()) {
Function<AttributeDescriptor, String> toString =
new Function<AttributeDescriptor, String>() {
@Override
public String apply(AttributeDescriptor input) {
return input.getLocalName();
}
};
SimpleFeature peek = peekingIt.peek();
Set<String> sourceAtts =
new HashSet<String>(
Lists.transform(peek.getFeatureType().getAttributeDescriptors(), toString));
Set<String> targetAtts =
new HashSet<String>(
Lists.transform(targetStore.getSchema().getAttributeDescriptors(), toString));
if (Sets.intersection(sourceAtts, targetAtts).isEmpty()) {
throw new GeoToolsOpException(
StatusCode.UNABLE_TO_ADD,
"No common attributes between source and target feature types");
}
}
filtered = peekingIt;
}
FeatureCollection<SimpleFeatureType, SimpleFeature> asFeatureCollection =
new BaseFeatureCollection<SimpleFeatureType, SimpleFeature>() {
@Override
public FeatureIterator<SimpleFeature> features() {
return new DelegateFeatureIterator<SimpleFeature>(filtered);
}
};
// add the feature collection to the feature store
final Transaction transaction;
if (transactional) {
transaction = new DefaultTransaction("create");
} else {
transaction = Transaction.AUTO_COMMIT;
}
try {
targetStore.setTransaction(transaction);
try {
targetStore.addFeatures(asFeatureCollection);
transaction.commit();
} catch (final Exception e) {
if (transactional) {
transaction.rollback();
}
Throwables.propagateIfInstanceOf(e, GeoToolsOpException.class);
throw new GeoToolsOpException(e, StatusCode.UNABLE_TO_ADD);
} finally {
transaction.close();
}
} catch (IOException e) {
throw new GeoToolsOpException(e, StatusCode.UNABLE_TO_ADD);
}
progressListener.complete();
return targetStore;
}
/**
* Retrieve objects from the remote repository, and update have/want lists accordingly.
* Specifically, any retrieved commits are removed from the want list and added to the have list,
* and any parents of those commits are removed from the have list (it only represents the most
* recent common commits.) Retrieved objects are added to the local repository, and the want/have
* lists are updated in-place.
*
* @param want a list of ObjectIds that need to be fetched
* @param have a list of ObjectIds that are in common with the remote repository
* @param progress
*/
private void fetchMoreData(
final List<ObjectId> want, final Set<ObjectId> have, final ProgressListener progress) {
final JsonObject message = createFetchMessage(want, have);
final URL resourceURL;
try {
resourceURL = new URL(repositoryURL.toString() + "/repo/batchobjects");
} catch (MalformedURLException e) {
throw Throwables.propagate(e);
}
final HttpURLConnection connection;
try {
final Gson gson = new Gson();
OutputStream out;
final Writer writer;
connection = (HttpURLConnection) resourceURL.openConnection();
connection.setDoOutput(true);
connection.setDoInput(true);
connection.addRequestProperty("Accept-Encoding", "gzip");
out = connection.getOutputStream();
writer = new OutputStreamWriter(out);
gson.toJson(message, writer);
writer.flush();
out.flush();
} catch (IOException e) {
throw Throwables.propagate(e);
}
final HttpUtils.ReportingInputStream in = HttpUtils.getResponseStream(connection);
BinaryPackedObjects unpacker = new BinaryPackedObjects(localRepository.objectDatabase());
BinaryPackedObjects.Callback callback =
new BinaryPackedObjects.Callback() {
@Override
public void callback(Supplier<RevObject> supplier) {
RevObject object = supplier.get();
progress.setProgress(progress.getProgress() + 1);
if (object instanceof RevCommit) {
RevCommit commit = (RevCommit) object;
want.remove(commit.getId());
have.removeAll(commit.getParentIds());
have.add(commit.getId());
} else if (object instanceof RevTag) {
RevTag tag = (RevTag) object;
want.remove(tag.getId());
have.remove(tag.getCommitId());
have.add(tag.getId());
}
}
};
Stopwatch sw = Stopwatch.createStarted();
IngestResults ingestResults = unpacker.ingest(in, callback);
sw.stop();
String msg =
String.format(
"Processed %,d objects. Inserted: %,d. Existing: %,d. Time: %s. Compressed size: %,d bytes. Uncompressed size: %,d bytes.",
ingestResults.total(),
ingestResults.getInserted(),
ingestResults.getExisting(),
sw,
in.compressedSize(),
in.unCompressedSize());
LOGGER.info(msg);
progress.setDescription(msg);
}