private void sendPackedObjects(
final List<ObjectId> toSend,
final Set<ObjectId> roots,
Deduplicator deduplicator,
final ProgressListener progress) {
Set<ObjectId> sent = new HashSet<ObjectId>();
while (!toSend.isEmpty()) {
try {
BinaryPackedObjects.Callback callback =
new BinaryPackedObjects.Callback() {
@Override
public void callback(Supplier<RevObject> supplier) {
RevObject object = supplier.get();
progress.setProgress(progress.getProgress() + 1);
if (object instanceof RevCommit) {
RevCommit commit = (RevCommit) object;
toSend.remove(commit.getId());
roots.removeAll(commit.getParentIds());
roots.add(commit.getId());
}
}
};
ObjectDatabase database = localRepository.objectDatabase();
BinaryPackedObjects packer = new BinaryPackedObjects(database);
ImmutableList<ObjectId> have = ImmutableList.copyOf(roots);
final boolean traverseCommits = false;
Stopwatch sw = Stopwatch.createStarted();
ObjectSerializingFactory serializer = DataStreamSerializationFactoryV1.INSTANCE;
SendObjectsConnectionFactory outFactory;
ObjectFunnel objectFunnel;
outFactory = new SendObjectsConnectionFactory(repositoryURL);
int pushBytesLimit = parsePushLimit();
objectFunnel = ObjectFunnels.newFunnel(outFactory, serializer, pushBytesLimit);
final long writtenObjectsCount =
packer.write(objectFunnel, toSend, have, sent, callback, traverseCommits, deduplicator);
objectFunnel.close();
sw.stop();
long compressedSize = outFactory.compressedSize;
long uncompressedSize = outFactory.uncompressedSize;
LOGGER.info(
String.format(
"HttpRemoteRepo: Written %,d objects."
+ " Time to process: %s."
+ " Compressed size: %,d bytes. Uncompressed size: %,d bytes.",
writtenObjectsCount, sw, compressedSize, uncompressedSize));
} catch (IOException e) {
Throwables.propagate(e);
}
}
}
/**
* Retrieve objects from the remote repository, and update have/want lists accordingly.
* Specifically, any retrieved commits are removed from the want list and added to the have list,
* and any parents of those commits are removed from the have list (it only represents the most
* recent common commits.) Retrieved objects are added to the local repository, and the want/have
* lists are updated in-place.
*
* @param want a list of ObjectIds that need to be fetched
* @param have a list of ObjectIds that are in common with the remote repository
* @param progress
*/
private void fetchMoreData(
final List<ObjectId> want, final Set<ObjectId> have, final ProgressListener progress) {
final JsonObject message = createFetchMessage(want, have);
final URL resourceURL;
try {
resourceURL = new URL(repositoryURL.toString() + "/repo/batchobjects");
} catch (MalformedURLException e) {
throw Throwables.propagate(e);
}
final HttpURLConnection connection;
try {
final Gson gson = new Gson();
OutputStream out;
final Writer writer;
connection = (HttpURLConnection) resourceURL.openConnection();
connection.setDoOutput(true);
connection.setDoInput(true);
connection.addRequestProperty("Accept-Encoding", "gzip");
out = connection.getOutputStream();
writer = new OutputStreamWriter(out);
gson.toJson(message, writer);
writer.flush();
out.flush();
} catch (IOException e) {
throw Throwables.propagate(e);
}
final HttpUtils.ReportingInputStream in = HttpUtils.getResponseStream(connection);
BinaryPackedObjects unpacker = new BinaryPackedObjects(localRepository.objectDatabase());
BinaryPackedObjects.Callback callback =
new BinaryPackedObjects.Callback() {
@Override
public void callback(Supplier<RevObject> supplier) {
RevObject object = supplier.get();
progress.setProgress(progress.getProgress() + 1);
if (object instanceof RevCommit) {
RevCommit commit = (RevCommit) object;
want.remove(commit.getId());
have.removeAll(commit.getParentIds());
have.add(commit.getId());
} else if (object instanceof RevTag) {
RevTag tag = (RevTag) object;
want.remove(tag.getId());
have.remove(tag.getCommitId());
have.add(tag.getId());
}
}
};
Stopwatch sw = Stopwatch.createStarted();
IngestResults ingestResults = unpacker.ingest(in, callback);
sw.stop();
String msg =
String.format(
"Processed %,d objects. Inserted: %,d. Existing: %,d. Time: %s. Compressed size: %,d bytes. Uncompressed size: %,d bytes.",
ingestResults.total(),
ingestResults.getInserted(),
ingestResults.getExisting(),
sw,
in.compressedSize(),
in.unCompressedSize());
LOGGER.info(msg);
progress.setDescription(msg);
}
