public void testPushReplicate() throws Exception {
// create mock sync gateway that will serve as a pull target and return random docs
MockDispatcher dispatcher = new MockDispatcher();
MockWebServer server = MockHelper.getMockWebServer(dispatcher);
dispatcher.setServerType(MockDispatcher.ServerType.SYNC_GW);
// fake checkpoint response 404
MockCheckpointGet mockCheckpointGet = new MockCheckpointGet();
dispatcher.enqueueResponse(MockHelper.PATH_REGEX_CHECKPOINT, mockCheckpointGet);
server.play();
Map<String, Object> replicateJsonMap = getPushReplicationParsedJson(server.getUrl("/db"));
Log.v(TAG, "map: " + replicateJsonMap);
Map<String, Object> result =
(Map<String, Object>) sendBody("POST", "/_replicate", replicateJsonMap, Status.OK, null);
Log.v(TAG, "result: " + result);
assertNotNull(result.get("session_id"));
boolean success = waitForReplicationToFinish();
assertTrue(success);
server.shutdown();
}
public void testPullReplicate() throws Exception {
// create mock sync gateway that will serve as a pull target and return random docs
int numMockDocsToServe = 0;
MockDispatcher dispatcher = new MockDispatcher();
MockWebServer server =
MockHelper.getPreloadedPullTargetMockCouchDB(dispatcher, numMockDocsToServe, 1);
dispatcher.setServerType(MockDispatcher.ServerType.COUCHDB);
server.setDispatcher(dispatcher);
server.play();
// kick off replication via REST api
Map<String, Object> replicateJsonMap = getPullReplicationParsedJson(server.getUrl("/db"));
Log.v(TAG, "map: " + replicateJsonMap);
Map<String, Object> result =
(Map<String, Object>) sendBody("POST", "/_replicate", replicateJsonMap, Status.OK, null);
Log.v(TAG, "result: " + result);
assertNotNull(result.get("session_id"));
// wait for replication to finish
boolean success = waitForReplicationToFinish();
assertTrue(success);
// cleanup
server.shutdown();
}
/** This method is called when a part's headers have been parsed, before its data is parsed. */
public void startedPart(Map headers) {
if (_docReader != null) {
throw new IllegalStateException("_docReader is already defined");
}
Log.v(Log.TAG_SYNC, "%s: Starting new document; headers =%s", this, headers);
Log.v(Log.TAG_SYNC, "%s: Starting new document; ID=%s", this, headers.get("X-Doc-Id"));
_docReader = new MultipartDocumentReader(_db);
_docReader.setHeaders(headers);
_docReader.startedPart(headers);
}
/** - (void) maybeCreateRemoteDB in CBL_Replicator.m */
@Override
@InterfaceAudience.Private
protected void maybeCreateRemoteDB() {
if (!createTarget) {
return;
}
creatingTarget = true;
Log.v(Log.TAG_SYNC, "Remote db might not exist; creating it...");
Future future =
sendAsyncRequest(
"PUT",
"",
null,
new RemoteRequestCompletionBlock() {
@Override
public void onCompletion(HttpResponse httpResponse, Object result, Throwable e) {
creatingTarget = false;
if (e != null
&& e instanceof HttpResponseException
&& ((HttpResponseException) e).getStatusCode() != 412) {
Log.e(Log.TAG_SYNC, this + ": Failed to create remote db", e);
setError(e);
triggerStopGraceful(); // this is fatal: no db to push to!
} else {
Log.v(Log.TAG_SYNC, "%s: Created remote db", this);
createTarget = false;
beginReplicating();
}
}
});
pendingFutures.add(future);
}
private void revisionFailed(RevisionInternal rev, Throwable throwable) {
if (!Utils.isTransientError(throwable)) {
Log.v(Log.TAG_SYNC, "%s: giving up on %s: %s", this, rev, throwable);
pendingSequences.removeSequence(rev.getSequence());
pauseOrResume();
}
completedChangesCount.getAndIncrement();
}
private void setPaused(boolean paused) {
Log.v(Log.TAG, "setPaused: " + paused);
synchronized (pausedObj) {
if (this.paused != paused) {
this.paused = paused;
pausedObj.notifyAll();
}
}
}
/** This method is called when a part is complete. */
public void finishedPart() {
Log.v(Log.TAG_SYNC, "%s: Finished document", this);
if (_docReader == null) {
throw new IllegalStateException("_docReader is not defined");
}
_docReader.finish();
_onDocument.onDocument(_docReader.getDocumentProperties());
_docReader = null;
}
private void waitIfPaused() {
synchronized (pausedObj) {
while (paused) {
Log.v(Log.TAG, "Waiting: " + paused);
try {
pausedObj.wait();
} catch (InterruptedException e) {
}
}
}
}
public void testFacebookToken() {
send("PUT", "/db", Status.CREATED, null);
Map<String, Object> doc1 = new HashMap<String, Object>();
doc1.put("email", "*****@*****.**");
doc1.put("remote_url", getReplicationURL().toExternalForm());
doc1.put("access_token", "fake_access_token");
Map<String, Object> result =
(Map<String, Object>) sendBody("POST", "/_facebook_token", doc1, Status.OK, null);
Log.v(TAG, String.format("result %s", result));
}
public void testPersonaAssertion() {
send("PUT", "/db", Status.CREATED, null);
Map<String, Object> doc1 = new HashMap<String, Object>();
String sampleAssertion =
"eyJhbGciOiJSUzI1NiJ9.eyJwdWJsaWMta2V5Ijp7ImFsZ29yaXRobSI6IkRTIiwieSI6ImNhNWJiYTYzZmI4MDQ2OGE0MjFjZjgxYTIzN2VlMDcwYTJlOTM4NTY0ODhiYTYzNTM0ZTU4NzJjZjllMGUwMDk0ZWQ2NDBlOGNhYmEwMjNkYjc5ODU3YjkxMzBlZGNmZGZiNmJiNTUwMWNjNTk3MTI1Y2NiMWQ1ZWQzOTVjZTMyNThlYjEwN2FjZTM1ODRiOWIwN2I4MWU5MDQ4NzhhYzBhMjFlOWZkYmRjYzNhNzNjOTg3MDAwYjk4YWUwMmZmMDQ4ODFiZDNiOTBmNzllYzVlNDU1YzliZjM3NzFkYjEzMTcxYjNkMTA2ZjM1ZDQyZmZmZjQ2ZWZiZDcwNjgyNWQiLCJwIjoiZmY2MDA0ODNkYjZhYmZjNWI0NWVhYjc4NTk0YjM1MzNkNTUwZDlmMWJmMmE5OTJhN2E4ZGFhNmRjMzRmODA0NWFkNGU2ZTBjNDI5ZDMzNGVlZWFhZWZkN2UyM2Q0ODEwYmUwMGU0Y2MxNDkyY2JhMzI1YmE4MWZmMmQ1YTViMzA1YThkMTdlYjNiZjRhMDZhMzQ5ZDM5MmUwMGQzMjk3NDRhNTE3OTM4MDM0NGU4MmExOGM0NzkzMzQzOGY4OTFlMjJhZWVmODEyZDY5YzhmNzVlMzI2Y2I3MGVhMDAwYzNmNzc2ZGZkYmQ2MDQ2MzhjMmVmNzE3ZmMyNmQwMmUxNyIsInEiOiJlMjFlMDRmOTExZDFlZDc5OTEwMDhlY2FhYjNiZjc3NTk4NDMwOWMzIiwiZyI6ImM1MmE0YTBmZjNiN2U2MWZkZjE4NjdjZTg0MTM4MzY5YTYxNTRmNGFmYTkyOTY2ZTNjODI3ZTI1Y2ZhNmNmNTA4YjkwZTVkZTQxOWUxMzM3ZTA3YTJlOWUyYTNjZDVkZWE3MDRkMTc1ZjhlYmY2YWYzOTdkNjllMTEwYjk2YWZiMTdjN2EwMzI1OTMyOWU0ODI5YjBkMDNiYmM3ODk2YjE1YjRhZGU1M2UxMzA4NThjYzM0ZDk2MjY5YWE4OTA0MWY0MDkxMzZjNzI0MmEzODg5NWM5ZDViY2NhZDRmMzg5YWYxZDdhNGJkMTM5OGJkMDcyZGZmYTg5NjIzMzM5N2EifSwicHJpbmNpcGFsIjp7ImVtYWlsIjoiamVuc0Btb29zZXlhcmQuY29tIn0sImlhdCI6MTM1ODI5NjIzNzU3NywiZXhwIjoxMzU4MzgyNjM3NTc3LCJpc3MiOiJsb2dpbi5wZXJzb25hLm9yZyJ9.RnDK118nqL2wzpLCVRzw1MI4IThgeWpul9jPl6ypyyxRMMTurlJbjFfs-BXoPaOem878G8-4D2eGWS6wd307k7xlPysevYPogfFWxK_eDHwkTq3Ts91qEDqrdV_JtgULC8c1LvX65E0TwW_GL_TM94g3CvqoQnGVxxoaMVye4ggvR7eOZjimWMzUuu4Lo9Z-VBHBj7XM0UMBie57CpGwH4_Wkv0V_LHZRRHKdnl9ISp_aGwfBObTcHG9v0P3BW9vRrCjihIn0SqOJQ9obl52rMf84GD4Lcy9NIktzfyka70xR9Sh7ALotW7rWywsTzMTu3t8AzMz2MJgGjvQmx49QA~eyJhbGciOiJEUzEyOCJ9.eyJleHAiOjEzNTgyOTY0Mzg0OTUsImF1ZCI6Imh0dHA6Ly9sb2NhbGhvc3Q6NDk4NC8ifQ.4FV2TrUQffDya0MOxOQlzJQbDNvCPF2sfTIJN7KOLvvlSFPknuIo5g";
doc1.put("assertion", sampleAssertion);
Map<String, Object> result =
(Map<String, Object>) sendBody("POST", "/_persona_assertion", doc1, Status.OK, null);
Log.v(TAG, String.format("result %s", result));
String email = (String) result.get("email");
assertEquals(email, "*****@*****.**");
}
// This invokes the tranformation block if one is installed and queues the resulting CBL_Revision
private void queueDownloadedRevision(RevisionInternal rev) {
if (revisionBodyTransformationBlock != null) {
// Add 'file' properties to attachments pointing to their bodies:
for (Map.Entry<String, Map<String, Object>> entry :
((Map<String, Map<String, Object>>) rev.getProperties().get("_attachments")).entrySet()) {
String name = entry.getKey();
Map<String, Object> attachment = entry.getValue();
attachment.remove("file");
if (attachment.get("follows") != null && attachment.get("data") == null) {
String filePath = db.fileForAttachmentDict(attachment).getPath();
if (filePath != null) attachment.put("file", filePath);
}
}
RevisionInternal xformed = transformRevision(rev);
if (xformed == null) {
Log.v(Log.TAG_SYNC, "%s: Transformer rejected revision %s", this, rev);
pendingSequences.removeSequence(rev.getSequence());
lastSequence = pendingSequences.getCheckpointedValue();
pauseOrResume();
return;
}
rev = xformed;
// Clean up afterwards
Map<String, Object> attachments =
(Map<String, Object>) rev.getProperties().get("_attachments");
for (Map.Entry<String, Map<String, Object>> entry :
((Map<String, Map<String, Object>>) rev.getProperties().get("_attachments")).entrySet()) {
Map<String, Object> attachment = entry.getValue();
attachment.remove("file");
}
}
if (rev != null && rev.getBody() != null) rev.getBody().compact();
downloadsToInsert.queueObject(rev);
}
/**
* in CBL_Puller.m - (void) changeTrackerReceivedSequence: (id)remoteSequenceID docID:
* (NSString*)docID revIDs: (NSArray*)revIDs deleted: (BOOL)deleted
*/
protected void processChangeTrackerChange(final Map<String, Object> change) {
String lastSequence = change.get("seq").toString();
String docID = (String) change.get("id");
if (docID == null) {
return;
}
if (!Document.isValidDocumentId(docID)) {
Log.w(Log.TAG_SYNC, "%s: Received invalid doc ID from _changes: %s", this, change);
return;
}
boolean deleted =
(change.containsKey("deleted") && ((Boolean) change.get("deleted")).equals(Boolean.TRUE));
List<Map<String, Object>> changes = (List<Map<String, Object>>) change.get("changes");
for (Map<String, Object> changeDict : changes) {
String revID = (String) changeDict.get("rev");
if (revID == null) {
continue;
}
PulledRevision rev = new PulledRevision(docID, revID, deleted);
// Remember its remote sequence ID (opaque), and make up a numeric sequence
// based on the order in which it appeared in the _changes feed:
rev.setRemoteSequenceID(lastSequence);
if (changes.size() > 1) rev.setConflicted(true);
Log.d(Log.TAG_SYNC, "%s: adding rev to inbox %s", this, rev);
Log.v(Log.TAG_SYNC, "%s: changeTrackerReceivedChange() incrementing changesCount by 1", this);
// this is purposefully done slightly different than the ios version
addToChangesCount(1);
addToInbox(rev);
}
pauseOrResume();
}
@Override
protected void executeRequest(HttpClient httpClient, HttpUriRequest request) {
Object fullBody = null;
Throwable error = null;
HttpResponse response = null;
try {
if (request.isAborted()) {
respondWithResult(
fullBody,
new Exception(String.format("%s: Request %s has been aborted", this, request)),
response);
return;
}
response = httpClient.execute(request);
try {
// add in cookies to global store
if (httpClient instanceof DefaultHttpClient) {
DefaultHttpClient defaultHttpClient = (DefaultHttpClient) httpClient;
clientFactory.addCookies(defaultHttpClient.getCookieStore().getCookies());
}
} catch (Exception e) {
Log.e(Log.TAG_REMOTE_REQUEST, "Unable to add in cookies to global store", e);
}
StatusLine status = response.getStatusLine();
if (status.getStatusCode() >= 300) {
Log.e(
Log.TAG_REMOTE_REQUEST,
"Got error status: %d for %s. Reason: %s",
status.getStatusCode(),
request,
status.getReasonPhrase());
error = new HttpResponseException(status.getStatusCode(), status.getReasonPhrase());
} else {
HttpEntity entity = null;
try {
entity = response.getEntity();
if (entity != null) {
InputStream inputStream = null;
try {
inputStream = entity.getContent();
Header contentTypeHeader = entity.getContentType();
if (contentTypeHeader != null) {
// multipart
if (contentTypeHeader.getValue().contains("multipart/")) {
Log.v(Log.TAG_SYNC, "contentTypeHeader = %s", contentTypeHeader.getValue());
_topReader = new MultipartReader(contentTypeHeader.getValue(), this);
byte[] buffer = new byte[BUF_LEN];
int numBytesRead = 0;
while ((numBytesRead = inputStream.read(buffer)) != -1) {
_topReader.appendData(buffer, 0, numBytesRead);
}
_topReader.finished();
respondWithResult(fullBody, error, response);
}
// non-multipart
else {
Log.v(
Log.TAG_SYNC,
"contentTypeHeader is not multipart = %s",
contentTypeHeader.getValue());
GZIPInputStream gzipStream = null;
try {
// decompress if contentEncoding is gzip
if (Utils.isGzip(entity)) {
gzipStream = new GZIPInputStream(inputStream);
fullBody = Manager.getObjectMapper().readValue(gzipStream, Object.class);
} else {
fullBody = Manager.getObjectMapper().readValue(inputStream, Object.class);
}
respondWithResult(fullBody, error, response);
} finally {
try {
if (gzipStream != null) {
gzipStream.close();
}
} catch (IOException e) {
}
gzipStream = null;
}
}
}
} finally {
try {
if (inputStream != null) {
inputStream.close();
}
} catch (IOException e) {
}
inputStream = null;
}
}
} finally {
if (entity != null) {
try {
entity.consumeContent();
} catch (IOException e) {
}
}
entity = null;
}
}
} catch (IOException e) {
Log.e(Log.TAG_REMOTE_REQUEST, "io exception", e);
error = e;
} catch (Exception e) {
Log.e(Log.TAG_REMOTE_REQUEST, "%s: executeRequest() Exception: ", e, this);
error = e;
} finally {
Log.v(Log.TAG_SYNC, "%s: BulkDownloader finally block. url: %s", this, url);
}
Log.v(
Log.TAG_SYNC,
"%s: BulkDownloader calling respondWithResult. url: %s, error: %s",
this,
url,
error);
respondWithResult(fullBody, error, response);
}
/**
* Post the revisions to the destination. "new_edits":false means that the server should use the
* given _rev IDs instead of making up new ones.
*
* <p>- (void) uploadBulkDocs: (NSArray*)docsToSend changes: (CBL_RevisionList*)changes in
* CBLRestPusher.m
*/
@InterfaceAudience.Private
protected void uploadBulkDocs(List<Object> docsToSend, final RevisionList changes) {
final int numDocsToSend = docsToSend.size();
if (numDocsToSend == 0) {
return;
}
Log.v(
Log.TAG_SYNC,
"%s: POSTing " + numDocsToSend + " revisions to _bulk_docs: %s",
PusherInternal.this,
docsToSend);
addToChangesCount(numDocsToSend);
Map<String, Object> bulkDocsBody = new HashMap<String, Object>();
bulkDocsBody.put("docs", docsToSend);
bulkDocsBody.put("new_edits", false);
CustomFuture future =
sendAsyncRequest(
"POST",
"/_bulk_docs",
bulkDocsBody,
new RemoteRequestCompletionBlock() {
@Override
public void onCompletion(HttpResponse httpResponse, Object result, Throwable e) {
if (e == null) {
Set<String> failedIDs = new HashSet<String>();
// _bulk_docs response is really an array, not a dictionary!
List<Map<String, Object>> items = (List) result;
for (Map<String, Object> item : items) {
Status status = statusFromBulkDocsResponseItem(item);
if (status.isError()) {
// One of the docs failed to save.
Log.w(Log.TAG_SYNC, "%s: _bulk_docs got an error: %s", item, this);
// 403/Forbidden means validation failed; don't treat it as an error
// because I did my job in sending the revision. Other statuses are
// actual replication errors.
if (status.getCode() != Status.FORBIDDEN) {
String docID = (String) item.get("id");
failedIDs.add(docID);
// TODO - port from iOS
// NSURL* url = docID ? [_remote URLByAppendingPathComponent: docID] : nil;
// error = CBLStatusToNSError(status, url);
}
}
}
// Remove from the pending list all the revs that didn't fail:
for (RevisionInternal revisionInternal : changes) {
if (!failedIDs.contains(revisionInternal.getDocID())) {
removePending(revisionInternal);
}
}
}
if (e != null) {
setError(e);
} else {
Log.v(Log.TAG_SYNC, "%s: POSTed to _bulk_docs", PusherInternal.this);
}
addToCompletedChangesCount(numDocsToSend);
}
});
future.setQueue(pendingFutures);
pendingFutures.add(future);
}
/** - (void) processInbox: (CBL_RevisionList*)changes in CBLRestPusher.m */
@Override
@InterfaceAudience.Private
protected void processInbox(final RevisionList changes) {
Log.v(Log.TAG_SYNC, "processInbox() changes=" + changes.size());
// Generate a set of doc/rev IDs in the JSON format that _revs_diff wants:
// <http://wiki.apache.org/couchdb/HttpPostRevsDiff>
Map<String, List<String>> diffs = new HashMap<String, List<String>>();
for (RevisionInternal rev : changes) {
String docID = rev.getDocID();
List<String> revs = diffs.get(docID);
if (revs == null) {
revs = new ArrayList<String>();
diffs.put(docID, revs);
}
revs.add(rev.getRevID());
addPending(rev);
}
// Call _revs_diff on the target db:
Log.v(Log.TAG_SYNC, "%s: posting to /_revs_diff", this);
CustomFuture future =
sendAsyncRequest(
"POST",
"/_revs_diff",
diffs,
new RemoteRequestCompletionBlock() {
@Override
public void onCompletion(HttpResponse httpResponse, Object response, Throwable e) {
Log.v(Log.TAG_SYNC, "%s: got /_revs_diff response", this);
Map<String, Object> results = (Map<String, Object>) response;
if (e != null) {
setError(e);
} else {
if (results.size() != 0) {
// Go through the list of local changes again, selecting the ones the
// destination server
// said were missing and mapping them to a JSON dictionary in the form
// _bulk_docs wants:
List<Object> docsToSend = new ArrayList<Object>();
RevisionList revsToSend = new RevisionList();
long bufferedSize = 0;
for (RevisionInternal rev : changes) {
// Is this revision in the server's 'missing' list?
Map<String, Object> properties = null;
Map<String, Object> revResults =
(Map<String, Object>) results.get(rev.getDocID());
if (revResults == null) {
removePending(rev);
continue;
}
List<String> revs = (List<String>) revResults.get("missing");
if (revs == null || !revs.contains(rev.getRevID())) {
removePending(rev);
continue;
}
// NOTE: force to load body by Database.loadRevisionBody()
// In SQLiteStore.loadRevisionBody() does not load data from database
// if sequence != 0 && body != null
rev.setSequence(0);
rev.setBody(null);
RevisionInternal loadedRev;
try {
loadedRev = db.loadRevisionBody(rev);
} catch (CouchbaseLiteException e1) {
Log.w(
Log.TAG_SYNC,
"%s Couldn't get local contents of %s",
rev,
PusherInternal.this);
continue;
}
RevisionInternal populatedRev = transformRevision(loadedRev);
loadedRev = null;
List<String> possibleAncestors =
(List<String>) revResults.get("possible_ancestors");
properties = new HashMap<String, Object>(populatedRev.getProperties());
Map<String, Object> revisions =
db.getRevisionHistoryDictStartingFromAnyAncestor(
populatedRev, possibleAncestors);
properties.put("_revisions", revisions);
populatedRev.setProperties(properties);
// Strip any attachments already known to the target db:
if (properties.containsKey("_attachments")) {
// Look for the latest common ancestor and stub out older attachments:
int minRevPos = findCommonAncestor(populatedRev, possibleAncestors);
Status status = new Status(Status.OK);
if (!db.expandAttachments(
populatedRev, minRevPos + 1, !dontSendMultipart, false, status)) {
Log.w(
Log.TAG_SYNC,
"%s: Couldn't expand attachments of %s",
this,
populatedRev);
continue;
}
properties = populatedRev.getProperties();
if (!dontSendMultipart && uploadMultipartRevision(populatedRev)) {
continue;
}
}
if (properties == null || !properties.containsKey("_id")) {
throw new IllegalStateException("properties must contain a document _id");
}
revsToSend.add(rev);
docsToSend.add(properties);
bufferedSize += JSONUtils.estimate(properties);
if (bufferedSize > kMaxBulkDocsObjectSize) {
uploadBulkDocs(docsToSend, revsToSend);
docsToSend = new ArrayList<Object>();
revsToSend = new RevisionList();
bufferedSize = 0;
}
}
// Post the revisions to the destination:
uploadBulkDocs(docsToSend, revsToSend);
} else {
// None of the revisions are new to the remote
for (RevisionInternal revisionInternal : changes) {
removePending(revisionInternal);
}
}
}
}
});
future.setQueue(pendingFutures);
pendingFutures.add(future);
pauseOrResume();
}
/** Process a bunch of remote revisions from the _changes feed at once */
@Override
@InterfaceAudience.Private
protected void processInbox(RevisionList inbox) {
Log.d(Log.TAG_SYNC, "processInbox called");
if (canBulkGet == null) {
canBulkGet = serverIsSyncGatewayVersion("0.81");
}
// Ask the local database which of the revs are not known to it:
String lastInboxSequence = ((PulledRevision) inbox.get(inbox.size() - 1)).getRemoteSequenceID();
int numRevisionsRemoved = 0;
try {
// findMissingRevisions is the local equivalent of _revs_diff. it looks at the
// array of revisions in "inbox" and removes the ones that already exist.
// So whatever's left in 'inbox'
// afterwards are the revisions that need to be downloaded.
numRevisionsRemoved = db.findMissingRevisions(inbox);
} catch (SQLException e) {
Log.e(Log.TAG_SYNC, String.format("%s failed to look up local revs", this), e);
inbox = null;
}
// introducing this to java version since inbox may now be null everywhere
int inboxCount = 0;
if (inbox != null) {
inboxCount = inbox.size();
}
if (numRevisionsRemoved > 0) {
Log.v(
Log.TAG_SYNC,
"%s: processInbox() setting changesCount to: %s",
this,
getChangesCount().get() - numRevisionsRemoved);
// May decrease the changesCount, to account for the revisions we just found out we don't need
// to get.
addToChangesCount(-1 * numRevisionsRemoved);
}
if (inboxCount == 0) {
// Nothing to do. Just bump the lastSequence.
Log.w(
Log.TAG_SYNC,
"%s no new remote revisions to fetch. add lastInboxSequence (%s) to pendingSequences (%s)",
this,
lastInboxSequence,
pendingSequences);
long seq = pendingSequences.addValue(lastInboxSequence);
pendingSequences.removeSequence(seq);
setLastSequence(pendingSequences.getCheckpointedValue());
pauseOrResume();
return;
}
Log.v(Log.TAG_SYNC, "%s: fetching %s remote revisions...", this, inboxCount);
// Dump the revs into the queue of revs to pull from the remote db:
for (int i = 0; i < inbox.size(); i++) {
PulledRevision rev = (PulledRevision) inbox.get(i);
if (canBulkGet || (rev.getGeneration() == 1 && !rev.isDeleted() && !rev.isConflicted())) {
bulkRevsToPull.add(rev);
} else {
queueRemoteRevision(rev);
}
rev.setSequence(pendingSequences.addValue(rev.getRemoteSequenceID()));
}
pullRemoteRevisions();
pauseOrResume();
}
public void waitForPendingFutures() {
if (waitingForPendingFutures) {
return;
}
synchronized (lockWaitForPendingFutures) {
waitingForPendingFutures = true;
Log.d(
Log.TAG_SYNC,
"[waitForPendingFutures()] STARTED - thread id: " + Thread.currentThread().getId());
try {
// wait for batcher's pending futures
if (batcher != null) {
Log.d(Log.TAG_SYNC, "batcher.waitForPendingFutures()");
// TODO: should we call batcher.flushAll(); here?
batcher.waitForPendingFutures();
Log.d(Log.TAG_SYNC, "/batcher.waitForPendingFutures()");
}
while (!pendingFutures.isEmpty()) {
Future future = pendingFutures.take();
try {
Log.d(Log.TAG_SYNC, "calling future.get() on %s", future);
future.get();
Log.d(Log.TAG_SYNC, "done calling future.get() on %s", future);
} catch (InterruptedException e) {
Log.e(Log.TAG_SYNC, "InterruptedException in Future.get()", e);
} catch (ExecutionException e) {
Log.e(Log.TAG_SYNC, "ExecutionException in Future.get()", e);
}
}
// since it's possible that in the process of waiting for pendingFutures,
// new items were added to the batcher, let's wait for the batcher to
// drain again.
if (batcher != null) {
Log.d(Log.TAG_SYNC, "batcher.waitForPendingFutures()");
batcher.waitForPendingFutures();
Log.d(Log.TAG_SYNC, "/batcher.waitForPendingFutures()");
}
// If pendingFutures queue is empty and state is RUNNING, fireTrigger to IDLE
// NOTE: in case of many documents sync, new Future tasks could be added into the queue.
// This is reason to check if queue is empty.
if (pendingFutures.isEmpty()) {
Log.v(Log.TAG_SYNC, "[waitForPendingFutures()] state=" + stateMachine.getState());
if (isContinuous()) {
// Make state IDLE
Log.v(
Log.TAG_SYNC,
"[waitForPendingFutures()] fireTrigger(ReplicationTrigger.WAITING_FOR_CHANGES);");
fireTrigger(ReplicationTrigger.WAITING_FOR_CHANGES);
} else {
// Make state STOPPING
triggerStopGraceful();
}
}
} catch (Exception e) {
Log.e(Log.TAG_SYNC, "Exception waiting for pending futures: %s", e);
} finally {
Log.d(
Log.TAG_SYNC,
"[waitForPendingFutures()] END - thread id: " + Thread.currentThread().getId());
waitingForPendingFutures = false;
}
}
}
// Get a bunch of revisions in one bulk request. Will use _bulk_get if possible.
protected void pullBulkRevisions(List<RevisionInternal> bulkRevs) {
int nRevs = bulkRevs.size();
if (nRevs == 0) {
return;
}
Log.v(Log.TAG_SYNC, "%s bulk-fetching %d remote revisions...", this, nRevs);
Log.v(Log.TAG_SYNC, "%s bulk-fetching remote revisions: %s", this, bulkRevs);
if (!canBulkGet) {
pullBulkWithAllDocs(bulkRevs);
return;
}
Log.v(Log.TAG_SYNC, "%s: POST _bulk_get", this);
final List<RevisionInternal> remainingRevs = new ArrayList<RevisionInternal>(bulkRevs);
++httpConnectionCount;
final BulkDownloader dl;
try {
dl =
new BulkDownloader(
workExecutor,
clientFactory,
remote,
bulkRevs,
db,
this.requestHeaders,
new BulkDownloader.BulkDownloaderDocumentBlock() {
public void onDocument(Map<String, Object> props) {
// Got a revision!
// Find the matching revision in 'remainingRevs' and get its sequence:
RevisionInternal rev;
if (props.get("_id") != null) {
rev = new RevisionInternal(props);
} else {
rev =
new RevisionInternal(
(String) props.get("id"), (String) props.get("rev"), false);
}
int pos = remainingRevs.indexOf(rev);
if (pos > -1) {
rev.setSequence(remainingRevs.get(pos).getSequence());
remainingRevs.remove(pos);
} else {
Log.w(Log.TAG_SYNC, "%s : Received unexpected rev rev", this);
}
if (props.get("_id") != null) {
// Add to batcher ... eventually it will be fed to -insertRevisions:.
queueDownloadedRevision(rev);
} else {
Status status = statusFromBulkDocsResponseItem(props);
error = new CouchbaseLiteException(status);
revisionFailed(rev, error);
}
}
},
new RemoteRequestCompletionBlock() {
public void onCompletion(HttpResponse httpResponse, Object result, Throwable e) {
// The entire _bulk_get is finished:
if (e != null) {
setError(e);
completedChangesCount.addAndGet(remainingRevs.size());
}
--httpConnectionCount;
// Start another task if there are still revisions waiting to be pulled:
pullRemoteRevisions();
}
});
} catch (Exception e) {
Log.e(Log.TAG_SYNC, "%s: pullBulkRevisions Exception: %s", this, e);
return;
}
dl.setAuthenticator(getAuthenticator());
// set compressed request - gzip
dl.setCompressedRequest(canSendCompressedRequests());
synchronized (remoteRequestExecutor) {
if (!remoteRequestExecutor.isShutdown()) {
Future future = remoteRequestExecutor.submit(dl);
pendingFutures.add(future);
}
}
}
public void testDocs() {
send("PUT", "/db", Status.CREATED, null);
// PUT:
Map<String, Object> doc1 = new HashMap<String, Object>();
doc1.put("message", "hello");
Map<String, Object> result =
(Map<String, Object>) sendBody("PUT", "/db/doc1", doc1, Status.CREATED, null);
String revID = (String) result.get("rev");
assertTrue(revID.startsWith("1-"));
// PUT to update:
doc1.put("message", "goodbye");
doc1.put("_rev", revID);
result = (Map<String, Object>) sendBody("PUT", "/db/doc1", doc1, Status.CREATED, null);
Log.v(TAG, String.format("PUT returned %s", result));
revID = (String) result.get("rev");
assertTrue(revID.startsWith("2-"));
doc1.put("_id", "doc1");
doc1.put("_rev", revID);
result = (Map<String, Object>) send("GET", "/db/doc1", Status.OK, doc1);
// Add more docs:
Map<String, Object> docX = new HashMap<String, Object>();
docX.put("message", "hello");
result = (Map<String, Object>) sendBody("PUT", "/db/doc3", docX, Status.CREATED, null);
String revID3 = (String) result.get("rev");
result = (Map<String, Object>) sendBody("PUT", "/db/doc2", docX, Status.CREATED, null);
String revID2 = (String) result.get("rev");
// _all_docs:
result = (Map<String, Object>) send("GET", "/db/_all_docs", Status.OK, null);
assertEquals(3, result.get("total_rows"));
assertEquals(0, result.get("offset"));
Map<String, Object> value1 = valueMapWithRev(revID);
Map<String, Object> value2 = valueMapWithRev(revID2);
Map<String, Object> value3 = valueMapWithRev(revID3);
Map<String, Object> row1 = new HashMap<String, Object>();
row1.put("id", "doc1");
row1.put("key", "doc1");
row1.put("value", value1);
Map<String, Object> row2 = new HashMap<String, Object>();
row2.put("id", "doc2");
row2.put("key", "doc2");
row2.put("value", value2);
Map<String, Object> row3 = new HashMap<String, Object>();
row3.put("id", "doc3");
row3.put("key", "doc3");
row3.put("value", value3);
List<Map<String, Object>> expectedRows = new ArrayList<Map<String, Object>>();
expectedRows.add(row1);
expectedRows.add(row2);
expectedRows.add(row3);
List<Map<String, Object>> rows = (List<Map<String, Object>>) result.get("rows");
assertEquals(expectedRows, rows);
// DELETE:
result =
(Map<String, Object>)
send("DELETE", String.format("/db/doc1?rev=%s", revID), Status.OK, null);
revID = (String) result.get("rev");
assertTrue(revID.startsWith("3-"));
send("GET", "/db/doc1", Status.NOT_FOUND, null);
// _changes:
List<Object> changes1 = new ArrayList<Object>();
changes1.add(valueMapWithRevNoConflictArray(revID));
List<Object> changes2 = new ArrayList<Object>();
changes2.add(valueMapWithRevNoConflictArray(revID2));
List<Object> changes3 = new ArrayList<Object>();
changes3.add(valueMapWithRevNoConflictArray(revID3));
Map<String, Object> result1 = new HashMap<String, Object>();
result1.put("id", "doc1");
result1.put("seq", 5);
result1.put("deleted", true);
result1.put("changes", changes1);
Map<String, Object> result2 = new HashMap<String, Object>();
result2.put("id", "doc2");
result2.put("seq", 4);
result2.put("changes", changes2);
Map<String, Object> result3 = new HashMap<String, Object>();
result3.put("id", "doc3");
result3.put("seq", 3);
result3.put("changes", changes3);
List<Object> results = new ArrayList<Object>();
results.add(result3);
results.add(result2);
results.add(result1);
Map<String, Object> expectedChanges = new HashMap<String, Object>();
expectedChanges.put("last_seq", 5);
expectedChanges.put("results", results);
send("GET", "/db/_changes", Status.OK, expectedChanges);
// _changes with ?since:
results.remove(result3);
results.remove(result2);
expectedChanges.put("results", results);
send("GET", "/db/_changes?since=4", Status.OK, expectedChanges);
results.remove(result1);
expectedChanges.put("results", results);
send("GET", "/db/_changes?since=5", Status.OK, expectedChanges);
// Put with _deleted to delete a doc:
Log.d(TAG, "Put with _deleted to delete a doc");
send("GET", "/db/doc5", Status.NOT_FOUND, null);
Map<String, Object> doc5 = new HashMap<String, Object>();
doc5.put("message", "hello5");
Map<String, Object> resultDoc5 =
(Map<String, Object>) sendBody("PUT", "/db/doc5", doc5, Status.CREATED, null);
String revIdDoc5 = (String) resultDoc5.get("rev");
assertTrue(revIdDoc5.startsWith("1-"));
doc5.put("_deleted", true);
doc5.put("_rev", revIdDoc5);
doc5.put("_id", "doc5");
result = (Map<String, Object>) sendBody("PUT", "/db/doc5", doc5, Status.OK, null);
send("GET", "/db/doc5", Status.NOT_FOUND, null);
Log.d(TAG, "Finished put with _deleted to delete a doc");
}
