--- a/mobile/android/tests/background/junit4/src/org/mozilla/gecko/sync/repositories/downloaders/BatchingDownloaderTest.java
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/gecko/sync/repositories/downloaders/BatchingDownloaderTest.java
@@ -1,37 +1,41 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.sync.repositories.downloaders;
+import android.net.Uri;
+import android.os.SystemClock;
import android.support.annotation.NonNull;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mozilla.gecko.background.testhelpers.TestRunner;
import org.mozilla.gecko.sync.CryptoRecord;
import org.mozilla.gecko.sync.InfoCollections;
import org.mozilla.gecko.sync.InfoConfiguration;
import org.mozilla.gecko.sync.net.AuthHeaderProvider;
import org.mozilla.gecko.sync.net.SyncResponse;
import org.mozilla.gecko.sync.net.SyncStorageCollectionRequest;
import org.mozilla.gecko.sync.net.SyncStorageResponse;
-import org.mozilla.gecko.sync.repositories.Repository;
+import org.mozilla.gecko.sync.repositories.RepositorySession;
import org.mozilla.gecko.sync.repositories.Server11Repository;
import org.mozilla.gecko.sync.repositories.Server11RepositorySession;
+import org.mozilla.gecko.sync.repositories.Repository;
import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionFetchRecordsDelegate;
import org.mozilla.gecko.sync.repositories.domain.Record;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
import ch.boye.httpclientandroidlib.ProtocolVersion;
import ch.boye.httpclientandroidlib.message.BasicHttpResponse;
import ch.boye.httpclientandroidlib.message.BasicStatusLine;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
@@ -39,17 +43,17 @@ import static org.junit.Assert.assertEqu
public class BatchingDownloaderTest {
private MockSever11Repository serverRepository;
private Server11RepositorySession repositorySession;
private MockSessionFetchRecordsDelegate sessionFetchRecordsDelegate;
private MockDownloader mockDownloader;
private String DEFAULT_COLLECTION_NAME = "dummyCollection";
private String DEFAULT_COLLECTION_URL = "http://dummy.url/";
private long DEFAULT_NEWER = 1;
- private String DEFAULT_SORT = "index";
+ private String DEFAULT_SORT = "oldest";
private String DEFAULT_IDS = "1";
private String DEFAULT_LMHEADER = "12345678";
class MockSessionFetchRecordsDelegate implements RepositorySessionFetchRecordsDelegate {
public boolean isFailure;
public boolean isFetched;
public boolean isSuccess;
public int batchesCompleted;
@@ -100,18 +104,18 @@ public class BatchingDownloaderTest {
public long newer;
public long limit;
public boolean full;
public String sort;
public String ids;
public String offset;
public boolean abort;
- public MockDownloader(Server11Repository repository, Server11RepositorySession repositorySession) {
- super(repository, repositorySession);
+ public MockDownloader(RepositorySession repositorySession, boolean allowMultipleBatches) {
+ super(null, Uri.EMPTY, SystemClock.elapsedRealtime() + TimeUnit.MINUTES.toMillis(30), allowMultipleBatches, repositorySession);
}
@Override
public void fetchWithParameters(long newer,
long batchLimit,
boolean full,
String sort,
String ids,
@@ -144,22 +148,17 @@ public class BatchingDownloaderTest {
return super.makeSyncStorageCollectionRequest(newer, batchLimit, full, sort, ids, offset);
}
}
class MockSever11Repository extends Server11Repository {
public MockSever11Repository(@NonNull String collection, @NonNull String storageURL,
AuthHeaderProvider authHeaderProvider, @NonNull InfoCollections infoCollections,
@NonNull InfoConfiguration infoConfiguration) throws URISyntaxException {
- super(collection, storageURL, authHeaderProvider, infoCollections, infoConfiguration);
- }
-
- @Override
- public long getDefaultTotalLimit() {
- return 200;
+ super(collection, SystemClock.elapsedRealtime() + TimeUnit.MINUTES.toMillis(30), storageURL, authHeaderProvider, infoCollections, infoConfiguration);
}
}
class MockRepositorySession extends Server11RepositorySession {
public boolean abort;
public MockRepositorySession(Repository repository) {
super(repository);
@@ -173,17 +172,17 @@ public class BatchingDownloaderTest {
@Before
public void setUp() throws Exception {
sessionFetchRecordsDelegate = new MockSessionFetchRecordsDelegate();
serverRepository = new MockSever11Repository(DEFAULT_COLLECTION_NAME, DEFAULT_COLLECTION_URL, null,
new InfoCollections(), new InfoConfiguration());
repositorySession = new Server11RepositorySession(serverRepository);
- mockDownloader = new MockDownloader(serverRepository, repositorySession);
+ mockDownloader = new MockDownloader(repositorySession, true);
}
@Test
public void testFlattenId() {
String[] emptyGuid = new String[]{};
String flatten = BatchingDownloader.flattenIDs(emptyGuid);
assertEquals("", flatten);
@@ -199,241 +198,123 @@ public class BatchingDownloaderTest {
multiGuid[0] = guid0;
multiGuid[1] = guid1;
multiGuid[2] = guid2;
flatten = BatchingDownloader.flattenIDs(multiGuid);
assertEquals("123456789abc,456789abc,789abc", flatten);
}
@Test
- public void testEncodeParam() throws Exception {
- String param = "123&123";
- String encodedParam = mockDownloader.encodeParam(param);
- assertEquals("123%26123", encodedParam);
- }
-
- @Test(expected=IllegalArgumentException.class)
- public void testOverTotalLimit() throws Exception {
- // Per-batch limits exceed total.
- Server11Repository repository = new Server11Repository(DEFAULT_COLLECTION_NAME, DEFAULT_COLLECTION_URL,
- null, new InfoCollections(), new InfoConfiguration()) {
- @Override
- public long getDefaultTotalLimit() {
- return 100;
- }
- @Override
- public long getDefaultBatchLimit() {
- return 200;
- }
- };
- MockDownloader mockDownloader = new MockDownloader(repository, repositorySession);
+ public void testBatchingTrivial() throws Exception {
+ MockDownloader mockDownloader = new MockDownloader(repositorySession, true);
assertNull(mockDownloader.getLastModified());
- mockDownloader.fetchSince(DEFAULT_NEWER, sessionFetchRecordsDelegate);
- }
+ // Number of records == batch limit.
+ final long BATCH_LIMIT = 100;
+ mockDownloader.fetchSince(sessionFetchRecordsDelegate, DEFAULT_NEWER, BATCH_LIMIT, DEFAULT_SORT);
- @Test
- public void testTotalLimit() throws Exception {
- // Total and per-batch limits are the same.
- Server11Repository repository = new Server11Repository(DEFAULT_COLLECTION_NAME, DEFAULT_COLLECTION_URL,
- null, new InfoCollections(), new InfoConfiguration()) {
- @Override
- public long getDefaultTotalLimit() {
- return 100;
- }
- @Override
- public long getDefaultBatchLimit() {
- return 100;
- }
- };
- MockDownloader mockDownloader = new MockDownloader(repository, repositorySession);
-
- assertNull(mockDownloader.getLastModified());
- mockDownloader.fetchSince(DEFAULT_NEWER, sessionFetchRecordsDelegate);
-
- SyncStorageResponse response = makeSyncStorageResponse(200, DEFAULT_LMHEADER, "100", "100");
+ SyncStorageResponse response = makeSyncStorageResponse(200, DEFAULT_LMHEADER, null, "100");
SyncStorageCollectionRequest request = new SyncStorageCollectionRequest(new URI(DEFAULT_COLLECTION_URL));
- long limit = repository.getDefaultBatchLimit();
mockDownloader.onFetchCompleted(response, sessionFetchRecordsDelegate, request,
- DEFAULT_NEWER, limit, true, DEFAULT_SORT, DEFAULT_IDS);
+ DEFAULT_NEWER, BATCH_LIMIT, true, DEFAULT_SORT, DEFAULT_IDS);
assertEquals(DEFAULT_LMHEADER, mockDownloader.getLastModified());
assertTrue(sessionFetchRecordsDelegate.isSuccess);
assertFalse(sessionFetchRecordsDelegate.isFetched);
assertFalse(sessionFetchRecordsDelegate.isFailure);
+ assertEquals(0, sessionFetchRecordsDelegate.batchesCompleted);
}
@Test
- public void testOverHalfOfTotalLimit() throws Exception {
- // Per-batch limit is just a bit lower than total.
- Server11Repository repository = new Server11Repository(DEFAULT_COLLECTION_NAME, DEFAULT_COLLECTION_URL,
- null, new InfoCollections(), new InfoConfiguration()) {
- @Override
- public long getDefaultTotalLimit() {
- return 100;
- }
- @Override
- public long getDefaultBatchLimit() {
- return 75;
- }
- };
- MockDownloader mockDownloader = new MockDownloader(repository, repositorySession);
+ public void testBatchingSingleBatchMode() throws Exception {
+ MockDownloader mockDownloader = new MockDownloader(repositorySession, false);
assertNull(mockDownloader.getLastModified());
- mockDownloader.fetchSince(DEFAULT_NEWER, sessionFetchRecordsDelegate);
-
- String offsetHeader = "75";
- String recordsHeader = "75";
- SyncStorageResponse response = makeSyncStorageResponse(200, DEFAULT_LMHEADER, offsetHeader, recordsHeader);
- SyncStorageCollectionRequest request = new SyncStorageCollectionRequest(new URI(DEFAULT_COLLECTION_URL));
- long limit = repository.getDefaultBatchLimit();
-
- mockDownloader.onFetchCompleted(response, sessionFetchRecordsDelegate, request, DEFAULT_NEWER,
- limit, true, DEFAULT_SORT, DEFAULT_IDS);
+ // Number of records > batch limit. But, we're only allowed to make one batch request.
+ final long BATCH_LIMIT = 100;
+ mockDownloader.fetchSince(sessionFetchRecordsDelegate, DEFAULT_NEWER, BATCH_LIMIT, DEFAULT_SORT);
- assertEquals(DEFAULT_LMHEADER, mockDownloader.getLastModified());
- // Verify the same parameters are used in the next fetch.
- assertSameParameters(mockDownloader, limit);
- assertEquals(offsetHeader, mockDownloader.offset);
- assertFalse(sessionFetchRecordsDelegate.isSuccess);
- assertFalse(sessionFetchRecordsDelegate.isFetched);
- assertFalse(sessionFetchRecordsDelegate.isFailure);
-
- // The next batch, we still have an offset token but we complete our fetch since we have reached the total limit.
- offsetHeader = "150";
- response = makeSyncStorageResponse(200, DEFAULT_LMHEADER, offsetHeader, recordsHeader);
- mockDownloader.onFetchCompleted(response, sessionFetchRecordsDelegate, request, DEFAULT_NEWER,
- limit, true, DEFAULT_SORT, DEFAULT_IDS);
+ String offsetHeader = "25";
+ String recordsHeader = "500";
+ SyncStorageResponse response = makeSyncStorageResponse(200, DEFAULT_LMHEADER, offsetHeader, recordsHeader);
+ SyncStorageCollectionRequest request = new SyncStorageCollectionRequest(new URI(DEFAULT_COLLECTION_URL));
+ mockDownloader.onFetchCompleted(response, sessionFetchRecordsDelegate, request,
+ DEFAULT_NEWER, BATCH_LIMIT, true, DEFAULT_SORT, DEFAULT_IDS);
assertEquals(DEFAULT_LMHEADER, mockDownloader.getLastModified());
assertTrue(sessionFetchRecordsDelegate.isSuccess);
assertFalse(sessionFetchRecordsDelegate.isFetched);
assertFalse(sessionFetchRecordsDelegate.isFailure);
+ assertEquals(0, sessionFetchRecordsDelegate.batchesCompleted);
}
@Test
- public void testHalfOfTotalLimit() throws Exception {
- // Per-batch limit is half of total.
- Server11Repository repository = new Server11Repository(DEFAULT_COLLECTION_NAME, DEFAULT_COLLECTION_URL,
- null, new InfoCollections(), new InfoConfiguration()) {
- @Override
- public long getDefaultTotalLimit() {
- return 100;
- }
- @Override
- public long getDefaultBatchLimit() {
- return 50;
- }
- };
- mockDownloader = new MockDownloader(repository, repositorySession);
+ public void testBatching() throws Exception {
+ final long BATCH_LIMIT = 25;
+ mockDownloader = new MockDownloader(repositorySession, true);
assertNull(mockDownloader.getLastModified());
- mockDownloader.fetchSince(DEFAULT_NEWER, sessionFetchRecordsDelegate);
+ mockDownloader.fetchSince(sessionFetchRecordsDelegate, DEFAULT_NEWER, BATCH_LIMIT, DEFAULT_SORT);
- String offsetHeader = "50";
- String recordsHeader = "50";
+ String offsetHeader = "25";
+ String recordsHeader = "25";
SyncStorageResponse response = makeSyncStorageResponse(200, DEFAULT_LMHEADER, offsetHeader, recordsHeader);
SyncStorageCollectionRequest request = new SyncStorageCollectionRequest(new URI(DEFAULT_COLLECTION_URL));
- long limit = repository.getDefaultBatchLimit();
mockDownloader.onFetchCompleted(response, sessionFetchRecordsDelegate, request, DEFAULT_NEWER,
- limit, true, DEFAULT_SORT, DEFAULT_IDS);
+ BATCH_LIMIT, true, DEFAULT_SORT, DEFAULT_IDS);
assertEquals(DEFAULT_LMHEADER, mockDownloader.getLastModified());
// Verify the same parameters are used in the next fetch.
- assertSameParameters(mockDownloader, limit);
+ assertSameParameters(mockDownloader, BATCH_LIMIT);
assertEquals(offsetHeader, mockDownloader.offset);
assertFalse(sessionFetchRecordsDelegate.isSuccess);
assertFalse(sessionFetchRecordsDelegate.isFetched);
assertFalse(sessionFetchRecordsDelegate.isFailure);
+ assertEquals(1, sessionFetchRecordsDelegate.batchesCompleted);
- // The next batch, we still have an offset token but we complete our fetch since we have reached the total limit.
- offsetHeader = "100";
+ // The next batch, we still have an offset token and has not exceed the total limit.
+ offsetHeader = "50";
response = makeSyncStorageResponse(200, DEFAULT_LMHEADER, offsetHeader, recordsHeader);
mockDownloader.onFetchCompleted(response, sessionFetchRecordsDelegate, request, DEFAULT_NEWER,
- limit, true, DEFAULT_SORT, DEFAULT_IDS);
+ BATCH_LIMIT, true, DEFAULT_SORT, DEFAULT_IDS);
+
+ assertEquals(DEFAULT_LMHEADER, mockDownloader.getLastModified());
+ // Verify the same parameters are used in the next fetch.
+ assertSameParameters(mockDownloader, BATCH_LIMIT);
+ assertEquals(offsetHeader, mockDownloader.offset);
+ assertFalse(sessionFetchRecordsDelegate.isSuccess);
+ assertFalse(sessionFetchRecordsDelegate.isFetched);
+ assertFalse(sessionFetchRecordsDelegate.isFailure);
+ assertEquals(2, sessionFetchRecordsDelegate.batchesCompleted);
+
+ // The next batch, we still have an offset token and has not exceed the total limit.
+ offsetHeader = "75";
+ response = makeSyncStorageResponse(200, DEFAULT_LMHEADER, offsetHeader, recordsHeader);
+ mockDownloader.onFetchCompleted(response, sessionFetchRecordsDelegate, request, DEFAULT_NEWER,
+ BATCH_LIMIT, true, DEFAULT_SORT, DEFAULT_IDS);
+
+ assertEquals(DEFAULT_LMHEADER, mockDownloader.getLastModified());
+ // Verify the same parameters are used in the next fetch.
+ assertSameParameters(mockDownloader, BATCH_LIMIT);
+ assertEquals(offsetHeader, mockDownloader.offset);
+ assertFalse(sessionFetchRecordsDelegate.isSuccess);
+ assertFalse(sessionFetchRecordsDelegate.isFetched);
+ assertFalse(sessionFetchRecordsDelegate.isFailure);
+ assertEquals(3, sessionFetchRecordsDelegate.batchesCompleted);
+
+ // No more offset token, so we complete batching.
+ response = makeSyncStorageResponse(200, DEFAULT_LMHEADER, null, recordsHeader);
+ mockDownloader.onFetchCompleted(response, sessionFetchRecordsDelegate, request, DEFAULT_NEWER,
+ BATCH_LIMIT, true, DEFAULT_SORT, DEFAULT_IDS);
assertEquals(DEFAULT_LMHEADER, mockDownloader.getLastModified());
assertTrue(sessionFetchRecordsDelegate.isSuccess);
assertFalse(sessionFetchRecordsDelegate.isFetched);
assertFalse(sessionFetchRecordsDelegate.isFailure);
- }
-
- @Test
- public void testFractionOfTotalLimit() throws Exception {
- // Per-batch limit is a small fraction of the total.
- Server11Repository repository = new Server11Repository(DEFAULT_COLLECTION_NAME, DEFAULT_COLLECTION_URL,
- null, new InfoCollections(), new InfoConfiguration()) {
- @Override
- public long getDefaultTotalLimit() {
- return 100;
- }
- @Override
- public long getDefaultBatchLimit() {
- return 25;
- }
- };
- mockDownloader = new MockDownloader(repository, repositorySession);
-
- assertNull(mockDownloader.getLastModified());
- mockDownloader.fetchSince(DEFAULT_NEWER, sessionFetchRecordsDelegate);
-
- String offsetHeader = "25";
- String recordsHeader = "25";
- SyncStorageResponse response = makeSyncStorageResponse(200, DEFAULT_LMHEADER, offsetHeader, recordsHeader);
- SyncStorageCollectionRequest request = new SyncStorageCollectionRequest(new URI(DEFAULT_COLLECTION_URL));
- long limit = repository.getDefaultBatchLimit();
- mockDownloader.onFetchCompleted(response, sessionFetchRecordsDelegate, request, DEFAULT_NEWER,
- limit, true, DEFAULT_SORT, DEFAULT_IDS);
-
- assertEquals(DEFAULT_LMHEADER, mockDownloader.getLastModified());
- // Verify the same parameters are used in the next fetch.
- assertSameParameters(mockDownloader, limit);
- assertEquals(offsetHeader, mockDownloader.offset);
- assertFalse(sessionFetchRecordsDelegate.isSuccess);
- assertFalse(sessionFetchRecordsDelegate.isFetched);
- assertFalse(sessionFetchRecordsDelegate.isFailure);
-
- // The next batch, we still have an offset token and has not exceed the total limit.
- offsetHeader = "50";
- response = makeSyncStorageResponse(200, DEFAULT_LMHEADER, offsetHeader, recordsHeader);
- mockDownloader.onFetchCompleted(response, sessionFetchRecordsDelegate, request, DEFAULT_NEWER,
- limit, true, DEFAULT_SORT, DEFAULT_IDS);
-
- assertEquals(DEFAULT_LMHEADER, mockDownloader.getLastModified());
- // Verify the same parameters are used in the next fetch.
- assertSameParameters(mockDownloader, limit);
- assertEquals(offsetHeader, mockDownloader.offset);
- assertFalse(sessionFetchRecordsDelegate.isSuccess);
- assertFalse(sessionFetchRecordsDelegate.isFetched);
- assertFalse(sessionFetchRecordsDelegate.isFailure);
-
- // The next batch, we still have an offset token and has not exceed the total limit.
- offsetHeader = "75";
- response = makeSyncStorageResponse(200, DEFAULT_LMHEADER, offsetHeader, recordsHeader);
- mockDownloader.onFetchCompleted(response, sessionFetchRecordsDelegate, request, DEFAULT_NEWER,
- limit, true, DEFAULT_SORT, DEFAULT_IDS);
-
- assertEquals(DEFAULT_LMHEADER, mockDownloader.getLastModified());
- // Verify the same parameters are used in the next fetch.
- assertSameParameters(mockDownloader, limit);
- assertEquals(offsetHeader, mockDownloader.offset);
- assertFalse(sessionFetchRecordsDelegate.isSuccess);
- assertFalse(sessionFetchRecordsDelegate.isFetched);
- assertFalse(sessionFetchRecordsDelegate.isFailure);
-
- // The next batch, we still have an offset token but we complete our fetch since we have reached the total limit.
- offsetHeader = "100";
- response = makeSyncStorageResponse(200, DEFAULT_LMHEADER, offsetHeader, recordsHeader);
- mockDownloader.onFetchCompleted(response, sessionFetchRecordsDelegate, request, DEFAULT_NEWER,
- limit, true, DEFAULT_SORT, DEFAULT_IDS);
-
- assertEquals(DEFAULT_LMHEADER, mockDownloader.getLastModified());
- assertTrue(sessionFetchRecordsDelegate.isSuccess);
- assertFalse(sessionFetchRecordsDelegate.isFetched);
- assertFalse(sessionFetchRecordsDelegate.isFailure);
+ assertEquals(3, sessionFetchRecordsDelegate.batchesCompleted);
}
@Test
public void testFailureLMChangedMultiBatch() throws Exception {
assertNull(mockDownloader.getLastModified());
String lmHeader = "12345678";
String offsetHeader = "100";
@@ -490,22 +371,38 @@ public class BatchingDownloaderTest {
assertFalse(sessionFetchRecordsDelegate.isSuccess);
assertFalse(sessionFetchRecordsDelegate.isFailure);
assertEquals(record, sessionFetchRecordsDelegate.record);
}
@Test
public void testAbortRequests() {
MockRepositorySession mockRepositorySession = new MockRepositorySession(serverRepository);
- BatchingDownloader downloader = new BatchingDownloader(serverRepository, mockRepositorySession);
+ BatchingDownloader downloader = new BatchingDownloader(null, Uri.EMPTY, SystemClock.elapsedRealtime() + TimeUnit.MINUTES.toMillis(30), true, mockRepositorySession);
assertFalse(mockRepositorySession.abort);
downloader.abortRequests();
assertTrue(mockRepositorySession.abort);
}
+ @Test
+ public void testBuildCollectionURI() {
+ try {
+ assertEquals("?full=1&newer=5000.000", BatchingDownloader.buildCollectionURI(Uri.EMPTY, true, 5000000L, -1, null, null, null).toString());
+ assertEquals("?newer=1230.000", BatchingDownloader.buildCollectionURI(Uri.EMPTY, false, 1230000L, -1, null, null, null).toString());
+ assertEquals("?newer=5000.000&limit=10", BatchingDownloader.buildCollectionURI(Uri.EMPTY, false, 5000000L, 10, null, null, null).toString());
+ assertEquals("?full=1&newer=5000.000&sort=index", BatchingDownloader.buildCollectionURI(Uri.EMPTY, true, 5000000L, 0, "index", null, null).toString());
+ assertEquals("?full=1&ids=123%2Cabc", BatchingDownloader.buildCollectionURI(Uri.EMPTY, true, -1L, -1, null, "123,abc", null).toString());
+
+ final Uri baseUri = Uri.parse("https://moztest.org/collection/");
+ assertEquals(baseUri + "?full=1&ids=123%2Cabc&offset=1234", BatchingDownloader.buildCollectionURI(baseUri, true, -1L, -1, null, "123,abc", "1234").toString());
+ } catch (URISyntaxException e) {
+ fail();
+ }
+ }
+
private void assertSameParameters(MockDownloader mockDownloader, long limit) {
assertEquals(DEFAULT_NEWER, mockDownloader.newer);
assertEquals(limit, mockDownloader.limit);
assertTrue(mockDownloader.full);
assertEquals(DEFAULT_SORT, mockDownloader.sort);
assertEquals(DEFAULT_IDS, mockDownloader.ids);
}
