--- a/mobile/android/base/android-services.mozbuild
+++ b/mobile/android/base/android-services.mozbuild
@@ -950,16 +950,17 @@ sync_java_files = [TOPSRCDIR + '/mobile/
'sync/PrefsBackoffHandler.java',
'sync/ReflowIsNecessaryException.java',
'sync/repositories/android/BookmarksDataAccessor.java',
'sync/repositories/android/BookmarksDeletionManager.java',
'sync/repositories/android/BookmarksInsertionManager.java',
'sync/repositories/android/BookmarksRepository.java',
'sync/repositories/android/BookmarksRepositorySession.java',
'sync/repositories/android/BookmarksSessionHelper.java',
+ 'sync/repositories/android/BookmarksValidationRepository.java',
'sync/repositories/android/BrowserContractHelpers.java',
'sync/repositories/android/CachedSQLiteOpenHelper.java',
'sync/repositories/android/ClientsDatabase.java',
'sync/repositories/android/ClientsDatabaseAccessor.java',
'sync/repositories/android/DataAccessor.java',
'sync/repositories/android/FennecTabsRepository.java',
'sync/repositories/android/FormHistoryRepositorySession.java',
'sync/repositories/android/HistoryDataAccessor.java',
@@ -1062,16 +1063,17 @@ sync_java_files = [TOPSRCDIR + '/mobile/
'sync/stage/GlobalSyncStage.java',
'sync/stage/HistoryServerSyncStage.java',
'sync/stage/NoSuchStageException.java',
'sync/stage/PasswordsServerSyncStage.java',
'sync/stage/RecentHistoryServerSyncStage.java',
'sync/stage/ServerSyncStage.java',
'sync/stage/SyncClientsEngineStage.java',
'sync/stage/UploadMetaGlobalStage.java',
+ 'sync/stage/ValidateBookmarksSyncStage.java',
'sync/stage/VersionedServerSyncStage.java',
'sync/SyncConfiguration.java',
'sync/SyncConfigurationException.java',
'sync/SyncConstants.java',
'sync/SyncDeadlineReachedException.java',
'sync/SyncException.java',
'sync/synchronizer/ConcurrentRecordConsumer.java',
'sync/synchronizer/RecordConsumer.java',
@@ -1090,14 +1092,18 @@ sync_java_files = [TOPSRCDIR + '/mobile/
'sync/SynchronizerConfiguration.java',
'sync/telemetry/TelemetryCollector.java',
'sync/telemetry/TelemetryContract.java',
'sync/telemetry/TelemetryStageCollector.java',
'sync/ThreadPool.java',
'sync/UnexpectedJSONException.java',
'sync/UnknownSynchronizerConfigurationVersionException.java',
'sync/Utils.java',
+ 'sync/validation/BookmarkValidationResults.java',
+ 'sync/validation/BookmarkValidator.java',
+ 'sync/validation/CollectionValidator.java',
+ 'sync/validation/ValidationResults.java',
'tokenserver/TokenServerClient.java',
'tokenserver/TokenServerClientDelegate.java',
'tokenserver/TokenServerException.java',
'tokenserver/TokenServerToken.java',
'util/PRNGFixes.java',
]]
--- a/mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetrySyncPingBuilder.java
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetrySyncPingBuilder.java
@@ -30,17 +30,17 @@ public class TelemetrySyncPingBuilder ex
public TelemetrySyncPingBuilder setStages(@NonNull final Serializable data) {
HashMap<String, TelemetryStageCollector> stages = castSyncData(data);
final JSONArray engines = new JSONArray();
for (String stageName : stages.keySet()) {
final TelemetryStageCollector stage = stages.get(stageName);
// Skip stages that did nothing.
- if (stage.inbound == 0 && stage.outbound == 0) {
+ if (stage.inbound == 0 && stage.outbound == 0 && stage.error == null && stage.validation == null) {
continue;
}
final ExtendedJSONObject stageJSON = new ExtendedJSONObject();
stageJSON.put("name", stageName);
stageJSON.put("took", stage.finished - stage.started);
@@ -78,16 +78,20 @@ public class TelemetrySyncPingBuilder ex
stageJSON.put("outgoing", outgoingJSON);
}
// We depend on the error builder from TelemetryCollector to produce the right schema.
// Spreading around our schema definition like that is awkward, but, alas, here we are.
if (stage.error != null) {
stageJSON.put("failureReason", stage.error);
}
+ // As above for validation too.
+ if (stage.validation != null) {
+ stageJSON.put("validation", stage.validation);
+ }
addUnchecked(engines, stageJSON);
}
payload.put("engines", engines);
return this;
}
public TelemetrySyncPingBuilder setUID(@NonNull String uid) {
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/GlobalSession.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/GlobalSession.java
@@ -39,16 +39,17 @@ import org.mozilla.gecko.sync.stage.Fetc
import org.mozilla.gecko.sync.stage.FetchMetaGlobalStage;
import org.mozilla.gecko.sync.stage.FormHistoryServerSyncStage;
import org.mozilla.gecko.sync.stage.GlobalSyncStage;
import org.mozilla.gecko.sync.stage.GlobalSyncStage.Stage;
import org.mozilla.gecko.sync.stage.NoSuchStageException;
import org.mozilla.gecko.sync.stage.PasswordsServerSyncStage;
import org.mozilla.gecko.sync.stage.SyncClientsEngineStage;
import org.mozilla.gecko.sync.stage.UploadMetaGlobalStage;
+import org.mozilla.gecko.sync.stage.ValidateBookmarksSyncStage;
import org.mozilla.gecko.sync.telemetry.TelemetryCollector;
import org.mozilla.gecko.sync.telemetry.TelemetryStageCollector;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
@@ -202,16 +203,17 @@ public class GlobalSession implements Ht
stages.put(Stage.syncTabs, new FennecTabsServerSyncStage());
stages.put(Stage.syncPasswords, new PasswordsServerSyncStage());
// Will only run if syncFullHistory stage never completed.
// Bug 1316110 tracks follow up work to improve efficiency of this stage.
stages.put(Stage.syncRecentHistory, new RecentHistoryServerSyncStage());
stages.put(Stage.syncBookmarks, new BookmarksServerSyncStage());
+ stages.put(Stage.validateBookmarks, new ValidateBookmarksSyncStage());
stages.put(Stage.syncFormHistory, new FormHistoryServerSyncStage());
stages.put(Stage.syncFullHistory, new HistoryServerSyncStage());
stages.put(Stage.uploadMetaGlobal, new UploadMetaGlobalStage());
stages.put(Stage.completed, new CompletedStage());
this.stages = Collections.unmodifiableMap(stages);
}
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/SyncConfiguration.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/SyncConfiguration.java
@@ -91,16 +91,17 @@ public class SyncConfiguration {
protected final AuthHeaderProvider authHeaderProvider;
public static final String PREF_PREFS_VERSION = "prefs.version";
public static final long CURRENT_PREFS_VERSION = 1;
public static final String CLIENTS_COLLECTION_TIMESTAMP = "serverClientsTimestamp"; // When the collection was touched.
public static final String CLIENT_RECORD_TIMESTAMP = "serverClientRecordTimestamp"; // When our record was touched.
public static final String MIGRATION_SENTINEL_CHECK_TIMESTAMP = "migrationSentinelCheckTimestamp"; // When we last looked in meta/fxa_credentials.
+ public static final String BOOKMARK_VALIDATION_CHECK_TIMESTAMP = "bookmarkValidationCheckTimestamp"; // Last time we considered performing validation
public static final String PREF_CLUSTER_URL = "clusterURL";
public static final String PREF_SYNC_ID = "syncID";
public static final String PREF_ENABLED_ENGINE_NAMES = "enabledEngineNames";
public static final String PREF_DECLINED_ENGINE_NAMES = "declinedEngineNames";
public static final String PREF_USER_SELECTED_ENGINES_TO_SYNC = "userSelectedEngines";
public static final String PREF_USER_SELECTED_ENGINES_TO_SYNC_TIMESTAMP = "userSelectedEnginesTimestamp";
@@ -453,16 +454,24 @@ public class SyncConfiguration {
public void persistLastMigrationSentinelCheckTimestamp(long timestamp) {
getEditor().putLong(SyncConfiguration.MIGRATION_SENTINEL_CHECK_TIMESTAMP, timestamp).commit();
}
public long getLastMigrationSentinelCheckTimestamp() {
return getPrefs().getLong(SyncConfiguration.MIGRATION_SENTINEL_CHECK_TIMESTAMP, 0L);
}
+ public void persistLastValidationCheckTimestamp(long timestamp) {
+ getEditor().putLong(SyncConfiguration.BOOKMARK_VALIDATION_CHECK_TIMESTAMP, timestamp).commit();
+ }
+
+ public long getLastValidationCheckTimestamp() {
+ return getPrefs().getLong(SyncConfiguration.BOOKMARK_VALIDATION_CHECK_TIMESTAMP, 0L);
+ }
+
public void purgeCryptoKeys() {
if (collectionKeys != null) {
collectionKeys.clear();
}
persistedCryptoKeys().purge();
}
public void purgeMetaGlobal() {
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/ConfigurableServer15Repository.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/ConfigurableServer15Repository.java
@@ -22,42 +22,44 @@ import org.mozilla.gecko.sync.stage.Serv
* @author rnewman
*
*/
public class ConfigurableServer15Repository extends Server15Repository {
private final String sortOrder;
private final long batchLimit;
private final ServerSyncStage.MultipleBatches multipleBatches;
private final ServerSyncStage.HighWaterMark highWaterMark;
-
+ private final boolean forceFullFetch;
public ConfigurableServer15Repository(
String collection,
long syncDeadline,
String storageURL,
AuthHeaderProvider authHeaderProvider,
InfoCollections infoCollections,
InfoConfiguration infoConfiguration,
long batchLimit,
String sort,
ServerSyncStage.MultipleBatches multipleBatches,
ServerSyncStage.HighWaterMark highWaterMark,
- RepositoryStateProvider stateProvider) throws URISyntaxException {
+ RepositoryStateProvider stateProvider,
+ boolean forceFullFetch) throws URISyntaxException {
super(
collection,
syncDeadline,
storageURL,
authHeaderProvider,
infoCollections,
infoConfiguration,
stateProvider
);
this.batchLimit = batchLimit;
this.sortOrder = sort;
this.multipleBatches = multipleBatches;
this.highWaterMark = highWaterMark;
+ this.forceFullFetch = forceFullFetch;
// Sanity check: let's ensure we're configured correctly. At this point in time, it doesn't make
// sense to use H.W.M. with a non-persistent state provider. This might change if we start retrying
// during a download in case of 412s.
if (!stateProvider.isPersistent() && highWaterMark.equals(ServerSyncStage.HighWaterMark.Enabled)) {
throw new IllegalArgumentException("Can not use H.W.M. with NonPersistentRepositoryStateProvider");
}
}
@@ -76,9 +78,15 @@ public class ConfigurableServer15Reposit
public boolean getAllowMultipleBatches() {
return multipleBatches.equals(ServerSyncStage.MultipleBatches.Enabled);
}
@Override
public boolean getAllowHighWaterMark() {
return highWaterMark.equals(ServerSyncStage.HighWaterMark.Enabled);
}
+
+ @Override
+ public boolean getFullFetchForced() {
+ return forceFullFetch;
+ }
+
}
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/Server15Repository.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/Server15Repository.java
@@ -106,16 +106,20 @@ public class Server15Repository extends
public boolean getAllowMultipleBatches() {
return true;
}
public boolean getAllowHighWaterMark() {
return false;
}
+ public boolean getFullFetchForced() {
+ return false;
+ }
+
/**
* A point in time by which this repository's session must complete fetch and store operations.
* Particularly pertinent for batching downloads performed by the session (should we fetch
* another batch?) and buffered repositories (do we have enough time to merge what we've downloaded?).
*/
public long getSyncDeadline() {
return syncDeadlineMillis;
}
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/Server15RepositorySession.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/Server15RepositorySession.java
@@ -110,16 +110,20 @@ public class Server15RepositorySession e
/**
* @return Repository's high-water-mark if it's available, its use is allowed by the repository,
* repository is set to fetch oldest-first, and it's greater than collection's last-synced timestamp.
* Otherwise, returns repository's last-synced timestamp.
*/
@Override
public long getLastSyncTimestamp() {
+ if (serverRepository.getFullFetchForced()) {
+ return 0;
+ }
+
if (!serverRepository.getAllowHighWaterMark() || !serverRepository.getSortOrder().equals("oldest")) {
return super.getLastSyncTimestamp();
}
final Long highWaterMark = serverRepository.stateProvider.getLong(
RepositoryStateProvider.KEY_HIGH_WATER_MARK);
// After a successful sync we expect that last-synced timestamp for a collection will be greater
new file mode 100644
--- /dev/null
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/android/BookmarksValidationRepository.java
@@ -0,0 +1,229 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.sync.repositories.android;
+
+
+
+import android.content.Context;
+import android.os.SystemClock;
+
+import org.mozilla.gecko.background.common.log.Logger;
+import org.mozilla.gecko.sync.ExtendedJSONObject;
+import org.mozilla.gecko.sync.delegates.ClientsDataDelegate;
+import org.mozilla.gecko.sync.repositories.InactiveSessionException;
+import org.mozilla.gecko.sync.repositories.InvalidSessionTransitionException;
+import org.mozilla.gecko.sync.repositories.NoStoreDelegateException;
+import org.mozilla.gecko.sync.repositories.Repository;
+import org.mozilla.gecko.sync.repositories.RepositorySession;
+import org.mozilla.gecko.sync.repositories.RepositorySessionBundle;
+import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionBeginDelegate;
+import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionCreationDelegate;
+import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionFetchRecordsDelegate;
+import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionFinishDelegate;
+import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionWipeDelegate;
+import org.mozilla.gecko.sync.repositories.domain.BookmarkRecord;
+import org.mozilla.gecko.sync.repositories.domain.Record;
+import org.mozilla.gecko.sync.telemetry.TelemetryStageCollector;
+import org.mozilla.gecko.sync.validation.BookmarkValidationResults;
+import org.mozilla.gecko.sync.validation.BookmarkValidator;
+
+import java.util.ArrayList;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+
+/**
+ * This repository fetches all records that are present on the client or server, and runs
+ * the BookmarkValidator on these, so that we can include that data in telemetry.
+ *
+ * This is obviously expensive, so it's worth noting that we don't run it frequently.
+ * @see org.mozilla.gecko.sync.stage.ValidateBookmarksSyncStage for the set of requirements
+ * that must be met in order to run validation.
+ *
+ * It's mostly concerned with the client's view of the world, and how client propagates that
+ * view outwards. That is why we're wrapping a regular bookmarks session here, and capturing
+ * any side-effects that its internal 'fetch' methods will have.
+ *
+ * We're not concerned with how client's view of the world is shaped - that is, we're
+ * not capturing record reconciliation here directly. These sorts of effects will
+ * (hopefully) be determined from validation results in aggregate.
+ *
+ * @see BookmarkValidationResults for the concrete set of problems it checks for.
+ */
+public class BookmarksValidationRepository extends Repository {
+ private static final String LOG_TAG = "BookmarksValidationRepository";
+
+ protected final ClientsDataDelegate clientsDataDelegate;
+ private final TelemetryStageCollector parentCollector;
+
+ public BookmarksValidationRepository(ClientsDataDelegate clientsDataDelegate, TelemetryStageCollector collector) {
+ this.clientsDataDelegate = clientsDataDelegate;
+ this.parentCollector = collector;
+ }
+
+ @Override
+ public void createSession(RepositorySessionCreationDelegate delegate, Context context) {
+ delegate.onSessionCreated(new BookmarksValidationRepositorySession(this, context));
+ }
+
+ public class BookmarksValidationRepositorySession extends RepositorySession {
+
+ private final ConcurrentLinkedQueue<BookmarkRecord> local = new ConcurrentLinkedQueue<>();
+ private final ConcurrentLinkedQueue<BookmarkRecord> remote = new ConcurrentLinkedQueue<>();
+ private long startTime;
+ private final BookmarksRepositorySession wrappedSession;
+
+ public BookmarksValidationRepositorySession(Repository r, Context context) {
+ super(r);
+ startTime = SystemClock.elapsedRealtime();
+ wrappedSession = new BookmarksRepositorySession(r, context);
+ }
+
+ @Override
+ public long getLastSyncTimestamp() {
+ return 0;
+ }
+
+ @Override
+ public void wipe(RepositorySessionWipeDelegate delegate) {
+ Logger.error(LOG_TAG, "wipe() called on bookmark validator");
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public synchronized boolean isActive() {
+ return this.wrappedSession.isActive();
+ }
+
+
+ @Override
+ public void begin(final RepositorySessionBeginDelegate delegate) throws InvalidSessionTransitionException {
+ wrappedSession.begin(new RepositorySessionBeginDelegate() {
+
+ @Override
+ public void onBeginFailed(Exception ex) {
+ delegate.onBeginFailed(ex);
+ }
+
+ @Override
+ public void onBeginSucceeded(RepositorySession session) {
+ delegate.onBeginSucceeded(BookmarksValidationRepositorySession.this);
+ }
+
+ @Override
+ public RepositorySessionBeginDelegate deferredBeginDelegate(ExecutorService executor) {
+ final RepositorySessionBeginDelegate deferred = delegate.deferredBeginDelegate(executor);
+ return new RepositorySessionBeginDelegate() {
+ @Override
+ public void onBeginSucceeded(RepositorySession session) {
+ if (wrappedSession != session) {
+ Logger.warn(LOG_TAG, "Got onBeginSucceeded for session " + session + ", not our inner session!");
+ }
+ deferred.onBeginSucceeded(BookmarksValidationRepositorySession.this);
+ }
+
+ @Override
+ public void onBeginFailed(Exception ex) {
+ deferred.onBeginFailed(ex);
+ }
+
+ @Override
+ public RepositorySessionBeginDelegate deferredBeginDelegate(ExecutorService executor) {
+ return this;
+ }
+ };
+ }
+ });
+ }
+
+ @Override
+ public void finish(final RepositorySessionFinishDelegate delegate) throws InactiveSessionException {
+ wrappedSession.finish(new RepositorySessionFinishDelegate() {
+ @Override
+ public void onFinishFailed(Exception ex) {
+ delegate.onFinishFailed(ex);
+ }
+
+ @Override
+ public void onFinishSucceeded(RepositorySession session, RepositorySessionBundle bundle) {
+ delegate.onFinishSucceeded(BookmarksValidationRepositorySession.this, bundle);
+ }
+
+ @Override
+ public RepositorySessionFinishDelegate deferredFinishDelegate(ExecutorService executor) {
+ return this;
+ }
+ });
+ }
+
+ private void validateForTelemetry() {
+ ArrayList<BookmarkRecord> localRecords = new ArrayList<>(local);
+ ArrayList<BookmarkRecord> remoteRecords = new ArrayList<>(remote);
+ BookmarkValidationResults results = BookmarkValidator.validateClientAgainstServer(localRecords, remoteRecords);
+ ExtendedJSONObject o = new ExtendedJSONObject();
+ o.put("took", SystemClock.elapsedRealtime() - startTime);
+ o.put("checked", Math.max(localRecords.size(), remoteRecords.size()));
+ o.put("problems", results.jsonSummary());
+ Logger.info(LOG_TAG, "Completed validation in " + (SystemClock.elapsedRealtime() - startTime) + " ms");
+ parentCollector.validation = o;
+ }
+
+ @Override
+ public void fetchModified(final RepositorySessionFetchRecordsDelegate delegate) {
+ wrappedSession.fetchAll(new RepositorySessionFetchRecordsDelegate() {
+ @Override
+ public void onFetchFailed(Exception ex) {
+ local.clear();
+ remote.clear();
+ delegate.onFetchFailed(ex);
+ }
+
+ @Override
+ public void onFetchedRecord(Record record) {
+ local.add((BookmarkRecord)record);
+ }
+
+ @Override
+ public void onFetchCompleted(long fetchEnd) {
+ validateForTelemetry();
+ delegate.onFetchCompleted(fetchEnd);
+ }
+
+ @Override
+ public void onBatchCompleted() {}
+
+ @Override
+ public RepositorySessionFetchRecordsDelegate deferredFetchDelegate(ExecutorService executor) {
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public void fetch(String[] guids, RepositorySessionFetchRecordsDelegate delegate) throws InactiveSessionException {
+ Logger.error(LOG_TAG, "fetch guids[] called on bookmark validator");
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void fetchAll(RepositorySessionFetchRecordsDelegate delegate) {
+ this.fetchModified(delegate);
+ }
+
+ @Override
+ public void store(Record record) throws NoStoreDelegateException {
+ remote.add((BookmarkRecord)record);
+ }
+
+ @Override
+ public void storeIncomplete() {
+ super.storeIncomplete();
+ }
+
+ @Override
+ public void storeDone() {
+ super.storeDone();
+ }
+ }
+}
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/BookmarksServerSyncStage.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/BookmarksServerSyncStage.java
@@ -76,17 +76,18 @@ public class BookmarksServerSyncStage ex
session.config.storageURL(),
session.getAuthHeaderProvider(),
session.config.infoCollections,
session.config.infoConfiguration,
BOOKMARKS_BATCH_LIMIT,
BOOKMARKS_SORT,
getAllowedMultipleBatches(),
getAllowedToUseHighWaterMark(),
- getRepositoryStateProvider()
+ getRepositoryStateProvider(),
+ false
);
}
@Override
protected Repository getLocalRepository() {
return new BufferingMiddlewareRepository(
session.getSyncDeadline(),
new MemoryBufferStorage(),
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/FormHistoryServerSyncStage.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/FormHistoryServerSyncStage.java
@@ -70,17 +70,18 @@ public class FormHistoryServerSyncStage
session.config.storageURL(),
session.getAuthHeaderProvider(),
session.config.infoCollections,
session.config.infoConfiguration,
FORM_HISTORY_BATCH_LIMIT,
FORM_HISTORY_SORT,
getAllowedMultipleBatches(),
getAllowedToUseHighWaterMark(),
- getRepositoryStateProvider()
+ getRepositoryStateProvider(),
+ false
);
}
@Override
protected Repository getLocalRepository() {
return new BufferingMiddlewareRepository(
session.getSyncDeadline(),
new MemoryBufferStorage(),
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/GlobalSyncStage.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/GlobalSyncStage.java
@@ -36,16 +36,18 @@ public interface GlobalSyncStage {
*/
syncTabs("tabs"),
syncPasswords("passwords"),
syncRecentHistory("recentHistory"),
syncBookmarks("bookmarks"),
syncFullHistory("history"),
syncFormHistory("forms"),
+ validateBookmarks,
+
uploadMetaGlobal,
completed;
// Maintain a mapping from names ("bookmarks") to Stage enumerations (syncBookmarks).
private static final Map<String, Stage> named = new HashMap<String, Stage>();
static {
for (Stage s : EnumSet.allOf(Stage.class)) {
if (s.getRepositoryName() != null) {
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/HistoryServerSyncStage.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/HistoryServerSyncStage.java
@@ -88,17 +88,18 @@ public class HistoryServerSyncStage exte
session.config.storageURL(),
session.getAuthHeaderProvider(),
session.config.infoCollections,
session.config.infoConfiguration,
HISTORY_BATCH_LIMIT,
HISTORY_SORT,
getAllowedMultipleBatches(),
getAllowedToUseHighWaterMark(),
- getRepositoryStateProvider()
+ getRepositoryStateProvider(),
+ false
);
}
@Override
protected RecordFactory getRecordFactory() {
return new HistoryRecordFactory();
}
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/RecentHistoryServerSyncStage.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/RecentHistoryServerSyncStage.java
@@ -89,17 +89,18 @@ public class RecentHistoryServerSyncStag
session.config.storageURL(),
session.getAuthHeaderProvider(),
session.config.infoCollections,
session.config.infoConfiguration,
HISTORY_BATCH_LIMIT,
HISTORY_SORT,
getAllowedMultipleBatches(),
getAllowedToUseHighWaterMark(),
- getRepositoryStateProvider());
+ getRepositoryStateProvider(),
+ false);
}
/**
* This stage is only enabled if full history session is enabled and did not complete a sync yet.
*/
@Override
public boolean isEnabled() throws MetaGlobalException {
final boolean historyStageEnabled = super.isEnabled();
new file mode 100644
--- /dev/null
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/ValidateBookmarksSyncStage.java
@@ -0,0 +1,204 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.sync.stage;
+
+
+import android.content.Context;
+import android.database.Cursor;
+import android.os.SystemClock;
+import android.support.annotation.Nullable;
+
+import org.mozilla.gecko.AppConstants;
+import org.mozilla.gecko.db.BrowserContract;
+import org.mozilla.gecko.sync.MetaGlobalException;
+import org.mozilla.gecko.sync.NonObjectJSONException;
+import org.mozilla.gecko.sync.SynchronizerConfiguration;
+import org.mozilla.gecko.sync.middleware.BufferingMiddlewareRepository;
+import org.mozilla.gecko.sync.middleware.MiddlewareRepository;
+import org.mozilla.gecko.sync.middleware.storage.MemoryBufferStorage;
+import org.mozilla.gecko.sync.repositories.ConfigurableServer15Repository;
+import org.mozilla.gecko.sync.repositories.RecordFactory;
+import org.mozilla.gecko.sync.repositories.Repository;
+import org.mozilla.gecko.sync.repositories.android.BookmarksValidationRepository;
+import org.mozilla.gecko.sync.repositories.android.BrowserContractHelpers;
+import org.mozilla.gecko.sync.repositories.domain.BookmarkRecordFactory;
+import org.mozilla.gecko.sync.repositories.domain.VersionConstants;
+import org.mozilla.gecko.sync.telemetry.TelemetryCollector;
+import org.mozilla.gecko.sync.telemetry.TelemetryStageCollector;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This sync stage runs bookmark validation for including in telemetry. Bookmark validation
+ * is fairly costly, so we don't want to run it frequently, and there are a number of
+ * checks to make sure it's a good idea to run it.
+ *
+ * We only even consider running validation once a day, and during that time we
+ * make a number of checks to determine if we should run it:
+ *
+ * - We only run validation (VALIDATION_PROBABILITY * 100)% of the time (based on Math.random())
+ * - We must be on nightly (may be relaxed eventually).
+ * - You must have have fewer than MAX_BOOKMARKS_COUNT bookmarks in your local database
+ * - There must be enough time left in the sync deadline (at least TIME_REQUIRED_TO_VALIDATE ms).
+ * - The bookmarks collection must have ran, reported telemetry, and not included an error in
+ * the telemetry stage.
+ */
+public class ValidateBookmarksSyncStage extends ServerSyncStage {
+ protected static final String LOG_TAG = "ValidateBookmarksStage";
+
+ private static final String BOOKMARKS_SORT = "oldest";
+ private static final long BOOKMARKS_BATCH_LIMIT = 5000;
+
+ // Maximum number of (local) bookmarks we have where we're still
+ // willing to run a validation.
+ private static final long MAX_BOOKMARKS_COUNT = 1000;
+
+ // Probability that, given all other requirements are met, we'll run a validation.
+ private static final double VALIDATION_PROBABILITY = 0.1;
+
+ // If we have less than this amount of time left, we don't bother validating.
+ // Currently 2 minutes, which would be an insane amount of time for a
+ // validation to take, but it's better to be conservative here.
+ private static final long TIME_REQUIRED_TO_VALIDATE = TimeUnit.MINUTES.toMillis(2);
+
+ // We only even attempt to validate this frequently
+ private static final long VALIDATION_INTERVAL = TimeUnit.DAYS.toMillis(1);
+
+ @Override
+ protected String getCollection() {
+ return "bookmarks";
+ }
+
+ @Override
+ protected String getEngineName() {
+ return "bookmarks";
+ }
+
+ @Override
+ public Integer getStorageVersion() {
+ return VersionConstants.BOOKMARKS_ENGINE_VERSION;
+ }
+
+ /**
+ * We can't validate without all of the records, so a HWM doesn't make sense.
+ *
+ * @return HighWaterMark.Disabled
+ */
+ @Override
+ protected HighWaterMark getAllowedToUseHighWaterMark() {
+ return HighWaterMark.Disabled;
+ }
+
+ /**
+ * Full batching is allowed, because we want all of the records.
+ *
+ * @return MultipleBatches.Enabled
+ */
+ @Override
+ protected MultipleBatches getAllowedMultipleBatches() {
+ return MultipleBatches.Enabled;
+ }
+ @Override
+ protected Repository getRemoteRepository() throws URISyntaxException {
+ return new ConfigurableServer15Repository(
+ getCollection(),
+ session.getSyncDeadline(),
+ session.config.storageURL(),
+ session.getAuthHeaderProvider(),
+ session.config.infoCollections,
+ session.config.infoConfiguration,
+ BOOKMARKS_BATCH_LIMIT,
+ BOOKMARKS_SORT,
+ getAllowedMultipleBatches(),
+ getAllowedToUseHighWaterMark(),
+ getRepositoryStateProvider(),
+ true
+ );
+ }
+
+ @Override
+ protected Repository getLocalRepository() {
+ TelemetryStageCollector bookmarkCollector =
+ this.telemetryStageCollector.getSyncCollector().collectorFor("bookmarks");
+ return new BookmarksValidationRepository(session.getClientsDelegate(), bookmarkCollector);
+ }
+
+ @Override
+ protected RecordFactory getRecordFactory() {
+ return new BookmarkRecordFactory();
+ }
+
+ private long getLocalBookmarkRecordCount() {
+ final Context context = session.getContext();
+ final Cursor cursor = context.getContentResolver().query(
+ BrowserContractHelpers.BOOKMARKS_CONTENT_URI,
+ new String[] {BrowserContract.Bookmarks._ID},
+ null, null, null
+ );
+ if (cursor == null) {
+ return -1;
+ }
+ try {
+ return cursor.getCount();
+ } finally {
+ cursor.close();
+ }
+ }
+
+ // This implements the logic described in the header comment.
+ private boolean shouldValidate() {
+ if (!AppConstants.NIGHTLY_BUILD) {
+ return false;
+ }
+ final long consideredValidationLast = session.config.getLastValidationCheckTimestamp();
+ final long now = System.currentTimeMillis();
+
+ if (now - consideredValidationLast < VALIDATION_INTERVAL) {
+ return false;
+ }
+
+ session.config.persistLastValidationCheckTimestamp(now);
+
+ if (Math.random() > VALIDATION_PROBABILITY) {
+ return false;
+ }
+
+ // Note that syncDeadline is relative to the elapsedRealtime clock, not `now`.
+ final long timeToSyncDeadline = session.getSyncDeadline() - SystemClock.elapsedRealtime();
+ if (timeToSyncDeadline < TIME_REQUIRED_TO_VALIDATE) {
+ return false;
+ }
+
+ // See if we'll have somewhere to store the validation results
+ TelemetryCollector syncCollector = telemetryStageCollector.getSyncCollector();
+ if (!syncCollector.hasCollectorFor("bookmarks")) {
+ return false;
+ }
+
+ // And make sure that bookmarks sync didn't hit an error.
+ TelemetryStageCollector stageCollector = syncCollector.collectorFor("bookmarks");
+ if (stageCollector.error != null) {
+ return false;
+ }
+ long count = getLocalBookmarkRecordCount();
+ if (count < 0 || count > MAX_BOOKMARKS_COUNT) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ protected boolean isEnabled() throws MetaGlobalException {
+ if (session == null || session.getContext() == null) {
+ return false;
+ }
+
+ return super.isEnabled() && shouldValidate();
+ }
+
+}
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/telemetry/TelemetryCollector.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/telemetry/TelemetryCollector.java
@@ -59,16 +59,20 @@ public class TelemetryCollector {
return stageCollectors.get(stageName);
}
final TelemetryStageCollector collector = new TelemetryStageCollector(this);
stageCollectors.put(stageName, collector);
return collector;
}
+ public boolean hasCollectorFor(@NonNull String stageName) {
+ return stageCollectors.containsKey(stageName);
+ }
+
public void setRestarted() {
this.didRestart = true;
}
public void setIDs(@NonNull String uid, @NonNull String deviceID) {
// We use hashed_fxa_uid from the token server as our UID.
this.hashedUID = uid;
try {
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/telemetry/TelemetryStageCollector.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/telemetry/TelemetryStageCollector.java
@@ -21,16 +21,17 @@ public class TelemetryStageCollector {
public volatile int inbound = 0;
public volatile int inboundStored = 0;
public volatile int inboundFailed = 0;
public volatile int outbound = 0;
public volatile int outboundStored = 0;
public volatile int outboundFailed = 0;
public volatile int reconciled = 0;
public volatile ExtendedJSONObject error = null;
+ public volatile ExtendedJSONObject validation = null;
public TelemetryStageCollector(TelemetryCollector syncCollector) {
this.syncCollector = syncCollector;
}
public TelemetryCollector getSyncCollector() {
return this.syncCollector;
}
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/validation/BookmarkValidationResults.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/validation/BookmarkValidationResults.java
@@ -1,16 +1,14 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.sync.validation;
-import org.json.simple.JSONArray;
-import org.json.simple.JSONObject;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
--- a/mobile/android/tests/background/junit4/src/org/mozilla/gecko/telemetry/pingbuilders/TelemetrySyncPingBuilderTest.java
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/gecko/telemetry/pingbuilders/TelemetrySyncPingBuilderTest.java
@@ -9,19 +9,23 @@ import android.os.Parcelable;
import org.json.JSONException;
import org.json.JSONObject;
import org.json.simple.JSONArray;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mozilla.gecko.background.testhelpers.TestRunner;
import org.mozilla.gecko.sync.ExtendedJSONObject;
+import org.mozilla.gecko.sync.telemetry.TelemetryStageCollector;
+import org.mozilla.gecko.sync.validation.BookmarkValidationResults;
+import org.mozilla.gecko.sync.validation.ValidationResults;
import org.mozilla.gecko.telemetry.TelemetryLocalPing;
import java.util.ArrayList;
+import java.util.HashMap;
import static org.junit.Assert.*;
@RunWith(TestRunner.class)
public class TelemetrySyncPingBuilderTest {
private TelemetrySyncPingBundleBuilderTest.MockTelemetryPingStore pingStore;
private TelemetrySyncPingBuilder builder;
@@ -55,16 +59,66 @@ public class TelemetrySyncPingBuilderTes
assertEquals("uid", payload.getString("uid"));
assertEquals(Long.valueOf(123L), payload.getLong("took"));
assertEquals("device-id", payload.getString("deviceID"));
assertTrue(payload.getLong("when") != null);
assertEquals(true, payload.getBoolean("restarted"));
}
@Test
+ public void testStage() throws Exception {
+ HashMap<String, TelemetryStageCollector> stages = new HashMap<>();
+ TelemetryStageCollector stage = new TelemetryStageCollector(null);
+ stage.validation = new ExtendedJSONObject();
+ stage.validation.put("took", 1L);
+ stage.validation.put("checked", 0L);
+ stage.validation.put("problems", new JSONArray());
+
+ stage.error = new ExtendedJSONObject();
+ stage.error.put("name", "unexpectederror");
+ stage.error.put("error", "test");
+ stage.started = 100;
+ stage.finished = 105;
+ stage.inbound = 5;
+ stage.inboundStored = 3;
+ stage.inboundFailed = 1;
+ stage.reconciled = 1;
+
+
+ stages.put("testing", stage);
+ TelemetryLocalPing localPing = builder
+ .setStages(stages)
+ .build();
+ ExtendedJSONObject payload = localPing.getPayload();
+
+ assertEquals(1, payload.getArray("engines").size());
+ ExtendedJSONObject engine = (ExtendedJSONObject)payload.getArray("engines").get(0);
+
+ assertEquals("testing", engine.getString("name"));
+ assertEquals(Long.valueOf(5L), engine.getLong("took"));
+
+ ExtendedJSONObject inbound = engine.getObject("incoming");
+ assertEquals(Integer.valueOf(stage.inbound), inbound.getIntegerSafely("applied"));
+ assertEquals(Integer.valueOf(stage.inboundStored), inbound.getIntegerSafely("succeeded"));
+ assertEquals(Integer.valueOf(stage.inboundFailed), inbound.getIntegerSafely("failed"));
+ assertEquals(Integer.valueOf(stage.reconciled), inbound.getIntegerSafely("reconciled"));
+
+ // TODO: Test outbound once bug 1389233 is addressed
+
+ ExtendedJSONObject error = engine.getObject("failureReason");
+ assertEquals("unexpectederror", error.getString("name"));
+ assertEquals("test", error.getString("error"));
+
+ ExtendedJSONObject validation = engine.getObject("validation");
+ assertEquals(stage.validation.getLong("took"), validation.getLong("took"));
+ assertEquals(stage.validation.getLong("checked"), validation.getLong("checked"));
+ assertEquals(0, stage.validation.getArray("problems").size());
+ }
+
+ @Test
public void testDevices() throws Exception {
ArrayList<Parcelable> devices = new ArrayList<>();
TelemetryLocalPing localPing = builder
.setDevices(devices)
.build();
ExtendedJSONObject payload = localPing.getPayload();
