Bug 1360359 - WIP Part 2: Simple round-tripping FormAutofill sync stage draft
authorGrigory Kruglov <gkruglov@mozilla.com>
Thu, 27 Apr 2017 20:56:52 -0400
changeset 569900 fef12de1388baf87cb78f15d1d9ead6a9eb977cd
parent 569899 0a88136718ce880f439de383dad0d429f8b4ad20
child 569901 647c4381f98c21c0a5202cd044c8b3de19d2479d
push id56303
push userbmo:gkruglov@mozilla.com
push dateFri, 28 Apr 2017 01:01:51 +0000
bugs1360359
milestone55.0a1
Bug 1360359 - WIP Part 2: Simple round-tripping FormAutofill sync stage MozReview-Commit-ID: 6JOmpjuRKDV
mobile/android/base/android-services.mozbuild
mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/android/BrowserContractHelpers.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/android/FormAutofillRepository.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/domain/FormAutofillRecord.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/domain/FormAutofillRecordFactory.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/domain/VersionConstants.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/FormAutofillServerSyncStage.java
--- a/mobile/android/base/android-services.mozbuild
+++ b/mobile/android/base/android-services.mozbuild
@@ -957,16 +957,17 @@ sync_java_files = [TOPSRCDIR + '/mobile/
     'sync/repositories/android/AndroidBrowserRepositorySession.java',
     'sync/repositories/android/BookmarksDeletionManager.java',
     'sync/repositories/android/BookmarksInsertionManager.java',
     'sync/repositories/android/BrowserContractHelpers.java',
     'sync/repositories/android/CachedSQLiteOpenHelper.java',
     'sync/repositories/android/ClientsDatabase.java',
     'sync/repositories/android/ClientsDatabaseAccessor.java',
     'sync/repositories/android/FennecTabsRepository.java',
+    'sync/repositories/android/FormAutofillRepository.java',
     'sync/repositories/android/FormHistoryRepositorySession.java',
     'sync/repositories/android/PasswordsRepositorySession.java',
     'sync/repositories/android/RepoUtils.java',
     'sync/repositories/android/VisitsHelper.java',
     'sync/repositories/BookmarkNeedsReparentingException.java',
     'sync/repositories/BookmarksRepository.java',
     'sync/repositories/ConfigurableServer15Repository.java',
     'sync/repositories/delegates/DeferrableRepositorySessionCreationDelegate.java',
@@ -981,16 +982,18 @@ sync_java_files = [TOPSRCDIR + '/mobile/
     'sync/repositories/delegates/RepositorySessionFinishDelegate.java',
     'sync/repositories/delegates/RepositorySessionGuidsSinceDelegate.java',
     'sync/repositories/delegates/RepositorySessionStoreDelegate.java',
     'sync/repositories/delegates/RepositorySessionWipeDelegate.java',
     'sync/repositories/domain/BookmarkRecord.java',
     'sync/repositories/domain/BookmarkRecordFactory.java',
     'sync/repositories/domain/ClientRecord.java',
     'sync/repositories/domain/ClientRecordFactory.java',
+    'sync/repositories/domain/FormAutofillRecord.java',
+    'sync/repositories/domain/FormAutofillRecordFactory.java',
     'sync/repositories/domain/FormHistoryRecord.java',
     'sync/repositories/domain/HistoryRecord.java',
     'sync/repositories/domain/HistoryRecordFactory.java',
     'sync/repositories/domain/PasswordRecord.java',
     'sync/repositories/domain/PasswordRecordFactory.java',
     'sync/repositories/domain/Record.java',
     'sync/repositories/domain/RecordParseException.java',
     'sync/repositories/domain/TabsRecord.java',
@@ -1050,16 +1053,17 @@ sync_java_files = [TOPSRCDIR + '/mobile/
     'sync/stage/AndroidBrowserRecentHistoryServerSyncStage.java',
     'sync/stage/CheckPreconditionsStage.java',
     'sync/stage/CompletedStage.java',
     'sync/stage/EnsureCrypto5KeysStage.java',
     'sync/stage/FennecTabsServerSyncStage.java',
     'sync/stage/FetchInfoCollectionsStage.java',
     'sync/stage/FetchInfoConfigurationStage.java',
     'sync/stage/FetchMetaGlobalStage.java',
+    'sync/stage/FormAutofillServerSyncStage.java',
     'sync/stage/FormHistoryServerSyncStage.java',
     'sync/stage/GlobalSyncStage.java',
     'sync/stage/NoSuchStageException.java',
     'sync/stage/PasswordsServerSyncStage.java',
     'sync/stage/ServerSyncStage.java',
     'sync/stage/SyncClientsEngineStage.java',
     'sync/stage/UploadMetaGlobalStage.java',
     'sync/SyncConfiguration.java',
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/android/BrowserContractHelpers.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/android/BrowserContractHelpers.java
@@ -37,16 +37,17 @@ public class BrowserContractHelpers exte
   public static final Uri SCHEMA_CONTENT_URI               = withSyncAndDeletedAndProfile(Schema.CONTENT_URI);
   public static final Uri PASSWORDS_CONTENT_URI            = withSyncAndDeletedAndProfile(Passwords.CONTENT_URI);
   public static final Uri DELETED_PASSWORDS_CONTENT_URI    = withSyncAndDeletedAndProfile(DeletedPasswords.CONTENT_URI);
   public static final Uri FORM_HISTORY_CONTENT_URI         = withSyncAndProfile(FormHistory.CONTENT_URI);
   public static final Uri DELETED_FORM_HISTORY_CONTENT_URI = withSyncAndProfile(DeletedFormHistory.CONTENT_URI);
   public static final Uri TABS_CONTENT_URI                 = withSyncAndProfile(Tabs.CONTENT_URI);
   public static final Uri CLIENTS_CONTENT_URI              = withSyncAndProfile(Clients.CONTENT_URI);
   public static final Uri LOGINS_CONTENT_URI               = withSyncAndProfile(Logins.CONTENT_URI);
+  public static final Uri FORM_AUTOFILL_CONTENT_URI        = withSyncAndProfile(FormAutofill.CONTENT_URI);
 
   public static final String[] PasswordColumns = new String[] {
     Passwords.ID,
     Passwords.HOSTNAME,
     Passwords.HTTP_REALM,
     Passwords.FORM_SUBMIT_URL,
     Passwords.USERNAME_FIELD,
     Passwords.PASSWORD_FIELD,
new file mode 100644
--- /dev/null
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/android/FormAutofillRepository.java
@@ -0,0 +1,173 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.sync.repositories.android;
+
+import android.content.ContentProviderClient;
+import android.content.ContentValues;
+import android.content.Context;
+import android.database.Cursor;
+import android.os.RemoteException;
+import android.util.Log;
+
+import org.json.JSONException;
+import org.mozilla.gecko.db.BrowserContract;
+import org.mozilla.gecko.sync.repositories.InactiveSessionException;
+import org.mozilla.gecko.sync.repositories.NoStoreDelegateException;
+import org.mozilla.gecko.sync.repositories.NullCursorException;
+import org.mozilla.gecko.sync.repositories.Repository;
+import org.mozilla.gecko.sync.repositories.RepositorySession;
+import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionCreationDelegate;
+import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionFetchRecordsDelegate;
+import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionGuidsSinceDelegate;
+import org.mozilla.gecko.sync.repositories.delegates.RepositorySessionWipeDelegate;
+import org.mozilla.gecko.sync.repositories.domain.FormAutofillRecord;
+import org.mozilla.gecko.sync.repositories.domain.Record;
+
+/**
+ * Simple, round-tripping repository and session which expects that its local data is never modified.
+ */
+public class FormAutofillRepository extends Repository {
+    @Override
+    public void createSession(RepositorySessionCreationDelegate delegate, Context context) {
+        try {
+            final FormAutofillRepositorySession repositorySession = new FormAutofillRepositorySession(this, context);
+            delegate.onSessionCreated(repositorySession);
+        } catch (Exception e) {
+            delegate.onSessionCreateFailed(e);
+        }
+    }
+
+    private static class FormAutofillRepositorySession extends RepositorySession {
+        private static final String LOG_TAG = "FormAutofillRepoSession";
+
+        private final RepoUtils.QueryHelper dataHelper;
+        private final ContentProviderClient dataClient;
+
+        FormAutofillRepositorySession(Repository repository, Context context) {
+            super(repository);
+
+            dataHelper = new RepoUtils.QueryHelper(
+                    context, BrowserContractHelpers.FORM_AUTOFILL_CONTENT_URI, LOG_TAG);
+            dataClient = context.getContentResolver()
+                    .acquireContentProviderClient(BrowserContract.FORM_AUTOFILL_AUTHORITY_URI);
+        }
+
+        @Override
+        public void fetchSince(final long timestamp, final RepositorySessionFetchRecordsDelegate delegate) {
+            delegateQueue.execute(new Runnable() {
+                @Override
+                public void run() {
+                    // We are not touching this repository's data locally, and so the data will not
+                    // change and is simply round-tripped. For the sake of efficiency, we only
+                    // return data on the first sync of this repository.
+                    if (timestamp > 0) {
+                        delegate.onFetchCompleted(now());
+                        return;
+                    }
+
+                    Cursor cursor = null;
+                    try {
+                        cursor = dataClient.query(
+                                BrowserContractHelpers.FORM_AUTOFILL_CONTENT_URI,
+                                null,
+                                null,
+                                null,
+                                null
+                        );
+
+                        if (cursor == null) {
+                            delegate.onFetchFailed(new NullCursorException(null));
+                            return;
+                        }
+
+                        while (cursor.moveToFirst()) {
+                            try {
+                                delegate.onFetchedRecord(
+                                        FormAutofillRecord.fromCursor(cursor)
+                                );
+                            } catch (JSONException e) {
+                                // If we tell the delegate that we failed, it will stop flow of records.
+                                // Ignoring a bad record seems like a sensible thing to do.
+                                Log.e(LOG_TAG, "Error parsing form autofill JSON record blob. Ignoring it.", e);
+                            }
+                        }
+
+                        delegate.onFetchCompleted(now());
+
+                    } catch (RemoteException e) {
+                        delegate.onFetchFailed(e);
+                    } finally {
+                        if (cursor != null) {
+                            cursor.close();
+                        }
+                    }
+                }
+            });
+        }
+
+        @Override
+        public void fetchAll(RepositorySessionFetchRecordsDelegate delegate) {
+            fetchSince(0, delegate);
+        }
+
+        @Override
+        public void store(final Record record) throws NoStoreDelegateException {
+            storeWorkQueue.execute(new Runnable() {
+                @Override
+                public void run() {
+                    final ContentValues values;
+
+                    try {
+                        values = FormAutofillRecord.toContentValues(
+                                (FormAutofillRecord) record
+                        );
+                    } catch (JSONException e) {
+                        storeDelegate.onRecordStoreFailed(e, record.guid);
+                        return;
+                    }
+
+                    // Content provider will insert or replace records for us, based on clashing GUIDs.
+                    try {
+                        dataClient.insert(BrowserContractHelpers.FORM_AUTOFILL_CONTENT_URI, values);
+                        storeDelegate.onRecordStoreSucceeded(record.guid);
+                    } catch (RemoteException e) {
+                        storeDelegate.onRecordStoreFailed(e, record.guid);
+                    }
+                }
+            });
+        }
+
+        @Override
+        public void wipe(RepositorySessionWipeDelegate delegate) {
+            try {
+                dataClient.delete(BrowserContractHelpers.FORM_AUTOFILL_CONTENT_URI, null, null);
+                delegate.onWipeSucceeded();
+            } catch (RemoteException e) {
+                delegate.onWipeFailed(e);
+            }
+        }
+
+        // No-op methods:
+        @Override
+        public void fetch(String[] guids, final RepositorySessionFetchRecordsDelegate delegate) throws InactiveSessionException {
+            delegateQueue.execute(new Runnable() {
+                @Override
+                public void run() {
+                    delegate.onFetchCompleted(now());
+                }
+            });
+        }
+
+        @Override
+        public void guidsSince(long timestamp, final RepositorySessionGuidsSinceDelegate delegate) {
+            delegateQueue.execute(new Runnable() {
+                @Override
+                public void run() {
+                    delegate.onGuidsSinceSucceeded(new String[] {});
+                }
+            });
+        }
+    }
+}
new file mode 100644
--- /dev/null
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/domain/FormAutofillRecord.java
@@ -0,0 +1,166 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.sync.repositories.domain;
+
+import android.content.ContentValues;
+import android.database.Cursor;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.mozilla.gecko.db.BrowserContract;
+import org.mozilla.gecko.sync.ExtendedJSONObject;
+import org.mozilla.gecko.sync.Utils;
+import org.mozilla.gecko.sync.repositories.android.BrowserContractHelpers;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+public class FormAutofillRecord extends Record {
+    // TODO perhaps just drop this separation. We're just round-tripping records, and performing
+    // zero merging at this point.
+    private final Map<String, String> knownValues = new HashMap<>(8);
+    private final Map<String, String> unknownValues = new HashMap<>();
+
+    private static final String COLLECTION_NAME = "formautofill";
+
+    private static final List<String> validKeys;
+
+    static {
+        String[] keys = new String[]{
+                "guid",
+                "organization",
+                "streetAddress",
+                "addressLevel1",
+                "addressLevel2",
+                "postalCode",
+                "country",
+                "tel",
+                "email"
+        };
+        validKeys = Arrays.asList(keys);
+    }
+
+    public FormAutofillRecord(String guid, String collection, long lastModified, boolean deleted) {
+        super(guid, collection, lastModified, deleted);
+    }
+
+    FormAutofillRecord() {
+        this(Utils.generateGuid(), COLLECTION_NAME, 0, false);
+    }
+
+    @Override
+    protected void populatePayload(ExtendedJSONObject payload) {
+        // Populate payload with known keys.
+        for (String key : knownValues.keySet()) {
+            putPayload(payload, key, knownValues.get(key));
+        }
+
+        // Round-trip unrecognized keys.
+        for (String key : unknownValues.keySet()) {
+            putPayload(payload, key, unknownValues.get(key));
+        }
+    }
+
+    @Override
+    protected void initFromPayload(ExtendedJSONObject payload) {
+        for (String key : payload.keySet()) {
+            if (validKeys.contains(key)) {
+                // TODO always strings?
+                knownValues.put(key, payload.getString(key));
+            } else {
+                // TODO use .get and work with Objects instead?
+                unknownValues.put(key, payload.getString(key));
+            }
+        }
+    }
+
+    @Override
+    public Record copyWithIDs(String guid, long androidID) {
+        FormAutofillRecord out = new FormAutofillRecord(guid, this.collection, this.lastModified, this.deleted);
+        out.androidID = androidID;
+        out.sortIndex = this.sortIndex;
+        out.ttl       = this.ttl;
+
+        out.knownValues.putAll(knownValues);
+        out.unknownValues.putAll(unknownValues);
+
+        return out;
+    }
+
+    public static FormAutofillRecord fromCursor(Cursor cursor) throws JSONException {
+        final int guidCol = cursor.getColumnIndexOrThrow(BrowserContract.FormAutofill.GUID);
+        final int jsonCol = cursor.getColumnIndexOrThrow(BrowserContract.FormAutofill.JSON);
+        final int deletedCol = cursor.getColumnIndexOrThrow(BrowserContract.FormAutofill.IS_DELETED);
+        final int dateModifiedCol = cursor.getColumnIndexOrThrow(BrowserContract.FormAutofill.DATE_MODIFIED);
+        final boolean isDeleted = cursor.getInt(deletedCol) == 1;
+
+        final FormAutofillRecord record = new FormAutofillRecord(
+                cursor.getString(guidCol),
+                COLLECTION_NAME,
+                cursor.getLong(dateModifiedCol),
+                isDeleted
+        );
+
+        final String jsonString = cursor.getString(jsonCol);
+        final JSONObject json = new JSONObject(jsonString);
+        final Iterator<String> keyIterator = json.keys();
+        while (keyIterator.hasNext()) {
+            final String key = keyIterator.next();
+            if (validKeys.contains(key)) {
+                record.knownValues.put(key, json.getString(key));
+            } else {
+                record.unknownValues.put(key, json.getString(key));
+            }
+        }
+
+        return record;
+    }
+
+    public static ContentValues toContentValues(FormAutofillRecord record) throws JSONException {
+        final ContentValues values = new ContentValues();
+
+        values.put(BrowserContractHelpers.FormAutofill.GUID, record.guid);
+
+        if (record.deleted) {
+            values.put(BrowserContractHelpers.FormAutofill.IS_DELETED, 1);
+        } else {
+            values.put(BrowserContractHelpers.FormAutofill.IS_DELETED, 0);
+        }
+
+        values.put(BrowserContract.FormAutofill.DATE_MODIFIED, record.lastModified);
+
+        // JSON blob
+        JSONObject json = new JSONObject(record.knownValues);
+
+        for (String key : record.unknownValues.keySet()) {
+            json.put(key, record.unknownValues.get(key));
+        }
+
+        return values;
+    }
+// Example record:
+//{
+//    guid,              // 12 character...
+//
+//    // profile
+//    organization,      // Company
+//    street-address,    // (Multiline)
+//    address-level2,    // City/Town
+//    address-level1,    // Province (Standardized code if possible)
+//    postal-code,
+//    country,           // ISO 3166
+//    tel,
+//    email,
+//
+//    // metadata
+//    timeCreated,      // in ms
+//    timeLastUsed,     // in ms
+//    timeLastModified, // in ms
+//    timesUsed
+//}
+}
new file mode 100644
--- /dev/null
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/domain/FormAutofillRecordFactory.java
@@ -0,0 +1,18 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.sync.repositories.domain;
+
+import org.mozilla.gecko.sync.CryptoRecord;
+import org.mozilla.gecko.sync.repositories.RecordFactory;
+
+
+public class FormAutofillRecordFactory extends RecordFactory {
+    @Override
+    public Record createRecord(Record record) {
+        FormAutofillRecord r = new FormAutofillRecord();
+        r.initFromEnvelope((CryptoRecord) record);
+        return r;
+    }
+}
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/domain/VersionConstants.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/repositories/domain/VersionConstants.java
@@ -6,9 +6,10 @@ package org.mozilla.gecko.sync.repositor
 
 public class VersionConstants {
   public static final int BOOKMARKS_ENGINE_VERSION = 2;
   public static final int CLIENTS_ENGINE_VERSION = 1;
   public static final int FORMS_ENGINE_VERSION = 1;
   public static final int HISTORY_ENGINE_VERSION = 1;
   public static final int PASSWORDS_ENGINE_VERSION = 1;
   public static final int TABS_ENGINE_VERSION = 1;
+  public static final int FORM_AUTOFILL_ENGINE_VERSION = 1;
 }
new file mode 100644
--- /dev/null
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/FormAutofillServerSyncStage.java
@@ -0,0 +1,67 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.sync.stage;
+
+import org.mozilla.gecko.sync.middleware.BufferingMiddlewareRepository;
+import org.mozilla.gecko.sync.middleware.storage.MemoryBufferStorage;
+import org.mozilla.gecko.sync.repositories.ConfigurableServer15Repository;
+import org.mozilla.gecko.sync.repositories.RecordFactory;
+import org.mozilla.gecko.sync.repositories.Repository;
+import org.mozilla.gecko.sync.repositories.android.FormAutofillRepository;
+import org.mozilla.gecko.sync.repositories.domain.FormAutofillRecordFactory;
+import org.mozilla.gecko.sync.repositories.domain.VersionConstants;
+
+import java.net.URISyntaxException;
+
+public class FormAutofillServerSyncStage extends ServerSyncStage {
+    private final long FORM_AUTOFILL_BATCH_LIMIT = 5000;
+    private final String FORM_AUTOFILL_SORT = "latest";
+
+    @Override
+    protected String getEngineName() {
+        return "formautofill";
+    }
+
+    @Override
+    protected String getCollection() {
+        return "formautofill";
+    }
+
+    @Override
+    public Integer getStorageVersion() {
+        return VersionConstants.FORM_AUTOFILL_ENGINE_VERSION;
+    }
+
+    @Override
+    protected Repository getLocalRepository() {
+        return new BufferingMiddlewareRepository(
+                session.getSyncDeadline(),
+                new MemoryBufferStorage(),
+                new FormAutofillRepository()
+        );
+    }
+
+    @Override
+    protected Repository getRemoteRepository() throws URISyntaxException {
+        return new ConfigurableServer15Repository(
+                getCollection(),
+                session.getSyncDeadline(),
+                session.config.storageURL(),
+                session.getAuthHeaderProvider(),
+                session.config.infoCollections,
+                session.config.infoConfiguration,
+                FORM_AUTOFILL_BATCH_LIMIT,
+                FORM_AUTOFILL_SORT,
+                getAllowedMultipleBatches(),
+                getAllowedToUseHighWaterMark(),
+                getRepositoryStateProvider()
+        );
+    }
+
+    @Override
+    protected RecordFactory getRecordFactory() {
+        return new FormAutofillRecordFactory();
+    }
+}