Bug 1346438 - Specify X-I-U-S header value while uploading meta/global r=nalexander,rnewman draft
authorGrigory Kruglov <gkruglov@mozilla.com>
Mon, 20 Mar 2017 16:40:13 -0700
changeset 501769 d9a1b49076aebcff7300c570bc8a4fd85a92d048
parent 498628 cef93bf5a0a15ab7a2dc96d0cee90a599a4eae76
child 501770 864678b5489e0a2f416f5196a97581bd7c902b61
push id50121
push usergkruglov@mozilla.com
push dateMon, 20 Mar 2017 23:41:58 +0000
reviewersnalexander, rnewman
bugs1346438
milestone55.0a1
Bug 1346438 - Specify X-I-U-S header value while uploading meta/global r=nalexander,rnewman We upload meta/global in three scenarios: - fresh start - when it was modified after a successful sync - when it was modified after an aborted sync Use X-I-U-S header to assert what we believe about meta/global's presence (during freshStart) and last-modified timestamp (in all other cases). We might encounter a concurrent modification condition, manifesting as a 412 error. If we see such an error: - on fresh start, we restart globalSession - on regular upload, we request a re-sync of all stages MozReview-Commit-ID: 3qyb6rUSOeY
mobile/android/services/src/main/java/org/mozilla/gecko/fxa/sync/FxAccountSyncAdapter.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/GlobalSession.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/InfoCollections.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/MetaGlobal.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/delegates/GlobalSessionCallback.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/FetchMetaGlobalStage.java
mobile/android/tests/background/junit3/src/org/mozilla/gecko/background/testhelpers/DefaultGlobalSessionCallback.java
mobile/android/tests/background/junit4/src/org/mozilla/android/sync/net/test/TestGlobalSession.java
mobile/android/tests/background/junit4/src/org/mozilla/android/sync/net/test/TestMetaGlobal.java
mobile/android/tests/background/junit4/src/org/mozilla/android/sync/test/helpers/MockGlobalSessionCallback.java
mobile/android/tests/background/junit4/src/org/mozilla/gecko/background/testhelpers/DefaultGlobalSessionCallback.java
mobile/android/tests/background/junit4/src/org/mozilla/gecko/sync/stage/test/TestFetchMetaGlobalStage.java
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/fxa/sync/FxAccountSyncAdapter.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/fxa/sync/FxAccountSyncAdapter.java
@@ -29,16 +29,17 @@ import org.mozilla.gecko.fxa.authenticat
 import org.mozilla.gecko.fxa.authenticator.FxAccountAuthenticator;
 import org.mozilla.gecko.fxa.login.FxAccountLoginStateMachine;
 import org.mozilla.gecko.fxa.login.Married;
 import org.mozilla.gecko.fxa.login.State;
 import org.mozilla.gecko.fxa.login.State.StateLabel;
 import org.mozilla.gecko.fxa.sync.FxAccountSyncDelegate.Result;
 import org.mozilla.gecko.sync.BackoffHandler;
 import org.mozilla.gecko.sync.GlobalSession;
+import org.mozilla.gecko.sync.MetaGlobal;
 import org.mozilla.gecko.sync.PrefsBackoffHandler;
 import org.mozilla.gecko.sync.SharedPreferencesClientsDataDelegate;
 import org.mozilla.gecko.sync.SyncConfiguration;
 import org.mozilla.gecko.sync.ThreadPool;
 import org.mozilla.gecko.sync.Utils;
 import org.mozilla.gecko.sync.crypto.KeyBundle;
 import org.mozilla.gecko.sync.delegates.GlobalSessionCallback;
 import org.mozilla.gecko.sync.delegates.ClientsDataDelegate;
@@ -127,16 +128,17 @@ public class FxAccountSyncAdapter extend
     /* package-local */ void requestFollowUpSync(String stage) {
       this.stageNamesForFollowUpSync.add(stage);
     }
 
     protected final Collection<String> stageNamesToSync;
 
     // Keeps track of incomplete stages during this sync that need to be re-synced once we're done.
     private final List<String> stageNamesForFollowUpSync = Collections.synchronizedList(new ArrayList<String>());
+    private boolean fullSyncNecessary = false;
 
     public SyncDelegate(BlockingQueue<Result> latch, SyncResult syncResult, AndroidFxAccount fxAccount, Collection<String> stageNamesToSync) {
       super(latch, syncResult);
       this.stageNamesToSync = Collections.unmodifiableCollection(stageNamesToSync);
     }
 
     public Collection<String> getStageNamesToSync() {
       return this.stageNamesToSync;
@@ -195,16 +197,24 @@ public class FxAccountSyncAdapter extend
      * Schedule an incomplete stage for a follow-up sync.
      */
     @Override
     public void handleIncompleteStage(Stage currentState,
                                       GlobalSession globalSession) {
       syncDelegate.requestFollowUpSync(currentState.getRepositoryName());
     }
 
+    /**
+     * Use with caution, as this will request an immediate follow-up sync of all stages.
+     */
+    @Override
+    public void handleFullSyncNecessary() {
+      syncDelegate.fullSyncNecessary = true;
+    }
+
     @Override
     public void handleSuccess(GlobalSession globalSession) {
       Logger.info(LOG_TAG, "Global session succeeded.");
 
       // Get the number of clients, so we can schedule the sync interval accordingly.
       try {
         int otherClientsCount = globalSession.getClientsDelegate().getClientsCount();
         Logger.debug(LOG_TAG, "" + otherClientsCount + " other client(s).");
@@ -449,16 +459,17 @@ public class FxAccountSyncAdapter extend
     });
 
     final BlockingQueue<Result> latch = new LinkedBlockingQueue<>(1);
 
     Collection<String> knownStageNames = SyncConfiguration.validEngineNames();
     Collection<String> stageNamesToSync = Utils.getStagesToSyncFromBundle(knownStageNames, extras);
 
     final SyncDelegate syncDelegate = new SyncDelegate(latch, syncResult, fxAccount, stageNamesToSync);
+    Result offeredResult = null;
 
     try {
       // This will be the same chunk of SharedPreferences that we pass through to GlobalSession/SyncConfiguration.
       final SharedPreferences sharedPrefs = fxAccount.getSyncPrefs();
 
       final BackoffHandler backgroundBackoffHandler = new PrefsBackoffHandler(sharedPrefs, "background");
       final BackoffHandler rateLimitBackoffHandler = new PrefsBackoffHandler(sharedPrefs, "rate");
 
@@ -586,37 +597,57 @@ public class FxAccountSyncAdapter extend
             fxAccount.fetchProfileJSON();
           } catch (Exception e) {
             syncDelegate.handleError(e);
             return;
           }
         }
       });
 
-      latch.take();
+      offeredResult = latch.take();
     } catch (Exception e) {
       Logger.error(LOG_TAG, "Got error syncing.", e);
       syncDelegate.handleError(e);
     } finally {
       fxAccount.releaseSharedAccountStateLock();
     }
 
+    lastSyncRealtimeMillis = SystemClock.elapsedRealtime();
+
+    // We got to this point without being offered a result, and so it's unwise to proceed with
+    // trying to sync stages again. Nothing else we can do but log an error.
+    if (offeredResult == null) {
+      Logger.error(LOG_TAG, "Did not receive a sync result from the delegate.");
+      return;
+    }
+
+    // Full sync (of all of stages) is necessary if we hit "concurrent modification" errors while
+    // uploading meta/global stage. This is considered both a rare and important event, so it's
+    // deemed safe and necessary to request an immediate sync, which will ignore any back-offs and
+    // will happen right away.
+    if (syncDelegate.fullSyncNecessary) {
+      Logger.info(LOG_TAG, "Syncing done. Full follow-up sync necessary, requesting immediate sync.");
+      fxAccount.requestImmediateSync(null, null);
+      return;
+    }
+
     // If there are any incomplete stages, request a follow-up sync. Otherwise, we're done.
     // Incomplete stage is:
     // - one that hit a 412 error during either upload or download of data, indicating that
     //   its collection has been modified remotely, or
     // - one that hit a sync deadline
     final String[] stagesToSyncAgain;
     synchronized (syncDelegate.stageNamesForFollowUpSync) {
       stagesToSyncAgain = syncDelegate.stageNamesForFollowUpSync.toArray(
               new String[syncDelegate.stageNamesForFollowUpSync.size()]
       );
     }
 
-    if (stagesToSyncAgain.length > 0) {
-      Logger.info(LOG_TAG, "Syncing done. Requesting an immediate follow-up sync.");
-      fxAccount.requestImmediateSync(stagesToSyncAgain, null);
-    } else {
+    if (stagesToSyncAgain.length == 0) {
       Logger.info(LOG_TAG, "Syncing done.");
+      return;
     }
-    lastSyncRealtimeMillis = SystemClock.elapsedRealtime();
+
+    // If there are any other stages marked as incomplete, request that they're synced again.
+    Logger.info(LOG_TAG, "Syncing done. Requesting an immediate follow-up sync.");
+    fxAccount.requestImmediateSync(stagesToSyncAgain, null);
   }
 }
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/GlobalSession.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/GlobalSession.java
@@ -1,15 +1,16 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.gecko.sync;
 
 import android.content.Context;
+import android.support.annotation.VisibleForTesting;
 
 import org.json.simple.JSONArray;
 import org.mozilla.gecko.background.common.log.Logger;
 import org.mozilla.gecko.sync.crypto.CryptoException;
 import org.mozilla.gecko.sync.crypto.KeyBundle;
 import org.mozilla.gecko.sync.delegates.ClientsDataDelegate;
 import org.mozilla.gecko.sync.delegates.FreshStartDelegate;
 import org.mozilla.gecko.sync.delegates.GlobalSessionCallback;
@@ -404,69 +405,91 @@ public class GlobalSession implements Ht
     updateMetaGlobalInPlace();
 
     Logger.debug(LOG_TAG, "Uploading updated meta/global record.");
     final Object monitor = new Object();
 
     Runnable doUpload = new Runnable() {
       @Override
       public void run() {
-        config.metaGlobal.upload(new MetaGlobalDelegate() {
-          @Override
-          public void handleSuccess(MetaGlobal global, SyncStorageResponse response) {
-            Logger.info(LOG_TAG, "Successfully uploaded updated meta/global record.");
-            // Engine changes are stored as diffs, so update enabled engines in config to match uploaded meta/global.
-            config.enabledEngineNames = config.metaGlobal.getEnabledEngineNames();
-            // Clear userSelectedEngines because they are updated in config and meta/global.
-            config.userSelectedEngines = null;
-
-            synchronized (monitor) {
-              monitor.notify();
-            }
-          }
-
-          @Override
-          public void handleMissing(MetaGlobal global, SyncStorageResponse response) {
-            Logger.warn(LOG_TAG, "Got 404 missing uploading updated meta/global record; shouldn't happen.  Ignoring.");
-            synchronized (monitor) {
-              monitor.notify();
-            }
-          }
-
-          @Override
-          public void handleFailure(SyncStorageResponse response) {
-            Logger.warn(LOG_TAG, "Failed to upload updated meta/global record; ignoring.");
-            synchronized (monitor) {
-              monitor.notify();
-            }
-          }
-
-          @Override
-          public void handleError(Exception e) {
-            Logger.warn(LOG_TAG, "Got exception trying to upload updated meta/global record; ignoring.", e);
-            synchronized (monitor) {
-              monitor.notify();
-            }
-          }
-        });
+        // During regular meta/global upload, set X-I-U-S to the last-modified value of meta/global
+        // in info/collections, to ensure we catch concurrent modifications by other clients.
+        Long lastModifiedTimestamp = config.infoCollections.getTimestamp("meta");
+        // Theoretically, meta/global's timestamp might be missing from info/collections.
+        // The safest thing in that case is to assert that meta/global hasn't been modified by other
+        // clients by setting X-I-U-S to 0.
+        // See Bug 1346438.
+        if (lastModifiedTimestamp == null) {
+          lastModifiedTimestamp = 0L;
+        }
+        config.metaGlobal.upload(lastModifiedTimestamp, makeMetaGlobalUploadDelegate(config, callback, monitor));
       }
     };
 
     final Thread upload = new Thread(doUpload);
     synchronized (monitor) {
       try {
         upload.start();
         monitor.wait();
         Logger.debug(LOG_TAG, "Uploaded updated meta/global record.");
       } catch (InterruptedException e) {
         Logger.error(LOG_TAG, "Uploading updated meta/global interrupted; continuing.");
       }
     }
   }
 
+  @VisibleForTesting
+  public static MetaGlobalDelegate makeMetaGlobalUploadDelegate(final SyncConfiguration config, final GlobalSessionCallback callback, final Object monitor) {
+    return new MetaGlobalDelegate() {
+      @Override
+      public void handleSuccess(MetaGlobal global, SyncStorageResponse response) {
+        Logger.info(LOG_TAG, "Successfully uploaded updated meta/global record.");
+        // Engine changes are stored as diffs, so update enabled engines in config to match uploaded meta/global.
+        config.enabledEngineNames = config.metaGlobal.getEnabledEngineNames();
+        // Clear userSelectedEngines because they are updated in config and meta/global.
+        config.userSelectedEngines = null;
+
+        synchronized (monitor) {
+          monitor.notify();
+        }
+      }
+
+      @Override
+      public void handleMissing(MetaGlobal global, SyncStorageResponse response) {
+        Logger.warn(LOG_TAG, "Got 404 missing uploading updated meta/global record; shouldn't happen.  Ignoring.");
+        synchronized (monitor) {
+          monitor.notify();
+        }
+      }
+
+      @Override
+      public void handleFailure(SyncStorageResponse response) {
+        Logger.warn(LOG_TAG, "Failed to upload updated meta/global record; ignoring.");
+
+        // If we encountered a concurrent modification while uploading meta/global, request that
+        // sync of all stages happens once we're done.
+        if (response.getStatusCode() == 412) {
+          callback.handleFullSyncNecessary();
+        }
+
+        synchronized (monitor) {
+          monitor.notify();
+        }
+      }
+
+      @Override
+      public void handleError(Exception e) {
+        Logger.warn(LOG_TAG, "Got exception trying to upload updated meta/global record; ignoring.", e);
+        synchronized (monitor) {
+          monitor.notify();
+        }
+      }
+    };
+  }
+
 
   public void abort(Exception e, String reason) {
     Logger.warn(LOG_TAG, "Aborting sync: " + reason, e);
     cleanUp();
     long existingBackoff = largestBackoffObserved.get();
     if (existingBackoff > 0) {
       callback.requestBackoff(existingBackoff);
     }
@@ -704,36 +727,56 @@ public class GlobalSession implements Ht
   public void processMissingMetaGlobal(MetaGlobal global) {
     freshStart();
   }
 
   /**
    * Do a fresh start then quietly finish the sync, starting another.
    */
   public void freshStart() {
-    final GlobalSession globalSession = this;
-    freshStart(this, new FreshStartDelegate() {
+    freshStart(this, makeFreshStartDelegate(this));
+  }
 
+  @VisibleForTesting
+  public static FreshStartDelegate makeFreshStartDelegate(final GlobalSession globalSession) {
+    return new FreshStartDelegate() {
       @Override
       public void onFreshStartFailed(Exception e) {
-        globalSession.abort(e, "Fresh start failed.");
+        if (!(e instanceof  HTTPFailureException)) {
+          globalSession.abort(e, "Fresh start failed.");
+          return;
+        }
+
+        if (((HTTPFailureException) e).response.getStatusCode() != 412) {
+          globalSession.abort(e, "Fresh start failed with non-412 status code.");
+          return;
+        }
+
+        // In case of a concurrent modification during a fresh start, restart global session.
+        try {
+          // We are not persisting SyncConfiguration at this point; we can't be sure of its state.
+          globalSession.restart();
+        } catch (AlreadySyncingException restartException) {
+          Logger.warn(LOG_TAG, "Got exception restarting sync after freshStart failure.", restartException);
+          globalSession.abort(restartException, "Got exception restarting sync after freshStart failure.");
+        }
       }
 
       @Override
       public void onFreshStart() {
         try {
           Logger.warn(LOG_TAG, "Fresh start succeeded; restarting global session.");
           globalSession.config.persistToPrefs();
           globalSession.restart();
         } catch (Exception e) {
           Logger.warn(LOG_TAG, "Got exception when restarting sync after freshStart.", e);
           globalSession.abort(e, "Got exception after freshStart.");
         }
       }
-    });
+    };
   }
 
   /**
    * Clean the server, aborting the current sync.
    * <p>
    * <ol>
    * <li>Wipe the server storage.</li>
    * <li>Reset all stages and purge cached state: (meta/global and crypto/keys records).</li>
@@ -757,21 +800,21 @@ public class GlobalSession implements Ht
 
         session.resetAllStages();
         session.config.purgeMetaGlobal();
         session.config.purgeCryptoKeys();
         session.config.persistToPrefs();
 
         Logger.info(LOG_TAG, "Uploading new meta/global with sync ID " + mg.syncID + ".");
 
-        // It would be good to set the X-If-Unmodified-Since header to `timestamp`
-        // for this PUT to ensure at least some level of transactionality.
-        // Unfortunately, the servers don't support it after a wipe right now
-        // (bug 693893), so we're going to defer this until bug 692700.
-        mg.upload(new MetaGlobalDelegate() {
+        // During a fresh start, set X-I-U-S to 0 to ensure we don't race with other clients.
+        // Since we are performing a fresh start, we are asserting that meta/global was not uploaded
+        // by other clients.
+        // See Bug 1346438.
+        mg.upload(0L, new MetaGlobalDelegate() {
           @Override
           public void handleSuccess(MetaGlobal uploadedGlobal, SyncStorageResponse uploadResponse) {
             Logger.info(LOG_TAG, "Uploaded new meta/global with sync ID " + uploadedGlobal.syncID + ".");
 
             // Generate new keys.
             CollectionKeys keys = null;
             try {
               keys = session.generateNewCryptoKeys();
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/InfoCollections.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/InfoCollections.java
@@ -1,14 +1,16 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.gecko.sync;
 
+import android.support.annotation.Nullable;
+
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 
 import org.mozilla.gecko.background.common.log.Logger;
 
 /**
@@ -62,16 +64,17 @@ public class InfoCollections {
   /**
    * Return the timestamp for the given collection, or null if the timestamps
    * have not been fetched or the given collection does not have a timestamp.
    *
    * @param collection
    *          The collection to inspect.
    * @return the timestamp in milliseconds since epoch.
    */
+  @Nullable
   public Long getTimestamp(String collection) {
     if (timestamps == null) {
       return null;
     }
     return timestamps.get(collection);
   }
 
   /**
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/MetaGlobal.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/MetaGlobal.java
@@ -17,17 +17,17 @@ import org.mozilla.gecko.background.comm
 import org.mozilla.gecko.sync.MetaGlobalException.MetaGlobalMalformedSyncIDException;
 import org.mozilla.gecko.sync.MetaGlobalException.MetaGlobalMalformedVersionException;
 import org.mozilla.gecko.sync.delegates.MetaGlobalDelegate;
 import org.mozilla.gecko.sync.net.AuthHeaderProvider;
 import org.mozilla.gecko.sync.net.SyncStorageRecordRequest;
 import org.mozilla.gecko.sync.net.SyncStorageRequestDelegate;
 import org.mozilla.gecko.sync.net.SyncStorageResponse;
 
-public class MetaGlobal implements SyncStorageRequestDelegate {
+public class MetaGlobal {
   private static final String LOG_TAG = "MetaGlobal";
   protected String metaURL;
 
   // Fields.
   protected ExtendedJSONObject  engines;
   protected JSONArray           declined;
   protected Long                storageVersion;
   protected String              syncID;
@@ -49,29 +49,28 @@ public class MetaGlobal implements SyncS
     this.authHeaderProvider = authHeaderProvider;
   }
 
   public void fetch(MetaGlobalDelegate delegate) {
     this.callback = delegate;
     try {
       this.isUploading = false;
       SyncStorageRecordRequest r = new SyncStorageRecordRequest(this.metaURL);
-      r.delegate = this;
+      r.delegate = new MetaUploadDelegate(this, null);
       r.get();
     } catch (URISyntaxException e) {
       this.callback.handleError(e);
     }
   }
 
-  public void upload(MetaGlobalDelegate callback) {
+  public void upload(long lastModifiedTimestamp, MetaGlobalDelegate callback) {
     try {
       this.isUploading = true;
       SyncStorageRecordRequest r = new SyncStorageRecordRequest(this.metaURL);
-
-      r.delegate = this;
+      r.delegate = new MetaUploadDelegate(this, lastModifiedTimestamp);
       this.callback = callback;
       r.put(this.asCryptoRecord());
     } catch (Exception e) {
       callback.handleError(e);
     }
   }
 
   protected ExtendedJSONObject asRecordContents() {
@@ -314,35 +313,16 @@ public class MetaGlobal implements SyncS
     this.syncID = syncID;
   }
 
   // SyncStorageRequestDelegate methods for fetching.
   public String credentials() {
     return null;
   }
 
-  @Override
-  public AuthHeaderProvider getAuthHeaderProvider() {
-    return authHeaderProvider;
-  }
-
-  @Override
-  public String ifUnmodifiedSince() {
-    return null;
-  }
-
-  @Override
-  public void handleRequestSuccess(SyncStorageResponse response) {
-    if (this.isUploading) {
-      this.handleUploadSuccess(response);
-    } else {
-      this.handleDownloadSuccess(response);
-    }
-  }
-
   private void handleUploadSuccess(SyncStorageResponse response) {
     this.callback.handleSuccess(this, response);
   }
 
   private void handleDownloadSuccess(SyncStorageResponse response) {
     if (response.wasSuccessful()) {
       try {
         CryptoRecord record = CryptoRecord.fromJSONRecord(response.jsonObjectBody());
@@ -351,22 +331,54 @@ public class MetaGlobal implements SyncS
       } catch (Exception e) {
         this.callback.handleError(e);
       }
       return;
     }
     this.callback.handleFailure(response);
   }
 
-  @Override
-  public void handleRequestFailure(SyncStorageResponse response) {
-    if (response.getStatusCode() == 404) {
-      this.callback.handleMissing(this, response);
-      return;
+  private static class MetaUploadDelegate implements SyncStorageRequestDelegate {
+    private final MetaGlobal metaGlobal;
+    private final Long ifUnmodifiedSinceTimestamp;
+
+    /* package-local */ MetaUploadDelegate(final MetaGlobal metaGlobal, final Long ifUnmodifiedSinceTimestamp) {
+      this.metaGlobal = metaGlobal;
+      this.ifUnmodifiedSinceTimestamp = ifUnmodifiedSinceTimestamp;
+    }
+
+    @Override
+    public AuthHeaderProvider getAuthHeaderProvider() {
+      return metaGlobal.authHeaderProvider;
+    }
+
+    @Override
+    public String ifUnmodifiedSince() {
+      if (ifUnmodifiedSinceTimestamp == null) {
+        return null;
+      }
+      return Utils.millisecondsToDecimalSecondsString(ifUnmodifiedSinceTimestamp);
     }
-    this.callback.handleFailure(response);
-  }
+
+    @Override
+    public void handleRequestSuccess(SyncStorageResponse response) {
+      if (metaGlobal.isUploading) {
+        metaGlobal.handleUploadSuccess(response);
+      } else {
+        metaGlobal.handleDownloadSuccess(response);
+      }
+    }
 
-  @Override
-  public void handleRequestError(Exception e) {
-    this.callback.handleError(e);
+    @Override
+    public void handleRequestFailure(SyncStorageResponse response) {
+      if (response.getStatusCode() == 404) {
+        metaGlobal.callback.handleMissing(metaGlobal, response);
+        return;
+      }
+      metaGlobal.callback.handleFailure(response);
+    }
+
+    @Override
+    public void handleRequestError(Exception e) {
+      metaGlobal.callback.handleError(e);
+    }
   }
 }
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/delegates/GlobalSessionCallback.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/delegates/GlobalSessionCallback.java
@@ -34,16 +34,17 @@ public interface GlobalSessionCallback {
    */
   void informMigrated(GlobalSession session);
 
   void handleAborted(GlobalSession globalSession, String reason);
   void handleError(GlobalSession globalSession, Exception ex);
   void handleSuccess(GlobalSession globalSession);
   void handleStageCompleted(Stage currentState, GlobalSession globalSession);
   void handleIncompleteStage(Stage currentState, GlobalSession globalSession);
+  void handleFullSyncNecessary();
 
   /**
    * Called when a {@link GlobalSession} wants to know if it should continue
    * to make storage requests.
    *
    * @return false if the session should make no further requests.
    */
   boolean shouldBackOffStorage();
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/FetchMetaGlobalStage.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/stage/FetchMetaGlobalStage.java
@@ -53,27 +53,28 @@ public class FetchMetaGlobalStage extend
   @Override
   public void execute() throws NoSuchStageException {
     InfoCollections infoCollections = session.config.infoCollections;
     if (infoCollections == null) {
       session.abort(null, "No info/collections set in FetchMetaGlobalStage.");
       return;
     }
 
-    long lastModified = session.config.persistedMetaGlobal().lastModified();
+    final long lastModified = session.config.persistedMetaGlobal().lastModified();
     if (!infoCollections.updateNeeded(META_COLLECTION, lastModified)) {
       // Try to use our local collection keys for this session.
       Logger.info(LOG_TAG, "Trying to use persisted meta/global for this session.");
       MetaGlobal global = session.config.persistedMetaGlobal().metaGlobal(session.config.metaURL(), session.getAuthHeaderProvider());
       if (global != null) {
         Logger.info(LOG_TAG, "Using persisted meta/global for this session.");
         session.processMetaGlobal(global); // Calls session.advance().
         return;
       }
       Logger.info(LOG_TAG, "Failed to use persisted meta/global for this session.");
     }
 
     // We need an update: fetch or upload meta/global as necessary.
+    // We assert when we believe meta/global was last modified via X-I-U-S.
     Logger.info(LOG_TAG, "Fetching fresh meta/global for this session.");
     MetaGlobal global = new MetaGlobal(session.config.metaURL(), session.getAuthHeaderProvider());
     global.fetch(new StageMetaGlobalDelegate(session));
   }
 }
--- a/mobile/android/tests/background/junit3/src/org/mozilla/gecko/background/testhelpers/DefaultGlobalSessionCallback.java
+++ b/mobile/android/tests/background/junit3/src/org/mozilla/gecko/background/testhelpers/DefaultGlobalSessionCallback.java
@@ -30,16 +30,21 @@ public class DefaultGlobalSessionCallbac
 
   @Override
   public void handleIncompleteStage(Stage currentState,
                                     GlobalSession globalSession) {
 
   }
 
   @Override
+  public void handleFullSyncNecessary() {
+
+  }
+
+  @Override
   public void handleAborted(GlobalSession globalSession, String reason) {
   }
 
   @Override
   public void handleError(GlobalSession globalSession, Exception ex) {
   }
 
   @Override
--- a/mobile/android/tests/background/junit4/src/org/mozilla/android/sync/net/test/TestGlobalSession.java
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/android/sync/net/test/TestGlobalSession.java
@@ -22,23 +22,26 @@ import org.mozilla.gecko.background.test
 import org.mozilla.gecko.background.testhelpers.MockPrefsGlobalSession;
 import org.mozilla.gecko.background.testhelpers.MockServerSyncStage;
 import org.mozilla.gecko.background.testhelpers.MockSharedPreferences;
 import org.mozilla.gecko.background.testhelpers.TestRunner;
 import org.mozilla.gecko.background.testhelpers.WaitHelper;
 import org.mozilla.gecko.sync.EngineSettings;
 import org.mozilla.gecko.sync.ExtendedJSONObject;
 import org.mozilla.gecko.sync.GlobalSession;
+import org.mozilla.gecko.sync.HTTPFailureException;
+import org.mozilla.gecko.sync.InfoCollections;
 import org.mozilla.gecko.sync.MetaGlobal;
 import org.mozilla.gecko.sync.NonObjectJSONException;
 import org.mozilla.gecko.sync.SyncConfiguration;
 import org.mozilla.gecko.sync.SyncConfigurationException;
 import org.mozilla.gecko.sync.Utils;
 import org.mozilla.gecko.sync.crypto.CryptoException;
 import org.mozilla.gecko.sync.crypto.KeyBundle;
+import org.mozilla.gecko.sync.delegates.MetaGlobalDelegate;
 import org.mozilla.gecko.sync.net.BaseResource;
 import org.mozilla.gecko.sync.net.BasicAuthHeaderProvider;
 import org.mozilla.gecko.sync.net.SyncStorageResponse;
 import org.mozilla.gecko.sync.repositories.domain.VersionConstants;
 import org.mozilla.gecko.sync.stage.AndroidBrowserBookmarksServerSyncStage;
 import org.mozilla.gecko.sync.stage.GlobalSyncStage;
 import org.mozilla.gecko.sync.stage.GlobalSyncStage.Stage;
 import org.mozilla.gecko.sync.stage.NoSuchStageException;
@@ -54,16 +57,17 @@ import java.util.List;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
 
 @RunWith(TestRunner.class)
 public class TestGlobalSession {
   private int          TEST_PORT                = HTTPServerTestHelper.getTestPort();
   private final String TEST_CLUSTER_URL         = "http://localhost:" + TEST_PORT;
   private final String TEST_USERNAME            = "johndoe";
   private final String TEST_PASSWORD            = "password";
   private final String TEST_SYNC_KEY            = "abcdeabcdeabcdeabcdeabcdea";
@@ -386,16 +390,17 @@ public class TestGlobalSession {
 
   @Test
   public void testUploadUpdatedMetaGlobal() throws Exception {
     // Set up session with meta/global.
     final MockGlobalSessionCallback callback = new MockGlobalSessionCallback();
     final GlobalSession session = MockPrefsGlobalSession.getSession(TEST_USERNAME, TEST_PASSWORD,
         new KeyBundle(TEST_USERNAME, TEST_SYNC_KEY), callback, null, null);
     session.config.metaGlobal = session.generateNewMetaGlobal();
+    session.config.infoCollections = mock(InfoCollections.class);
     session.enginesToUpdate.clear();
 
     // Set enabledEngines in meta/global, including a "new engine."
     String[] origEngines = new String[] { "bookmarks", "clients", "forms", "history", "tabs", "new-engine" };
 
     ExtendedJSONObject origEnginesJSONObject = new ExtendedJSONObject();
     for (String engineName : origEngines) {
       EngineSettings mockEngineSettings = new EngineSettings(Utils.generateGuid(), Integer.valueOf(0));
@@ -428,13 +433,58 @@ public class TestGlobalSession {
       expected.remove(name);
     }
     for (String name : toAdd) {
       expected.add(name);
     }
     assertEquals(expected, session.config.metaGlobal.getEnabledEngineNames());
   }
 
+  @Test
+  public void testUploadMetaGlobalDelegate412() {
+    final Object monitor = new Object();
+    final MockGlobalSessionCallback callback = new MockGlobalSessionCallback();
+    MetaGlobalDelegate metaGlobalDelegate = GlobalSession.makeMetaGlobalUploadDelegate(
+            mock(SyncConfiguration.class),
+            callback,
+            monitor
+    );
+
+    metaGlobalDelegate.handleFailure(makeSyncStorageResponse(412));
+
+    assertTrue(callback.calledFullSyncNecessary);
+  }
+
+  @Test
+  public void testUploadMetaGlobalDelegateNon412() {
+    final Object monitor = new Object();
+    final MockGlobalSessionCallback callback = new MockGlobalSessionCallback();
+    MetaGlobalDelegate metaGlobalDelegate = GlobalSession.makeMetaGlobalUploadDelegate(
+            mock(SyncConfiguration.class),
+            callback,
+            monitor
+    );
+
+    metaGlobalDelegate.handleFailure(makeSyncStorageResponse(400));
+
+    assertFalse(callback.calledFullSyncNecessary);
+  }
+
   public void testStageAdvance() {
     assertEquals(GlobalSession.nextStage(Stage.idle), Stage.checkPreconditions);
     assertEquals(GlobalSession.nextStage(Stage.completed), Stage.idle);
   }
+
+  public static HTTPFailureException makeHttpFailureException(int statusCode) {
+    return new HTTPFailureException(makeSyncStorageResponse(statusCode));
+  }
+
+  public static SyncStorageResponse makeSyncStorageResponse(int statusCode) {
+    // \\( >.<)//
+    return new SyncStorageResponse(
+            new BasicHttpResponse(
+                    new BasicStatusLine(
+                            new ProtocolVersion("HTTP", 1, 1), statusCode, null
+                    )
+            )
+    );
+  }
 }
--- a/mobile/android/tests/background/junit4/src/org/mozilla/android/sync/net/test/TestMetaGlobal.java
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/android/sync/net/test/TestMetaGlobal.java
@@ -291,17 +291,17 @@ public class TestMetaGlobal {
   }
 
 
   public MockMetaGlobalFetchDelegate doUpload(final MetaGlobal global) {
     final MockMetaGlobalFetchDelegate delegate = new MockMetaGlobalFetchDelegate();
     WaitHelper.getTestWaiter().performWait(WaitHelper.onThreadRunnable(new Runnable() {
       @Override
       public void run() {
-        global.upload(delegate);
+        global.upload(0L, delegate);
       }
     }));
 
     return delegate;
   }
 
   @Test
   public void testUpload() {
--- a/mobile/android/tests/background/junit4/src/org/mozilla/android/sync/test/helpers/MockGlobalSessionCallback.java
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/android/sync/test/helpers/MockGlobalSessionCallback.java
@@ -4,16 +4,17 @@
 package org.mozilla.android.sync.test.helpers;
 
 import org.mozilla.gecko.background.testhelpers.WaitHelper;
 import org.mozilla.gecko.sync.GlobalSession;
 import org.mozilla.gecko.sync.delegates.GlobalSessionCallback;
 import org.mozilla.gecko.sync.stage.GlobalSyncStage.Stage;
 
 import java.net.URI;
+import java.util.ArrayList;
 
 import static org.junit.Assert.assertEquals;
 
 /**
  * A callback for use with a GlobalSession that records what happens for later
  * inspection.
  *
  * This callback is expected to be used from within the friendly confines of a
@@ -30,16 +31,18 @@ public class MockGlobalSessionCallback i
   public Exception calledErrorException = null;
   public boolean calledAborted = false;
   public boolean calledRequestBackoff = false;
   public boolean calledInformUnauthorizedResponse = false;
   public boolean calledInformUpgradeRequiredResponse = false;
   public boolean calledInformMigrated = false;
   public URI calledInformUnauthorizedResponseClusterURL = null;
   public long weaveBackoff = -1;
+  public boolean calledFullSyncNecessary = false;
+  public ArrayList<String> incompleteStages = new ArrayList<>();
 
   @Override
   public void handleSuccess(GlobalSession globalSession) {
     this.calledSuccess = true;
     assertEquals(0, this.stageCounter);
     this.testWaiter().performNotify();
   }
 
@@ -54,17 +57,22 @@ public class MockGlobalSessionCallback i
     this.calledError = true;
     this.calledErrorException = ex;
     this.testWaiter().performNotify();
   }
 
   @Override
   public void handleIncompleteStage(Stage currentState,
                                     GlobalSession globalSession) {
+    this.incompleteStages.add(currentState.getRepositoryName());
+  }
 
+  @Override
+  public void handleFullSyncNecessary() {
+    this.calledFullSyncNecessary = true;
   }
 
   @Override
   public void handleStageCompleted(Stage currentState,
            GlobalSession globalSession) {
     stageCounter--;
   }
 
--- a/mobile/android/tests/background/junit4/src/org/mozilla/gecko/background/testhelpers/DefaultGlobalSessionCallback.java
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/gecko/background/testhelpers/DefaultGlobalSessionCallback.java
@@ -46,12 +46,17 @@ public class DefaultGlobalSessionCallbac
 
   @Override
   public void handleIncompleteStage(Stage currentState,
                                     GlobalSession globalSession) {
 
   }
 
   @Override
+  public void handleFullSyncNecessary() {
+
+  }
+
+  @Override
   public boolean shouldBackOffStorage() {
     return false;
   }
 }
--- a/mobile/android/tests/background/junit4/src/org/mozilla/gecko/sync/stage/test/TestFetchMetaGlobalStage.java
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/gecko/sync/stage/test/TestFetchMetaGlobalStage.java
@@ -4,50 +4,59 @@
 package org.mozilla.gecko.sync.stage.test;
 
 import android.os.SystemClock;
 
 import org.json.simple.JSONArray;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.mozilla.android.sync.net.test.TestGlobalSession;
 import org.mozilla.android.sync.net.test.TestMetaGlobal;
 import org.mozilla.android.sync.test.helpers.HTTPServerTestHelper;
 import org.mozilla.android.sync.test.helpers.MockGlobalSessionCallback;
 import org.mozilla.android.sync.test.helpers.MockServer;
 import org.mozilla.gecko.background.testhelpers.MockGlobalSession;
 import org.mozilla.gecko.background.testhelpers.TestRunner;
 import org.mozilla.gecko.background.testhelpers.WaitHelper;
 import org.mozilla.gecko.sync.AlreadySyncingException;
 import org.mozilla.gecko.sync.CollectionKeys;
 import org.mozilla.gecko.sync.CryptoRecord;
 import org.mozilla.gecko.sync.ExtendedJSONObject;
 import org.mozilla.gecko.sync.GlobalSession;
+import org.mozilla.gecko.sync.HTTPFailureException;
 import org.mozilla.gecko.sync.InfoCollections;
 import org.mozilla.gecko.sync.MetaGlobal;
 import org.mozilla.gecko.sync.NonObjectJSONException;
 import org.mozilla.gecko.sync.SyncConfigurationException;
 import org.mozilla.gecko.sync.crypto.CryptoException;
 import org.mozilla.gecko.sync.crypto.KeyBundle;
 import org.mozilla.gecko.sync.delegates.FreshStartDelegate;
+import org.mozilla.gecko.sync.delegates.GlobalSessionCallback;
 import org.mozilla.gecko.sync.delegates.KeyUploadDelegate;
 import org.mozilla.gecko.sync.delegates.WipeServerDelegate;
 import org.mozilla.gecko.sync.net.AuthHeaderProvider;
+import org.mozilla.gecko.sync.net.SyncStorageResponse;
 import org.mozilla.gecko.sync.stage.FetchMetaGlobalStage;
 import org.mozilla.gecko.sync.stage.GlobalSyncStage.Stage;
 import org.simpleframework.http.Request;
 import org.simpleframework.http.Response;
 
 import java.io.IOException;
 import java.net.URI;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import ch.boye.httpclientandroidlib.HttpResponse;
+import ch.boye.httpclientandroidlib.ProtocolVersion;
+import ch.boye.httpclientandroidlib.message.BasicHttpResponse;
+import ch.boye.httpclientandroidlib.message.BasicStatusLine;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 @RunWith(TestRunner.class)
 public class TestFetchMetaGlobalStage {
   @SuppressWarnings("unused")
   private static final String  LOG_TAG          = "TestMetaGlobalStage";
@@ -64,103 +73,115 @@ public class TestFetchMetaGlobalStage {
   private final String TEST_INFO_COLLECTIONS_JSON = "{}";
 
   private static final String  TEST_SYNC_ID         = "testSyncID";
   private static final long    TEST_STORAGE_VERSION = GlobalSession.STORAGE_VERSION;
 
   private InfoCollections infoCollections;
   private KeyBundle syncKeyBundle;
   private MockGlobalSessionCallback callback;
-  private GlobalSession session;
-
-  private boolean calledRequiresUpgrade = false;
-  private boolean calledProcessMissingMetaGlobal = false;
-  private boolean calledFreshStart = false;
-  private boolean calledWipeServer = false;
-  private boolean calledUploadKeys = false;
-  private boolean calledResetAllStages = false;
+  private LocalMockGlobalSession session;
 
   private static void assertSameContents(JSONArray expected, Set<String> actual) {
     assertEquals(expected.size(), actual.size());
     for (Object o : expected) {
       assertTrue(actual.contains(o));
     }
   }
 
+  private class LocalMockGlobalSession extends MockGlobalSession {
+    private boolean calledRequiresUpgrade = false;
+    private boolean calledProcessMissingMetaGlobal = false;
+    private boolean calledFreshStart = false;
+    private boolean calledWipeServer = false;
+    private boolean calledUploadKeys = false;
+    private boolean calledResetAllStages = false;
+    private boolean calledRestart = false;
+    private boolean calledAbort = false;
+
+    public LocalMockGlobalSession(String username, String password, KeyBundle keyBundle, GlobalSessionCallback callback) throws SyncConfigurationException, IllegalArgumentException, NonObjectJSONException, IOException {
+      super(username, password, keyBundle, callback);
+    }
+
+    @Override
+    protected void prepareStages() {
+      super.prepareStages();
+      withStage(Stage.fetchMetaGlobal, new FetchMetaGlobalStage());
+    }
+
+    @Override
+    public void requiresUpgrade() {
+      calledRequiresUpgrade = true;
+      this.abort(null, "Requires upgrade");
+    }
+
+    @Override
+    public void processMissingMetaGlobal(MetaGlobal mg) {
+      calledProcessMissingMetaGlobal = true;
+      this.abort(null, "Missing meta/global");
+    }
+
+    // Don't really uploadKeys.
+    @Override
+    public void uploadKeys(CollectionKeys keys, long lastModified, KeyUploadDelegate keyUploadDelegate) {
+      calledUploadKeys = true;
+      keyUploadDelegate.onKeysUploaded();
+    }
+
+    // On fresh start completed, just stop.
+    @Override
+    public void freshStart() {
+      calledFreshStart = true;
+      freshStart(this, new FreshStartDelegate() {
+        @Override
+        public void onFreshStartFailed(Exception e) {
+          WaitHelper.getTestWaiter().performNotify(e);
+        }
+
+        @Override
+        public void onFreshStart() {
+          WaitHelper.getTestWaiter().performNotify();
+        }
+      });
+    }
+
+    // Don't really wipeServer.
+    @Override
+    protected void wipeServer(final AuthHeaderProvider authHeaderProvider, final WipeServerDelegate wipeDelegate) {
+      calledWipeServer = true;
+      wipeDelegate.onWiped(System.currentTimeMillis());
+    }
+
+    @Override
+    protected void restart() throws AlreadySyncingException {
+      calledRestart = true;
+      WaitHelper.getTestWaiter().performNotify();
+    }
+
+    @Override
+    public void abort(Exception e, String reason) {
+      calledAbort = true;
+      super.abort(e, reason);
+    }
+
+    // Don't really resetAllStages.
+    @Override
+    public void resetAllStages() {
+      calledResetAllStages = true;
+    }
+  }
+
   @Before
   public void setUp() throws Exception {
-    calledRequiresUpgrade = false;
-    calledProcessMissingMetaGlobal = false;
-    calledFreshStart = false;
-    calledWipeServer = false;
-    calledUploadKeys = false;
-    calledResetAllStages = false;
-
     // Set info collections to not have crypto.
     infoCollections = new InfoCollections(new ExtendedJSONObject(TEST_INFO_COLLECTIONS_JSON));
 
     syncKeyBundle = new KeyBundle(TEST_USERNAME, TEST_SYNC_KEY);
     callback = new MockGlobalSessionCallback();
-    session = new MockGlobalSession(TEST_USERNAME, TEST_PASSWORD,
-      syncKeyBundle, callback) {
-      @Override
-      protected void prepareStages() {
-        super.prepareStages();
-        withStage(Stage.fetchMetaGlobal, new FetchMetaGlobalStage());
-      }
-
-      @Override
-      public void requiresUpgrade() {
-        calledRequiresUpgrade = true;
-        this.abort(null, "Requires upgrade");
-      }
-
-      @Override
-      public void processMissingMetaGlobal(MetaGlobal mg) {
-        calledProcessMissingMetaGlobal = true;
-        this.abort(null, "Missing meta/global");
-      }
-
-      // Don't really uploadKeys.
-      @Override
-      public void uploadKeys(CollectionKeys keys, KeyUploadDelegate keyUploadDelegate) {
-        calledUploadKeys = true;
-        keyUploadDelegate.onKeysUploaded();
-      }
-
-      // On fresh start completed, just stop.
-      @Override
-      public void freshStart() {
-        calledFreshStart = true;
-        freshStart(this, new FreshStartDelegate() {
-          @Override
-          public void onFreshStartFailed(Exception e) {
-            WaitHelper.getTestWaiter().performNotify(e);
-          }
-
-          @Override
-          public void onFreshStart() {
-            WaitHelper.getTestWaiter().performNotify();
-          }
-        });
-      }
-
-      // Don't really wipeServer.
-      @Override
-      protected void wipeServer(final AuthHeaderProvider authHeaderProvider, final WipeServerDelegate wipeDelegate) {
-        calledWipeServer = true;
-        wipeDelegate.onWiped(System.currentTimeMillis());
-      }
-
-      // Don't really resetAllStages.
-      @Override
-      public void resetAllStages() {
-        calledResetAllStages = true;
-      }
-    };
+    session = new LocalMockGlobalSession(TEST_USERNAME, TEST_PASSWORD, syncKeyBundle, callback);
     session.config.setClusterURL(new URI(TEST_CLUSTER_URL));
     session.config.infoCollections = infoCollections;
   }
 
   protected void doSession(MockServer server) {
     data.startHTTPServer(server);
     WaitHelper.getTestWaiter().performWait(WaitHelper.onThreadRunnable(new Runnable() {
       @Override
@@ -180,17 +201,17 @@ public class TestFetchMetaGlobalStage {
     MetaGlobal mg = new MetaGlobal(null, null);
     mg.setSyncID(TEST_SYNC_ID);
     mg.setStorageVersion(Long.valueOf(TEST_STORAGE_VERSION + 1));
 
     MockServer server = new MockServer(200, mg.asCryptoRecord().toJSONString());
     doSession(server);
 
     assertEquals(true, callback.calledError);
-    assertTrue(calledRequiresUpgrade);
+    assertTrue(session.calledRequiresUpgrade);
   }
 
   @SuppressWarnings("unchecked")
   private JSONArray makeTestDeclinedArray() {
     final JSONArray declined = new JSONArray();
     declined.add("foobar");
     return declined;
   }
@@ -212,18 +233,18 @@ public class TestFetchMetaGlobalStage {
     // Set declined engines in the server object.
     final JSONArray testingDeclinedEngines = makeTestDeclinedArray();
     mg.setDeclinedEngineNames(testingDeclinedEngines);
 
     MockServer server = new MockServer(200, mg.asCryptoRecord().toJSONString());
     doSession(server);
 
     assertTrue(callback.calledSuccess);
-    assertFalse(calledProcessMissingMetaGlobal);
-    assertFalse(calledResetAllStages);
+    assertFalse(session.calledProcessMissingMetaGlobal);
+    assertFalse(session.calledResetAllStages);
     assertEquals(TEST_SYNC_ID, session.config.metaGlobal.getSyncID());
     assertEquals(TEST_STORAGE_VERSION, session.config.metaGlobal.getStorageVersion().longValue());
     assertEquals(TEST_SYNC_ID, session.config.syncID);
 
     // Declined engines propagate from the server meta/global.
     final Set<String> actual = session.config.metaGlobal.getDeclinedEngineNames();
     assertSameContents(testingDeclinedEngines, actual);
   }
@@ -245,18 +266,18 @@ public class TestFetchMetaGlobalStage {
     // Set declined engines in the server object.
     final JSONArray testingDeclinedEngines = makeTestDeclinedArray();
     mg.setDeclinedEngineNames(testingDeclinedEngines);
 
     MockServer server = new MockServer(200, mg.asCryptoRecord().toJSONString());
     doSession(server);
 
     assertEquals(true, callback.calledSuccess);
-    assertFalse(calledProcessMissingMetaGlobal);
-    assertTrue(calledResetAllStages);
+    assertFalse(session.calledProcessMissingMetaGlobal);
+    assertTrue(session.calledResetAllStages);
     assertEquals(TEST_SYNC_ID, session.config.metaGlobal.getSyncID());
     assertEquals(TEST_STORAGE_VERSION, session.config.metaGlobal.getStorageVersion().longValue());
     assertEquals(TEST_SYNC_ID, session.config.syncID);
 
     // Declined engines propagate from the server meta/global.
     final Set<String> actual = session.config.metaGlobal.getDeclinedEngineNames();
     assertSameContents(testingDeclinedEngines, actual);
   }
@@ -294,41 +315,41 @@ public class TestFetchMetaGlobalStage {
   }
 
   @Test
   public void testFetchMissing() throws Exception {
     MockServer server = new MockServer(404, "missing");
     doSession(server);
 
     assertEquals(true, callback.calledError);
-    assertTrue(calledProcessMissingMetaGlobal);
+    assertTrue(session.calledProcessMissingMetaGlobal);
   }
 
   /**
    * Empty payload object has no syncID or storageVersion and should call freshStart.
    * @throws Exception
    */
   @Test
   public void testFetchEmptyPayload() throws Exception {
     MockServer server = new MockServer(200, TestMetaGlobal.TEST_META_GLOBAL_EMPTY_PAYLOAD_RESPONSE);
     doSession(server);
 
-    assertTrue(calledFreshStart);
+    assertTrue(session.calledFreshStart);
   }
 
   /**
    * No payload means no syncID or storageVersion and therefore we should call freshStart.
    * @throws Exception
    */
   @Test
   public void testFetchNoPayload() throws Exception {
     MockServer server = new MockServer(200, TestMetaGlobal.TEST_META_GLOBAL_NO_PAYLOAD_RESPONSE);
     doSession(server);
 
-    assertTrue(calledFreshStart);
+    assertTrue(session.calledFreshStart);
   }
 
   /**
    * Malformed payload is a server response issue, not a meta/global record
    * issue. This should error out of the sync.
    * @throws Exception
    */
   @Test
@@ -379,16 +400,67 @@ public class TestFetchMetaGlobalStage {
           // We shouldn't be trying to download anything after uploading meta/global.
           mgDownloaded.set(true);
         }
         this.handle(request, response, 404, "missing");
       }
     };
     doFreshStart(server);
 
-    assertTrue(this.calledFreshStart);
-    assertTrue(this.calledWipeServer);
-    assertTrue(this.calledUploadKeys);
+    assertTrue(session.calledFreshStart);
+    assertTrue(session.calledWipeServer);
+    assertTrue(session.calledUploadKeys);
     assertTrue(mgUploaded.get());
     assertFalse(mgDownloaded.get());
     assertEquals(GlobalSession.STORAGE_VERSION, uploadedMg.getStorageVersion().longValue());
   }
+
+  @Test
+  public void testFreshStartDelegateSuccess() {
+    final FreshStartDelegate freshStartDelegate = GlobalSession.makeFreshStartDelegate(session);
+
+    WaitHelper.getTestWaiter().performWait(WaitHelper.onThreadRunnable(
+            new Runnable() {
+              @Override
+              public void run() {
+                freshStartDelegate.onFreshStart();
+              }
+            }
+    ));
+
+    assertTrue(session.calledRestart);
+    assertFalse(session.calledAbort);
+  }
+
+  @Test
+  public void testFreshStartDelegate412() {
+    final FreshStartDelegate freshStartDelegate = GlobalSession.makeFreshStartDelegate(session);
+
+    WaitHelper.getTestWaiter().performWait(WaitHelper.onThreadRunnable(
+            new Runnable() {
+              @Override
+              public void run() {
+                freshStartDelegate.onFreshStartFailed(TestGlobalSession.makeHttpFailureException(412));
+              }
+            }
+    ));
+
+    assertTrue(session.calledRestart);
+    assertFalse(session.calledAbort);
+  }
+
+  @Test
+  public void testFreshStartDelegateNon412() {
+    final FreshStartDelegate freshStartDelegate = GlobalSession.makeFreshStartDelegate(session);
+
+    WaitHelper.getTestWaiter().performWait(WaitHelper.onThreadRunnable(
+            new Runnable() {
+              @Override
+              public void run() {
+                freshStartDelegate.onFreshStartFailed(TestGlobalSession.makeHttpFailureException(400));
+              }
+            }
+    ));
+
+    assertFalse(session.calledRestart);
+    assertTrue(session.calledAbort);
+  }
 }