--- a/services/sync/tests/unit/sync_ping_schema.json
+++ b/services/sync/tests/unit/sync_ping_schema.json
@@ -23,17 +23,17 @@
         "didLogin": { "type": "boolean" },
         "when": { "type": "integer" },
         "uid": {
           "type": "string",
           "pattern": "^[0-9a-f]{32}$"
         },
         "devices": {
           "type": "array",
-          "items": { "$ref": "#/definitions/engine" }
+          "items": { "$ref": "#/definitions/device" }
         },
         "deviceID": {
           "type": "string",
           "pattern": "^[0-9a-f]{64}$"
         },
         "status": {
           "type": "object",
           "anyOf": [

--- a/services/sync/tps/extensions/tps/resource/tps.jsm
+++ b/services/sync/tps/extensions/tps/resource/tps.jsm
@@ -14,16 +14,17 @@ const {classes: Cc, interfaces: Ci, util
 var module = this;
 
 // Global modules
 Cu.import("resource://gre/modules/Log.jsm");
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/AppConstants.jsm");
 Cu.import("resource://gre/modules/PlacesUtils.jsm");
+Cu.import("resource://gre/modules/FileUtils.jsm");
 Cu.import("resource://services-common/async.js");
 Cu.import("resource://services-sync/constants.js");
 Cu.import("resource://services-sync/main.js");
 Cu.import("resource://services-sync/util.js");
 Cu.import("resource://services-sync/telemetry.js");
 Cu.import("resource://services-sync/bookmark_validator.js");
 Cu.import("resource://services-sync/engines/passwords.js");
 Cu.import("resource://services-sync/engines/forms.js");
@@ -44,16 +45,21 @@ Cu.import("resource://tps/modules/window
 var hh = Cc["@mozilla.org/network/protocol;1?name=http"]
          .getService(Ci.nsIHttpProtocolHandler);
 var prefs = Cc["@mozilla.org/preferences-service;1"]
             .getService(Ci.nsIPrefBranch);
 
 var mozmillInit = {};
 Cu.import('resource://mozmill/driver/mozmill.js', mozmillInit);
 
+XPCOMUtils.defineLazyGetter(this, "fileProtocolHandler", () => {
+  let fileHandler = Services.io.getProtocolHandler("file");
+  return fileHandler.QueryInterface(Ci.nsIFileProtocolHandler);
+});
+
 // Options for wiping data during a sync
 const SYNC_RESET_CLIENT = "resetClient";
 const SYNC_WIPE_CLIENT  = "wipeClient";
 const SYNC_WIPE_REMOTE  = "wipeRemote";
 
 // Actions a test can perform
 const ACTION_ADD                = "add";
 const ACTION_DELETE             = "delete";
@@ -739,24 +745,28 @@ var TPS = {
           this.ValidatePasswords();
         }
         if (this.shouldValidateForms) {
           this.ValidateForms();
         }
         if (this.shouldValidateAddons) {
           this.ValidateAddons();
         }
+        // Force this early so that we run the validation and detect missing pings
+        // *before* we start shutting down, since if we do it after, the python
+        // code won't notice the failure.
+        SyncTelemetry.shutdown();
         // we're all done
         Logger.logInfo("test phase " + this._currentPhase + ": " +
                        (this._errors ? "FAIL" : "PASS"));
         this._phaseFinished = true;
         this.quit();
         return;
       }
-
+      this.seconds_since_epoch = prefs.getIntPref("tps.seconds_since_epoch", 0);
       if (this.seconds_since_epoch)
         this._usSinceEpoch = this.seconds_since_epoch * 1000 * 1000;
       else {
         this.DumpError("seconds-since-epoch not set");
         return;
       }
 
       let phase = this._phaselist[this._currentPhase];
@@ -780,16 +790,60 @@ var TPS = {
       } else {
         this.DumpError("RunNextTestAction failed", e);
       }
       return;
     }
     this.RunNextTestAction();
   },
 
+  _getFileRelativeToSourceRoot(testFileURL, relativePath) {
+    let file = fileProtocolHandler.getFileFromURLSpec(testFileURL);
+    let root = file // <root>/services/sync/tests/tps/test_foo.js
+      .parent // <root>/services/sync/tests/tps
+      .parent // <root>/services/sync/tests
+      .parent // <root>/services/sync
+      .parent // <root>/services
+      .parent // <root>
+      ;
+    root.appendRelativePath(relativePath);
+    return root;
+  },
+
+  // Attempt to load the sync_ping_schema.json and initialize `this.pingValidator`
+  // based on the source of the tps file. Assumes that it's at "../unit/sync_ping_schema.json"
+  // relative to the directory the tps test file (testFile) is contained in.
+  _tryLoadPingSchema(testFile) {
+    try {
+      let schemaFile = this._getFileRelativeToSourceRoot(testFile,
+        "services/sync/tests/unit/sync_ping_schema.json");
+
+      let stream = Cc["@mozilla.org/network/file-input-stream;1"]
+                   .createInstance(Ci.nsIFileInputStream);
+
+      let jsonReader = Cc["@mozilla.org/dom/json;1"]
+                       .createInstance(Components.interfaces.nsIJSON);
+
+      stream.init(schemaFile, FileUtils.MODE_RDONLY, FileUtils.PERMS_FILE, 0);
+      let schema = jsonReader.decodeFromStream(stream, stream.available());
+      Logger.logInfo("Successfully loaded schema")
+
+      // Importing resource://testing-common/* isn't possible from within TPS,
+      // so we load Ajv manually.
+      let ajvFile = this._getFileRelativeToSourceRoot(testFile, "testing/modules/ajv-4.1.1.js");
+      let ajvURL = fileProtocolHandler.getURLSpecFromFile(ajvFile);
+      let ns = {};
+      Cu.import(ajvURL, ns);
+      let ajv = new ns.Ajv({ async: "co*" });
+      this.pingValidator = ajv.compile(schema);
+    } catch (e) {
+      this.DumpError(`Failed to load ping schema and AJV relative to "${testFile}".`, e);
+    }
+  },
+
   /**
    * Runs a single test phase.
    *
    * This is the main entry point for each phase of a test. The TPS command
    * line driver loads this module and calls into the function with the
    * arguments from the command line.
    *
    * When a phase is executed, the file is loaded as JavaScript into the
@@ -848,25 +902,29 @@ var TPS = {
 
   /**
    * Executes a single test phase.
    *
    * This is called by RunTestPhase() after the environment is validated.
    */
   _executeTestPhase: function _executeTestPhase(file, phase, settings) {
     try {
+      this.config = JSON.parse(prefs.getCharPref('tps.config'));
       // parse the test file
       Services.scriptloader.loadSubScript(file, this);
       this._currentPhase = phase;
       if (this._currentPhase.startsWith("cleanup-")) {
         let profileToClean = Cc["@mozilla.org/toolkit/profile-service;1"]
                              .getService(Ci.nsIToolkitProfileService)
                              .selectedProfile.name;
         this.phases[this._currentPhase] = profileToClean;
         this.Phase(this._currentPhase, [[this.Cleanup]]);
+      } else {
+        // Don't bother doing this for cleanup phases.
+        this._tryLoadPingSchema(file);
       }
       let this_phase = this._phaselist[this._currentPhase];
 
       if (this_phase == undefined) {
         this.DumpError("invalid phase " + this._currentPhase);
         return;
       }
 
@@ -890,34 +948,16 @@ var TPS = {
           }
         }
       }
       Logger.logInfo("Starting phase " + this._currentPhase);
 
       Logger.logInfo("setting client.name to " + this.phases[this._currentPhase]);
       Weave.Svc.Prefs.set("client.name", this.phases[this._currentPhase]);
 
-      // If a custom server was specified, set it now
-      if (this.config["serverURL"]) {
-        Weave.Service.serverURL = this.config.serverURL;
-        prefs.setCharPref('tps.serverURL', this.config.serverURL);
-      }
-
-      // Store account details as prefs so they're accessible to the Mozmill
-      // framework.
-      if (this.fxaccounts_enabled) {
-        prefs.setCharPref('tps.account.username', this.config.fx_account.username);
-        prefs.setCharPref('tps.account.password', this.config.fx_account.password);
-      }
-      else {
-        prefs.setCharPref('tps.account.username', this.config.sync_account.username);
-        prefs.setCharPref('tps.account.password', this.config.sync_account.password);
-        prefs.setCharPref('tps.account.passphrase', this.config.sync_account.passphrase);
-      }
-
       this._interceptSyncTelemetry();
 
       // start processing the test actions
       this._currentAction = 0;
     }
     catch(e) {
       this.DumpError("_executeTestPhase failed", e);
       return;
@@ -937,25 +977,37 @@ var TPS = {
       } catch (e) {
         self.DumpError("Error when generating sync telemetry", e);
       }
     };
     SyncTelemetry.submit = record => {
       Logger.logInfo("Intercepted sync telemetry submission: " + JSON.stringify(record));
       this._syncsReportedViaTelemetry += record.syncs.length + (record.discarded || 0);
       if (record.discarded) {
-        Logger.AssertTrue(record.syncs.length == SyncTelemetry.maxPayloadCount,
-                          "Syncs discarded from ping before maximum payload count reached");
+        if (record.syncs.length != SyncTelemetry.maxPayloadCount) {
+          this.DumpError("Syncs discarded from ping before maximum payload count reached");
+        }
       }
       // If this is the shutdown ping, check and see that the telemetry saw all the syncs.
       if (record.why === "shutdown") {
         // If we happen to sync outside of tps manually causing it, its not an
         // error in the telemetry, so we only complain if we didn't see all of them.
-        Logger.AssertTrue(this._syncsReportedViaTelemetry >= this._syncCount,
-                          `Telemetry missed syncs: Saw ${this._syncsReportedViaTelemetry}, should have >= ${this._syncCount}.`);
+        if (this._syncsReportedViaTelemetry < this._syncCount) {
+          this.DumpError(`Telemetry missed syncs: Saw ${this._syncsReportedViaTelemetry}, should have >= ${this._syncCount}.`);
+        }
+      }
+      if (!record.syncs.length) {
+        // Note: we're overwriting submit, so this is called even for pings that
+        // may have no data (which wouldn't be submitted to telemetry and would
+        // fail validation).
+        return;
+      }
+      if (!this.pingValidator(record)) {
+        // Note that we already logged the record.
+        this.DumpError("Sync ping validation failed with errors: " + JSON.stringify(this.pingValidator.errors));
       }
     };
   },
 
   /**
    * Register a single phase with the test harness.
    *
    * This is called when loading individual test files.

--- a/testing/tps/tps/testrunner.py
+++ b/testing/tps/tps/testrunner.py
@@ -233,22 +233,17 @@ class TPSTestRunner(object):
         f = open(testpath, 'r')
         testcontent = f.read()
         f.close()
         try:
             test = json.loads(testcontent)
         except:
             test = json.loads(testcontent[testcontent.find('{'):testcontent.find('}') + 1])
 
-        testcontent += 'var config = %s;\n' % json.dumps(self.config, indent=2)
-        testcontent += 'var seconds_since_epoch = %d;\n' % int(time.time())
-
-        tmpfile = TempFile(prefix='tps_test_')
-        tmpfile.write(testcontent)
-        tmpfile.close()
+        self.preferences['tps.seconds_since_epoch'] = int(time.time())
 
         # generate the profiles defined in the test, and a list of test phases
         profiles = {}
         phaselist = []
         for phase in test:
             profilename = test[phase]
 
             # create the profile if necessary
@@ -256,17 +251,17 @@ class TPSTestRunner(object):
                 profiles[profilename] = Profile(preferences = self.preferences,
                                                 addons = self.extensions)
 
             # create the test phase
             phaselist.append(TPSTestPhase(
                 phase,
                 profiles[profilename],
                 testname,
-                tmpfile.filename,
+                testpath,
                 self.logfile,
                 self.env,
                 self.firefoxRunner,
                 self.log,
                 ignore_unused_engines=self.ignore_unused_engines))
 
         # sort the phase list by name
         phaselist = sorted(phaselist, key=lambda phase: phase.phase)
@@ -278,17 +273,17 @@ class TPSTestRunner(object):
             if phase.status != 'PASS':
                 failed = True
                 break;
 
         for profilename in profiles:
             cleanup_phase = TPSTestPhase(
                 'cleanup-' + profilename,
                 profiles[profilename], testname,
-                tmpfile.filename,
+                testpath,
                 self.logfile,
                 self.env,
                 self.firefoxRunner,
                 self.log)
 
             cleanup_phase.run()
             if cleanup_phase.status != 'PASS':
                 failed = True
@@ -369,16 +364,18 @@ class TPSTestRunner(object):
             self.preferences.update({'services.sync.username': "dummy"})
 
         if self.debug:
             self.preferences.update(self.debug_preferences)
 
         if 'preferences' in self.config:
             self.preferences.update(self.config['preferences'])
 
+        self.preferences['tps.config'] = json.dumps(self.config)
+
     def run_tests(self):
         # delete the logfile if it already exists
         if os.access(self.logfile, os.F_OK):
             os.remove(self.logfile)
 
         # Copy the system env variables, and update them for custom settings
         self.env = os.environ.copy()
         self.env.update(self.extra_env)