Bug 1425909 - Enable adding scalars in artifact builds without rebuilding Firefox. r?chutten, froydnj draft
authorAlessio Placitelli <alessio.placitelli@gmail.com>
Thu, 18 Jan 2018 18:25:01 +0100
changeset 750456 e133d0f69592c6f69ca984fc9494f28c3c7f4341
parent 749389 7b46ef2ae1412b15ed45e7d2367ca491344729f7
child 750457 8ca11065a8cde49bfc4bc64c57e9c78e4e29f156
push id97663
push useralessio.placitelli@gmail.com
push dateFri, 02 Feb 2018 09:42:15 +0000
reviewerschutten, froydnj
bugs1425909
milestone60.0a1
Bug 1425909 - Enable adding scalars in artifact builds without rebuilding Firefox. r?chutten, froydnj This patch enables generating a JSON file that mirrors the scalar definitions in Scalars.yaml. On local developer builds, this file is loaded when Firefox starts to register all the scalars. If some change was introduced in the definition files, the new scalar will be dynamically added. The JSON definition file will be regenerated every time an artifact build is performed or the build faster command is invoked. MozReview-Commit-ID: Do3WjE38aIK
toolkit/components/telemetry/ScalarInfo.h
toolkit/components/telemetry/Telemetry.cpp
toolkit/components/telemetry/TelemetryController.jsm
toolkit/components/telemetry/TelemetryScalar.cpp
toolkit/components/telemetry/TelemetryScalar.h
toolkit/components/telemetry/gen_scalar_data.py
toolkit/components/telemetry/moz.build
toolkit/components/telemetry/nsITelemetry.idl
toolkit/components/telemetry/parse_scalars.py
toolkit/components/telemetry/tests/unit/test_TelemetryScalars_buildFaster.js
toolkit/components/telemetry/tests/unit/xpcshell.ini
--- a/toolkit/components/telemetry/ScalarInfo.h
+++ b/toolkit/components/telemetry/ScalarInfo.h
@@ -18,24 +18,26 @@ namespace {
 /**
  * Base scalar information, common to both "static" and dynamic scalars.
  */
 struct BaseScalarInfo {
   uint32_t kind;
   uint32_t dataset;
   mozilla::Telemetry::Common::RecordedProcessType record_in_processes;
   bool keyed;
+  bool builtin;
 
   BaseScalarInfo(uint32_t aKind, uint32_t aDataset,
                  mozilla::Telemetry::Common::RecordedProcessType aRecordInProcess,
-                 bool aKeyed)
+                 bool aKeyed, bool aBuiltin = true)
     : kind(aKind)
     , dataset(aDataset)
     , record_in_processes(aRecordInProcess)
     , keyed(aKeyed)
+    , builtin(aBuiltin)
   {}
   virtual ~BaseScalarInfo() {}
 
   virtual const char *name() const = 0;
   virtual const char *expiration() const = 0;
 };
 
 /**
--- a/toolkit/components/telemetry/Telemetry.cpp
+++ b/toolkit/components/telemetry/Telemetry.cpp
@@ -1722,17 +1722,25 @@ TelemetryImpl::SnapshotKeyedScalars(unsi
                                                aResult);
 }
 
 NS_IMETHODIMP
 TelemetryImpl::RegisterScalars(const nsACString& aCategoryName,
                                JS::Handle<JS::Value> aScalarData,
                                JSContext* cx)
 {
-  return TelemetryScalar::RegisterScalars(aCategoryName, aScalarData, cx);
+  return TelemetryScalar::RegisterScalars(aCategoryName, aScalarData, false, cx);
+}
+
+NS_IMETHODIMP
+TelemetryImpl::RegisterBuiltinScalars(const nsACString& aCategoryName,
+                                      JS::Handle<JS::Value> aScalarData,
+                                      JSContext* cx)
+{
+  return TelemetryScalar::RegisterScalars(aCategoryName, aScalarData, true, cx);
 }
 
 NS_IMETHODIMP
 TelemetryImpl::ClearScalars()
 {
   TelemetryScalar::ClearScalars();
   return NS_OK;
 }
--- a/toolkit/components/telemetry/TelemetryController.jsm
+++ b/toolkit/components/telemetry/TelemetryController.jsm
@@ -62,16 +62,17 @@ XPCOMUtils.defineLazyModuleGetters(this,
   UpdateUtils: "resource://gre/modules/UpdateUtils.jsm",
   TelemetryArchive: "resource://gre/modules/TelemetryArchive.jsm",
   TelemetrySession: "resource://gre/modules/TelemetrySession.jsm",
   TelemetrySend: "resource://gre/modules/TelemetrySend.jsm",
   TelemetryReportingPolicy: "resource://gre/modules/TelemetryReportingPolicy.jsm",
   TelemetryModules: "resource://gre/modules/TelemetryModules.jsm",
   UpdatePing: "resource://gre/modules/UpdatePing.jsm",
   TelemetryHealthPing: "resource://gre/modules/TelemetryHealthPing.jsm",
+  OS: "resource://gre/modules/osfile.jsm",
 });
 
 /**
  * Setup Telemetry logging. This function also gets called when loggin related
  * preferences change.
  */
 var gLogger = null;
 var gLogAppenderDump = null;
@@ -152,16 +153,23 @@ this.TelemetryController = Object.freeze
   /**
    * Used only for testing purposes.
    */
   testSetupContent() {
     return Impl.setupContentTelemetry(true);
   },
 
   /**
+   * Used only for testing purposes.
+   */
+  testPromiseJsProbeRegistration() {
+    return Promise.resolve(Impl._probeRegistrationPromise);
+  },
+
+  /**
    * Send a notification.
    */
   observe(aSubject, aTopic, aData) {
     return Impl.observe(aSubject, aTopic, aData);
   },
 
   /**
    * Submit ping payloads to Telemetry. This will assemble a complete ping, adding
@@ -317,16 +325,18 @@ var Impl = {
   // After this barrier, clients can not submit Telemetry pings anymore.
   _shutdownBarrier: new AsyncShutdown.Barrier("TelemetryController: Waiting for clients."),
   // This is a private barrier blocked by pending async ping activity (sending & saving).
   _connectionsBarrier: new AsyncShutdown.Barrier("TelemetryController: Waiting for pending ping activity"),
   // This is true when running in the test infrastructure.
   _testMode: false,
   // The task performing the delayed sending of the "new-profile" ping.
   _delayedNewPingTask: null,
+  // The promise used to wait for the JS probe registration (dynamic builtin).
+  _probeRegistrationPromise: null,
 
   get _log() {
     if (!this._logger) {
       this._logger = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX);
     }
 
     return this._logger;
   },
@@ -674,16 +684,21 @@ var Impl = {
       return this._delayedInitTaskDeferred.promise;
     }
 
     if (this._initialized && !this._testMode) {
       this._log.error("setupTelemetry - already initialized");
       return Promise.resolve();
     }
 
+    // Enable adding scalars in artifact builds and build faster modes.
+    // The function is async: we intentionally don't wait for it to complete
+    // as we don't want to delay startup.
+    this._probeRegistrationPromise = this.registerJsProbes();
+
     // This will trigger displaying the datachoices infobar.
     TelemetryReportingPolicy.setup();
 
     if (!this.enableTelemetryRecording()) {
       this._log.config("setupChromeProcess - Telemetry recording is disabled, skipping Chrome process setup.");
       return Promise.resolve();
     }
 
@@ -1033,9 +1048,69 @@ var Impl = {
     };
     // TODO: we need to be smarter about when to send the ping (and save the
     // state to file). |requestIdleCallback| is currently only accessible
     // through DOM. See bug 1361996.
     await TelemetryController.submitExternalPing("new-profile", payload, options)
                              .then(() => TelemetrySession.markNewProfilePingSent(),
                                    e => this._log.error("sendNewProfilePing - failed to submit new-profile ping", e));
   },
+
+  /**
+   * Register 'dynamic builtin' probes from the JSON definition files.
+   * This is needed to support adding new probes in developer builds
+   * without rebuilding the whole codebase.
+   *
+   * This is not meant to be used outside of local developer builds.
+   */
+  async registerJsProbes() {
+    // We don't support this outside of developer builds.
+    if (AppConstants.MOZILLA_OFFICIAL && !this._testMode) {
+      return;
+    }
+
+    this._log.trace("registerJsProbes - registering builtin JS probes");
+
+    // Load the scalar probes JSON file.
+    const scalarProbeFilename = "ScalarArtifactDefinitions.json";
+    let scalarProbeFile = Services.dirsvc.get("GreBinD", Ci.nsIFile);
+    scalarProbeFile.append(scalarProbeFilename);
+    if (!scalarProbeFile.exists()) {
+      this._log.trace("registerJsProbes - no scalar builtin JS probes");
+      return;
+    }
+
+    // Load the file off the disk.
+    let scalarJSProbes = {};
+    try {
+      let fileContent = await OS.File.read(scalarProbeFile.path, { encoding: "utf-8" });
+      scalarJSProbes = JSON.parse(fileContent, (property, value) => {
+        // Fixup the "kind" property: it's a string, and we need the constant
+        // coming from nsITelemetry.
+        if (property !== "kind" || typeof value != "string") {
+          return value;
+        }
+
+        let newValue;
+        switch (value) {
+          case "nsITelemetry::SCALAR_TYPE_COUNT":
+            newValue = Telemetry.SCALAR_TYPE_COUNT;
+            break;
+          case "nsITelemetry::SCALAR_TYPE_BOOLEAN":
+            newValue = Telemetry.SCALAR_TYPE_BOOLEAN;
+            break;
+          case "nsITelemetry::SCALAR_TYPE_STRING":
+            newValue = Telemetry.SCALAR_TYPE_STRING;
+            break;
+        }
+        return newValue;
+      });
+    } catch (ex) {
+      this._log.error(`registerJsProbes - there was an error loading {$scalarProbeFilename}`,
+                      ex);
+    }
+
+    // Register the builtin probes.
+    for (let category in scalarJSProbes) {
+      Telemetry.registerBuiltinScalars(category, scalarJSProbes[category]);
+    }
+  },
 };
--- a/toolkit/components/telemetry/TelemetryScalar.cpp
+++ b/toolkit/components/telemetry/TelemetryScalar.cpp
@@ -132,23 +132,24 @@ struct ScalarKey {
  * Scalar information for dynamic definitions.
  */
 struct DynamicScalarInfo : BaseScalarInfo {
   nsCString mDynamicName;
   bool mDynamicExpiration;
 
   DynamicScalarInfo(uint32_t aKind, bool aRecordOnRelease,
                     bool aExpired, const nsACString& aName,
-                    bool aKeyed)
+                    bool aKeyed, bool aBuiltin)
     : BaseScalarInfo(aKind,
                      aRecordOnRelease ?
                      nsITelemetry::DATASET_RELEASE_CHANNEL_OPTOUT :
                      nsITelemetry::DATASET_RELEASE_CHANNEL_OPTIN,
                      RecordedProcessType::All,
-                     aKeyed)
+                     aKeyed,
+                     aBuiltin)
     , mDynamicName(aName)
     , mDynamicExpiration(aExpired)
   {}
 
   // The following functions will read the stored text
   // instead of looking it up in the statically generated
   // tables.
   const char *name() const override;
@@ -849,16 +850,20 @@ bool gCanRecordExtended;
 ScalarMapType gScalarNameIDMap(kScalarCount);
 
 // The (Process Id -> (Scalar ID -> Scalar Object)) map. This is a nsClassHashtable,
 // it owns the scalar instances and takes care of deallocating them when they are
 // removed from the map.
 ProcessesScalarsMapType gScalarStorageMap;
 // As above, for the keyed scalars.
 ProcessesKeyedScalarsMapType gKeyedScalarStorageMap;
+// Provide separate storage for "dynamic builtin" plain and keyed scalars,
+// needed to support "build faster" in local developer builds.
+ProcessesScalarsMapType gDynamicBuiltinScalarStorageMap;
+ProcessesKeyedScalarsMapType gDynamicBuiltinKeyedScalarStorageMap;
 
 } // namespace
 
 ////////////////////////////////////////////////////////////////////////
 ////////////////////////////////////////////////////////////////////////
 //
 // PRIVATE: Function that may call JS code.
 
@@ -1095,32 +1100,40 @@ internal_GetScalarByEnum(const StaticMut
 {
   if (!internal_IsValidId(lock, aId)) {
     MOZ_ASSERT(false, "Requested a scalar with an invalid id.");
     return NS_ERROR_INVALID_ARG;
   }
 
   const BaseScalarInfo &info = internal_GetScalarInfo(lock, aId);
 
-  // Dynamic scalars fixup: they are always stored in the "dynamic" process.
-  if (aId.dynamic) {
+  // Dynamic scalars fixup: they are always stored in the "dynamic" process,
+  // unless they are part of the "builtin" Firefox probes. Please note that
+  // "dynamic builtin" probes are meant to support "artifact" and "build faster"
+  // builds.
+  if (aId.dynamic && !info.builtin) {
     aProcessStorage = ProcessID::Dynamic;
   }
 
   ScalarBase* scalar = nullptr;
   ScalarStorageMapType* scalarStorage = nullptr;
   // Initialize the scalar storage to the parent storage. This will get
   // set to the child storage if needed.
   uint32_t storageId = static_cast<uint32_t>(aProcessStorage);
 
+  // Put dynamic-builtin scalars (used to support "build faster") in a
+  // separate storage.
+  ProcessesScalarsMapType& processStorage =
+    (aId.dynamic && info.builtin) ? gDynamicBuiltinScalarStorageMap : gScalarStorageMap;
+
   // Get the process-specific storage or create one if it's not
   // available.
-  if (!gScalarStorageMap.Get(storageId, &scalarStorage)) {
+  if (!processStorage.Get(storageId, &scalarStorage)) {
     scalarStorage = new ScalarStorageMapType();
-    gScalarStorageMap.Put(storageId, scalarStorage);
+    processStorage.Put(storageId, scalarStorage);
   }
 
   // Check if the scalar is already allocated in the parent or in the child storage.
   if (scalarStorage->Get(aId.id, &scalar)) {
     // Dynamic scalars can expire at any time during the session (e.g. an
     // add-on was updated). Check if it expired.
     if (aId.dynamic) {
       const DynamicScalarInfo& dynInfo = static_cast<const DynamicScalarInfo&>(info);
@@ -1252,32 +1265,40 @@ internal_GetKeyedScalarByEnum(const Stat
 {
   if (!internal_IsValidId(lock, aId)) {
     MOZ_ASSERT(false, "Requested a keyed scalar with an invalid id.");
     return NS_ERROR_INVALID_ARG;
   }
 
   const BaseScalarInfo &info = internal_GetScalarInfo(lock, aId);
 
-  // Dynamic scalars fixup: they are always stored in the "dynamic" process.
-  if (aId.dynamic) {
+  // Dynamic scalars fixup: they are always stored in the "dynamic" process,
+  // unless they are part of the "builtin" Firefox probes. Please note that
+  // "dynamic builtin" probes are meant to support "artifact" and "build faster"
+  // builds.
+  if (aId.dynamic && !info.builtin) {
     aProcessStorage = ProcessID::Dynamic;
   }
 
   KeyedScalar* scalar = nullptr;
   KeyedScalarStorageMapType* scalarStorage = nullptr;
   // Initialize the scalar storage to the parent storage. This will get
   // set to the child storage if needed.
   uint32_t storageId = static_cast<uint32_t>(aProcessStorage);
 
+  // Put dynamic-builtin scalars (used to support "build faster") in a
+  // separate storage.
+  ProcessesKeyedScalarsMapType& processStorage =
+    (aId.dynamic && info.builtin) ? gDynamicBuiltinKeyedScalarStorageMap : gKeyedScalarStorageMap;
+
   // Get the process-specific storage or create one if it's not
   // available.
-  if (!gKeyedScalarStorageMap.Get(storageId, &scalarStorage)) {
+  if (!processStorage.Get(storageId, &scalarStorage)) {
     scalarStorage = new KeyedScalarStorageMapType();
-    gKeyedScalarStorageMap.Put(storageId, scalarStorage);
+    processStorage.Put(storageId, scalarStorage);
   }
 
   if (scalarStorage->Get(aId.id, &scalar)) {
     *aRet = scalar;
     return NS_OK;
   }
 
   if (IsExpiredVersion(info.expiration())) {
@@ -1420,17 +1441,17 @@ internal_RegisterScalars(const StaticMut
     gDynamicScalarInfo = new nsTArray<DynamicScalarInfo>();
   }
 
   for (auto scalarInfo : scalarInfos) {
     // Allow expiring scalars that were already registered.
     CharPtrEntryType *existingKey = gScalarNameIDMap.GetEntry(scalarInfo.name());
     if (existingKey) {
       // Change the scalar to expired if needed.
-      if (scalarInfo.mDynamicExpiration) {
+      if (scalarInfo.mDynamicExpiration && !scalarInfo.builtin) {
         DynamicScalarInfo& scalarData = (*gDynamicScalarInfo)[existingKey->mData.id];
         scalarData.mDynamicExpiration = true;
       }
       continue;
     }
 
     gDynamicScalarInfo->AppendElement(scalarInfo);
     uint32_t scalarId = gDynamicScalarInfo->Length() - 1;
@@ -1483,16 +1504,18 @@ void
 TelemetryScalar::DeInitializeGlobalState()
 {
   StaticMutexAutoLock locker(gTelemetryScalarsMutex);
   gCanRecordBase = false;
   gCanRecordExtended = false;
   gScalarNameIDMap.Clear();
   gScalarStorageMap.Clear();
   gKeyedScalarStorageMap.Clear();
+  gDynamicBuiltinScalarStorageMap.Clear();
+  gDynamicBuiltinKeyedScalarStorageMap.Clear();
   gDynamicScalarInfo = nullptr;
   gInitDone = false;
 }
 
 void
 TelemetryScalar::SetCanRecordBase(bool b)
 {
   StaticMutexAutoLock locker(gTelemetryScalarsMutex);
@@ -2128,51 +2151,73 @@ TelemetryScalar::CreateSnapshots(unsigne
   }
 
   // Only lock the mutex while accessing our data, without locking any JS related code.
   typedef mozilla::Pair<const char*, nsCOMPtr<nsIVariant>> DataPair;
   typedef nsTArray<DataPair> ScalarArray;
   nsDataHashtable<ProcessIDHashKey, ScalarArray> scalarsToReflect;
   {
     StaticMutexAutoLock locker(gTelemetryScalarsMutex);
-    // Iterate the scalars in gScalarStorageMap. The storage may contain empty or yet to be
-    // initialized scalars from all the supported processes.
-    for (auto iter = gScalarStorageMap.Iter(); !iter.Done(); iter.Next()) {
-      ScalarStorageMapType* scalarStorage = static_cast<ScalarStorageMapType*>(iter.Data());
-      ScalarArray& processScalars = scalarsToReflect.GetOrInsert(iter.Key());
-
-      // Are we in the "Dynamic" process?
-      bool isDynamicProcess = ProcessID::Dynamic == static_cast<ProcessID>(iter.Key());
-
-      // Iterate each available child storage.
-      for (auto childIter = scalarStorage->Iter(); !childIter.Done(); childIter.Next()) {
-        ScalarBase* scalar = static_cast<ScalarBase*>(childIter.Data());
-
-        // Get the informations for this scalar.
-        const BaseScalarInfo& info =
-          internal_GetScalarInfo(locker, ScalarKey{childIter.Key(),
-                                 isDynamicProcess});
-
-        // Serialize the scalar if it's in the desired dataset.
-        if (IsInDataset(info.dataset, aDataset)) {
-          // Get the scalar value.
-          nsCOMPtr<nsIVariant> scalarValue;
-          nsresult rv = scalar->GetValue(scalarValue);
-          if (NS_FAILED(rv)) {
-            return rv;
+
+    // The snapshotting function is the same for both static and dynamic builtin scalars.
+    // We can use the same function and store the scalars in the same output storage.
+    auto snapshotter = [aDataset, &locker, &scalarsToReflect]
+                       (ProcessesScalarsMapType& aProcessStorage, bool aIsBuiltinDynamic)
+                       -> nsresult
+    {
+      // Iterate the scalars in aProcessStorage. The storage may contain empty or yet to be
+      // initialized scalars from all the supported processes.
+      for (auto iter = aProcessStorage.Iter(); !iter.Done(); iter.Next()) {
+        ScalarStorageMapType* scalarStorage = static_cast<ScalarStorageMapType*>(iter.Data());
+        ScalarArray& processScalars = scalarsToReflect.GetOrInsert(iter.Key());
+
+        // Are we in the "Dynamic" process?
+        bool isDynamicProcess = ProcessID::Dynamic == static_cast<ProcessID>(iter.Key());
+
+        // Iterate each available child storage.
+        for (auto childIter = scalarStorage->Iter(); !childIter.Done(); childIter.Next()) {
+          ScalarBase* scalar = static_cast<ScalarBase*>(childIter.Data());
+
+          // Get the informations for this scalar.
+          const BaseScalarInfo& info =
+            internal_GetScalarInfo(locker, ScalarKey{childIter.Key(),
+                                   aIsBuiltinDynamic ? true : isDynamicProcess});
+
+          // Serialize the scalar if it's in the desired dataset.
+          if (IsInDataset(info.dataset, aDataset)) {
+            // Get the scalar value.
+            nsCOMPtr<nsIVariant> scalarValue;
+            nsresult rv = scalar->GetValue(scalarValue);
+            if (NS_FAILED(rv)) {
+              return rv;
+            }
+            // Append it to our list.
+            processScalars.AppendElement(mozilla::MakePair(info.name(), scalarValue));
           }
-          // Append it to our list.
-          processScalars.AppendElement(mozilla::MakePair(info.name(), scalarValue));
         }
       }
+      return NS_OK;
+    };
+
+    // Take a snapshot of the scalars.
+    nsresult rv = snapshotter(gScalarStorageMap, false);
+    if (NS_FAILED(rv)) {
+      return rv;
+    }
+
+    // And a snapshot of the dynamic builtin ones.
+    rv = snapshotter(gDynamicBuiltinScalarStorageMap, true);
+    if (NS_FAILED(rv)) {
+      return rv;
     }
 
     if (aClearScalars) {
       // The map already takes care of freeing the allocated memory.
       gScalarStorageMap.Clear();
+      gDynamicBuiltinScalarStorageMap.Clear();
     }
   }
 
   // Reflect it to JS.
   for (auto iter = scalarsToReflect.Iter(); !iter.Done(); iter.Next()) {
     ScalarArray& processScalars = iter.Data();
     const char* processName = GetNameForProcessID(ProcessID(iter.Key()));
 
@@ -2236,51 +2281,71 @@ TelemetryScalar::CreateKeyedSnapshots(un
   }
 
   // Only lock the mutex while accessing our data, without locking any JS related code.
   typedef mozilla::Pair<const char*, nsTArray<KeyedScalar::KeyValuePair>> DataPair;
   typedef nsTArray<DataPair> ScalarArray;
   nsDataHashtable<ProcessIDHashKey, ScalarArray> scalarsToReflect;
   {
     StaticMutexAutoLock locker(gTelemetryScalarsMutex);
-    // Iterate the scalars in gKeyedScalarStorageMap. The storage may contain empty or yet
-    // to be initialized scalars from all the supported processes.
-    for (auto iter = gKeyedScalarStorageMap.Iter(); !iter.Done(); iter.Next()) {
-      KeyedScalarStorageMapType* scalarStorage =
-        static_cast<KeyedScalarStorageMapType*>(iter.Data());
-      ScalarArray& processScalars = scalarsToReflect.GetOrInsert(iter.Key());
-
-      // Are we in the "Dynamic" process?
-      bool isDynamicProcess = ProcessID::Dynamic == static_cast<ProcessID>(iter.Key());
-
-      for (auto childIter = scalarStorage->Iter(); !childIter.Done(); childIter.Next()) {
-        KeyedScalar* scalar = static_cast<KeyedScalar*>(childIter.Data());
-
-        // Get the informations for this scalar.
-        const BaseScalarInfo& info =
-          internal_GetScalarInfo(locker, ScalarKey{childIter.Key(),
-                                 isDynamicProcess});
-
-        // Serialize the scalar if it's in the desired dataset.
-        if (IsInDataset(info.dataset, aDataset)) {
-          // Get the keys for this scalar.
-          nsTArray<KeyedScalar::KeyValuePair> scalarKeyedData;
-          nsresult rv = scalar->GetValue(scalarKeyedData);
-          if (NS_FAILED(rv)) {
-            return rv;
+
+    auto snapshotter = [aDataset, &locker, &scalarsToReflect]
+                       (ProcessesKeyedScalarsMapType& aProcessStorage,
+                        bool aIsBuiltinDynamic) -> nsresult
+    {
+      // Iterate the scalars in aProcessStorage. The storage may contain empty or yet
+      // to be initialized scalars from all the supported processes.
+      for (auto iter = aProcessStorage.Iter(); !iter.Done(); iter.Next()) {
+        KeyedScalarStorageMapType* scalarStorage =
+          static_cast<KeyedScalarStorageMapType*>(iter.Data());
+        ScalarArray& processScalars = scalarsToReflect.GetOrInsert(iter.Key());
+
+        // Are we in the "Dynamic" process?
+        bool isDynamicProcess = ProcessID::Dynamic == static_cast<ProcessID>(iter.Key());
+
+        for (auto childIter = scalarStorage->Iter(); !childIter.Done(); childIter.Next()) {
+          KeyedScalar* scalar = static_cast<KeyedScalar*>(childIter.Data());
+
+          // Get the informations for this scalar.
+          const BaseScalarInfo& info =
+            internal_GetScalarInfo(locker, ScalarKey{childIter.Key(),
+                                   aIsBuiltinDynamic ? true : isDynamicProcess});
+
+          // Serialize the scalar if it's in the desired dataset.
+          if (IsInDataset(info.dataset, aDataset)) {
+            // Get the keys for this scalar.
+            nsTArray<KeyedScalar::KeyValuePair> scalarKeyedData;
+            nsresult rv = scalar->GetValue(scalarKeyedData);
+            if (NS_FAILED(rv)) {
+              return rv;
+            }
+            // Append it to our list.
+            processScalars.AppendElement(mozilla::MakePair(info.name(), scalarKeyedData));
           }
-          // Append it to our list.
-          processScalars.AppendElement(mozilla::MakePair(info.name(), scalarKeyedData));
         }
       }
+      return NS_OK;
+    };
+
+    // Take a snapshot of the scalars.
+    nsresult rv = snapshotter(gKeyedScalarStorageMap, false);
+    if (NS_FAILED(rv)) {
+      return rv;
+    }
+
+    // And a snapshot of the dynamic builtin ones.
+    rv = snapshotter(gDynamicBuiltinKeyedScalarStorageMap, true);
+    if (NS_FAILED(rv)) {
+      return rv;
     }
 
     if (aClearScalars) {
       // The map already takes care of freeing the allocated memory.
       gKeyedScalarStorageMap.Clear();
+      gDynamicBuiltinKeyedScalarStorageMap.Clear();
     }
   }
 
   // Reflect it to JS.
   for (auto iter = scalarsToReflect.Iter(); !iter.Done(); iter.Next()) {
     ScalarArray& processScalars = iter.Data();
     const char* processName = GetNameForProcessID(ProcessID(iter.Key()));
 
@@ -2328,16 +2393,17 @@ TelemetryScalar::CreateKeyedSnapshots(un
   }
 
   return NS_OK;
 }
 
 nsresult
 TelemetryScalar::RegisterScalars(const nsACString& aCategoryName,
                                  JS::Handle<JS::Value> aScalarData,
+                                 bool aBuiltin,
                                  JSContext* cx)
 {
   MOZ_ASSERT(XRE_IsParentProcess(),
              "Dynamic scalars should only be created in the parent process.");
 
   if (!IsValidIdentifierString(aCategoryName, kMaximumCategoryNameLength, true, false)) {
     JS_ReportErrorASCII(cx, "Invalid category name %s.",
                         PromiseFlatCString(aCategoryName).get());
@@ -2426,17 +2492,17 @@ TelemetryScalar::RegisterScalars(const n
         return NS_ERROR_FAILURE;
       }
       expired = static_cast<bool>(value.toBoolean());
     }
 
     // We defer the actual registration here in case any other event description is invalid.
     // In that case we don't need to roll back any partial registration.
     newScalarInfos.AppendElement(DynamicScalarInfo{
-      kind, recordOnRelease, expired, fullName, keyed
+      kind, recordOnRelease, expired, fullName, keyed, aBuiltin
     });
   }
 
   // Register the dynamic definition on the parent process.
   {
     StaticMutexAutoLock locker(gTelemetryScalarsMutex);
     ::internal_RegisterScalars(locker, newScalarInfos);
 
@@ -2456,47 +2522,52 @@ TelemetryScalar::ClearScalars()
   MOZ_ASSERT(XRE_IsParentProcess(), "Scalars should only be cleared in the parent process.");
   if (!XRE_IsParentProcess()) {
     return;
   }
 
   StaticMutexAutoLock locker(gTelemetryScalarsMutex);
   gScalarStorageMap.Clear();
   gKeyedScalarStorageMap.Clear();
+  gDynamicBuiltinScalarStorageMap.Clear();
+  gDynamicBuiltinKeyedScalarStorageMap.Clear();
 }
 
 size_t
 TelemetryScalar::GetMapShallowSizesOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf)
 {
   StaticMutexAutoLock locker(gTelemetryScalarsMutex);
   return gScalarNameIDMap.ShallowSizeOfExcludingThis(aMallocSizeOf);
 }
 
 size_t
 TelemetryScalar::GetScalarSizesOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf)
 {
   StaticMutexAutoLock locker(gTelemetryScalarsMutex);
   size_t n = 0;
-  // Account for scalar data coming from parent and child processes.
-  for (auto iter = gScalarStorageMap.Iter(); !iter.Done(); iter.Next()) {
-    ScalarStorageMapType* scalarStorage = static_cast<ScalarStorageMapType*>(iter.Data());
-    for (auto childIter = scalarStorage->Iter(); !childIter.Done(); childIter.Next()) {
-      ScalarBase* scalar = static_cast<ScalarBase*>(childIter.Data());
-      n += scalar->SizeOfIncludingThis(aMallocSizeOf);
+
+  auto getSizeOf = [aMallocSizeOf](auto &storageMap)
+  {
+    size_t partial = 0;
+    for (auto iter = storageMap.Iter(); !iter.Done(); iter.Next()) {
+      auto scalarStorage = iter.UserData();
+      for (auto childIter = scalarStorage->Iter(); !childIter.Done(); childIter.Next()) {
+        auto scalar = childIter.UserData();
+        partial += scalar->SizeOfIncludingThis(aMallocSizeOf);
+      }
     }
-  }
-  // Also account for keyed scalar data coming from parent and child processes.
-  for (auto iter = gKeyedScalarStorageMap.Iter(); !iter.Done(); iter.Next()) {
-    KeyedScalarStorageMapType* scalarStorage =
-      static_cast<KeyedScalarStorageMapType*>(iter.Data());
-    for (auto childIter = scalarStorage->Iter(); !childIter.Done(); childIter.Next()) {
-      KeyedScalar* scalar = static_cast<KeyedScalar*>(childIter.Data());
-      n += scalar->SizeOfIncludingThis(aMallocSizeOf);
-    }
-  }
+    return partial;
+  };
+
+  // Account for all the storage used for the different scalar types.
+  n += getSizeOf(gScalarStorageMap);
+  n += getSizeOf(gKeyedScalarStorageMap);
+  n += getSizeOf(gDynamicBuiltinScalarStorageMap);
+  n += getSizeOf(gDynamicBuiltinKeyedScalarStorageMap);
+
   return n;
 }
 
 void
 TelemetryScalar::UpdateChildData(ProcessID aProcessType,
                                  const nsTArray<mozilla::Telemetry::ScalarAction>& aScalarActions)
 {
   MOZ_ASSERT(XRE_IsParentProcess(),
@@ -2761,16 +2832,17 @@ TelemetryScalar::AddDynamicScalarDefinit
   // Populate the definitions array before acquiring the lock.
   for (auto def : aDefs) {
     bool recordOnRelease = def.dataset == nsITelemetry::DATASET_RELEASE_CHANNEL_OPTOUT;
     dynamicStubs.AppendElement(DynamicScalarInfo{
       def.type,
       recordOnRelease,
       def.expired,
       def.name,
-      def.keyed});
+      def.keyed,
+      false /* builtin */});
   }
 
   {
     StaticMutexAutoLock locker(gTelemetryScalarsMutex);
     internal_RegisterScalars(locker, dynamicStubs);
   }
 }
--- a/toolkit/components/telemetry/TelemetryScalar.h
+++ b/toolkit/components/telemetry/TelemetryScalar.h
@@ -51,17 +51,17 @@ void SetMaximum(mozilla::Telemetry::Scal
 
 // Keyed C++ API Endpoints.
 void Add(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
 void Set(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
 void Set(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, bool aValue);
 void SetMaximum(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
 
 nsresult RegisterScalars(const nsACString& aCategoryName, JS::Handle<JS::Value> aScalarData,
-                         JSContext* cx);
+                         bool aBuiltin, JSContext* cx);
 
 // Only to be used for testing.
 void ClearScalars();
 
 size_t GetMapShallowSizesOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf);
 size_t GetScalarSizesOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf);
 
 void UpdateChildData(mozilla::Telemetry::ProcessID aProcessType,
--- a/toolkit/components/telemetry/gen_scalar_data.py
+++ b/toolkit/components/telemetry/gen_scalar_data.py
@@ -1,18 +1,20 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 # Write out scalar information for C++.  The scalars are defined
 # in a file provided as a command-line argument.
 
 from __future__ import print_function
+from collections import OrderedDict
 from shared_telemetry_utils import StringTable, static_assert, ParserError
 
+import json
 import parse_scalars
 import sys
 
 # The banner/text at the top of the generated file.
 banner = """/* This file is auto-generated, only for internal use in TelemetryScalar.h,
    see gen_scalar_data.py. */
 """
 
@@ -73,27 +75,59 @@ def write_scalar_tables(scalars, output)
     print("};", file=output)
 
     string_table_name = "gScalarsStringTable"
     string_table.writeDefinition(output, string_table_name)
     static_assert(output, "sizeof(%s) <= UINT32_MAX" % string_table_name,
                   "index overflow")
 
 
-def main(output, *filenames):
-    # Load the scalars first.
+def parse_scalar_definitions(filenames):
     if len(filenames) > 1:
         raise Exception('We don\'t support loading from more than one file.')
 
     try:
-        scalars = parse_scalars.load_scalars(filenames[0])
+        return parse_scalars.load_scalars(filenames[0])
     except ParserError as ex:
         print("\nError processing scalars:\n" + str(ex) + "\n")
         sys.exit(1)
 
+
+def generate_JSON_definitions(output, *filenames):
+    """ Write the scalar definitions to a JSON file.
+
+    :param output: the file to write the content to.
+    :param filenames: a list of filenames provided by the build system.
+           We only support a single file.
+    """
+    scalars = parse_scalar_definitions(filenames)
+
+    scalar_definitions = OrderedDict()
+    for scalar in scalars:
+        category = scalar.category
+
+        if category not in scalar_definitions:
+            scalar_definitions[category] = OrderedDict()
+
+        scalar_definitions[category][scalar.name] = OrderedDict({
+            'kind': scalar.nsITelemetry_kind,
+            'keyed': scalar.keyed,
+            'record_on_release': True if scalar.dataset == 'opt-out' else False,
+            # We don't expire dynamic-builtin scalars: they're only meant for
+            # use in local developer builds anyway. They will expire when rebuilding.
+            'expired': False,
+        })
+
+    json.dump(scalar_definitions, output)
+
+
+def main(output, *filenames):
+    # Load the scalars first.
+    scalars = parse_scalar_definitions(filenames)
+
     # Write the scalar data file.
     print(banner, file=output)
     print(file_header, file=output)
     write_scalar_tables(scalars, output)
     print(file_footer, file=output)
 
 
 if __name__ == '__main__':
--- a/toolkit/components/telemetry/moz.build
+++ b/toolkit/components/telemetry/moz.build
@@ -105,16 +105,17 @@ TESTING_JS_MODULES += [
   'tests/unit/TelemetryArchiveTesting.jsm',
 ]
 
 PYTHON_UNITTEST_MANIFESTS += [
     'tests/python/python.ini',
 ]
 
 GENERATED_FILES = [
+    'ScalarArtifactDefinitions.json',
     'TelemetryEventData.h',
     'TelemetryEventEnums.h',
     'TelemetryHistogramData.inc',
     'TelemetryHistogramEnums.h',
     'TelemetryProcessData.h',
     'TelemetryProcessEnums.h',
     'TelemetryScalarData.h',
     'TelemetryScalarEnums.h',
@@ -143,16 +144,25 @@ scalar_files = [
 scalar_data = GENERATED_FILES['TelemetryScalarData.h']
 scalar_data.script = 'gen_scalar_data.py'
 scalar_data.inputs = scalar_files
 
 scalar_enums = GENERATED_FILES['TelemetryScalarEnums.h']
 scalar_enums.script = 'gen_scalar_enum.py'
 scalar_enums.inputs = scalar_files
 
+# Generate the JSON scalar definitions. They will only be
+# used in artifact or "build faster" builds.
+scalar_json_data = GENERATED_FILES['ScalarArtifactDefinitions.json']
+scalar_json_data.script = 'gen_scalar_data.py:generate_JSON_definitions'
+scalar_json_data.inputs = scalar_files
+
+# Move the scalars JSON file to the directory where the Firefox binary is.
+FINAL_TARGET_FILES += ['!ScalarArtifactDefinitions.json']
+
 # Generate event files.
 event_files = [
     'Events.yaml',
 ]
 
 event_data = GENERATED_FILES['TelemetryEventData.h']
 event_data.script = 'gen_event_data.py'
 event_data.inputs = event_files
--- a/toolkit/components/telemetry/nsITelemetry.idl
+++ b/toolkit/components/telemetry/nsITelemetry.idl
@@ -537,12 +537,23 @@ interface nsITelemetry : nsISupports
    *                                             Defaults to false.
    * @param aScalarData.<name>.expired Optional, whether this scalar entry is expired. This allows
    *                                   recording it without error, but it will be discarded. Defaults to false.
    */
   [implicit_jscontext]
   void registerScalars(in ACString aCategoryName, in jsval aScalarData);
 
   /**
+   * Parent process only. Register dynamic builtin scalars. The parameters
+   * have the same meaning as the usual |registerScalars| function.
+   *
+   * This function is only meant to be used to support the "artifact build"/
+   * "built faster" developers by allowing to add new scalars without rebuilding
+   * the C++ components including the headers files.
+   */
+  [implicit_jscontext]
+  void registerBuiltinScalars(in ACString aCategoryName, in jsval aScalarData);
+
+  /**
    * Resets all the stored events. This is intended to be only used in tests.
    */
   void clearEvents();
 };
--- a/toolkit/components/telemetry/parse_scalars.py
+++ b/toolkit/components/telemetry/parse_scalars.py
@@ -194,16 +194,21 @@ class ScalarType:
         # using the deprecated format 'N.Na1'. Those scripts set
         # self._strict_type_checks to false.
         expires = definition.get('expires')
         if not utils.validate_expiration_version(expires) and self._strict_type_checks:
             raise ParserError('{} - invalid expires: {}.\nSee: {}#required-fields'
                               .format(self._name, expires, BASE_DOC_URL))
 
     @property
+    def category(self):
+        """Get the category name"""
+        return self._category_name
+
+    @property
     def name(self):
         """Get the scalar name"""
         return self._name
 
     @property
     def label(self):
         """Get the scalar label generated from the scalar and category names."""
         return self._category_name + '.' + self._name
new file mode 100644
--- /dev/null
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryScalars_buildFaster.js
@@ -0,0 +1,137 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/
+*/
+
+const UINT_SCALAR = "telemetry.test.unsigned_int_kind";
+const STRING_SCALAR = "telemetry.test.string_kind";
+const BOOLEAN_SCALAR = "telemetry.test.boolean_kind";
+const KEYED_UINT_SCALAR = "telemetry.test.keyed_unsigned_int";
+
+ChromeUtils.import("resource://services-common/utils.js");
+
+/**
+ * Return the path to the definitions file for the scalars.
+ */
+function getDefinitionsPath() {
+  // Write the scalar definition to the spec file in the binary directory.
+  let definitionFile = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile);
+  definitionFile = Services.dirsvc.get("GreBinD", Ci.nsIFile);
+  definitionFile.append("ScalarArtifactDefinitions.json");
+  return definitionFile.path;
+}
+
+add_task(async function test_setup() {
+  do_get_profile();
+});
+
+add_task({
+    // The test needs to write a file, and that fails in tests on Android.
+    // We don't really need the Android coverage, so skip on Android.
+    skip_if: () => AppConstants.platform == "android"
+  }, async function test_invalidJSON() {
+  const INVALID_JSON = "{ invalid,JSON { {1}";
+  const FILE_PATH = getDefinitionsPath();
+
+  // Write a corrupted JSON file.
+  await OS.File.writeAtomic(FILE_PATH, INVALID_JSON, { encoding: "utf-8", noOverwrite: false });
+
+  // Simulate Firefox startup. This should not throw!
+  await TelemetryController.testSetup();
+  await TelemetryController.testPromiseJsProbeRegistration();
+
+  // Cleanup.
+  await TelemetryController.testShutdown();
+  await OS.File.remove(FILE_PATH);
+});
+
+add_task({
+    // The test needs to write a file, and that fails in tests on Android.
+    // We don't really need the Android coverage, so skip on Android.
+    skip_if: () => AppConstants.platform == "android"
+  }, async function test_dynamicBuiltin() {
+  const DYNAMIC_SCALAR_SPEC =  {
+    "telemetry.test": {
+      "builtin_dynamic": {
+        "kind": "nsITelemetry::SCALAR_TYPE_COUNT",
+        "expired": false,
+        "record_on_release": false,
+        "keyed": false
+      },
+      "builtin_dynamic_other": {
+        "kind": "nsITelemetry::SCALAR_TYPE_BOOLEAN",
+        "expired": false,
+        "record_on_release": false,
+        "keyed": false
+      }
+    }
+  };
+
+  Telemetry.clearScalars();
+
+  // Let's write to the definition file to also cover the file
+  // loading part.
+  const FILE_PATH = getDefinitionsPath();
+  await CommonUtils.writeJSON(DYNAMIC_SCALAR_SPEC, FILE_PATH);
+
+  // Start TelemetryController to trigger loading the specs.
+  await TelemetryController.testReset();
+  await TelemetryController.testPromiseJsProbeRegistration();
+
+  // Store to that scalar.
+  const TEST_SCALAR1 = "telemetry.test.builtin_dynamic";
+  const TEST_SCALAR2 = "telemetry.test.builtin_dynamic_other";
+  Telemetry.scalarSet(TEST_SCALAR1, 3785);
+  Telemetry.scalarSet(TEST_SCALAR2, true);
+
+  // Check the values we tried to store.
+  const scalars =
+    Telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, false).parent;
+
+  // Check that they are serialized to the correct format.
+  Assert.equal(typeof(scalars[TEST_SCALAR1]), "number",
+               TEST_SCALAR1 + " must be serialized to the correct format.");
+  Assert.ok(Number.isInteger(scalars[TEST_SCALAR1]),
+               TEST_SCALAR1 + " must be a finite integer.");
+  Assert.equal(scalars[TEST_SCALAR1], 3785,
+               TEST_SCALAR1 + " must have the correct value.");
+  Assert.equal(typeof(scalars[TEST_SCALAR2]), "boolean",
+               TEST_SCALAR2 + " must be serialized to the correct format.");
+  Assert.equal(scalars[TEST_SCALAR2], true,
+               TEST_SCALAR2 + " must have the correct value.");
+
+  // Clean up.
+  await TelemetryController.testShutdown();
+  await OS.File.remove(FILE_PATH);
+});
+
+add_task(async function test_keyedDynamicBuiltin() {
+  Telemetry.clearScalars();
+
+  // Register the built-in scalars (let's not take the I/O hit).
+  Telemetry.registerBuiltinScalars("telemetry.test", {
+    "builtin_dynamic_keyed": {
+      "kind": Ci.nsITelemetry.SCALAR_TYPE_COUNT,
+      "expired": false,
+      "record_on_release": false,
+      "keyed": true
+    }
+  });
+
+  // Store to that scalar.
+  const TEST_SCALAR1 = "telemetry.test.builtin_dynamic_keyed";
+  Telemetry.keyedScalarSet(TEST_SCALAR1, "test-key", 3785);
+
+  // Check the values we tried to store.
+  const scalars =
+    Telemetry.snapshotKeyedScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, false).parent;
+
+  // Check that they are serialized to the correct format.
+  Assert.equal(typeof(scalars[TEST_SCALAR1]), "object",
+               TEST_SCALAR1 + " must be a keyed scalar.");
+  Assert.equal(typeof(scalars[TEST_SCALAR1]["test-key"]), "number",
+               TEST_SCALAR1 + " must be serialized to the correct format.");
+  Assert.ok(Number.isInteger(scalars[TEST_SCALAR1]["test-key"]),
+               TEST_SCALAR1 + " must be a finite integer.");
+  Assert.equal(scalars[TEST_SCALAR1]["test-key"], 3785,
+               TEST_SCALAR1 + " must have the correct value.");
+});
--- a/toolkit/components/telemetry/tests/unit/xpcshell.ini
+++ b/toolkit/components/telemetry/tests/unit/xpcshell.ini
@@ -59,16 +59,17 @@ skip-if = os == "android"
 skip-if = os == "android" # Disabled due to crashes (see bug 1331366)
 tags = addons
 [test_ChildScalars.js]
 skip-if = os == "android" # Disabled due to crashes (see bug 1331366)
 [test_TelemetryReportingPolicy.js]
 skip-if = os == "android" # Disabled due to crashes (see bug 1367762)
 tags = addons
 [test_TelemetryScalars.js]
+[test_TelemetryScalars_buildFaster.js]
 [test_TelemetryTimestamps.js]
 skip-if = toolkit == 'android'
 [test_TelemetryCaptureStack.js]
 [test_TelemetryEvents.js]
 [test_ChildEvents.js]
 skip-if = os == "android" # Disabled due to crashes (see bug 1331366)
 [test_TelemetryModules.js]
 [test_PingSender.js]