--- a/toolkit/components/telemetry/ScalarInfo.h
+++ b/toolkit/components/telemetry/ScalarInfo.h
@@ -18,24 +18,26 @@ namespace {
/**
* Base scalar information, common to both "static" and dynamic scalars.
*/
struct BaseScalarInfo {
uint32_t kind;
uint32_t dataset;
mozilla::Telemetry::Common::RecordedProcessType record_in_processes;
bool keyed;
+ bool builtin;
BaseScalarInfo(uint32_t aKind, uint32_t aDataset,
mozilla::Telemetry::Common::RecordedProcessType aRecordInProcess,
- bool aKeyed)
+ bool aKeyed, bool aBuiltin = true)
: kind(aKind)
, dataset(aDataset)
, record_in_processes(aRecordInProcess)
, keyed(aKeyed)
+ , builtin(aBuiltin)
{}
virtual ~BaseScalarInfo() {}
virtual const char *name() const = 0;
virtual const char *expiration() const = 0;
};
/**
--- a/toolkit/components/telemetry/Telemetry.cpp
+++ b/toolkit/components/telemetry/Telemetry.cpp
@@ -1722,17 +1722,25 @@ TelemetryImpl::SnapshotKeyedScalars(unsi
aResult);
}
NS_IMETHODIMP
TelemetryImpl::RegisterScalars(const nsACString& aCategoryName,
JS::Handle<JS::Value> aScalarData,
JSContext* cx)
{
- return TelemetryScalar::RegisterScalars(aCategoryName, aScalarData, cx);
+ return TelemetryScalar::RegisterScalars(aCategoryName, aScalarData, false, cx);
+}
+
+NS_IMETHODIMP
+TelemetryImpl::RegisterBuiltinScalars(const nsACString& aCategoryName,
+ JS::Handle<JS::Value> aScalarData,
+ JSContext* cx)
+{
+ return TelemetryScalar::RegisterScalars(aCategoryName, aScalarData, true, cx);
}
NS_IMETHODIMP
TelemetryImpl::ClearScalars()
{
TelemetryScalar::ClearScalars();
return NS_OK;
}
--- a/toolkit/components/telemetry/TelemetryController.jsm
+++ b/toolkit/components/telemetry/TelemetryController.jsm
@@ -62,16 +62,17 @@ XPCOMUtils.defineLazyModuleGetters(this,
UpdateUtils: "resource://gre/modules/UpdateUtils.jsm",
TelemetryArchive: "resource://gre/modules/TelemetryArchive.jsm",
TelemetrySession: "resource://gre/modules/TelemetrySession.jsm",
TelemetrySend: "resource://gre/modules/TelemetrySend.jsm",
TelemetryReportingPolicy: "resource://gre/modules/TelemetryReportingPolicy.jsm",
TelemetryModules: "resource://gre/modules/TelemetryModules.jsm",
UpdatePing: "resource://gre/modules/UpdatePing.jsm",
TelemetryHealthPing: "resource://gre/modules/TelemetryHealthPing.jsm",
+ OS: "resource://gre/modules/osfile.jsm",
});
/**
* Setup Telemetry logging. This function also gets called when loggin related
* preferences change.
*/
var gLogger = null;
var gLogAppenderDump = null;
@@ -152,16 +153,23 @@ this.TelemetryController = Object.freeze
/**
* Used only for testing purposes.
*/
testSetupContent() {
return Impl.setupContentTelemetry(true);
},
/**
+ * Used only for testing purposes.
+ */
+ testPromiseJsProbeRegistration() {
+ return Promise.resolve(Impl._probeRegistrationPromise);
+ },
+
+ /**
* Send a notification.
*/
observe(aSubject, aTopic, aData) {
return Impl.observe(aSubject, aTopic, aData);
},
/**
* Submit ping payloads to Telemetry. This will assemble a complete ping, adding
@@ -317,16 +325,18 @@ var Impl = {
// After this barrier, clients can not submit Telemetry pings anymore.
_shutdownBarrier: new AsyncShutdown.Barrier("TelemetryController: Waiting for clients."),
// This is a private barrier blocked by pending async ping activity (sending & saving).
_connectionsBarrier: new AsyncShutdown.Barrier("TelemetryController: Waiting for pending ping activity"),
// This is true when running in the test infrastructure.
_testMode: false,
// The task performing the delayed sending of the "new-profile" ping.
_delayedNewPingTask: null,
+ // The promise used to wait for the JS probe registration (dynamic builtin).
+ _probeRegistrationPromise: null,
get _log() {
if (!this._logger) {
this._logger = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX);
}
return this._logger;
},
@@ -674,16 +684,21 @@ var Impl = {
return this._delayedInitTaskDeferred.promise;
}
if (this._initialized && !this._testMode) {
this._log.error("setupTelemetry - already initialized");
return Promise.resolve();
}
+ // Enable adding scalars in artifact builds and build faster modes.
+ // The function is async: we intentionally don't wait for it to complete
+ // as we don't want to delay startup.
+ this._probeRegistrationPromise = this.registerJsProbes();
+
// This will trigger displaying the datachoices infobar.
TelemetryReportingPolicy.setup();
if (!this.enableTelemetryRecording()) {
this._log.config("setupChromeProcess - Telemetry recording is disabled, skipping Chrome process setup.");
return Promise.resolve();
}
@@ -1033,9 +1048,69 @@ var Impl = {
};
// TODO: we need to be smarter about when to send the ping (and save the
// state to file). |requestIdleCallback| is currently only accessible
// through DOM. See bug 1361996.
await TelemetryController.submitExternalPing("new-profile", payload, options)
.then(() => TelemetrySession.markNewProfilePingSent(),
e => this._log.error("sendNewProfilePing - failed to submit new-profile ping", e));
},
+
+ /**
+ * Register 'dynamic builtin' probes from the JSON definition files.
+ * This is needed to support adding new probes in developer builds
+ * without rebuilding the whole codebase.
+ *
+ * This is not meant to be used outside of local developer builds.
+ */
+ async registerJsProbes() {
+ // We don't support this outside of developer builds.
+ if (AppConstants.MOZILLA_OFFICIAL && !this._testMode) {
+ return;
+ }
+
+ this._log.trace("registerJsProbes - registering builtin JS probes");
+
+ // Load the scalar probes JSON file.
+ const scalarProbeFilename = "ScalarArtifactDefinitions.json";
+ let scalarProbeFile = Services.dirsvc.get("GreBinD", Ci.nsIFile);
+ scalarProbeFile.append(scalarProbeFilename);
+ if (!scalarProbeFile.exists()) {
+ this._log.trace("registerJsProbes - no scalar builtin JS probes");
+ return;
+ }
+
+ // Load the file off the disk.
+ let scalarJSProbes = {};
+ try {
+ let fileContent = await OS.File.read(scalarProbeFile.path, { encoding: "utf-8" });
+ scalarJSProbes = JSON.parse(fileContent, (property, value) => {
+ // Fixup the "kind" property: it's a string, and we need the constant
+ // coming from nsITelemetry.
+ if (property !== "kind" || typeof value != "string") {
+ return value;
+ }
+
+ let newValue;
+ switch (value) {
+ case "nsITelemetry::SCALAR_TYPE_COUNT":
+ newValue = Telemetry.SCALAR_TYPE_COUNT;
+ break;
+ case "nsITelemetry::SCALAR_TYPE_BOOLEAN":
+ newValue = Telemetry.SCALAR_TYPE_BOOLEAN;
+ break;
+ case "nsITelemetry::SCALAR_TYPE_STRING":
+ newValue = Telemetry.SCALAR_TYPE_STRING;
+ break;
+ }
+ return newValue;
+ });
+ } catch (ex) {
+ this._log.error(`registerJsProbes - there was an error loading {$scalarProbeFilename}`,
+ ex);
+ }
+
+ // Register the builtin probes.
+ for (let category in scalarJSProbes) {
+ Telemetry.registerBuiltinScalars(category, scalarJSProbes[category]);
+ }
+ },
};
--- a/toolkit/components/telemetry/TelemetryScalar.cpp
+++ b/toolkit/components/telemetry/TelemetryScalar.cpp
@@ -132,23 +132,24 @@ struct ScalarKey {
* Scalar information for dynamic definitions.
*/
struct DynamicScalarInfo : BaseScalarInfo {
nsCString mDynamicName;
bool mDynamicExpiration;
DynamicScalarInfo(uint32_t aKind, bool aRecordOnRelease,
bool aExpired, const nsACString& aName,
- bool aKeyed)
+ bool aKeyed, bool aBuiltin)
: BaseScalarInfo(aKind,
aRecordOnRelease ?
nsITelemetry::DATASET_RELEASE_CHANNEL_OPTOUT :
nsITelemetry::DATASET_RELEASE_CHANNEL_OPTIN,
RecordedProcessType::All,
- aKeyed)
+ aKeyed,
+ aBuiltin)
, mDynamicName(aName)
, mDynamicExpiration(aExpired)
{}
// The following functions will read the stored text
// instead of looking it up in the statically generated
// tables.
const char *name() const override;
@@ -849,16 +850,20 @@ bool gCanRecordExtended;
ScalarMapType gScalarNameIDMap(kScalarCount);
// The (Process Id -> (Scalar ID -> Scalar Object)) map. This is a nsClassHashtable,
// it owns the scalar instances and takes care of deallocating them when they are
// removed from the map.
ProcessesScalarsMapType gScalarStorageMap;
// As above, for the keyed scalars.
ProcessesKeyedScalarsMapType gKeyedScalarStorageMap;
+// Provide separate storage for "dynamic builtin" plain and keyed scalars,
+// needed to support "build faster" in local developer builds.
+ProcessesScalarsMapType gDynamicBuiltinScalarStorageMap;
+ProcessesKeyedScalarsMapType gDynamicBuiltinKeyedScalarStorageMap;
} // namespace
////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
//
// PRIVATE: Function that may call JS code.
@@ -1095,32 +1100,40 @@ internal_GetScalarByEnum(const StaticMut
{
if (!internal_IsValidId(lock, aId)) {
MOZ_ASSERT(false, "Requested a scalar with an invalid id.");
return NS_ERROR_INVALID_ARG;
}
const BaseScalarInfo &info = internal_GetScalarInfo(lock, aId);
- // Dynamic scalars fixup: they are always stored in the "dynamic" process.
- if (aId.dynamic) {
+ // Dynamic scalars fixup: they are always stored in the "dynamic" process,
+ // unless they are part of the "builtin" Firefox probes. Please note that
+ // "dynamic builtin" probes are meant to support "artifact" and "build faster"
+ // builds.
+ if (aId.dynamic && !info.builtin) {
aProcessStorage = ProcessID::Dynamic;
}
ScalarBase* scalar = nullptr;
ScalarStorageMapType* scalarStorage = nullptr;
// Initialize the scalar storage to the parent storage. This will get
// set to the child storage if needed.
uint32_t storageId = static_cast<uint32_t>(aProcessStorage);
+ // Put dynamic-builtin scalars (used to support "build faster") in a
+ // separate storage.
+ ProcessesScalarsMapType& processStorage =
+ (aId.dynamic && info.builtin) ? gDynamicBuiltinScalarStorageMap : gScalarStorageMap;
+
// Get the process-specific storage or create one if it's not
// available.
- if (!gScalarStorageMap.Get(storageId, &scalarStorage)) {
+ if (!processStorage.Get(storageId, &scalarStorage)) {
scalarStorage = new ScalarStorageMapType();
- gScalarStorageMap.Put(storageId, scalarStorage);
+ processStorage.Put(storageId, scalarStorage);
}
// Check if the scalar is already allocated in the parent or in the child storage.
if (scalarStorage->Get(aId.id, &scalar)) {
// Dynamic scalars can expire at any time during the session (e.g. an
// add-on was updated). Check if it expired.
if (aId.dynamic) {
const DynamicScalarInfo& dynInfo = static_cast<const DynamicScalarInfo&>(info);
@@ -1252,32 +1265,40 @@ internal_GetKeyedScalarByEnum(const Stat
{
if (!internal_IsValidId(lock, aId)) {
MOZ_ASSERT(false, "Requested a keyed scalar with an invalid id.");
return NS_ERROR_INVALID_ARG;
}
const BaseScalarInfo &info = internal_GetScalarInfo(lock, aId);
- // Dynamic scalars fixup: they are always stored in the "dynamic" process.
- if (aId.dynamic) {
+ // Dynamic scalars fixup: they are always stored in the "dynamic" process,
+ // unless they are part of the "builtin" Firefox probes. Please note that
+ // "dynamic builtin" probes are meant to support "artifact" and "build faster"
+ // builds.
+ if (aId.dynamic && !info.builtin) {
aProcessStorage = ProcessID::Dynamic;
}
KeyedScalar* scalar = nullptr;
KeyedScalarStorageMapType* scalarStorage = nullptr;
// Initialize the scalar storage to the parent storage. This will get
// set to the child storage if needed.
uint32_t storageId = static_cast<uint32_t>(aProcessStorage);
+ // Put dynamic-builtin scalars (used to support "build faster") in a
+ // separate storage.
+ ProcessesKeyedScalarsMapType& processStorage =
+ (aId.dynamic && info.builtin) ? gDynamicBuiltinKeyedScalarStorageMap : gKeyedScalarStorageMap;
+
// Get the process-specific storage or create one if it's not
// available.
- if (!gKeyedScalarStorageMap.Get(storageId, &scalarStorage)) {
+ if (!processStorage.Get(storageId, &scalarStorage)) {
scalarStorage = new KeyedScalarStorageMapType();
- gKeyedScalarStorageMap.Put(storageId, scalarStorage);
+ processStorage.Put(storageId, scalarStorage);
}
if (scalarStorage->Get(aId.id, &scalar)) {
*aRet = scalar;
return NS_OK;
}
if (IsExpiredVersion(info.expiration())) {
@@ -1420,17 +1441,17 @@ internal_RegisterScalars(const StaticMut
gDynamicScalarInfo = new nsTArray<DynamicScalarInfo>();
}
for (auto scalarInfo : scalarInfos) {
// Allow expiring scalars that were already registered.
CharPtrEntryType *existingKey = gScalarNameIDMap.GetEntry(scalarInfo.name());
if (existingKey) {
// Change the scalar to expired if needed.
- if (scalarInfo.mDynamicExpiration) {
+ if (scalarInfo.mDynamicExpiration && !scalarInfo.builtin) {
DynamicScalarInfo& scalarData = (*gDynamicScalarInfo)[existingKey->mData.id];
scalarData.mDynamicExpiration = true;
}
continue;
}
gDynamicScalarInfo->AppendElement(scalarInfo);
uint32_t scalarId = gDynamicScalarInfo->Length() - 1;
@@ -1483,16 +1504,18 @@ void
TelemetryScalar::DeInitializeGlobalState()
{
StaticMutexAutoLock locker(gTelemetryScalarsMutex);
gCanRecordBase = false;
gCanRecordExtended = false;
gScalarNameIDMap.Clear();
gScalarStorageMap.Clear();
gKeyedScalarStorageMap.Clear();
+ gDynamicBuiltinScalarStorageMap.Clear();
+ gDynamicBuiltinKeyedScalarStorageMap.Clear();
gDynamicScalarInfo = nullptr;
gInitDone = false;
}
void
TelemetryScalar::SetCanRecordBase(bool b)
{
StaticMutexAutoLock locker(gTelemetryScalarsMutex);
@@ -2128,51 +2151,73 @@ TelemetryScalar::CreateSnapshots(unsigne
}
// Only lock the mutex while accessing our data, without locking any JS related code.
typedef mozilla::Pair<const char*, nsCOMPtr<nsIVariant>> DataPair;
typedef nsTArray<DataPair> ScalarArray;
nsDataHashtable<ProcessIDHashKey, ScalarArray> scalarsToReflect;
{
StaticMutexAutoLock locker(gTelemetryScalarsMutex);
- // Iterate the scalars in gScalarStorageMap. The storage may contain empty or yet to be
- // initialized scalars from all the supported processes.
- for (auto iter = gScalarStorageMap.Iter(); !iter.Done(); iter.Next()) {
- ScalarStorageMapType* scalarStorage = static_cast<ScalarStorageMapType*>(iter.Data());
- ScalarArray& processScalars = scalarsToReflect.GetOrInsert(iter.Key());
-
- // Are we in the "Dynamic" process?
- bool isDynamicProcess = ProcessID::Dynamic == static_cast<ProcessID>(iter.Key());
-
- // Iterate each available child storage.
- for (auto childIter = scalarStorage->Iter(); !childIter.Done(); childIter.Next()) {
- ScalarBase* scalar = static_cast<ScalarBase*>(childIter.Data());
-
- // Get the informations for this scalar.
- const BaseScalarInfo& info =
- internal_GetScalarInfo(locker, ScalarKey{childIter.Key(),
- isDynamicProcess});
-
- // Serialize the scalar if it's in the desired dataset.
- if (IsInDataset(info.dataset, aDataset)) {
- // Get the scalar value.
- nsCOMPtr<nsIVariant> scalarValue;
- nsresult rv = scalar->GetValue(scalarValue);
- if (NS_FAILED(rv)) {
- return rv;
+
+ // The snapshotting function is the same for both static and dynamic builtin scalars.
+ // We can use the same function and store the scalars in the same output storage.
+ auto snapshotter = [aDataset, &locker, &scalarsToReflect]
+ (ProcessesScalarsMapType& aProcessStorage, bool aIsBuiltinDynamic)
+ -> nsresult
+ {
+ // Iterate the scalars in aProcessStorage. The storage may contain empty or yet to be
+ // initialized scalars from all the supported processes.
+ for (auto iter = aProcessStorage.Iter(); !iter.Done(); iter.Next()) {
+ ScalarStorageMapType* scalarStorage = static_cast<ScalarStorageMapType*>(iter.Data());
+ ScalarArray& processScalars = scalarsToReflect.GetOrInsert(iter.Key());
+
+ // Are we in the "Dynamic" process?
+ bool isDynamicProcess = ProcessID::Dynamic == static_cast<ProcessID>(iter.Key());
+
+ // Iterate each available child storage.
+ for (auto childIter = scalarStorage->Iter(); !childIter.Done(); childIter.Next()) {
+ ScalarBase* scalar = static_cast<ScalarBase*>(childIter.Data());
+
+ // Get the informations for this scalar.
+ const BaseScalarInfo& info =
+ internal_GetScalarInfo(locker, ScalarKey{childIter.Key(),
+ aIsBuiltinDynamic ? true : isDynamicProcess});
+
+ // Serialize the scalar if it's in the desired dataset.
+ if (IsInDataset(info.dataset, aDataset)) {
+ // Get the scalar value.
+ nsCOMPtr<nsIVariant> scalarValue;
+ nsresult rv = scalar->GetValue(scalarValue);
+ if (NS_FAILED(rv)) {
+ return rv;
+ }
+ // Append it to our list.
+ processScalars.AppendElement(mozilla::MakePair(info.name(), scalarValue));
}
- // Append it to our list.
- processScalars.AppendElement(mozilla::MakePair(info.name(), scalarValue));
}
}
+ return NS_OK;
+ };
+
+ // Take a snapshot of the scalars.
+ nsresult rv = snapshotter(gScalarStorageMap, false);
+ if (NS_FAILED(rv)) {
+ return rv;
+ }
+
+ // And a snapshot of the dynamic builtin ones.
+ rv = snapshotter(gDynamicBuiltinScalarStorageMap, true);
+ if (NS_FAILED(rv)) {
+ return rv;
}
if (aClearScalars) {
// The map already takes care of freeing the allocated memory.
gScalarStorageMap.Clear();
+ gDynamicBuiltinScalarStorageMap.Clear();
}
}
// Reflect it to JS.
for (auto iter = scalarsToReflect.Iter(); !iter.Done(); iter.Next()) {
ScalarArray& processScalars = iter.Data();
const char* processName = GetNameForProcessID(ProcessID(iter.Key()));
@@ -2236,51 +2281,71 @@ TelemetryScalar::CreateKeyedSnapshots(un
}
// Only lock the mutex while accessing our data, without locking any JS related code.
typedef mozilla::Pair<const char*, nsTArray<KeyedScalar::KeyValuePair>> DataPair;
typedef nsTArray<DataPair> ScalarArray;
nsDataHashtable<ProcessIDHashKey, ScalarArray> scalarsToReflect;
{
StaticMutexAutoLock locker(gTelemetryScalarsMutex);
- // Iterate the scalars in gKeyedScalarStorageMap. The storage may contain empty or yet
- // to be initialized scalars from all the supported processes.
- for (auto iter = gKeyedScalarStorageMap.Iter(); !iter.Done(); iter.Next()) {
- KeyedScalarStorageMapType* scalarStorage =
- static_cast<KeyedScalarStorageMapType*>(iter.Data());
- ScalarArray& processScalars = scalarsToReflect.GetOrInsert(iter.Key());
-
- // Are we in the "Dynamic" process?
- bool isDynamicProcess = ProcessID::Dynamic == static_cast<ProcessID>(iter.Key());
-
- for (auto childIter = scalarStorage->Iter(); !childIter.Done(); childIter.Next()) {
- KeyedScalar* scalar = static_cast<KeyedScalar*>(childIter.Data());
-
- // Get the informations for this scalar.
- const BaseScalarInfo& info =
- internal_GetScalarInfo(locker, ScalarKey{childIter.Key(),
- isDynamicProcess});
-
- // Serialize the scalar if it's in the desired dataset.
- if (IsInDataset(info.dataset, aDataset)) {
- // Get the keys for this scalar.
- nsTArray<KeyedScalar::KeyValuePair> scalarKeyedData;
- nsresult rv = scalar->GetValue(scalarKeyedData);
- if (NS_FAILED(rv)) {
- return rv;
+
+ auto snapshotter = [aDataset, &locker, &scalarsToReflect]
+ (ProcessesKeyedScalarsMapType& aProcessStorage,
+ bool aIsBuiltinDynamic) -> nsresult
+ {
+ // Iterate the scalars in aProcessStorage. The storage may contain empty or yet
+ // to be initialized scalars from all the supported processes.
+ for (auto iter = aProcessStorage.Iter(); !iter.Done(); iter.Next()) {
+ KeyedScalarStorageMapType* scalarStorage =
+ static_cast<KeyedScalarStorageMapType*>(iter.Data());
+ ScalarArray& processScalars = scalarsToReflect.GetOrInsert(iter.Key());
+
+ // Are we in the "Dynamic" process?
+ bool isDynamicProcess = ProcessID::Dynamic == static_cast<ProcessID>(iter.Key());
+
+ for (auto childIter = scalarStorage->Iter(); !childIter.Done(); childIter.Next()) {
+ KeyedScalar* scalar = static_cast<KeyedScalar*>(childIter.Data());
+
+ // Get the informations for this scalar.
+ const BaseScalarInfo& info =
+ internal_GetScalarInfo(locker, ScalarKey{childIter.Key(),
+ aIsBuiltinDynamic ? true : isDynamicProcess});
+
+ // Serialize the scalar if it's in the desired dataset.
+ if (IsInDataset(info.dataset, aDataset)) {
+ // Get the keys for this scalar.
+ nsTArray<KeyedScalar::KeyValuePair> scalarKeyedData;
+ nsresult rv = scalar->GetValue(scalarKeyedData);
+ if (NS_FAILED(rv)) {
+ return rv;
+ }
+ // Append it to our list.
+ processScalars.AppendElement(mozilla::MakePair(info.name(), scalarKeyedData));
}
- // Append it to our list.
- processScalars.AppendElement(mozilla::MakePair(info.name(), scalarKeyedData));
}
}
+ return NS_OK;
+ };
+
+ // Take a snapshot of the scalars.
+ nsresult rv = snapshotter(gKeyedScalarStorageMap, false);
+ if (NS_FAILED(rv)) {
+ return rv;
+ }
+
+ // And a snapshot of the dynamic builtin ones.
+ rv = snapshotter(gDynamicBuiltinKeyedScalarStorageMap, true);
+ if (NS_FAILED(rv)) {
+ return rv;
}
if (aClearScalars) {
// The map already takes care of freeing the allocated memory.
gKeyedScalarStorageMap.Clear();
+ gDynamicBuiltinKeyedScalarStorageMap.Clear();
}
}
// Reflect it to JS.
for (auto iter = scalarsToReflect.Iter(); !iter.Done(); iter.Next()) {
ScalarArray& processScalars = iter.Data();
const char* processName = GetNameForProcessID(ProcessID(iter.Key()));
@@ -2328,16 +2393,17 @@ TelemetryScalar::CreateKeyedSnapshots(un
}
return NS_OK;
}
nsresult
TelemetryScalar::RegisterScalars(const nsACString& aCategoryName,
JS::Handle<JS::Value> aScalarData,
+ bool aBuiltin,
JSContext* cx)
{
MOZ_ASSERT(XRE_IsParentProcess(),
"Dynamic scalars should only be created in the parent process.");
if (!IsValidIdentifierString(aCategoryName, kMaximumCategoryNameLength, true, false)) {
JS_ReportErrorASCII(cx, "Invalid category name %s.",
PromiseFlatCString(aCategoryName).get());
@@ -2426,17 +2492,17 @@ TelemetryScalar::RegisterScalars(const n
return NS_ERROR_FAILURE;
}
expired = static_cast<bool>(value.toBoolean());
}
// We defer the actual registration here in case any other event description is invalid.
// In that case we don't need to roll back any partial registration.
newScalarInfos.AppendElement(DynamicScalarInfo{
- kind, recordOnRelease, expired, fullName, keyed
+ kind, recordOnRelease, expired, fullName, keyed, aBuiltin
});
}
// Register the dynamic definition on the parent process.
{
StaticMutexAutoLock locker(gTelemetryScalarsMutex);
::internal_RegisterScalars(locker, newScalarInfos);
@@ -2456,47 +2522,52 @@ TelemetryScalar::ClearScalars()
MOZ_ASSERT(XRE_IsParentProcess(), "Scalars should only be cleared in the parent process.");
if (!XRE_IsParentProcess()) {
return;
}
StaticMutexAutoLock locker(gTelemetryScalarsMutex);
gScalarStorageMap.Clear();
gKeyedScalarStorageMap.Clear();
+ gDynamicBuiltinScalarStorageMap.Clear();
+ gDynamicBuiltinKeyedScalarStorageMap.Clear();
}
size_t
TelemetryScalar::GetMapShallowSizesOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf)
{
StaticMutexAutoLock locker(gTelemetryScalarsMutex);
return gScalarNameIDMap.ShallowSizeOfExcludingThis(aMallocSizeOf);
}
size_t
TelemetryScalar::GetScalarSizesOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf)
{
StaticMutexAutoLock locker(gTelemetryScalarsMutex);
size_t n = 0;
- // Account for scalar data coming from parent and child processes.
- for (auto iter = gScalarStorageMap.Iter(); !iter.Done(); iter.Next()) {
- ScalarStorageMapType* scalarStorage = static_cast<ScalarStorageMapType*>(iter.Data());
- for (auto childIter = scalarStorage->Iter(); !childIter.Done(); childIter.Next()) {
- ScalarBase* scalar = static_cast<ScalarBase*>(childIter.Data());
- n += scalar->SizeOfIncludingThis(aMallocSizeOf);
+
+ auto getSizeOf = [aMallocSizeOf](auto &storageMap)
+ {
+ size_t partial = 0;
+ for (auto iter = storageMap.Iter(); !iter.Done(); iter.Next()) {
+ auto scalarStorage = iter.UserData();
+ for (auto childIter = scalarStorage->Iter(); !childIter.Done(); childIter.Next()) {
+ auto scalar = childIter.UserData();
+ partial += scalar->SizeOfIncludingThis(aMallocSizeOf);
+ }
}
- }
- // Also account for keyed scalar data coming from parent and child processes.
- for (auto iter = gKeyedScalarStorageMap.Iter(); !iter.Done(); iter.Next()) {
- KeyedScalarStorageMapType* scalarStorage =
- static_cast<KeyedScalarStorageMapType*>(iter.Data());
- for (auto childIter = scalarStorage->Iter(); !childIter.Done(); childIter.Next()) {
- KeyedScalar* scalar = static_cast<KeyedScalar*>(childIter.Data());
- n += scalar->SizeOfIncludingThis(aMallocSizeOf);
- }
- }
+ return partial;
+ };
+
+ // Account for all the storage used for the different scalar types.
+ n += getSizeOf(gScalarStorageMap);
+ n += getSizeOf(gKeyedScalarStorageMap);
+ n += getSizeOf(gDynamicBuiltinScalarStorageMap);
+ n += getSizeOf(gDynamicBuiltinKeyedScalarStorageMap);
+
return n;
}
void
TelemetryScalar::UpdateChildData(ProcessID aProcessType,
const nsTArray<mozilla::Telemetry::ScalarAction>& aScalarActions)
{
MOZ_ASSERT(XRE_IsParentProcess(),
@@ -2761,16 +2832,17 @@ TelemetryScalar::AddDynamicScalarDefinit
// Populate the definitions array before acquiring the lock.
for (auto def : aDefs) {
bool recordOnRelease = def.dataset == nsITelemetry::DATASET_RELEASE_CHANNEL_OPTOUT;
dynamicStubs.AppendElement(DynamicScalarInfo{
def.type,
recordOnRelease,
def.expired,
def.name,
- def.keyed});
+ def.keyed,
+ false /* builtin */});
}
{
StaticMutexAutoLock locker(gTelemetryScalarsMutex);
internal_RegisterScalars(locker, dynamicStubs);
}
}
--- a/toolkit/components/telemetry/TelemetryScalar.h
+++ b/toolkit/components/telemetry/TelemetryScalar.h
@@ -51,17 +51,17 @@ void SetMaximum(mozilla::Telemetry::Scal
// Keyed C++ API Endpoints.
void Add(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
void Set(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
void Set(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, bool aValue);
void SetMaximum(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
nsresult RegisterScalars(const nsACString& aCategoryName, JS::Handle<JS::Value> aScalarData,
- JSContext* cx);
+ bool aBuiltin, JSContext* cx);
// Only to be used for testing.
void ClearScalars();
size_t GetMapShallowSizesOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf);
size_t GetScalarSizesOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf);
void UpdateChildData(mozilla::Telemetry::ProcessID aProcessType,
--- a/toolkit/components/telemetry/gen_scalar_data.py
+++ b/toolkit/components/telemetry/gen_scalar_data.py
@@ -1,18 +1,20 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Write out scalar information for C++. The scalars are defined
# in a file provided as a command-line argument.
from __future__ import print_function
+from collections import OrderedDict
from shared_telemetry_utils import StringTable, static_assert, ParserError
+import json
import parse_scalars
import sys
# The banner/text at the top of the generated file.
banner = """/* This file is auto-generated, only for internal use in TelemetryScalar.h,
see gen_scalar_data.py. */
"""
@@ -73,27 +75,59 @@ def write_scalar_tables(scalars, output)
print("};", file=output)
string_table_name = "gScalarsStringTable"
string_table.writeDefinition(output, string_table_name)
static_assert(output, "sizeof(%s) <= UINT32_MAX" % string_table_name,
"index overflow")
-def main(output, *filenames):
- # Load the scalars first.
+def parse_scalar_definitions(filenames):
if len(filenames) > 1:
raise Exception('We don\'t support loading from more than one file.')
try:
- scalars = parse_scalars.load_scalars(filenames[0])
+ return parse_scalars.load_scalars(filenames[0])
except ParserError as ex:
print("\nError processing scalars:\n" + str(ex) + "\n")
sys.exit(1)
+
+def generate_JSON_definitions(output, *filenames):
+ """ Write the scalar definitions to a JSON file.
+
+ :param output: the file to write the content to.
+ :param filenames: a list of filenames provided by the build system.
+ We only support a single file.
+ """
+ scalars = parse_scalar_definitions(filenames)
+
+ scalar_definitions = OrderedDict()
+ for scalar in scalars:
+ category = scalar.category
+
+ if category not in scalar_definitions:
+ scalar_definitions[category] = OrderedDict()
+
+ scalar_definitions[category][scalar.name] = OrderedDict({
+ 'kind': scalar.nsITelemetry_kind,
+ 'keyed': scalar.keyed,
+ 'record_on_release': True if scalar.dataset == 'opt-out' else False,
+ # We don't expire dynamic-builtin scalars: they're only meant for
+ # use in local developer builds anyway. They will expire when rebuilding.
+ 'expired': False,
+ })
+
+ json.dump(scalar_definitions, output)
+
+
+def main(output, *filenames):
+ # Load the scalars first.
+ scalars = parse_scalar_definitions(filenames)
+
# Write the scalar data file.
print(banner, file=output)
print(file_header, file=output)
write_scalar_tables(scalars, output)
print(file_footer, file=output)
if __name__ == '__main__':
--- a/toolkit/components/telemetry/moz.build
+++ b/toolkit/components/telemetry/moz.build
@@ -105,16 +105,17 @@ TESTING_JS_MODULES += [
'tests/unit/TelemetryArchiveTesting.jsm',
]
PYTHON_UNITTEST_MANIFESTS += [
'tests/python/python.ini',
]
GENERATED_FILES = [
+ 'ScalarArtifactDefinitions.json',
'TelemetryEventData.h',
'TelemetryEventEnums.h',
'TelemetryHistogramData.inc',
'TelemetryHistogramEnums.h',
'TelemetryProcessData.h',
'TelemetryProcessEnums.h',
'TelemetryScalarData.h',
'TelemetryScalarEnums.h',
@@ -143,16 +144,25 @@ scalar_files = [
scalar_data = GENERATED_FILES['TelemetryScalarData.h']
scalar_data.script = 'gen_scalar_data.py'
scalar_data.inputs = scalar_files
scalar_enums = GENERATED_FILES['TelemetryScalarEnums.h']
scalar_enums.script = 'gen_scalar_enum.py'
scalar_enums.inputs = scalar_files
+# Generate the JSON scalar definitions. They will only be
+# used in artifact or "build faster" builds.
+scalar_json_data = GENERATED_FILES['ScalarArtifactDefinitions.json']
+scalar_json_data.script = 'gen_scalar_data.py:generate_JSON_definitions'
+scalar_json_data.inputs = scalar_files
+
+# Move the scalars JSON file to the directory where the Firefox binary is.
+FINAL_TARGET_FILES += ['!ScalarArtifactDefinitions.json']
+
# Generate event files.
event_files = [
'Events.yaml',
]
event_data = GENERATED_FILES['TelemetryEventData.h']
event_data.script = 'gen_event_data.py'
event_data.inputs = event_files
--- a/toolkit/components/telemetry/nsITelemetry.idl
+++ b/toolkit/components/telemetry/nsITelemetry.idl
@@ -537,12 +537,23 @@ interface nsITelemetry : nsISupports
* Defaults to false.
* @param aScalarData.<name>.expired Optional, whether this scalar entry is expired. This allows
* recording it without error, but it will be discarded. Defaults to false.
*/
[implicit_jscontext]
void registerScalars(in ACString aCategoryName, in jsval aScalarData);
/**
+ * Parent process only. Register dynamic builtin scalars. The parameters
+ * have the same meaning as the usual |registerScalars| function.
+ *
+ * This function is only meant to be used to support the "artifact build"/
+ * "built faster" developers by allowing to add new scalars without rebuilding
+ * the C++ components including the headers files.
+ */
+ [implicit_jscontext]
+ void registerBuiltinScalars(in ACString aCategoryName, in jsval aScalarData);
+
+ /**
* Resets all the stored events. This is intended to be only used in tests.
*/
void clearEvents();
};
--- a/toolkit/components/telemetry/parse_scalars.py
+++ b/toolkit/components/telemetry/parse_scalars.py
@@ -194,16 +194,21 @@ class ScalarType:
# using the deprecated format 'N.Na1'. Those scripts set
# self._strict_type_checks to false.
expires = definition.get('expires')
if not utils.validate_expiration_version(expires) and self._strict_type_checks:
raise ParserError('{} - invalid expires: {}.\nSee: {}#required-fields'
.format(self._name, expires, BASE_DOC_URL))
@property
+ def category(self):
+ """Get the category name"""
+ return self._category_name
+
+ @property
def name(self):
"""Get the scalar name"""
return self._name
@property
def label(self):
"""Get the scalar label generated from the scalar and category names."""
return self._category_name + '.' + self._name
new file mode 100644
--- /dev/null
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryScalars_buildFaster.js
@@ -0,0 +1,137 @@
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/
+*/
+
+const UINT_SCALAR = "telemetry.test.unsigned_int_kind";
+const STRING_SCALAR = "telemetry.test.string_kind";
+const BOOLEAN_SCALAR = "telemetry.test.boolean_kind";
+const KEYED_UINT_SCALAR = "telemetry.test.keyed_unsigned_int";
+
+ChromeUtils.import("resource://services-common/utils.js");
+
+/**
+ * Return the path to the definitions file for the scalars.
+ */
+function getDefinitionsPath() {
+ // Write the scalar definition to the spec file in the binary directory.
+ let definitionFile = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsIFile);
+ definitionFile = Services.dirsvc.get("GreBinD", Ci.nsIFile);
+ definitionFile.append("ScalarArtifactDefinitions.json");
+ return definitionFile.path;
+}
+
+add_task(async function test_setup() {
+ do_get_profile();
+});
+
+add_task({
+ // The test needs to write a file, and that fails in tests on Android.
+ // We don't really need the Android coverage, so skip on Android.
+ skip_if: () => AppConstants.platform == "android"
+ }, async function test_invalidJSON() {
+ const INVALID_JSON = "{ invalid,JSON { {1}";
+ const FILE_PATH = getDefinitionsPath();
+
+ // Write a corrupted JSON file.
+ await OS.File.writeAtomic(FILE_PATH, INVALID_JSON, { encoding: "utf-8", noOverwrite: false });
+
+ // Simulate Firefox startup. This should not throw!
+ await TelemetryController.testSetup();
+ await TelemetryController.testPromiseJsProbeRegistration();
+
+ // Cleanup.
+ await TelemetryController.testShutdown();
+ await OS.File.remove(FILE_PATH);
+});
+
+add_task({
+ // The test needs to write a file, and that fails in tests on Android.
+ // We don't really need the Android coverage, so skip on Android.
+ skip_if: () => AppConstants.platform == "android"
+ }, async function test_dynamicBuiltin() {
+ const DYNAMIC_SCALAR_SPEC = {
+ "telemetry.test": {
+ "builtin_dynamic": {
+ "kind": "nsITelemetry::SCALAR_TYPE_COUNT",
+ "expired": false,
+ "record_on_release": false,
+ "keyed": false
+ },
+ "builtin_dynamic_other": {
+ "kind": "nsITelemetry::SCALAR_TYPE_BOOLEAN",
+ "expired": false,
+ "record_on_release": false,
+ "keyed": false
+ }
+ }
+ };
+
+ Telemetry.clearScalars();
+
+ // Let's write to the definition file to also cover the file
+ // loading part.
+ const FILE_PATH = getDefinitionsPath();
+ await CommonUtils.writeJSON(DYNAMIC_SCALAR_SPEC, FILE_PATH);
+
+ // Start TelemetryController to trigger loading the specs.
+ await TelemetryController.testReset();
+ await TelemetryController.testPromiseJsProbeRegistration();
+
+ // Store to that scalar.
+ const TEST_SCALAR1 = "telemetry.test.builtin_dynamic";
+ const TEST_SCALAR2 = "telemetry.test.builtin_dynamic_other";
+ Telemetry.scalarSet(TEST_SCALAR1, 3785);
+ Telemetry.scalarSet(TEST_SCALAR2, true);
+
+ // Check the values we tried to store.
+ const scalars =
+ Telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, false).parent;
+
+ // Check that they are serialized to the correct format.
+ Assert.equal(typeof(scalars[TEST_SCALAR1]), "number",
+ TEST_SCALAR1 + " must be serialized to the correct format.");
+ Assert.ok(Number.isInteger(scalars[TEST_SCALAR1]),
+ TEST_SCALAR1 + " must be a finite integer.");
+ Assert.equal(scalars[TEST_SCALAR1], 3785,
+ TEST_SCALAR1 + " must have the correct value.");
+ Assert.equal(typeof(scalars[TEST_SCALAR2]), "boolean",
+ TEST_SCALAR2 + " must be serialized to the correct format.");
+ Assert.equal(scalars[TEST_SCALAR2], true,
+ TEST_SCALAR2 + " must have the correct value.");
+
+ // Clean up.
+ await TelemetryController.testShutdown();
+ await OS.File.remove(FILE_PATH);
+});
+
+add_task(async function test_keyedDynamicBuiltin() {
+ Telemetry.clearScalars();
+
+ // Register the built-in scalars (let's not take the I/O hit).
+ Telemetry.registerBuiltinScalars("telemetry.test", {
+ "builtin_dynamic_keyed": {
+ "kind": Ci.nsITelemetry.SCALAR_TYPE_COUNT,
+ "expired": false,
+ "record_on_release": false,
+ "keyed": true
+ }
+ });
+
+ // Store to that scalar.
+ const TEST_SCALAR1 = "telemetry.test.builtin_dynamic_keyed";
+ Telemetry.keyedScalarSet(TEST_SCALAR1, "test-key", 3785);
+
+ // Check the values we tried to store.
+ const scalars =
+ Telemetry.snapshotKeyedScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, false).parent;
+
+ // Check that they are serialized to the correct format.
+ Assert.equal(typeof(scalars[TEST_SCALAR1]), "object",
+ TEST_SCALAR1 + " must be a keyed scalar.");
+ Assert.equal(typeof(scalars[TEST_SCALAR1]["test-key"]), "number",
+ TEST_SCALAR1 + " must be serialized to the correct format.");
+ Assert.ok(Number.isInteger(scalars[TEST_SCALAR1]["test-key"]),
+ TEST_SCALAR1 + " must be a finite integer.");
+ Assert.equal(scalars[TEST_SCALAR1]["test-key"], 3785,
+ TEST_SCALAR1 + " must have the correct value.");
+});
--- a/toolkit/components/telemetry/tests/unit/xpcshell.ini
+++ b/toolkit/components/telemetry/tests/unit/xpcshell.ini
@@ -59,16 +59,17 @@ skip-if = os == "android"
skip-if = os == "android" # Disabled due to crashes (see bug 1331366)
tags = addons
[test_ChildScalars.js]
skip-if = os == "android" # Disabled due to crashes (see bug 1331366)
[test_TelemetryReportingPolicy.js]
skip-if = os == "android" # Disabled due to crashes (see bug 1367762)
tags = addons
[test_TelemetryScalars.js]
+[test_TelemetryScalars_buildFaster.js]
[test_TelemetryTimestamps.js]
skip-if = toolkit == 'android'
[test_TelemetryCaptureStack.js]
[test_TelemetryEvents.js]
[test_ChildEvents.js]
skip-if = os == "android" # Disabled due to crashes (see bug 1331366)
[test_TelemetryModules.js]
[test_PingSender.js]