Bug 1467705 - Refactor the histogram packing routines in TelemetryUtils. r?chutten draft
authorAlessio Placitelli <alessio.placitelli@gmail.com>
Fri, 08 Jun 2018 18:44:31 +0200
changeset 807507 6f93c5128eed6ca9606998015c5f13a6b468dcaf
parent 807395 91db0c695f0272f00bf92c81c471a85101056d96
child 807508 b3a24f55ace2cc89d5e91ea08b4f80569a2dcf63
push id113131
push useralessio.placitelli@gmail.com
push dateThu, 14 Jun 2018 18:52:31 +0000
reviewerschutten
bugs1467705
milestone62.0a1
Bug 1467705 - Refactor the histogram packing routines in TelemetryUtils. r?chutten Firefox Desktop is not sending raw histograms out, in pings. Instead, it processes the raw histograms and packs them in a sparser representation. GeckoView will need to do the same, so share the code rather than copy paste it. MozReview-Commit-ID: IW3tSHC12fc
toolkit/components/telemetry/TelemetrySession.jsm
toolkit/components/telemetry/TelemetryUtils.jsm
--- a/toolkit/components/telemetry/TelemetrySession.jsm
+++ b/toolkit/components/telemetry/TelemetrySession.jsm
@@ -807,113 +807,32 @@ var Impl = {
     ret.activeTicks = activeTicks;
 
     ret.pingsOverdue = TelemetrySend.overduePingsCount;
 
     return ret;
   },
 
   /**
-   * When reflecting a histogram into JS, Telemetry hands us an object
-   * with the following properties:
-   *
-   * - min, max, histogram_type, sum, sum_squares_{lo,hi}: simple integers;
-   * - counts: array of counts for histogram buckets;
-   * - ranges: array of calculated bucket sizes.
-   *
-   * This format is not straightforward to read and potentially bulky
-   * with lots of zeros in the counts array.  Packing histograms makes
-   * raw histograms easier to read and compresses the data a little bit.
-   *
-   * Returns an object:
-   * { range: [min, max], bucket_count: <number of buckets>,
-   *   histogram_type: <histogram_type>, sum: <sum>,
-   *   values: { bucket1: count1, bucket2: count2, ... } }
-   */
-  packHistogram: function packHistogram(hgram) {
-    let r = hgram.ranges;
-    let c = hgram.counts;
-    let retgram = {
-      range: [r[1], r[r.length - 1]],
-      bucket_count: r.length,
-      histogram_type: hgram.histogram_type,
-      values: {},
-      sum: hgram.sum
-    };
-
-    let first = true;
-    let last = 0;
-
-    for (let i = 0; i < c.length; i++) {
-      let value = c[i];
-      if (!value)
-        continue;
-
-      // add a lower bound
-      if (i && first) {
-        retgram.values[r[i - 1]] = 0;
-      }
-      first = false;
-      last = i + 1;
-      retgram.values[r[i]] = value;
-    }
-
-    // add an upper bound
-    if (last && last < c.length)
-      retgram.values[r[last]] = 0;
-    return retgram;
-  },
-
-  /**
    * Get the type of the dataset that needs to be collected, based on the preferences.
    * @return {Integer} A value from nsITelemetry.DATASET_*.
    */
   getDatasetType() {
     return Telemetry.canRecordExtended ? Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN
                                        : Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTOUT;
   },
 
   getHistograms: function getHistograms(clearSubsession) {
     let hls = Telemetry.snapshotHistograms(this.getDatasetType(), clearSubsession);
-    let ret = {};
-
-    for (let [process, histograms] of Object.entries(hls)) {
-      ret[process] = {};
-      for (let [name, value] of Object.entries(histograms)) {
-        if (this._testing || !name.startsWith("TELEMETRY_TEST_")) {
-          ret[process][name] = this.packHistogram(value);
-        }
-      }
-    }
-
-    return ret;
+    return TelemetryUtils.packHistograms(hls, this._testing);
   },
 
   getKeyedHistograms(clearSubsession) {
     let khs = Telemetry.snapshotKeyedHistograms(this.getDatasetType(), clearSubsession);
-    let ret = {};
-
-    for (let [process, histograms] of Object.entries(khs)) {
-      ret[process] = {};
-      for (let [name, value] of Object.entries(histograms)) {
-        if (this._testing || !name.startsWith("TELEMETRY_TEST_")) {
-          let keys = Object.keys(value);
-          if (keys.length == 0) {
-            // Skip empty keyed histogram
-            continue;
-          }
-          ret[process][name] = {};
-          for (let [key, hgram] of Object.entries(value)) {
-            ret[process][name][key] = this.packHistogram(hgram);
-          }
-        }
-      }
-    }
-
-    return ret;
+    return TelemetryUtils.packKeyedHistograms(khs, this._testing);
   },
 
   /**
    * Get a snapshot of the scalars and clear them.
    * @param {subsession} If true, then we collect the data for a subsession.
    * @param {clearSubsession} If true, we  need to clear the subsession.
    * @param {keyed} Take a snapshot of keyed or non keyed scalars.
    * @return {Object} The scalar data as a Javascript object, including the
--- a/toolkit/components/telemetry/TelemetryUtils.jsm
+++ b/toolkit/components/telemetry/TelemetryUtils.jsm
@@ -19,16 +19,67 @@ const PREF_TELEMETRY_ENABLED = "toolkit.
 const IS_CONTENT_PROCESS = (function() {
   // We cannot use Services.appinfo here because in telemetry xpcshell tests,
   // appinfo is initially unavailable, and becomes available only later on.
   // eslint-disable-next-line mozilla/use-services
   let runtime = Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULRuntime);
   return runtime.processType == Ci.nsIXULRuntime.PROCESS_TYPE_CONTENT;
 })();
 
+/**
+ * When reflecting a histogram into JS, Telemetry hands us an object
+ * with the following properties:
+ *
+ * - min, max, histogram_type, sum, sum_squares_{lo,hi}: simple integers;
+ * - counts: array of counts for histogram buckets;
+ * - ranges: array of calculated bucket sizes.
+ *
+ * This format is not straightforward to read and potentially bulky
+ * with lots of zeros in the counts array.  Packing histograms makes
+ * raw histograms easier to read and compresses the data a little bit.
+ *
+ * Returns an object:
+ * { range: [min, max], bucket_count: <number of buckets>,
+ *   histogram_type: <histogram_type>, sum: <sum>,
+ *   values: { bucket1: count1, bucket2: count2, ... } }
+ */
+function packHistogram(hgram) {
+  let r = hgram.ranges;
+  let c = hgram.counts;
+  let retgram = {
+    range: [r[1], r[r.length - 1]],
+    bucket_count: r.length,
+    histogram_type: hgram.histogram_type,
+    values: {},
+    sum: hgram.sum
+  };
+
+  let first = true;
+  let last = 0;
+
+  for (let i = 0; i < c.length; i++) {
+    let value = c[i];
+    if (!value)
+      continue;
+
+    // add a lower bound
+    if (i && first) {
+      retgram.values[r[i - 1]] = 0;
+    }
+    first = false;
+    last = i + 1;
+    retgram.values[r[i]] = value;
+  }
+
+  // add an upper bound
+  if (last && last < c.length)
+    retgram.values[r[last]] = 0;
+  return retgram;
+}
+
 var TelemetryUtils = {
   Preferences: Object.freeze({
     // General Preferences
     ArchiveEnabled: "toolkit.telemetry.archive.enabled",
     CachedClientId: "toolkit.telemetry.cachedClientID",
     FirstRun: "toolkit.telemetry.reportingpolicy.firstRun",
     FirstShutdownPingEnabled: "toolkit.telemetry.firstShutdownPing.enabled",
     HealthPingEnabled: "toolkit.telemetry.healthping.enabled",
@@ -221,9 +272,97 @@ var TelemetryUtils = {
     const isReleaseCandidateOnBeta =
       AppConstants.MOZ_UPDATE_CHANNEL === "release" &&
       Services.prefs.getCharPref("app.update.channel", null) === "beta";
     Services.telemetry.canRecordBase = true;
     Services.telemetry.canRecordExtended = isPrereleaseChannel ||
       isReleaseCandidateOnBeta ||
       Services.prefs.getBoolPref(this.Preferences.OverridePreRelease, false);
   },
+
+  /**
+   * Converts histograms from the raw to the packed format.
+   * This additionally filters TELEMETRY_TEST_ histograms.
+   *
+   * @param {Object} snapshot - The histogram snapshot.
+   * @param {Boolean} [testingMode=false] - Whether or not testing histograms
+   *        should be filtered.
+   * @returns {Object}
+   *
+   * {
+   *  "<process>": {
+   *    "<histogram>": {
+   *      range: [min, max],
+   *      bucket_count: <number of buckets>,
+   *      histogram_type: <histogram_type>,
+   *      sum: <sum>,
+   *      values: { bucket1: count1, bucket2: count2, ... }
+   *    },
+   *   ..
+   *   },
+   *  ..
+   * }
+   */
+  packHistograms(snapshot, testingMode = false) {
+    let ret = {};
+
+    for (let [process, histograms] of Object.entries(snapshot)) {
+      ret[process] = {};
+      for (let [name, value] of Object.entries(histograms)) {
+        if (testingMode || !name.startsWith("TELEMETRY_TEST_")) {
+          ret[process][name] = packHistogram(value);
+        }
+      }
+    }
+
+    return ret;
+  },
+
+  /**
+   * Converts keyed histograms from the raw to the packed format.
+   * This additionally filters TELEMETRY_TEST_ histograms and skips
+   * empty keyed histograms.
+   *
+   * @param {Object} snapshot - The keyed histogram snapshot.
+   * @param {Boolean} [testingMode=false] - Whether or not testing histograms should
+   *        be filtered.
+   * @returns {Object}
+   *
+   * {
+   *  "<process>": {
+   *    "<histogram>": {
+   *      "<key>": {
+   *        range: [min, max],
+   *        bucket_count: <number of buckets>,
+   *        histogram_type: <histogram_type>,
+   *        sum: <sum>,
+   *        values: { bucket1: count1, bucket2: count2, ... }
+   *      },
+   *      ..
+   *    },
+   *   ..
+   *   },
+   *  ..
+   * }
+   */
+  packKeyedHistograms(snapshot, testingMode = false) {
+    let ret = {};
+
+    for (let [process, histograms] of Object.entries(snapshot)) {
+      ret[process] = {};
+      for (let [name, value] of Object.entries(histograms)) {
+        if (testingMode || !name.startsWith("TELEMETRY_TEST_")) {
+          let keys = Object.keys(value);
+          if (keys.length == 0) {
+            // Skip empty keyed histogram
+            continue;
+          }
+          ret[process][name] = {};
+          for (let [key, hgram] of Object.entries(value)) {
+            ret[process][name][key] = packHistogram(hgram);
+          }
+        }
+      }
+    }
+
+    return ret;
+  },
 };