Bug 1244160 - Create json-schema for build telemetry data r?gps draft
authorDan Minor <dminor@mozilla.com>
Tue, 02 Feb 2016 09:32:49 -0500
changeset 333152 8bd77cecc3bbf07c6095258587c74510562d440c
parent 332881 789a12291942763bc1e3a89f97e0b82dc1c9d00b
child 514648 4ad25eefa972afc95da9628c56067743b7e663ba
push id11274
push userdminor@mozilla.com
push dateMon, 22 Feb 2016 19:25:14 +0000
reviewersgps
bugs1244160
milestone47.0a1
Bug 1244160 - Create json-schema for build telemetry data r?gps This adds a simple schema for build telemetry data. We can make it more restrictive once we have a better feeling for what kind of data we want to submit. This also moves more common data about the system to the telemetry handler. We leave psutil derivied information in the resource usage data as not every system will have psutil installed. MozReview-Commit-ID: CFRq1Ow6AOf
build/mach_bootstrap.py
build/telemetry-schema.json
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/resources/html-build-viewer/index.html
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -249,18 +249,36 @@ def bootstrap(topsrcdir, mozilla_dir=Non
         outgoing_dir = os.path.join(telemetry_dir, 'outgoing')
         try:
             os.mkdir(outgoing_dir)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
         # Add common metadata to help submit sorted data later on.
-        # For now, we'll just record the mach command that was invoked.
         data['argv'] = sys.argv
+        data.setdefault('system', {}).update(dict(
+            architecture=list(platform.architecture()),
+            machine=platform.machine(),
+            python_version=platform.python_version(),
+            release=platform.release(),
+            system=platform.system(),
+            version=platform.version(),
+        ))
+
+        if platform.system() == 'Linux':
+            dist = list(platform.linux_distribution())
+            data['system']['linux_distribution'] = dist
+        elif platform.system() == 'Windows':
+            win32_ver=list((platform.win32_ver())),
+            data['system']['win32_ver'] = win32_ver
+        elif platform.system() == 'Darwin':
+            # mac version is a special Cupertino snowflake
+            r, v, m = platform.mac_ver()
+            data['system']['mac_ver'] = [r, list(v), m]
 
         with open(os.path.join(outgoing_dir, str(uuid.uuid4()) + '.json'),
                   'w') as f:
             json.dump(data, f, sort_keys=True)
 
     def should_skip_dispatch(context, handler):
         # The user is performing a maintenance command.
         if handler.name in ('bootstrap', 'doctor', 'mach-commands', 'mercurial-setup'):
new file mode 100644
--- /dev/null
+++ b/build/telemetry-schema.json
@@ -0,0 +1,24 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "argv": {"type": "array"},
+    "system": {
+      "type": "object",
+      "properties": {
+        "architecture": {"type": "array"},
+        "linux_distribution": {"type": "array"},
+        "mac_ver": {"type": "array"},
+        "machine": {"type": "string"},
+        "python_version": {"type": "string"},
+        "release": {"type": "string"},
+        "system": {"type": "string"},
+        "version": {"type": "string"},
+        "win_ver": {"type": "array"}
+      },
+      "required": ["architecture", "machine", "python_version",
+                   "release", "system", "version"]
+    }
+  },
+  "required": ["argv", "system"]
+}
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -372,17 +372,17 @@ class BuildMonitor(MozbuildObject):
         if excessive is not None and (sin or sout):
             sin /= 1048576
             sout /= 1048576
             self.log(logging.WARNING, 'swap_activity',
                 {'sin': sin, 'sout': sout},
                 'Swap in/out (MB): {sin}/{sout}')
 
         o = dict(
-            version=2,
+            version=3,
             argv=sys.argv,
             start=self.start_time,
             end=self.end_time,
             duration=self.end_time - self.start_time,
             resources=[],
             cpu_percent=cpu_percent,
             cpu_times=cpu_times,
             io=io,
@@ -406,51 +406,33 @@ class BuildMonitor(MozbuildObject):
                 swap=list(usage.swap),
             )
 
             self.tiers.add_resources_to_dict(entry, start=usage.start,
                     end=usage.end)
 
             o['resources'].append(entry)
 
-        # TODO: it would be nice to collect data on the storage device as well
-        o['system'] = dict(
-            architecture=list(platform.architecture()),
-            machine=platform.machine(),
-            python_version=platform.python_version(),
-            release=platform.release(),
-            system=platform.system(),
-            version=platform.version(),
-        )
 
         # If the imports for this file ran before the in-tree virtualenv
         # was bootstrapped (for instance, for a clobber build in automation),
         # psutil might not be available.
         #
         # Treat psutil as optional to avoid an outright failure to log resources
+        # TODO: it would be nice to collect data on the storage device as well
         # in this case.
+        o['system'] = {}
         if psutil:
             o['system'].update(dict(
                 logical_cpu_count=psutil.cpu_count(),
                 physical_cpu_count=psutil.cpu_count(logical=False),
                 swap_total=psutil.swap_memory()[0],
                 vmem_total=psutil.virtual_memory()[0],
             ))
 
-        if platform.system() == 'Linux':
-            dist = list(platform.linux_distribution())
-            o['system']['linux_distribution'] = dist
-        elif platform.system() == 'Windows':
-            win32_ver=list((platform.win32_ver())),
-            o['system']['win32_ver'] = win32_ver
-        elif platform.system() == 'Darwin':
-            # mac version is a special Cupertino snowflake
-            r, v, m = platform.mac_ver()
-            o['system']['mac_ver'] = [r, list(v), m]
-
         return o
 
     def _log_resource_usage(self, prefix, m_type, duration, cpu_percent,
         cpu_times, io, extra_params={}):
 
         params = dict(
             duration=duration,
             cpu_percent=cpu_percent,
--- a/python/mozbuild/mozbuild/resources/html-build-viewer/index.html
+++ b/python/mozbuild/mozbuild/resources/html-build-viewer/index.html
@@ -51,17 +51,17 @@ svg {
   <body>
     <script>
 var currentResources;
 
 /**
  * Interface for a build resources JSON file.
  */
 function BuildResources(data) {
-  if (data.version != 1 && data.version != 2) {
+  if (data.version < 1 || data.version > 3) {
     throw new Error("Unsupported version of the JSON format: " + data.version);
   }
 
   this.resources = [];
 
   var cpu_fields = data.cpu_times_fields;
   var io_fields = data.io_fields;
   var virt_fields = data.virt_fields;