Bug 1272541 - Expose more data from SystemResourceMonitor.as_dict; r?ahal draft
authorGregory Szorc <gps@mozilla.com>
Thu, 12 May 2016 18:03:16 -0700
changeset 366639 09eabe207f3fe9c3241d9b46a029b85940fdfc5c
parent 366627 d82860087d47d10791757d56e289076800318ae1
child 366640 ab27b1e96a7ff5be65314747380be7bbbc2f8b23
push id18033
push userbmo:gps@mozilla.com
push dateFri, 13 May 2016 01:48:48 +0000
reviewersahal
bugs1272541
milestone49.0a1
Bug 1272541 - Expose more data from SystemResourceMonitor.as_dict; r?ahal We add some system information including processor count and memory sizes. We also add an "overall" section describing total resource usage. This (surprisingly) wasn't defined. This commit is the first in a series to reconcile the differences between the JSON format in mozsystemmonitor and what `mach build` writes so we can write a single tool to visualize the data. MozReview-Commit-ID: 9mdbKxeV9Ta
testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py
testing/mozbase/mozsystemmonitor/mozsystemmonitor/test/test_resource_monitor.py
--- a/testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py
+++ b/testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py
@@ -33,26 +33,30 @@ class PsutilStub(object):
             return self.pcputimes(0, 0)
     def disk_io_counters(self):
         return self.sdiskio(0, 0, 0, 0, 0, 0)
     def swap_memory(self):
         return self.sswap(0, 0, 0, 0, 0, 0)
     def virtual_memory(self):
         return self.svmem(0, 0, 0, 0, 0, 0, 0, 0, 0)
 
+
 # psutil will raise NotImplementedError if the platform is not supported.
 try:
     import psutil
+    have_psutil = True
 except Exception:
     try:
         # The PsutilStub should get us time intervals, at least
         psutil = PsutilStub()
     except Exception:
         psutil = None
 
+    have_psutil = False
+
 from contextlib import contextmanager
 
 def get_disk_io_counters():
     try:
         io_counters = psutil.disk_io_counters()
     except RuntimeError:
         io_counters = []
 
@@ -530,28 +534,32 @@ class SystemResourceMonitor(object):
 
         return max(values)
 
     def as_dict(self):
         """Convert the recorded data to a dict, suitable for serialization.
 
         The returned dict has the following keys:
 
-          version - Integer version number being rendered. Currently 1.
+          version - Integer version number being rendered. Currently 2.
           cpu_times_fields - A list of the names of the CPU times fields.
           io_fields - A list of the names of the I/O fields.
           virt_fields - A list of the names of the virtual memory fields.
           swap_fields - A list of the names of the swap memory fields.
           samples - A list of dicts containing low-level measurements.
           events - A list of lists representing point events. The inner list
             has 2 elements, the float wall time of the event and the string
             event name.
           phases - A list of dicts describing phases. Each phase looks a lot
             like an entry from samples (see below). Some phases may not have
             data recorded against them, so some keys may be None.
+          overall - A dict representing overall resource usage. This resembles
+            a sample entry.
+          system - Contains additional information about the system including
+            number of processors and amount of memory.
 
         Each entry in the sample list is a dict with the following keys:
 
           start - Float wall time this measurement began on.
           end - Float wall time this measurement ended on.
           io - List of numerics for I/O values.
           virt - List of numerics for virtual memory values.
           swap - List of numerics for swap memory values.
@@ -562,39 +570,53 @@ class SystemResourceMonitor(object):
           cpu_times_sum - List of floats representing the sum of CPU times
             across all cores.
           cpu_times_total - Float representing the sum of all CPU times across
             all cores. This is useful for calculating the percent in each CPU
             time.
         """
 
         o = dict(
-            version=1,
+            version=2,
             cpu_times_fields=list(self._cpu_times_type._fields),
             io_fields=list(self._io_type._fields),
             virt_fields=list(self._virt_type._fields),
             swap_fields=list(self._swap_type._fields),
             samples=[],
             phases=[],
+            system={},
         )
 
         def populate_derived(e):
             if e['cpu_percent_cores']:
                 e['cpu_percent_mean'] = sum(e['cpu_percent_cores']) / \
                     len(e['cpu_percent_cores'])
             else:
                 e['cpu_percent_mean'] = None
 
             if e['cpu_times']:
                 e['cpu_times_sum'] = [0.0] * self._cpu_times_len
                 for i in range(0, self._cpu_times_len):
                     e['cpu_times_sum'][i] = sum(core[i] for core in e['cpu_times'])
 
                 e['cpu_times_total'] = sum(e['cpu_times_sum'])
 
+        def phase_entry(name, start, end):
+            e = dict(
+                name=name,
+                start=start,
+                end=end,
+                duration=end - start,
+                cpu_percent_cores=self.aggregate_cpu_percent(phase=name),
+                cpu_times=[list(c) for c in
+                           self.aggregate_cpu_times(phase=name)],
+                io=list(self.aggregate_io(phase=name)),
+            )
+            populate_derived(e)
+            return e
 
         for m in self.measurements:
             e = dict(
                 start=m.start,
                 end=m.end,
                 io=list(m.io),
                 virt=list(m.virt),
                 swap=list(m.swap),
@@ -604,31 +626,29 @@ class SystemResourceMonitor(object):
 
             populate_derived(e)
             o['samples'].append(e)
 
         if o['samples']:
             o['start'] = o['samples'][0]['start']
             o['end'] = o['samples'][-1]['end']
             o['duration'] = o['end'] - o['start']
+            o['overall'] = phase_entry(None, o['start'], o['end'])
         else:
             o['start'] = None
             o['end'] = None
             o['duration'] = None
+            o['overall'] = None
 
         o['events'] = [list(ev) for ev in self.events]
 
         for phase, v in self.phases.items():
-            e = dict(
-                name=phase,
-                start=v[0],
-                end=v[1],
-                duration=v[1] - v[0],
-                cpu_percent_cores=self.aggregate_cpu_percent(phase=phase),
-                cpu_times=[list(c) for c in
-                    self.aggregate_cpu_times(phase=phase)],
-                io=list(self.aggregate_io(phase=phase)),
-            )
+            o['phases'].append(phase_entry(phase, v[0], v[1]))
 
-            populate_derived(e)
-            o['phases'].append(e)
+        if have_psutil:
+            o['system'].update(dict(
+                cpu_logical_count=psutil.cpu_count(logical=True),
+                cpu_physical_count=psutil.cpu_count(logical=False),
+                swap_total=psutil.swap_memory()[0],
+                vmem_total=psutil.virtual_memory()[0],
+            ))
 
         return o
--- a/testing/mozbase/mozsystemmonitor/mozsystemmonitor/test/test_resource_monitor.py
+++ b/testing/mozbase/mozsystemmonitor/mozsystemmonitor/test/test_resource_monitor.py
@@ -164,11 +164,16 @@ class TestResourceMonitor(unittest.TestC
         monitor.finish_phase('phase1')
         time.sleep(0.2)
         monitor.finish_phase('phase2')
         time.sleep(0.4)
         monitor.stop()
 
         d = monitor.as_dict()
 
-        self.assertEqual(d['version'], 1)
+        self.assertEqual(d['version'], 2)
         self.assertEqual(len(d['events']), 2)
         self.assertEqual(len(d['phases']), 2)
+        self.assertIn('system', d)
+        self.assertIsInstance(d['system'], dict)
+        self.assertIsInstance(d['overall'], dict)
+        self.assertIn('duration', d['overall'])
+        self.assertIn('cpu_times', d['overall'])