Bug 1239296 - add post_dispatch_handler hook to mach r?gps draft
authorDan Minor <dminor@mozilla.com>
Tue, 09 Feb 2016 10:09:17 -0500
changeset 329831 38793cfe0c9a5add554c4dfbf81dc4dfb3362ec0
parent 329830 19c3ef89f4eb704f6cf8cf0da75bb9c4c3061199
child 514042 6eff5f2e3d012b6069dbbbd6c5994e780704ed4d
push id10616
push userdminor@mozilla.com
push dateTue, 09 Feb 2016 16:18:53 +0000
reviewersgps
bugs1239296
milestone47.0a1
Bug 1239296 - add post_dispatch_handler hook to mach r?gps This adds a post_dispatch_handler hook that will be called after each mach invocation and uses it to submit data to telemetry.
build/mach_bootstrap.py
python/mach/mach/registrar.py
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import print_function, unicode_literals
 
 import errno
 import json
 import os
 import platform
+import random
 import sys
 import time
 import uuid
 import __builtin__
 
 from types import ModuleType
 
 
@@ -175,16 +176,24 @@ CATEGORIES = {
     'disabled': {
         'short': 'Disabled',
         'long': 'The disabled commands are hidden by default. Use -v to display them. These commands are unavailable for your current context, run "mach <command>" to see why.',
         'priority': 0,
     }
 }
 
 
+# Server to which to submit telemetry data
+BUILD_TELEMETRY_SERVER = 'http://52.88.27.118/build-metrics-dev'
+
+
+# We submit data to telemetry approximately every this many mach invocations
+TELEMETRY_SUBMISSION_FREQUENCY = 10
+
+
 def get_state_dir():
     """Obtain the path to a directory to hold state.
 
     Returns a tuple of the path and a bool indicating whether the value came
     from an environment variable.
     """
     state_user_dir = os.path.expanduser('~/.mozbuild')
     state_env_dir = os.environ.get('MOZBUILD_STATE_PATH', None)
@@ -238,44 +247,50 @@ def bootstrap(topsrcdir, mozilla_dir=Non
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
         # Add common metadata to help submit sorted data later on.
         # For now, we'll just record the mach command that was invoked.
         data['argv'] = sys.argv
 
-        with open(os.path.join(outgoing_dir, str(uuid.uuid4())), 'w') as f:
+        with open(os.path.join(outgoing_dir, str(uuid.uuid4()) + '.json'),
+                  'w') as f:
             json.dump(data, f, sort_keys=True)
 
+    def should_skip_dispatch(context, handler):
+        # The user is performing a maintenance command.
+        if handler.name in ('bootstrap', 'doctor', 'mach-commands', 'mercurial-setup'):
+            return True
+
+        # We are running in automation.
+        if 'MOZ_AUTOMATION' in os.environ or 'TASK_ID' in os.environ:
+            return True
+
+        # The environment is likely a machine invocation.
+        if sys.stdin.closed or not sys.stdin.isatty():
+            return True
+
+        return False
+
     def pre_dispatch_handler(context, handler, args):
         """Perform global checks before command dispatch.
 
         Currently, our goal is to ensure developers periodically run
         `mach mercurial-setup` (when applicable) to ensure their Mercurial
         tools are up to date.
         """
         # Don't do anything when...
-
-        # The user is performing a maintenance command.
-        if handler.name in ('bootstrap', 'doctor', 'mach-commands', 'mercurial-setup'):
-            return
-
-        # We are running in automation.
-        if 'MOZ_AUTOMATION' in os.environ or 'TASK_ID' in os.environ:
+        if should_skip_dispatch(context, handler):
             return
 
         # We are a curmudgeon who has found this undocumented variable.
         if 'I_PREFER_A_SUBOPTIMAL_MERCURIAL_EXPERIENCE' in os.environ:
             return
 
-        # The environment is likely a machine invocation.
-        if sys.stdin.closed or not sys.stdin.isatty():
-            return
-
         # Mercurial isn't managing this source checkout.
         if not os.path.exists(os.path.join(topsrcdir, '.hg')):
             return
 
         state_dir = get_state_dir()[0]
         last_check_path = os.path.join(state_dir, 'mercurial',
                                        'setup.lastcheck')
 
@@ -286,16 +301,78 @@ def bootstrap(topsrcdir, mozilla_dir=Non
             if e.errno != errno.ENOENT:
                 raise
 
         # No last run file means mercurial-setup has never completed.
         if mtime is None:
             print(NO_MERCURIAL_SETUP.format(mach=sys.argv[0]), file=sys.stderr)
             sys.exit(2)
 
+    def post_dispatch_handler(context, handler, args):
+        """Perform global operations after command dispatch.
+
+
+        For now,  we will use this to handle build system telemetry.
+        """
+        # Don't do anything when...
+        if should_skip_dispatch(context, handler):
+            return
+
+        # We have not opted-in to telemetry
+        if 'BUILD_SYSTEM_TELEMETRY' not in os.environ:
+            return
+
+        # Every n-th operation
+        if random.randint(1, TELEMETRY_SUBMISSION_FREQUENCY) != 1:
+            return
+
+        # No data to work with anyway
+        outgoing = os.path.join(get_state_dir()[0], 'telemetry', 'outgoing')
+        if not os.path.isdir(outgoing):
+            return
+
+        # We can't import requests until after it has been added during the
+        # bootstrapping below.
+        import requests
+
+        submitted = os.path.join(get_state_dir()[0], 'telemetry', 'submitted')
+        try:
+            os.mkdir(submitted)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+
+        session = requests.Session()
+        for filename in os.listdir(outgoing):
+            path = os.path.join(outgoing, filename)
+            if os.path.isdir(path) or not path.endswith('.json'):
+                continue
+            with open(path, 'r') as f:
+                data = f.read()
+                r = session.post(BUILD_TELEMETRY_SERVER, data=data,
+                                 headers={'Content-Type': 'application/json'})
+                # TODO: some of these errors are likely not recoverable, as
+                # written, we'll retry indefinitely
+                if r.status_code != 200:
+                    print('Error posting to telemetry: %s %s' %
+                          (r.status_code, r.text))
+                    continue
+
+            os.rename(os.path.join(outgoing, filename),
+                      os.path.join(submitted, filename))
+
+        session.close()
+
+        # Discard submitted data that is >= 30 days old
+        now = time.time()
+        for filename in os.listdir(submitted):
+            ctime = os.stat(os.path.join(submitted, filename)).st_ctime
+            if now - ctime >= 60*60*24*30:
+                os.remove(os.path.join(submitted, filename))
+
     def populate_context(context, key=None):
         if key is None:
             return
         if key == 'state_dir':
             state_dir, is_environ = get_state_dir()
             if is_environ:
                 if not os.path.exists(state_dir):
                     print('Creating global state directory from environment variable: %s'
@@ -325,16 +402,19 @@ def bootstrap(topsrcdir, mozilla_dir=Non
             return topsrcdir
 
         if key == 'pre_dispatch_handler':
             return pre_dispatch_handler
 
         if key == 'telemetry_handler':
             return telemetry_handler
 
+        if key == 'post_dispatch_handler':
+            return post_dispatch_handler
+
         raise AttributeError(key)
 
     mach = mach.main.Mach(os.getcwd())
     mach.populate_context_handler = populate_context
 
     for category, meta in CATEGORIES.items():
         mach.define_category(category, meta['short'], meta['long'],
             meta['priority'])
--- a/python/mach/mach/registrar.py
+++ b/python/mach/mach/registrar.py
@@ -86,16 +86,22 @@ class MachRegistrar(object):
         if debug_command:
             import pdb
             result = pdb.runcall(fn, **kwargs)
         else:
             result = fn(**kwargs)
 
         result = result or 0
         assert isinstance(result, (int, long))
+
+        if context:
+            postrun = getattr(context, 'post_dispatch_handler', None)
+            if postrun:
+                postrun(context, handler, args=kwargs)
+
         return result
 
     def dispatch(self, name, context=None, argv=None, **kwargs):
         """Dispatch/run a command.
 
         Commands can use this to call other commands.
         """
         # TODO handler.subcommand_handlers are ignored