--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -3,16 +3,17 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function, unicode_literals
import errno
import json
import os
import platform
+import random
import sys
import time
import uuid
import __builtin__
from types import ModuleType
@@ -175,16 +176,24 @@ CATEGORIES = {
'disabled': {
'short': 'Disabled',
'long': 'The disabled commands are hidden by default. Use -v to display them. These commands are unavailable for your current context, run "mach <command>" to see why.',
'priority': 0,
}
}
+# Server to which to submit telemetry data
+BUILD_TELEMETRY_SERVER = 'http://52.88.27.118/build-metrics-dev'
+
+
+# We submit data to telemetry approximately every this many mach invocations
+TELEMETRY_SUBMISSION_FREQUENCY = 10
+
+
def get_state_dir():
"""Obtain the path to a directory to hold state.
Returns a tuple of the path and a bool indicating whether the value came
from an environment variable.
"""
state_user_dir = os.path.expanduser('~/.mozbuild')
state_env_dir = os.environ.get('MOZBUILD_STATE_PATH', None)
@@ -238,44 +247,50 @@ def bootstrap(topsrcdir, mozilla_dir=Non
except OSError as e:
if e.errno != errno.EEXIST:
raise
# Add common metadata to help submit sorted data later on.
# For now, we'll just record the mach command that was invoked.
data['argv'] = sys.argv
- with open(os.path.join(outgoing_dir, str(uuid.uuid4())), 'w') as f:
+ with open(os.path.join(outgoing_dir, str(uuid.uuid4()) + '.json'),
+ 'w') as f:
json.dump(data, f, sort_keys=True)
+ def should_skip_dispatch(context, handler):
+ # The user is performing a maintenance command.
+ if handler.name in ('bootstrap', 'doctor', 'mach-commands', 'mercurial-setup'):
+ return True
+
+ # We are running in automation.
+ if 'MOZ_AUTOMATION' in os.environ or 'TASK_ID' in os.environ:
+ return True
+
+ # The environment is likely a machine invocation.
+ if sys.stdin.closed or not sys.stdin.isatty():
+ return True
+
+ return False
+
def pre_dispatch_handler(context, handler, args):
"""Perform global checks before command dispatch.
Currently, our goal is to ensure developers periodically run
`mach mercurial-setup` (when applicable) to ensure their Mercurial
tools are up to date.
"""
# Don't do anything when...
-
- # The user is performing a maintenance command.
- if handler.name in ('bootstrap', 'doctor', 'mach-commands', 'mercurial-setup'):
- return
-
- # We are running in automation.
- if 'MOZ_AUTOMATION' in os.environ or 'TASK_ID' in os.environ:
+ if should_skip_dispatch(context, handler):
return
# We are a curmudgeon who has found this undocumented variable.
if 'I_PREFER_A_SUBOPTIMAL_MERCURIAL_EXPERIENCE' in os.environ:
return
- # The environment is likely a machine invocation.
- if sys.stdin.closed or not sys.stdin.isatty():
- return
-
# Mercurial isn't managing this source checkout.
if not os.path.exists(os.path.join(topsrcdir, '.hg')):
return
state_dir = get_state_dir()[0]
last_check_path = os.path.join(state_dir, 'mercurial',
'setup.lastcheck')
@@ -286,16 +301,78 @@ def bootstrap(topsrcdir, mozilla_dir=Non
if e.errno != errno.ENOENT:
raise
# No last run file means mercurial-setup has never completed.
if mtime is None:
print(NO_MERCURIAL_SETUP.format(mach=sys.argv[0]), file=sys.stderr)
sys.exit(2)
+ def post_dispatch_handler(context, handler, args):
+ """Perform global operations after command dispatch.
+
+
+ For now, we will use this to handle build system telemetry.
+ """
+ # Don't do anything when...
+ if should_skip_dispatch(context, handler):
+ return
+
+ # We have not opted-in to telemetry
+ if 'BUILD_SYSTEM_TELEMETRY' not in os.environ:
+ return
+
+ # Every n-th operation
+ if random.randint(1, TELEMETRY_SUBMISSION_FREQUENCY) != 1:
+ return
+
+ # No data to work with anyway
+ outgoing = os.path.join(get_state_dir()[0], 'telemetry', 'outgoing')
+ if not os.path.isdir(outgoing):
+ return
+
+ # We can't import requests until after it has been added during the
+ # bootstrapping below.
+ import requests
+
+ submitted = os.path.join(get_state_dir()[0], 'telemetry', 'submitted')
+ try:
+ os.mkdir(submitted)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
+ session = requests.Session()
+ for filename in os.listdir(outgoing):
+ path = os.path.join(outgoing, filename)
+ if os.path.isdir(path) or not path.endswith('.json'):
+ continue
+ with open(path, 'r') as f:
+ data = f.read()
+ r = session.post(BUILD_TELEMETRY_SERVER, data=data,
+ headers={'Content-Type': 'application/json'})
+ # TODO: some of these errors are likely not recoverable, as
+ # written, we'll retry indefinitely
+ if r.status_code != 200:
+ print('Error posting to telemetry: %s %s' %
+ (r.status_code, r.text))
+ continue
+
+ os.rename(os.path.join(outgoing, filename),
+ os.path.join(submitted, filename))
+
+ session.close()
+
+ # Discard submitted data that is >= 30 days old
+ now = time.time()
+ for filename in os.listdir(submitted):
+ ctime = os.stat(os.path.join(submitted, filename)).st_ctime
+ if now - ctime >= 60*60*24*30:
+ os.remove(os.path.join(submitted, filename))
+
def populate_context(context, key=None):
if key is None:
return
if key == 'state_dir':
state_dir, is_environ = get_state_dir()
if is_environ:
if not os.path.exists(state_dir):
print('Creating global state directory from environment variable: %s'
@@ -325,16 +402,19 @@ def bootstrap(topsrcdir, mozilla_dir=Non
return topsrcdir
if key == 'pre_dispatch_handler':
return pre_dispatch_handler
if key == 'telemetry_handler':
return telemetry_handler
+ if key == 'post_dispatch_handler':
+ return post_dispatch_handler
+
raise AttributeError(key)
mach = mach.main.Mach(os.getcwd())
mach.populate_context_handler = populate_context
for category, meta in CATEGORIES.items():
mach.define_category(category, meta['short'], meta['long'],
meta['priority'])
--- a/python/mach/mach/registrar.py
+++ b/python/mach/mach/registrar.py
@@ -86,16 +86,22 @@ class MachRegistrar(object):
if debug_command:
import pdb
result = pdb.runcall(fn, **kwargs)
else:
result = fn(**kwargs)
result = result or 0
assert isinstance(result, (int, long))
+
+ if context:
+ postrun = getattr(context, 'post_dispatch_handler', None)
+ if postrun:
+ postrun(context, handler, args=kwargs)
+
return result
def dispatch(self, name, context=None, argv=None, **kwargs):
"""Dispatch/run a command.
Commands can use this to call other commands.
"""
# TODO handler.subcommand_handlers are ignored