Bug 1334209 - Activate mozharness virtualenv in-process from one click loaner mach commands, r?jmaher draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 31 Jan 2017 11:53:36 -0500
changeset 469142 5af5906eee7cd1052a01afde463bf4599a58fe59
parent 469044 9e7b1041929fccc06f6fad91cf66b9edcdfc0129
child 544103 b032b9d19ed849df4b4427c630eb34b8a19aaeb7
push id43625
push userahalberstadt@mozilla.com
push dateWed, 01 Feb 2017 18:58:17 +0000
reviewersjmaher
bugs1334209
milestone54.0a1
Bug 1334209 - Activate mozharness virtualenv in-process from one click loaner mach commands, r?jmaher Previously the run-wizard script would add a command to source the virtualenv in ~/.bashrc after mozharness finished setting things up. This is fragile, assumes people are using bash, etc. Plus it appeared to intermittently fail for some users. Instead, this activates the virtualenv directly from individual mach commands that need it. This guarantees we will always be using the virtualenv if required (and won't be using it if not). The 'activate_this.py' script is invoked the same way that we do it for in-tree mach commands: https://dxr.mozilla.org/mozilla-central/rev/9c06e744b1befb3a2e2fdac7414ce18220774a1d/python/mozbuild/mozbuild/virtualenv.py#456 MozReview-Commit-ID: CfcoiVJXQTl
layout/tools/reftest/mach_test_package_commands.py
taskcluster/scripts/tester/run-wizard
testing/marionette/mach_test_package_commands.py
testing/mochitest/mach_test_package_commands.py
testing/tools/mach_test_package_bootstrap.py
testing/xpcshell/mach_test_package_commands.py
--- a/layout/tools/reftest/mach_test_package_commands.py
+++ b/layout/tools/reftest/mach_test_package_commands.py
@@ -83,10 +83,11 @@ class ReftestCommands(object):
 
     def __init__(self, context):
         self.context = context
 
     @Command('reftest', category='testing',
              description='Run the reftest harness.',
              parser=setup_argument_parser)
     def reftest(self, **kwargs):
+        self.context.activate_mozharness_venv()
         kwargs['suite'] = 'reftest'
         return run_reftest(self.context, **kwargs)
--- a/taskcluster/scripts/tester/run-wizard
+++ b/taskcluster/scripts/tester/run-wizard
@@ -53,23 +53,16 @@ def setup():
     build_dir = os.path.expanduser(os.path.join('~', 'workspace', 'build'))
     mach_src = os.path.join(build_dir, 'tests', 'mach')
     mach_dest = os.path.expanduser(os.path.join('~', 'bin', 'mach'))
 
     if os.path.exists(mach_dest):
         os.remove(mach_dest)
     os.symlink(mach_src, mach_dest)
 
-    activate = os.path.join(build_dir, 'venv', 'bin', 'activate')
-    if os.path.isfile(activate):
-        # TODO Support other shells
-        bashrc = os.path.expanduser(os.path.join('~', '.bashrc'))
-        with open(bashrc, 'ab') as f:
-            f.write(". {}".format(activate))
-
     print("""
 Mozharness has finished downloading the build and tests to:
 {}
 
 A limited mach environment has also been set up and added to the $PATH, but
 it may be missing the command you need. To see a list of commands, run:
     $ mach help
 """.lstrip().format(build_dir))
--- a/testing/marionette/mach_test_package_commands.py
+++ b/testing/marionette/mach_test_package_commands.py
@@ -59,9 +59,10 @@ class MachCommands(object):
         self.context = context
 
     @Command(
         'marionette-test', category='testing',
         description='Run a Marionette test (Check UI or the internal JavaScript '
                     'using marionette).',
         parser=setup_marionette_argument_parser)
     def run_marionette_test(self, **kwargs):
+        self.context.activate_mozharness_venv()
         return run_marionette(self.context, **kwargs)
--- a/testing/mochitest/mach_test_package_commands.py
+++ b/testing/mochitest/mach_test_package_commands.py
@@ -88,9 +88,10 @@ class MochitestCommands(object):
 
     def __init__(self, context):
         self.context = context
 
     @Command('mochitest', category='testing',
              description='Run the mochitest harness.',
              parser=setup_argument_parser)
     def mochitest(self, **kwargs):
+        self.context.activate_mozharness_venv()
         return run_mochitest(self.context, **kwargs)
--- a/testing/tools/mach_test_package_bootstrap.py
+++ b/testing/tools/mach_test_package_bootstrap.py
@@ -71,28 +71,57 @@ CATEGORIES = {
         'long': 'The disabled commands are hidden by default. Use -v to display them. '
                 'These commands are unavailable for your current context, '
                 'run "mach <command>" to see why.',
         'priority': 0,
     }
 }
 
 
+IS_WIN = sys.platform in ('win32', 'cygwin')
+
+
 def ancestors(path, depth=0):
     """Emit the parent directories of a path."""
     count = 1
     while path and count != depth:
         yield path
         newpath = os.path.dirname(path)
         if newpath == path:
             break
         path = newpath
         count += 1
 
 
+def activate_mozharness_venv(context):
+    """Activate the mozharness virtualenv in-process."""
+    venv = os.path.join(context.mozharness_workdir,
+                        context.mozharness_config.get('virtualenv_path', 'venv'))
+
+    if not os.path.isdir(venv):
+        print("No mozharness virtualenv detected at '{}'.".format(venv))
+        return 1
+
+    venv_bin = os.path.join(venv, 'Scripts' if IS_WIN else 'bin')
+    activate_path = os.path.join(venv_bin, 'activate_this.py')
+
+    execfile(activate_path, dict(__file__=activate_path))
+
+    if isinstance(os.environ['PATH'], unicode):
+        os.environ['PATH'] = os.environ['PATH'].encode('utf-8')
+
+    # sys.executable is used by mochitest-media to start the websocketprocessbridge,
+    # for some reason it doesn't get set when calling `activate_this.py` so set it
+    # here instead.
+    binary = 'python'
+    if IS_WIN:
+        binary += '.exe'
+    sys.executable = os.path.join(venv_bin, binary)
+
+
 def find_firefox(context):
     """Try to automagically find the firefox binary."""
     import mozinstall
     search_paths = []
 
     # Check for a mozharness setup
     config = context.mozharness_config
     if config and 'binary_path' in config:
@@ -171,16 +200,19 @@ def bootstrap(test_package_root):
                         return json.load(f)
             return {}
 
         if key == 'mozharness_workdir':
             config = context.mozharness_config
             if config:
                 return os.path.join(config['base_work_dir'], config['work_dir'])
 
+        if key == 'activate_mozharness_venv':
+            return types.MethodType(activate_mozharness_venv, context)
+
     mach = mach.main.Mach(os.getcwd())
     mach.populate_context_handler = populate_context
 
     for category, meta in CATEGORIES.items():
         mach.define_category(category, meta['short'], meta['long'],
                              meta['priority'])
 
     for path in MACH_MODULES:
--- a/testing/xpcshell/mach_test_package_commands.py
+++ b/testing/xpcshell/mach_test_package_commands.py
@@ -56,9 +56,10 @@ class MochitestCommands(object):
 
     def __init__(self, context):
         self.context = context
 
     @Command('xpcshell-test', category='testing',
              description='Run the xpcshell harness.',
              parser=parser_desktop)
     def xpcshell(self, **kwargs):
+        self.context.activate_mozharness_venv()
         return run_xpcshell(self.context, **kwargs)