Bug 1416052 - Check clobber state from Python; r?nalexander draft
authorGregory Szorc <gps@mozilla.com>
Thu, 09 Nov 2017 23:13:59 -0800
changeset 697422 1749a04f0cb4252c1c7ae0c141429c42efd5126d
parent 697347 b5eff35144984f5be78faa0c446b625482ceefd7
child 697423 c14544c1746821bc43803ee15da3c5a20c13e14d
push id88993
push userbmo:gps@mozilla.com
push dateTue, 14 Nov 2017 00:01:59 +0000
reviewersnalexander
bugs1416052
milestone59.0a1
Bug 1416052 - Check clobber state from Python; r?nalexander The clobber logic is already written in Python. Now that we always use mach in front of client.mk, we can check the clobber state before we execute client.mk. Since we always check the clobber state, we can remove the CLOBBER files from various dependencies in client.mk. The clobberer code should ensure everything is in a good state. The refactor of the clobber Python code required some changes to its testing. We drop some support for verifying output strings. But testing this correctly would require a bit of effort. I don't think it is worth it. MozReview-Commit-ID: 69CoImCgtNm
client.mk
python/mozbuild/mozbuild/controller/building.py
python/mozbuild/mozbuild/controller/clobber.py
python/mozbuild/mozbuild/test/controller/test_clobber.py
--- a/client.mk
+++ b/client.mk
@@ -53,20 +53,16 @@ MOZCONFIG_CONTENT := $(subst ||,$(CR),$(
 include $(OBJDIR)/.mozconfig-client-mk
 
 # As '||' was used as a newline separator, it means it's not occurring in
 # lines themselves. It can thus safely be used to replaces normal spaces,
 # to then replace newlines with normal spaces. This allows to get a list
 # of mozconfig output lines.
 MOZCONFIG_OUT_LINES := $(subst $(CR), ,$(subst $(NULL) $(NULL),||,$(MOZCONFIG_CONTENT)))
 
-ifdef AUTOCLOBBER
-export AUTOCLOBBER=1
-endif
-
 ifdef MOZ_PARALLEL_BUILD
   MOZ_MAKE_FLAGS := $(filter-out -j%,$(MOZ_MAKE_FLAGS))
   MOZ_MAKE_FLAGS += -j$(MOZ_PARALLEL_BUILD)
 endif
 
 # Automatically add -jN to make flags if not defined. N defaults to number of cores.
 ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS)))
   cores=$(shell $(PYTHON) -c 'import multiprocessing; print(multiprocessing.cpu_count())')
@@ -96,17 +92,17 @@ build::
 
 ifndef MACH
 $(error client.mk must be used via `mach`. Try running \
 `./mach $(firstword $(MAKECMDGOALS) $(.DEFAULT_GOAL))`)
 endif
 
 # For now, only output "export" lines and lines containing UPLOAD_EXTRA_FILES.
 MOZCONFIG_MK_LINES := $(filter export||% UPLOAD_EXTRA_FILES% %UPLOAD_EXTRA_FILES%,$(MOZCONFIG_OUT_LINES))
-$(OBJDIR)/.mozconfig.mk: $(TOPSRCDIR)/client.mk $(FOUND_MOZCONFIG) $(OBJDIR)/CLOBBER
+$(OBJDIR)/.mozconfig.mk: $(TOPSRCDIR)/client.mk $(FOUND_MOZCONFIG)
 	$(if $(MOZCONFIG_MK_LINES),( $(foreach line,$(MOZCONFIG_MK_LINES), echo '$(subst ||, ,$(line))';) )) > $@
 
 # Include that makefile so that it is created. This should not actually change
 # the environment since MOZCONFIG_CONTENT, which MOZCONFIG_OUT_LINES derives
 # from, has already been eval'ed.
 include $(OBJDIR)/.mozconfig.mk
 
 # Print out any options loaded from mozconfig.
@@ -148,17 +144,16 @@ EXTRA_CONFIG_DEPS := \
 $(CONFIGURES): %: %.in $(EXTRA_CONFIG_DEPS)
 	@echo Generating $@
 	cp -f $< $@
 	chmod +x $@
 
 CONFIG_STATUS_DEPS := \
   $(wildcard $(TOPSRCDIR)/*/confvars.sh) \
   $(CONFIGURES) \
-  $(TOPSRCDIR)/CLOBBER \
   $(TOPSRCDIR)/nsprpub/configure \
   $(TOPSRCDIR)/config/milestone.txt \
   $(TOPSRCDIR)/browser/config/version.txt \
   $(TOPSRCDIR)/browser/config/version_display.txt \
   $(TOPSRCDIR)/build/virtualenv_packages.txt \
   $(TOPSRCDIR)/python/mozbuild/mozbuild/virtualenv.py \
   $(TOPSRCDIR)/testing/mozbase/packages.txt \
   $(OBJDIR)/.mozconfig.json \
@@ -176,24 +171,19 @@ CONFIGURE_ENV_ARGS += \
 #   $(TOPSRCDIR) will set @srcdir@ to "."; otherwise, it is set to the full
 #   path of $(TOPSRCDIR).
 ifeq ($(TOPSRCDIR),$(OBJDIR))
   CONFIGURE = ./configure
 else
   CONFIGURE = $(TOPSRCDIR)/configure
 endif
 
-$(OBJDIR)/CLOBBER: $(TOPSRCDIR)/CLOBBER
-	$(PYTHON) $(TOPSRCDIR)/config/pythonpath.py -I $(TOPSRCDIR)/testing/mozbase/mozfile \
-	    $(TOPSRCDIR)/python/mozbuild/mozbuild/controller/clobber.py $(TOPSRCDIR) $(OBJDIR)
-
 configure-files: $(CONFIGURES)
 
 configure-preqs = \
-  $(OBJDIR)/CLOBBER \
   configure-files \
   save-mozconfig \
   $(OBJDIR)/.mozconfig.json \
   $(NULL)
 
 CREATE_MOZCONFIG_JSON = $(shell $(TOPSRCDIR)/mach environment --format=json -o $(OBJDIR)/.mozconfig.json)
 # Force CREATE_MOZCONFIG_JSON above to be resolved, without side effects in
 # case the result is non empty, and allowing an override on the make command
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -1,15 +1,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, unicode_literals
 
 import getpass
+import io
 import json
 import logging
 import os
 import subprocess
 import sys
 import time
 import which
 
@@ -27,16 +28,19 @@ try:
 except Exception:
     psutil = None
 
 from mach.mixin.logging import LoggingMixin
 from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
 
 import mozpack.path as mozpath
 
+from .clobber import (
+    Clobberer,
+)
 from ..base import (
     BuildEnvironmentNotFoundException,
     MozbuildObject,
 )
 from ..backend import (
     get_backend_class,
 )
 from ..testing import (
@@ -1324,16 +1328,20 @@ class BuildDriver(MozbuildObject):
     def _run_client_mk(self, target=None, line_handler=None, jobs=0,
                        verbose=None, keep_going=False, append_env=None):
         append_env = dict(append_env or {})
         append_env['TOPSRCDIR'] = self.topsrcdir
 
         append_env['CONFIG_GUESS'] = self.resolve_config_guess()
 
         mozconfig = self.mozconfig
+
+        if self._check_clobber(mozconfig, os.environ):
+            return 1
+
         mozconfig_client_mk = os.path.join(self.topobjdir,
                                            '.mozconfig-client-mk')
         with FileAvoidWrite(mozconfig_client_mk) as fh:
             for arg in mozconfig['make_extra'] or []:
                 fh.write(arg)
                 fh.write(b'\n')
             if mozconfig['make_flags']:
                 fh.write(b'MOZ_MAKE_FLAGS=%s\n' % b' '.join(mozconfig['make_flags']))
@@ -1354,8 +1362,37 @@ class BuildDriver(MozbuildObject):
                               print_directory=False,
                               target=target,
                               line_handler=line_handler,
                               log=False,
                               num_jobs=jobs,
                               silent=not verbose,
                               keep_going=keep_going,
                               append_env=append_env)
+
+    def _check_clobber(self, mozconfig, env):
+        auto_clobber = any([
+            env.get('AUTOCLOBBER', False),
+            (mozconfig['env'] or {}).get('added', {}).get('AUTOCLOBBER', False),
+            'AUTOCLOBBER=1' in (mozconfig['make_extra'] or []),
+        ])
+
+        clobberer = Clobberer(self.topsrcdir, self.topobjdir)
+        clobber_output = io.BytesIO()
+        res = clobberer.maybe_do_clobber(os.getcwd(), auto_clobber,
+                                         clobber_output)
+        clobber_output.seek(0)
+        for line in clobber_output.readlines():
+            self.log(logging.WARNING, 'clobber',
+                     {'msg': line.rstrip()}, '{msg}')
+
+        clobber_required, clobber_performed, clobber_message = res
+        if not clobber_required or clobber_performed:
+            if clobber_performed and env.get('TINDERBOX_OUTPUT'):
+                self.log(logging.WARNING, 'clobber',
+                         {'msg': 'TinderboxPrint: auto clobber'}, '{msg}')
+        else:
+            for line in clobber_message.splitlines():
+                self.log(logging.WARNING, 'clobber',
+                         {'msg': line.rstrip()}, '{msg}')
+            return True
+
+        return False
--- a/python/mozbuild/mozbuild/controller/clobber.py
+++ b/python/mozbuild/mozbuild/controller/clobber.py
@@ -199,39 +199,8 @@ class Clobberer(object):
             return True, False, self._message(
                 'Error when automatically clobbering: ' + str(error))
 
     def _message(self, reason):
         lines = [' ' + line for line in self.clobber_cause()]
 
         return CLOBBER_MESSAGE.format(clobber_reason='\n'.join(lines),
             no_reason='  ' + reason, clobber_file=self.obj_clobber)
-
-
-def main(args, env, cwd, fh=sys.stderr):
-    if len(args) != 2:
-        print('Usage: clobber.py topsrcdir topobjdir', file=fh)
-        return 1
-
-    topsrcdir, topobjdir = args
-
-    if not os.path.isabs(topsrcdir):
-        topsrcdir = os.path.abspath(topsrcdir)
-
-    if not os.path.isabs(topobjdir):
-        topobjdir = os.path.abspath(topobjdir)
-
-    auto = True if env.get('AUTOCLOBBER', False) else False
-    clobber = Clobberer(topsrcdir, topobjdir)
-    required, performed, message = clobber.maybe_do_clobber(cwd, auto, fh)
-
-    if not required or performed:
-        if performed and env.get('TINDERBOX_OUTPUT'):
-            print('TinderboxPrint: auto clobber', file=fh)
-        return 0
-
-    print(message, file=fh)
-    return 1
-
-
-if __name__ == '__main__':
-    sys.exit(main(sys.argv[1:], os.environ, os.getcwd(), sys.stdout))
-
--- a/python/mozbuild/mozbuild/test/controller/test_clobber.py
+++ b/python/mozbuild/mozbuild/test/controller/test_clobber.py
@@ -8,18 +8,25 @@ import os
 import shutil
 import tempfile
 import unittest
 
 from StringIO import StringIO
 
 from mozunit import main
 
-from mozbuild.controller.clobber import Clobberer
-from mozbuild.controller.clobber import main as clobber
+from mozbuild.base import (
+    MozbuildObject,
+)
+from mozbuild.controller.building import (
+    BuildDriver,
+)
+from mozbuild.controller.clobber import (
+    Clobberer,
+)
 
 
 class TestClobberer(unittest.TestCase):
     def setUp(self):
         self._temp_dirs = []
 
         return unittest.TestCase.setUp(self)
 
@@ -188,26 +195,26 @@ class TestClobberer(unittest.TestCase):
         old_time = os.path.getmtime(os.path.join(topsrcdir, 'CLOBBER')) - 60
         os.utime(obj_clobber, (old_time, old_time))
 
         # Check auto clobber is off by default
         env = dict(os.environ)
         if env.get('AUTOCLOBBER', False):
             del env['AUTOCLOBBER']
 
-        s = StringIO()
-        status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s)
-        self.assertEqual(status, 1)
-        self.assertIn('Automatic clobbering is not enabled', s.getvalue())
+        mbo = MozbuildObject(topsrcdir, None, None, topobjdir)
+        build = mbo._spawn(BuildDriver)
+
+        status = build._check_clobber(build.mozconfig, env)
+
+        self.assertEqual(status, True)
         self.assertTrue(os.path.exists(dummy_file))
 
         # Check auto clobber opt-in works
         env['AUTOCLOBBER'] = '1'
 
-        s = StringIO()
-        status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s)
-        self.assertEqual(status, 0)
-        self.assertIn('Successfully completed auto clobber', s.getvalue())
+        status = build._check_clobber(build.mozconfig, env)
+        self.assertFalse(status)
         self.assertFalse(os.path.exists(dummy_file))
 
 
 if __name__ == '__main__':
     main()