Bug 1416052 - Check clobber state from Python; r?nalexander
The clobber logic is already written in Python. Now that we
always use mach in front of client.mk, we can check the clobber
state before we execute client.mk.
Since we always check the clobber state, we can remove the
CLOBBER files from various dependencies in client.mk. The
clobberer code should ensure everything is in a good state.
The refactor of the clobber Python code required some changes to
its testing. We drop some support for verifying output strings.
But testing this correctly would require a bit of effort. I don't
think it is worth it.
MozReview-Commit-ID: 69CoImCgtNm
--- a/client.mk
+++ b/client.mk
@@ -53,20 +53,16 @@ MOZCONFIG_CONTENT := $(subst ||,$(CR),$(
include $(OBJDIR)/.mozconfig-client-mk
# As '||' was used as a newline separator, it means it's not occurring in
# lines themselves. It can thus safely be used to replaces normal spaces,
# to then replace newlines with normal spaces. This allows to get a list
# of mozconfig output lines.
MOZCONFIG_OUT_LINES := $(subst $(CR), ,$(subst $(NULL) $(NULL),||,$(MOZCONFIG_CONTENT)))
-ifdef AUTOCLOBBER
-export AUTOCLOBBER=1
-endif
-
ifdef MOZ_PARALLEL_BUILD
MOZ_MAKE_FLAGS := $(filter-out -j%,$(MOZ_MAKE_FLAGS))
MOZ_MAKE_FLAGS += -j$(MOZ_PARALLEL_BUILD)
endif
# Automatically add -jN to make flags if not defined. N defaults to number of cores.
ifeq (,$(findstring -j,$(MOZ_MAKE_FLAGS)))
cores=$(shell $(PYTHON) -c 'import multiprocessing; print(multiprocessing.cpu_count())')
@@ -96,17 +92,17 @@ build::
ifndef MACH
$(error client.mk must be used via `mach`. Try running \
`./mach $(firstword $(MAKECMDGOALS) $(.DEFAULT_GOAL))`)
endif
# For now, only output "export" lines and lines containing UPLOAD_EXTRA_FILES.
MOZCONFIG_MK_LINES := $(filter export||% UPLOAD_EXTRA_FILES% %UPLOAD_EXTRA_FILES%,$(MOZCONFIG_OUT_LINES))
-$(OBJDIR)/.mozconfig.mk: $(TOPSRCDIR)/client.mk $(FOUND_MOZCONFIG) $(OBJDIR)/CLOBBER
+$(OBJDIR)/.mozconfig.mk: $(TOPSRCDIR)/client.mk $(FOUND_MOZCONFIG)
$(if $(MOZCONFIG_MK_LINES),( $(foreach line,$(MOZCONFIG_MK_LINES), echo '$(subst ||, ,$(line))';) )) > $@
# Include that makefile so that it is created. This should not actually change
# the environment since MOZCONFIG_CONTENT, which MOZCONFIG_OUT_LINES derives
# from, has already been eval'ed.
include $(OBJDIR)/.mozconfig.mk
# Print out any options loaded from mozconfig.
@@ -148,17 +144,16 @@ EXTRA_CONFIG_DEPS := \
$(CONFIGURES): %: %.in $(EXTRA_CONFIG_DEPS)
@echo Generating $@
cp -f $< $@
chmod +x $@
CONFIG_STATUS_DEPS := \
$(wildcard $(TOPSRCDIR)/*/confvars.sh) \
$(CONFIGURES) \
- $(TOPSRCDIR)/CLOBBER \
$(TOPSRCDIR)/nsprpub/configure \
$(TOPSRCDIR)/config/milestone.txt \
$(TOPSRCDIR)/browser/config/version.txt \
$(TOPSRCDIR)/browser/config/version_display.txt \
$(TOPSRCDIR)/build/virtualenv_packages.txt \
$(TOPSRCDIR)/python/mozbuild/mozbuild/virtualenv.py \
$(TOPSRCDIR)/testing/mozbase/packages.txt \
$(OBJDIR)/.mozconfig.json \
@@ -176,24 +171,19 @@ CONFIGURE_ENV_ARGS += \
# $(TOPSRCDIR) will set @srcdir@ to "."; otherwise, it is set to the full
# path of $(TOPSRCDIR).
ifeq ($(TOPSRCDIR),$(OBJDIR))
CONFIGURE = ./configure
else
CONFIGURE = $(TOPSRCDIR)/configure
endif
-$(OBJDIR)/CLOBBER: $(TOPSRCDIR)/CLOBBER
- $(PYTHON) $(TOPSRCDIR)/config/pythonpath.py -I $(TOPSRCDIR)/testing/mozbase/mozfile \
- $(TOPSRCDIR)/python/mozbuild/mozbuild/controller/clobber.py $(TOPSRCDIR) $(OBJDIR)
-
configure-files: $(CONFIGURES)
configure-preqs = \
- $(OBJDIR)/CLOBBER \
configure-files \
save-mozconfig \
$(OBJDIR)/.mozconfig.json \
$(NULL)
CREATE_MOZCONFIG_JSON = $(shell $(TOPSRCDIR)/mach environment --format=json -o $(OBJDIR)/.mozconfig.json)
# Force CREATE_MOZCONFIG_JSON above to be resolved, without side effects in
# case the result is non empty, and allowing an override on the make command
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -1,15 +1,16 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, unicode_literals
import getpass
+import io
import json
import logging
import os
import subprocess
import sys
import time
import which
@@ -27,16 +28,19 @@ try:
except Exception:
psutil = None
from mach.mixin.logging import LoggingMixin
from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
import mozpack.path as mozpath
+from .clobber import (
+ Clobberer,
+)
from ..base import (
BuildEnvironmentNotFoundException,
MozbuildObject,
)
from ..backend import (
get_backend_class,
)
from ..testing import (
@@ -1324,16 +1328,20 @@ class BuildDriver(MozbuildObject):
def _run_client_mk(self, target=None, line_handler=None, jobs=0,
verbose=None, keep_going=False, append_env=None):
append_env = dict(append_env or {})
append_env['TOPSRCDIR'] = self.topsrcdir
append_env['CONFIG_GUESS'] = self.resolve_config_guess()
mozconfig = self.mozconfig
+
+ if self._check_clobber(mozconfig, os.environ):
+ return 1
+
mozconfig_client_mk = os.path.join(self.topobjdir,
'.mozconfig-client-mk')
with FileAvoidWrite(mozconfig_client_mk) as fh:
for arg in mozconfig['make_extra'] or []:
fh.write(arg)
fh.write(b'\n')
if mozconfig['make_flags']:
fh.write(b'MOZ_MAKE_FLAGS=%s\n' % b' '.join(mozconfig['make_flags']))
@@ -1354,8 +1362,37 @@ class BuildDriver(MozbuildObject):
print_directory=False,
target=target,
line_handler=line_handler,
log=False,
num_jobs=jobs,
silent=not verbose,
keep_going=keep_going,
append_env=append_env)
+
+ def _check_clobber(self, mozconfig, env):
+ auto_clobber = any([
+ env.get('AUTOCLOBBER', False),
+ (mozconfig['env'] or {}).get('added', {}).get('AUTOCLOBBER', False),
+ 'AUTOCLOBBER=1' in (mozconfig['make_extra'] or []),
+ ])
+
+ clobberer = Clobberer(self.topsrcdir, self.topobjdir)
+ clobber_output = io.BytesIO()
+ res = clobberer.maybe_do_clobber(os.getcwd(), auto_clobber,
+ clobber_output)
+ clobber_output.seek(0)
+ for line in clobber_output.readlines():
+ self.log(logging.WARNING, 'clobber',
+ {'msg': line.rstrip()}, '{msg}')
+
+ clobber_required, clobber_performed, clobber_message = res
+ if not clobber_required or clobber_performed:
+ if clobber_performed and env.get('TINDERBOX_OUTPUT'):
+ self.log(logging.WARNING, 'clobber',
+ {'msg': 'TinderboxPrint: auto clobber'}, '{msg}')
+ else:
+ for line in clobber_message.splitlines():
+ self.log(logging.WARNING, 'clobber',
+ {'msg': line.rstrip()}, '{msg}')
+ return True
+
+ return False
--- a/python/mozbuild/mozbuild/controller/clobber.py
+++ b/python/mozbuild/mozbuild/controller/clobber.py
@@ -199,39 +199,8 @@ class Clobberer(object):
return True, False, self._message(
'Error when automatically clobbering: ' + str(error))
def _message(self, reason):
lines = [' ' + line for line in self.clobber_cause()]
return CLOBBER_MESSAGE.format(clobber_reason='\n'.join(lines),
no_reason=' ' + reason, clobber_file=self.obj_clobber)
-
-
-def main(args, env, cwd, fh=sys.stderr):
- if len(args) != 2:
- print('Usage: clobber.py topsrcdir topobjdir', file=fh)
- return 1
-
- topsrcdir, topobjdir = args
-
- if not os.path.isabs(topsrcdir):
- topsrcdir = os.path.abspath(topsrcdir)
-
- if not os.path.isabs(topobjdir):
- topobjdir = os.path.abspath(topobjdir)
-
- auto = True if env.get('AUTOCLOBBER', False) else False
- clobber = Clobberer(topsrcdir, topobjdir)
- required, performed, message = clobber.maybe_do_clobber(cwd, auto, fh)
-
- if not required or performed:
- if performed and env.get('TINDERBOX_OUTPUT'):
- print('TinderboxPrint: auto clobber', file=fh)
- return 0
-
- print(message, file=fh)
- return 1
-
-
-if __name__ == '__main__':
- sys.exit(main(sys.argv[1:], os.environ, os.getcwd(), sys.stdout))
-
--- a/python/mozbuild/mozbuild/test/controller/test_clobber.py
+++ b/python/mozbuild/mozbuild/test/controller/test_clobber.py
@@ -8,18 +8,25 @@ import os
import shutil
import tempfile
import unittest
from StringIO import StringIO
from mozunit import main
-from mozbuild.controller.clobber import Clobberer
-from mozbuild.controller.clobber import main as clobber
+from mozbuild.base import (
+ MozbuildObject,
+)
+from mozbuild.controller.building import (
+ BuildDriver,
+)
+from mozbuild.controller.clobber import (
+ Clobberer,
+)
class TestClobberer(unittest.TestCase):
def setUp(self):
self._temp_dirs = []
return unittest.TestCase.setUp(self)
@@ -188,26 +195,26 @@ class TestClobberer(unittest.TestCase):
old_time = os.path.getmtime(os.path.join(topsrcdir, 'CLOBBER')) - 60
os.utime(obj_clobber, (old_time, old_time))
# Check auto clobber is off by default
env = dict(os.environ)
if env.get('AUTOCLOBBER', False):
del env['AUTOCLOBBER']
- s = StringIO()
- status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s)
- self.assertEqual(status, 1)
- self.assertIn('Automatic clobbering is not enabled', s.getvalue())
+ mbo = MozbuildObject(topsrcdir, None, None, topobjdir)
+ build = mbo._spawn(BuildDriver)
+
+ status = build._check_clobber(build.mozconfig, env)
+
+ self.assertEqual(status, True)
self.assertTrue(os.path.exists(dummy_file))
# Check auto clobber opt-in works
env['AUTOCLOBBER'] = '1'
- s = StringIO()
- status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s)
- self.assertEqual(status, 0)
- self.assertIn('Successfully completed auto clobber', s.getvalue())
+ status = build._check_clobber(build.mozconfig, env)
+ self.assertFalse(status)
self.assertFalse(os.path.exists(dummy_file))
if __name__ == '__main__':
main()