bug 1416891 - allow LOCALIZED_FILES to contain objdir paths as long as they're also from LOCALIZED_GENERATED_FILES. r=nalexander draft
authorTed Mielczarek <ted@mielczarek.org>
Fri, 17 Nov 2017 12:16:20 -0500
changeset 705774 7d6cf3bf6f49c4a9d650d1f0f041634f82be027f
parent 705773 6e1ef09a316f3ae5ab6eafd4068c01210bbda79b
child 705775 3febcff2afe9821bcdbc4becc47ae22b7940b44e
push id91576
push userbmo:ted@mielczarek.org
push dateThu, 30 Nov 2017 16:56:58 +0000
reviewersnalexander
bugs1416891
milestone59.0a1
bug 1416891 - allow LOCALIZED_FILES to contain objdir paths as long as they're also from LOCALIZED_GENERATED_FILES. r=nalexander LOCALIZED_FILES and LOCALIZED_GENERATED_FILES are analogs of FINAL_TARGET_FILES and GENERATED_FILES, but they receive special handling in the recursive make backend so that l10n repacks work properly. To this end, we support using the output of LOCALIZED_GENERATED_FILES in LOCALIZED_FILES, but not mixing localized with non-localized targets. MozReview-Commit-ID: GCJAUfUG8OZ
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/frontend/data/localized-files-from-generated/moz.build
python/mozbuild/mozbuild/test/frontend/data/localized-files-not-localized-generated/moz.build
python/mozbuild/mozbuild/test/frontend/data/localized-generated-files-final-target-files/moz.build
python/mozbuild/mozbuild/test/frontend/test_emitter.py
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -1427,20 +1427,22 @@ VARIABLES = {
         ``FINAL_TARGET_FILES``. For a build with ``--enable-ui-locale``,
         the file will be taken from ``$LOCALE_SRCDIR``, with the leading
         ``en-US`` removed. For a l10n repack of an en-US build, the file
         will be taken from the first location where it exists from:
         * the merged locale directory if it exists
         * ``$LOCALE_SRCDIR`` with the leading ``en-US`` removed
         * the in-tree en-US location
 
-        Paths specified here must be relative to the source directory and must
-        include a leading ``en-US``. Wildcards are allowed, and will be
-        expanded at the time of locale packaging to match files in the
-        locale directory.
+        Source directory paths specified here must must include a leading ``en-US``.
+        Wildcards are allowed, and will be expanded at the time of locale packaging to match
+        files in the locale directory.
+
+        Object directory paths are allowed here only if the path matches an entry in
+        ``LOCALIZED_GENERATED_FILES``.
 
         Files that are missing from a locale will typically have the en-US
         version used, but for wildcard expansions only files from the
         locale directory will be used, even if that means no files will
         be copied.
 
         Example::
 
@@ -1471,16 +1473,19 @@ VARIABLES = {
 
         Refer to the documentation of ``GENERATED_FILES``; for the most part things work the same.
         The two major differences are:
         1. The function in the Python script will be passed an additional keyword argument `locale`
            which provides the locale in use, i.e. ``en-US``.
         2. The ``inputs`` list may contain paths to files that will be taken from the locale
            source directory (see ``LOCALIZED_FILES`` for a discussion of the specifics). Paths
            in ``inputs`` starting with ``en-US/`` are considered localized files.
+
+        To place the generated output file in a specific location, list its objdir path in
+        ``LOCALIZED_FILES``.
         """),
 
     'OBJDIR_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list,
         """List of files to be installed anywhere in the objdir. Use sparingly.
 
         ``OBJDIR_FILES`` is similar to FINAL_TARGET_FILES, but it allows copying
         anywhere in the object directory. This is intended for various one-off
         cases, not for general use. If you wish to add entries to OBJDIR_FILES,
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -1009,19 +1009,22 @@ class TreeMetadataEmitter(LoggingMixin):
             if (context.config.substs.get('MOZ_DEBUG') and
                     not context.config.substs.get('MOZ_NO_DEBUG_RTL')):
                 rtl_flag += 'd'
             computed_flags.resolve_flags('RTL', [rtl_flag])
             if not context.config.substs.get('CROSS_COMPILE'):
                 computed_host_flags.resolve_flags('RTL', [rtl_flag])
 
         generated_files = set()
+        localized_generated_files = set()
         for obj in self._process_generated_files(context):
             for f in obj.outputs:
                 generated_files.add(f)
+                if obj.localized:
+                    localized_generated_files.add(f)
             yield obj
 
         for path in context['CONFIGURE_SUBST_FILES']:
             sub = self._create_substitution(ConfigFileSubstitution, context,
                 path)
             generated_files.add(str(sub.relpath))
             yield sub
 
@@ -1104,27 +1107,27 @@ class TreeMetadataEmitter(LoggingMixin):
                     components.extend(files)
                 if base == 'defaults/pref':
                     has_prefs = True
                 if mozpath.split(base)[0] == 'res':
                     has_resources = True
                 for f in files:
                     if (var in ('FINAL_TARGET_PP_FILES',
                                 'OBJDIR_PP_FILES',
-                                'LOCALIZED_FILES',
                                 'LOCALIZED_PP_FILES') and
                         not isinstance(f, SourcePath)):
                         raise SandboxValidationError(
                                 ('Only source directory paths allowed in ' +
                                  '%s: %s')
                                 % (var, f,), context)
-                    if var.startswith('LOCALIZED_') and not f.startswith('en-US/'):
-                        raise SandboxValidationError(
-                                '%s paths must start with `en-US/`: %s'
-                                % (var, f,), context)
+                    if var.startswith('LOCALIZED_'):
+                        if isinstance(f, SourcePath) and not f.startswith('en-US/'):
+                            raise SandboxValidationError(
+                                    '%s paths must start with `en-US/`: %s'
+                                    % (var, f,), context)
                     if not isinstance(f, ObjDirPath):
                         path = f.full_path
                         if '*' not in path and not os.path.exists(path):
                             raise SandboxValidationError(
                                 'File listed in %s does not exist: %s'
                                 % (var, path), context)
                     else:
                         # TODO: Bug 1254682 - The '/' check is to allow
@@ -1132,16 +1135,31 @@ class TreeMetadataEmitter(LoggingMixin):
                         # which is done occasionally for tests. However, it
                         # means we don't fail early if the file isn't actually
                         # created by the other moz.build file.
                         if f.target_basename not in generated_files and '/' not in f:
                             raise SandboxValidationError(
                                 ('Objdir file listed in %s not in ' +
                                  'GENERATED_FILES: %s') % (var, f), context)
 
+                        if var.startswith('LOCALIZED_'):
+                            # Further require that LOCALIZED_FILES are from
+                            # LOCALIZED_GENERATED_FILES.
+                            if f.target_basename not in localized_generated_files:
+                                raise SandboxValidationError(
+                                    ('Objdir file listed in %s not in ' +
+                                     'LOCALIZED_GENERATED_FILES: %s') % (var, f), context)
+                        else:
+                            # Additionally, don't allow LOCALIZED_GENERATED_FILES to be used
+                            # in anything *but* LOCALIZED_FILES.
+                            if f.target_basename in localized_generated_files:
+                                raise SandboxValidationError(
+                                    ('Outputs of LOCALIZED_GENERATED_FILES cannot be used in %s: ' +
+                                     '%s') % (var, f), context)
+
             # Addons (when XPI_NAME is defined) and Applications (when
             # DIST_SUBDIR is defined) use a different preferences directory
             # (default/preferences) from the one the GRE uses (defaults/pref).
             # Hence, we move the files from the latter to the former in that
             # case.
             if has_prefs and (context.get('XPI_NAME') or
                               context.get('DIST_SUBDIR')):
                 all_files.defaults.preferences += all_files.defaults.pref
copy from python/mozbuild/mozbuild/test/frontend/data/localized-generated-files/moz.build
copy to python/mozbuild/mozbuild/test/frontend/data/localized-files-from-generated/moz.build
--- a/python/mozbuild/mozbuild/test/frontend/data/localized-generated-files/moz.build
+++ b/python/mozbuild/mozbuild/test/frontend/data/localized-files-from-generated/moz.build
@@ -1,5 +1,6 @@
 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
 # Any copyright is dedicated to the Public Domain.
 # http://creativecommons.org/publicdomain/zero/1.0/
 
-LOCALIZED_GENERATED_FILES += [ 'abc.ini', ('bar', 'baz') ]
+LOCALIZED_GENERATED_FILES += [ 'abc.ini' ]
+LOCALIZED_FILES += [ '!abc.ini' ]
copy from python/mozbuild/mozbuild/test/frontend/data/localized-generated-files/moz.build
copy to python/mozbuild/mozbuild/test/frontend/data/localized-files-not-localized-generated/moz.build
--- a/python/mozbuild/mozbuild/test/frontend/data/localized-generated-files/moz.build
+++ b/python/mozbuild/mozbuild/test/frontend/data/localized-files-not-localized-generated/moz.build
@@ -1,5 +1,6 @@
 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
 # Any copyright is dedicated to the Public Domain.
 # http://creativecommons.org/publicdomain/zero/1.0/
 
-LOCALIZED_GENERATED_FILES += [ 'abc.ini', ('bar', 'baz') ]
+GENERATED_FILES += [ 'abc.ini' ]
+LOCALIZED_FILES += [ '!abc.ini' ]
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/localized-generated-files-final-target-files/moz.build
@@ -0,0 +1,6 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+LOCALIZED_GENERATED_FILES += [ 'abc.ini' ]
+FINAL_TARGET_FILES += [ '!abc.ini' ]
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -483,16 +483,46 @@ class TestEmitterBasic(unittest.TestCase
         expected = ['abc.ini', ('bar', 'baz'), ]
         for o, f in zip(objs, expected):
             expected_filename = f if isinstance(f, tuple) else (f,)
             self.assertEqual(o.outputs, expected_filename)
             self.assertEqual(o.script, None)
             self.assertEqual(o.method, None)
             self.assertEqual(o.inputs, [])
 
+    def test_localized_files_from_generated(self):
+        """Test that using LOCALIZED_GENERATED_FILES and then putting the output in
+        LOCALIZED_FILES as an objdir path works.
+        """
+        reader = self.reader('localized-files-from-generated')
+        objs = self.read_topsrcdir(reader)
+
+        self.assertEqual(len(objs), 2)
+        self.assertIsInstance(objs[0], GeneratedFile)
+        self.assertIsInstance(objs[1], LocalizedFiles)
+
+    def test_localized_files_not_localized_generated(self):
+        """Test that using GENERATED_FILES and then putting the output in
+        LOCALIZED_FILES as an objdir path produces an error.
+        """
+        reader = self.reader('localized-files-not-localized-generated')
+        with self.assertRaisesRegexp(SandboxValidationError,
+            'Objdir file listed in LOCALIZED_FILES not in LOCALIZED_GENERATED_FILES:'):
+            objs = self.read_topsrcdir(reader)
+
+
+    def test_localized_generated_files_final_target_files(self):
+        """Test that using LOCALIZED_GENERATED_FILES and then putting the output in
+        FINAL_TARGET_FILES as an objdir path produces an error.
+        """
+        reader = self.reader('localized-generated-files-final-target-files')
+        with self.assertRaisesRegexp(SandboxValidationError,
+            'Outputs of LOCALIZED_GENERATED_FILES cannot be used in FINAL_TARGET_FILES:'):
+            objs = self.read_topsrcdir(reader)
+
     def test_generated_files_method_names(self):
         reader = self.reader('generated-files-method-names')
         objs = self.read_topsrcdir(reader)
 
         self.assertEqual(len(objs), 2)
         for o in objs:
             self.assertIsInstance(o, GeneratedFile)