bug 1416891 - add LOCALIZED_GENERATED_FILES to the moz.build sandbox. r=nalexander draft
authorTed Mielczarek <ted@mielczarek.org>
Thu, 16 Nov 2017 13:44:14 -0500
changeset 705773 6e1ef09a316f3ae5ab6eafd4068c01210bbda79b
parent 705036 3f6b9aaed8cd57954e0c960cde06d25228196456
child 705774 7d6cf3bf6f49c4a9d650d1f0f041634f82be027f
push id91576
push userbmo:ted@mielczarek.org
push dateThu, 30 Nov 2017 16:56:58 +0000
reviewersnalexander
bugs1416891
milestone59.0a1
bug 1416891 - add LOCALIZED_GENERATED_FILES to the moz.build sandbox. r=nalexander This change adds LOCALIZED_GENERATED_FILES, which emits GeneratedFile objects just like GENERATED_FILES. It also adds a `localized` field to GeneratedFile which will be `True` for objects emitted from LOCALIZED_GENERATED_FILES. MozReview-Commit-ID: 3iWGLMkbF2C
python/mozbuild/mozbuild/frontend/context.py
python/mozbuild/mozbuild/frontend/data.py
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/frontend/data/localized-generated-files/moz.build
python/mozbuild/mozbuild/test/frontend/test_emitter.py
--- a/python/mozbuild/mozbuild/frontend/context.py
+++ b/python/mozbuild/mozbuild/frontend/context.py
@@ -896,16 +896,21 @@ DependentTestsEntry = ContextDerivedType
                                                 ('tags', OrderedStringList),
                                                 ('flavors', OrderedTestFlavorList))
 BugzillaComponent = TypedNamedTuple('BugzillaComponent',
                         [('product', unicode), ('component', unicode)])
 SchedulingComponents = ContextDerivedTypedRecord(
         ('inclusive', TypedList(unicode, StrictOrderingOnAppendList)),
         ('exclusive', TypedList(unicode, StrictOrderingOnAppendList)))
 
+GeneratedFilesList = StrictOrderingOnAppendListWithFlagsFactory({
+    'script': unicode,
+    'inputs': list,
+    'flags': list, })
+
 
 class Files(SubContext):
     """Metadata attached to files.
 
     It is common to want to annotate files with metadata, such as which
     Bugzilla component tracks issues with certain files. This sub-context is
     where we stick that metadata.
 
@@ -1277,20 +1282,17 @@ VARIABLES = {
         """Source code files that can be compiled together.
 
         This variable contains a list of source code files to compile,
         that can be concatenated all together and built as a single source
         file. This can help make the build faster and reduce the debug info
         size.
         """),
 
-    'GENERATED_FILES': (StrictOrderingOnAppendListWithFlagsFactory({
-                'script': unicode,
-                'inputs': list,
-                'flags': list, }), list,
+    'GENERATED_FILES': (GeneratedFilesList, list,
         """Generic generated files.
 
         This variable contains a list of files for the build system to
         generate at export time. The generation method may be declared
         with optional ``script``, ``inputs`` and ``flags`` attributes on
         individual entries.
         If the optional ``script`` attribute is not present on an entry, it
         is assumed that rules for generating the file are present in
@@ -1457,16 +1459,30 @@ VARIABLES = {
 
     'LOCALIZED_PP_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list,
         """Like ``LOCALIZED_FILES``, with preprocessing.
 
         Note that the ``AB_CD`` define is available and expands to the current
         locale being packaged, as with preprocessed entries in jar manifests.
         """),
 
+    'LOCALIZED_GENERATED_FILES': (GeneratedFilesList, list,
+        """Like ``GENERATED_FILES``, but for files whose content varies based on the locale in use.
+
+        For simple cases of text substitution, prefer ``LOCALIZED_PP_FILES``.
+
+        Refer to the documentation of ``GENERATED_FILES``; for the most part things work the same.
+        The two major differences are:
+        1. The function in the Python script will be passed an additional keyword argument `locale`
+           which provides the locale in use, i.e. ``en-US``.
+        2. The ``inputs`` list may contain paths to files that will be taken from the locale
+           source directory (see ``LOCALIZED_FILES`` for a discussion of the specifics). Paths
+           in ``inputs`` starting with ``en-US/`` are considered localized files.
+        """),
+
     'OBJDIR_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list,
         """List of files to be installed anywhere in the objdir. Use sparingly.
 
         ``OBJDIR_FILES`` is similar to FINAL_TARGET_FILES, but it allows copying
         anywhere in the object directory. This is intended for various one-off
         cases, not for general use. If you wish to add entries to OBJDIR_FILES,
         please consult a build peer.
         """),
--- a/python/mozbuild/mozbuild/frontend/data.py
+++ b/python/mozbuild/mozbuild/frontend/data.py
@@ -1051,25 +1051,27 @@ class GeneratedFile(ContextDerived):
     """Represents a generated file."""
 
     __slots__ = (
         'script',
         'method',
         'outputs',
         'inputs',
         'flags',
+        'localized',
     )
 
-    def __init__(self, context, script, method, outputs, inputs, flags=()):
+    def __init__(self, context, script, method, outputs, inputs, flags=(), localized=False):
         ContextDerived.__init__(self, context)
         self.script = script
         self.method = method
         self.outputs = outputs if isinstance(outputs, tuple) else (outputs,)
         self.inputs = inputs
         self.flags = flags
+        self.localized = localized
 
 
 class AndroidResDirs(ContextDerived):
     """Represents Android resource directories."""
 
     __slots__ = (
         'paths',
     )
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -1271,55 +1271,57 @@ class TreeMetadataEmitter(LoggingMixin):
     def _process_generated_files(self, context):
         for path in context['CONFIGURE_DEFINE_FILES']:
             script = mozpath.join(mozpath.dirname(mozpath.dirname(__file__)),
                                   'action', 'process_define_files.py')
             yield GeneratedFile(context, script, 'process_define_file',
                                 unicode(path),
                                 [Path(context, path + '.in')])
 
-        generated_files = context.get('GENERATED_FILES')
-        if not generated_files:
+        generated_files = context.get('GENERATED_FILES') or []
+        localized_generated_files = context.get('LOCALIZED_GENERATED_FILES') or []
+        if not (generated_files or localized_generated_files):
             return
 
-        for f in generated_files:
-            flags = generated_files[f]
-            outputs = f
-            inputs = []
-            if flags.script:
-                method = "main"
-                script = SourcePath(context, flags.script).full_path
+        for (localized, gen) in ((False, generated_files), (True, localized_generated_files)):
+            for f in gen:
+                flags = gen[f]
+                outputs = f
+                inputs = []
+                if flags.script:
+                    method = "main"
+                    script = SourcePath(context, flags.script).full_path
 
-                # Deal with cases like "C:\\path\\to\\script.py:function".
-                if '.py:' in script:
-                    script, method = script.rsplit('.py:', 1)
-                    script += '.py'
+                    # Deal with cases like "C:\\path\\to\\script.py:function".
+                    if '.py:' in script:
+                        script, method = script.rsplit('.py:', 1)
+                        script += '.py'
 
-                if not os.path.exists(script):
-                    raise SandboxValidationError(
-                        'Script for generating %s does not exist: %s'
-                        % (f, script), context)
-                if os.path.splitext(script)[1] != '.py':
-                    raise SandboxValidationError(
-                        'Script for generating %s does not end in .py: %s'
-                        % (f, script), context)
+                    if not os.path.exists(script):
+                        raise SandboxValidationError(
+                            'Script for generating %s does not exist: %s'
+                            % (f, script), context)
+                    if os.path.splitext(script)[1] != '.py':
+                        raise SandboxValidationError(
+                            'Script for generating %s does not end in .py: %s'
+                            % (f, script), context)
 
-                for i in flags.inputs:
-                    p = Path(context, i)
-                    if (isinstance(p, SourcePath) and
-                            not os.path.exists(p.full_path)):
-                        raise SandboxValidationError(
-                            'Input for generating %s does not exist: %s'
-                            % (f, p.full_path), context)
-                    inputs.append(p)
-            else:
-                script = None
-                method = None
-            yield GeneratedFile(context, script, method, outputs, inputs,
-                                flags.flags)
+                    for i in flags.inputs:
+                        p = Path(context, i)
+                        if (isinstance(p, SourcePath) and
+                                not os.path.exists(p.full_path)):
+                            raise SandboxValidationError(
+                                'Input for generating %s does not exist: %s'
+                                % (f, p.full_path), context)
+                        inputs.append(p)
+                else:
+                    script = None
+                    method = None
+                yield GeneratedFile(context, script, method, outputs, inputs,
+                                    flags.flags, localized=localized)
 
     def _process_test_manifests(self, context):
         for prefix, info in TEST_MANIFESTS.items():
             for path, manifest in context.get('%s_MANIFESTS' % prefix, []):
                 for obj in self._process_test_manifest(context, info, path, manifest):
                     yield obj
 
         for flavor in REFTEST_FLAVORS:
copy from python/mozbuild/mozbuild/test/frontend/data/generated-files/moz.build
copy to python/mozbuild/mozbuild/test/frontend/data/localized-generated-files/moz.build
--- a/python/mozbuild/mozbuild/test/frontend/data/generated-files/moz.build
+++ b/python/mozbuild/mozbuild/test/frontend/data/localized-generated-files/moz.build
@@ -1,5 +1,5 @@
 # -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
 # Any copyright is dedicated to the Public Domain.
 # http://creativecommons.org/publicdomain/zero/1.0/
 
-GENERATED_FILES += [ 'bar.c', 'foo.c', ('xpidllex.py', 'xpidlyacc.py'), ]
+LOCALIZED_GENERATED_FILES += [ 'abc.ini', ('bar', 'baz') ]
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -456,25 +456,43 @@ class TestEmitterBasic(unittest.TestCase
 
     def test_generated_files(self):
         reader = self.reader('generated-files')
         objs = self.read_topsrcdir(reader)
 
         self.assertEqual(len(objs), 3)
         for o in objs:
             self.assertIsInstance(o, GeneratedFile)
+            self.assertFalse(o.localized)
 
         expected = ['bar.c', 'foo.c', ('xpidllex.py', 'xpidlyacc.py'), ]
         for o, f in zip(objs, expected):
             expected_filename = f if isinstance(f, tuple) else (f,)
             self.assertEqual(o.outputs, expected_filename)
             self.assertEqual(o.script, None)
             self.assertEqual(o.method, None)
             self.assertEqual(o.inputs, [])
 
+    def test_localized_generated_files(self):
+        reader = self.reader('localized-generated-files')
+        objs = self.read_topsrcdir(reader)
+
+        self.assertEqual(len(objs), 2)
+        for o in objs:
+            self.assertIsInstance(o, GeneratedFile)
+            self.assertTrue(o.localized)
+
+        expected = ['abc.ini', ('bar', 'baz'), ]
+        for o, f in zip(objs, expected):
+            expected_filename = f if isinstance(f, tuple) else (f,)
+            self.assertEqual(o.outputs, expected_filename)
+            self.assertEqual(o.script, None)
+            self.assertEqual(o.method, None)
+            self.assertEqual(o.inputs, [])
+
     def test_generated_files_method_names(self):
         reader = self.reader('generated-files-method-names')
         objs = self.read_topsrcdir(reader)
 
         self.assertEqual(len(objs), 2)
         for o in objs:
             self.assertIsInstance(o, GeneratedFile)