Bug 1417684 - Functionality for evaluating file-based mtime dependencies; r?build draft
authorGregory Szorc <gps@mozilla.com>
Wed, 15 Nov 2017 17:53:28 -0800
changeset 700678 fecd12663790d43ec098f417a16d01efb770b764
parent 700576 5c48b5edfc4ca945a2eaa5896454f3f4efa9052a
child 700679 c0b8fd0f84167547473a865c6744a5eebc426eb9
push id89935
push userbmo:gps@mozilla.com
push dateMon, 20 Nov 2017 19:05:20 +0000
reviewersbuild
bugs1417684
milestone59.0a1
Bug 1417684 - Functionality for evaluating file-based mtime dependencies; r?build We want to get rid of client.mk and drive the build system from Python instead of make. This requires reinventing aspects of a build system to determine if targets are up to date. And since our initial intent is to port logic from client.mk to Python, we must implement aspects of make's logic to Python. Make works by looking at target/file presence and mtime to determine if it is up to date. In this commit, we implement some utility Python code for evaluating a file-based "is up to date" check using mtimes. I figured it was worth making this code generic instead of inlining a bunch of I/O operations in to-be-written Python code. If nothing else, it makes the final Python code easier to read. And, who knows, perhaps we'll reuse this dependency checking code in other operations in the future. MozReview-Commit-ID: 3rmUQJMehMn
python/mozbuild/mozbuild/dependsutils.py
python/mozbuild/mozbuild/test/python.ini
python/mozbuild/mozbuild/test/test_dependsutils.py
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/dependsutils.py
@@ -0,0 +1,66 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Functionality for dependency evaluation.
+
+from __future__ import absolute_import, unicode_literals
+
+import errno
+import os
+
+
+def is_file_target_current(target_files, depends_files):
+    """Evaluate file dependencies and determine if up to date.
+
+    Given an iterable of output files and an iterable of input files, determine
+    if the outputs are up-to-date by looking at file mtimes.
+
+    This essentially emulates the behavior of GNU make.
+
+    Returns True if all the following conditions are met:
+
+    * All targets and dependencies exist
+    * The mtime of all targets is newer than the newest mtime of dependencies.
+    """
+    if not target_files:
+        raise ValueError('must specify targets')
+
+    stats = batch_stat(target_files + depends_files)
+
+    if None in stats.values():
+        return False
+
+    if not depends_files:
+        return True
+
+    oldest_target = min(stats[p].st_mtime for p in target_files)
+    newest_depend = max(stats[p].st_mtime for p in depends_files)
+
+    # The resolution of system clocks and filesystem metadata doesn't guarantee
+    # that a touch after another will have a strictly greater mtime value. e.g.
+    # if filesystem granularity is only 1.0s, multiple touches could occur in
+    # the same second and have the same reported mtime value. We therefore treat
+    # equivalent mtimes as meaning the target is up to date.
+    return oldest_target >= newest_depend
+
+
+def batch_stat(paths):
+    """Stat multiple filesystem paths.
+
+    Returns a dict mapping input paths to their stat result. Values will be
+    None if file does not exist.
+    """
+    res = {}
+
+    for path in paths:
+        try:
+            st = os.stat(path)
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                raise
+            st = None
+
+        res[path] = st
+
+    return res
--- a/python/mozbuild/mozbuild/test/python.ini
+++ b/python/mozbuild/mozbuild/test/python.ini
@@ -26,16 +26,17 @@
 [frontend/test_context.py]
 [frontend/test_emitter.py]
 [frontend/test_namespaces.py]
 [frontend/test_reader.py]
 [frontend/test_sandbox.py]
 [test_artifacts.py]
 [test_base.py]
 [test_containers.py]
+[test_dependsutils.py]
 [test_dotproperties.py]
 [test_expression.py]
 [test_jarmaker.py]
 [test_line_endings.py]
 [test_makeutil.py]
 [test_mozconfig.py]
 [test_mozinfo.py]
 [test_preprocessor.py]
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/test_dependsutils.py
@@ -0,0 +1,142 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import os
+import shutil
+import tempfile
+import time
+import unittest
+
+from mozunit import (
+    main,
+)
+
+from mozbuild.dependsutils import (
+    is_file_target_current,
+)
+
+
+class TestDependsUtils(unittest.TestCase):
+    def setUp(self):
+        self._temp_dir = tempfile.mkdtemp()
+        self._now = int(time.time())
+
+    def tearDown(self):
+        shutil.rmtree(self._temp_dir)
+
+    def p(self, *paths):
+        return os.path.join(self._temp_dir, *paths)
+
+    def touch_older(self, p):
+        """Touch a file so it is older than "now"."""
+        with open(p, 'a'):
+            pass
+        os.utime(p, (self._now - 10.0, self._now - 10.0))
+
+    def touch_now(self, p):
+        """Touch a file so it is modified "now"."""
+        with open(p, 'a'):
+            pass
+        os.utime(p, (self._now, self._now))
+
+    def touch_future(self, p):
+        """Touch a file so it is modified in the future."""
+        with open(p, 'a'):
+            pass
+        os.utime(p, (self._now + 10.0, self._now + 10.0))
+
+    def test_no_depends(self):
+        """Tests when there are no dependency files."""
+        targets = [self.p('missing1')]
+
+        # Single target.
+        self.assertFalse(is_file_target_current(targets, []))
+
+        # Multiple targets.
+        targets.append(self.p('missing2'))
+        self.assertFalse(is_file_target_current(targets, []))
+
+        # Multiple targets, some existing.
+        self.touch_older(targets[0])
+        self.assertFalse(is_file_target_current(targets, []))
+
+        # Multiple targets, all existing.
+        self.touch_older(targets[1])
+        self.assertTrue(is_file_target_current(targets, []))
+
+    def test_single_target(self):
+        targets = [self.p('target1')]
+        depends = [self.p('depends1')]
+
+        # Missing target and depends.
+        self.assertFalse(is_file_target_current(targets, depends))
+
+        # Missing target.
+        self.touch_now(depends[0])
+        self.assertFalse(is_file_target_current(targets, depends))
+
+        # Target exists but is older.
+        self.touch_older(targets[0])
+        self.assertFalse(is_file_target_current(targets, depends))
+
+        # Target exists and has same time.
+        self.touch_now(targets[0])
+        self.assertTrue(is_file_target_current(targets, depends))
+
+        # Target exists and is newer.
+        self.touch_future(targets[0])
+        self.assertTrue(is_file_target_current(targets, depends))
+
+    def test_multiple_targets(self):
+        targets = [self.p('target1'), self.p('target2')]
+        depends = [self.p('depends1')]
+
+        # Missing target and depends.
+        self.assertFalse(is_file_target_current(targets, depends))
+
+        # Depends present, no target.
+        self.touch_now(depends[0])
+        self.assertFalse(is_file_target_current(targets, depends))
+
+        # Single target present.
+        self.touch_future(targets[0])
+        self.assertFalse(is_file_target_current(targets, depends))
+
+        # Both targets present. 1 older.
+        self.touch_older(targets[1])
+        self.assertFalse(is_file_target_current(targets, depends))
+
+        # Both targets present. 1 same age.
+        self.touch_now(targets[1])
+        self.assertTrue(is_file_target_current(targets, depends))
+
+        # Both targets present, both newer.
+        self.touch_future(targets[1])
+        self.assertTrue(is_file_target_current(targets, depends))
+
+    def test_multiple_depends(self):
+        targets = [self.p('target1'), self.p('target2')]
+        depends = [self.p('depends1'), self.p('depends2')]
+
+        # Everything missing.
+        self.assertFalse(is_file_target_current(targets, depends))
+
+        # 1 depends present.
+        self.touch_now(depends[0])
+        self.assertFalse(is_file_target_current(targets, depends))
+
+        # All targets in future. Missing 1 depends.
+        self.touch_future(targets[0])
+        self.touch_future(targets[1])
+        self.assertFalse(is_file_target_current(targets, depends))
+
+        # All depends present.
+        self.touch_now(depends[1])
+        self.assertTrue(is_file_target_current(targets, depends))
+
+
+if __name__ == '__main__':
+    main()