Bug 1417684 - Functionality for evaluating file-based mtime dependencies; r?build
We want to get rid of client.mk and drive the build system from
Python instead of make. This requires reinventing aspects of a
build system to determine if targets are up to date. And since our
initial intent is to port logic from client.mk to Python, we must
implement aspects of make's logic to Python.
Make works by looking at target/file presence and mtime to determine
if it is up to date. In this commit, we implement some utility Python
code for evaluating a file-based "is up to date" check using mtimes.
I figured it was worth making this code generic instead of inlining
a bunch of I/O operations in to-be-written Python code. If nothing
else, it makes the final Python code easier to read. And, who knows,
perhaps we'll reuse this dependency checking code in other operations
in the future.
MozReview-Commit-ID: 3rmUQJMehMn
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/dependsutils.py
@@ -0,0 +1,66 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Functionality for dependency evaluation.
+
+from __future__ import absolute_import, unicode_literals
+
+import errno
+import os
+
+
+def is_file_target_current(target_files, depends_files):
+ """Evaluate file dependencies and determine if up to date.
+
+ Given an iterable of output files and an iterable of input files, determine
+ if the outputs are up-to-date by looking at file mtimes.
+
+ This essentially emulates the behavior of GNU make.
+
+ Returns True if all the following conditions are met:
+
+ * All targets and dependencies exist
+ * The mtime of all targets is newer than the newest mtime of dependencies.
+ """
+ if not target_files:
+ raise ValueError('must specify targets')
+
+ stats = batch_stat(target_files + depends_files)
+
+ if None in stats.values():
+ return False
+
+ if not depends_files:
+ return True
+
+ oldest_target = min(stats[p].st_mtime for p in target_files)
+ newest_depend = max(stats[p].st_mtime for p in depends_files)
+
+ # The resolution of system clocks and filesystem metadata doesn't guarantee
+ # that a touch after another will have a strictly greater mtime value. e.g.
+ # if filesystem granularity is only 1.0s, multiple touches could occur in
+ # the same second and have the same reported mtime value. We therefore treat
+ # equivalent mtimes as meaning the target is up to date.
+ return oldest_target >= newest_depend
+
+
+def batch_stat(paths):
+ """Stat multiple filesystem paths.
+
+ Returns a dict mapping input paths to their stat result. Values will be
+ None if file does not exist.
+ """
+ res = {}
+
+ for path in paths:
+ try:
+ st = os.stat(path)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ st = None
+
+ res[path] = st
+
+ return res
--- a/python/mozbuild/mozbuild/test/python.ini
+++ b/python/mozbuild/mozbuild/test/python.ini
@@ -26,16 +26,17 @@
[frontend/test_context.py]
[frontend/test_emitter.py]
[frontend/test_namespaces.py]
[frontend/test_reader.py]
[frontend/test_sandbox.py]
[test_artifacts.py]
[test_base.py]
[test_containers.py]
+[test_dependsutils.py]
[test_dotproperties.py]
[test_expression.py]
[test_jarmaker.py]
[test_line_endings.py]
[test_makeutil.py]
[test_mozconfig.py]
[test_mozinfo.py]
[test_preprocessor.py]
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/test_dependsutils.py
@@ -0,0 +1,142 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import os
+import shutil
+import tempfile
+import time
+import unittest
+
+from mozunit import (
+ main,
+)
+
+from mozbuild.dependsutils import (
+ is_file_target_current,
+)
+
+
+class TestDependsUtils(unittest.TestCase):
+ def setUp(self):
+ self._temp_dir = tempfile.mkdtemp()
+ self._now = int(time.time())
+
+ def tearDown(self):
+ shutil.rmtree(self._temp_dir)
+
+ def p(self, *paths):
+ return os.path.join(self._temp_dir, *paths)
+
+ def touch_older(self, p):
+ """Touch a file so it is older than "now"."""
+ with open(p, 'a'):
+ pass
+ os.utime(p, (self._now - 10.0, self._now - 10.0))
+
+ def touch_now(self, p):
+ """Touch a file so it is modified "now"."""
+ with open(p, 'a'):
+ pass
+ os.utime(p, (self._now, self._now))
+
+ def touch_future(self, p):
+ """Touch a file so it is modified in the future."""
+ with open(p, 'a'):
+ pass
+ os.utime(p, (self._now + 10.0, self._now + 10.0))
+
+ def test_no_depends(self):
+ """Tests when there are no dependency files."""
+ targets = [self.p('missing1')]
+
+ # Single target.
+ self.assertFalse(is_file_target_current(targets, []))
+
+ # Multiple targets.
+ targets.append(self.p('missing2'))
+ self.assertFalse(is_file_target_current(targets, []))
+
+ # Multiple targets, some existing.
+ self.touch_older(targets[0])
+ self.assertFalse(is_file_target_current(targets, []))
+
+ # Multiple targets, all existing.
+ self.touch_older(targets[1])
+ self.assertTrue(is_file_target_current(targets, []))
+
+ def test_single_target(self):
+ targets = [self.p('target1')]
+ depends = [self.p('depends1')]
+
+ # Missing target and depends.
+ self.assertFalse(is_file_target_current(targets, depends))
+
+ # Missing target.
+ self.touch_now(depends[0])
+ self.assertFalse(is_file_target_current(targets, depends))
+
+ # Target exists but is older.
+ self.touch_older(targets[0])
+ self.assertFalse(is_file_target_current(targets, depends))
+
+ # Target exists and has same time.
+ self.touch_now(targets[0])
+ self.assertTrue(is_file_target_current(targets, depends))
+
+ # Target exists and is newer.
+ self.touch_future(targets[0])
+ self.assertTrue(is_file_target_current(targets, depends))
+
+ def test_multiple_targets(self):
+ targets = [self.p('target1'), self.p('target2')]
+ depends = [self.p('depends1')]
+
+ # Missing target and depends.
+ self.assertFalse(is_file_target_current(targets, depends))
+
+ # Depends present, no target.
+ self.touch_now(depends[0])
+ self.assertFalse(is_file_target_current(targets, depends))
+
+ # Single target present.
+ self.touch_future(targets[0])
+ self.assertFalse(is_file_target_current(targets, depends))
+
+ # Both targets present. 1 older.
+ self.touch_older(targets[1])
+ self.assertFalse(is_file_target_current(targets, depends))
+
+ # Both targets present. 1 same age.
+ self.touch_now(targets[1])
+ self.assertTrue(is_file_target_current(targets, depends))
+
+ # Both targets present, both newer.
+ self.touch_future(targets[1])
+ self.assertTrue(is_file_target_current(targets, depends))
+
+ def test_multiple_depends(self):
+ targets = [self.p('target1'), self.p('target2')]
+ depends = [self.p('depends1'), self.p('depends2')]
+
+ # Everything missing.
+ self.assertFalse(is_file_target_current(targets, depends))
+
+ # 1 depends present.
+ self.touch_now(depends[0])
+ self.assertFalse(is_file_target_current(targets, depends))
+
+ # All targets in future. Missing 1 depends.
+ self.touch_future(targets[0])
+ self.touch_future(targets[1])
+ self.assertFalse(is_file_target_current(targets, depends))
+
+ # All depends present.
+ self.touch_now(depends[1])
+ self.assertTrue(is_file_target_current(targets, depends))
+
+
+if __name__ == '__main__':
+ main()