Bug 1356652 - Copy files for PGO server to temp directory; r?glandium draft
authorGregory Szorc <gps@mozilla.com>
Thu, 24 Aug 2017 18:48:11 -0700
changeset 652627 34ebae9a934dfce4e57d140f67d41eb45fe94a2c
parent 652626 82ca75a90c31e80a1777039b3a7ed5140de23ef3
child 728129 b74844e496eb299c38e0a05e58763c84de836b57
push id76100
push usergszorc@mozilla.com
push dateFri, 25 Aug 2017 01:48:32 +0000
reviewersglandium
bugs1356652
milestone57.0a1
Bug 1356652 - Copy files for PGO server to temp directory; r?glandium The impetus behind this change is to facilitate accessing files from across the repository. Previously, files needed to be in build/pgo. Alternatively, we could map different filesystem paths to different URI paths. I like keeping things static and simple. And feeding some mozpack FileFinder results into a FileCopier is pretty simple. A side-effect of copying all the files is that they will likely be in the page cache when we start and profile Firefox. This should help eliminate I/O wait loading these files as a potential source of variance in PGO profiles. I'd like to think that profiling is smart enough to skip over functions waiting on a kernel I/O call to complete and this isn't an issue. But PGO has surprised us before. MozReview-Commit-ID: 7yrJZ8QEFiN
build/pgo/profileserver.py
--- a/build/pgo/profileserver.py
+++ b/build/pgo/profileserver.py
@@ -6,37 +6,67 @@
 
 import json
 import os
 
 from buildconfig import substs
 from mozbuild.base import MozbuildObject
 from mozfile import TemporaryDirectory
 from mozhttpd import MozHttpd
+import mozpack.path as mozpath
+from mozpack.copier import FileCopier
+from mozpack.files import FileFinder
 from mozprofile import FirefoxProfile, Profile, Preferences
 from mozprofile.permissions import ServerLocations
 from mozrunner import FirefoxRunner, CLI
 
 PORT = 8888
 
+# base, pattern, dest of file copies to establish the docroot for the HTTP
+# server.
+HTTP_SERVER_COPIES = [
+    ('build/pgo', '**', '',)
+]
+
+
 if __name__ == '__main__':
     cli = CLI()
     debug_args, interactive = cli.debugger_arguments()
 
     build = MozbuildObject.from_environment()
-    httpd = MozHttpd(port=PORT,
-                     docroot=os.path.join(build.topsrcdir, "build", "pgo"))
-    httpd.start(block=False)
+
+    with TemporaryDirectory() as temp_dir:
+        # Copy all files to the docroot. This allows us to grab files from
+        # across the repo without resorting to dynamic path routing in the
+        # HTTP server.
+        doc_root = os.path.join(temp_dir, 'docroot')
+        os.mkdir(doc_root)
+
+        copier = FileCopier()
+        for base, pattern, dest in HTTP_SERVER_COPIES:
+            finder = FileFinder(base)
+            for p, f in finder.find(pattern):
+                if dest:
+                    p = mozpath.join(dest, p)
 
-    locations = ServerLocations()
-    locations.add_host(host='127.0.0.1',
-                       port=PORT,
-                       options='primary,privileged')
+                copier.add(p, f)
+
+        copier.copy(doc_root)
+
+        httpd = MozHttpd(port=PORT,
+                         docroot=doc_root)
+        httpd.start(block=False)
 
-    with TemporaryDirectory() as profilePath:
+        locations = ServerLocations()
+        locations.add_host(host='127.0.0.1',
+                           port=PORT,
+                           options='primary,privileged')
+
+        profilePath = os.path.join(temp_dir, 'profile')
+
         # TODO: refactor this into mozprofile
         prefpath = os.path.join(
             build.topsrcdir, "testing", "profiles", "prefs_general.js")
         prefs = {}
         prefs.update(Preferences.read_prefs(prefpath))
         interpolation = {"server": "%s:%d" % httpd.httpd.server_address,
                          "OOP": "false"}
         prefs = json.loads(json.dumps(prefs) % interpolation)