Bug 1255179 - Url Finder adition. draft
authorGreg Mierzwinski <gmierz2@outlook.com>
Fri, 25 Mar 2016 13:00:57 -0400
changeset 355017 4a11bf93515429adcc280fd0fc4a3e7ef0a8d1d6
parent 355016 0e83c87d5cd62843947fbf438e30bf7fb1be5428
child 519127 011be23c99edc9f93806935ae074564310fb3033
push id16211
push userbmo:gmierz2@outlook.com
push dateThu, 21 Apr 2016 23:55:54 +0000
bugs1255179
milestone48.0a1
Bug 1255179 - Url Finder adition. This contains the url finder class which is responsible for mapping the URL's to local source file locations. It was taken from lcov_rewriter.py. MozReview-Commit-ID: 5Cj3d3Vuycq * * * Bug 1255179 - Url Finder fix. Add flags needed to fix 'self._populate_chrome(extra_chrome_manifests)'. * * * Bug 1255179 - Remove uses of extra.manifests file. Left a TODO to fix this feature in the future. * * * Bug 1255179 - Remove populate_chrome method.
python/mozbuild/mozbuild/codecoverage/url_finder.py
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/codecoverage/url_finder.py
@@ -0,0 +1,216 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from argparse import ArgumentParser
+import json
+import os
+import re
+import sys
+import urlparse
+
+from mozpack.copier import FileRegistry
+from mozpack.files import PreprocessedFile
+from mozpack.manifests import InstallManifest
+from mozpack.chrome.manifest import parse_manifest
+from chrome_map import ChromeManifestHandler
+from mozpack.chrome.manifest import ManifestContent
+from mozpack.chrome.flags import Flags, Flag
+
+
+import buildconfig
+import mozpack.path as mozpath
+
+class UrlFinderError(Exception):
+    pass
+
+class UrlFinder(object):
+    # Given a "chrome://" or "resource://" url, uses data from the UrlMapBackend
+    # and install manifests to find a path to the source file and the corresponding
+    # (potentially pre-processed) file in the objdir.
+    # TODO: Fix self._populate_chrome and/or extra_chrome_manifests to have source file urls
+    # mapped to system paths correctly.
+    # Missed Sample: resource://testing-common/TestUtils.jsm
+
+    def __init__(self, appdir, gredir):
+        # Normalized for "startswith" checks below.
+        self.topobjdir = mozpath.normpath(buildconfig.topobjdir)
+
+        # Cached entries, so potentially pricey searches only happen once per url.
+        self._final_mapping = {}
+
+        info_file = os.path.join(self.topobjdir, 'chrome-map.json')
+
+        try:
+            with open(info_file) as fh:
+                url_prefixes, overrides, install_info = json.load(fh)
+        except IOError:
+            print("Error reading %s. Run |./mach build-backend -b ChromeMap| to "
+                  "populate the ChromeMap backend." % info_file)
+            raise
+
+        # These are added dynamically in nsIResProtocolHandler, we might
+        # need to get them at run time.
+        if "resource:///" not in url_prefixes:
+            url_prefixes["resource:///"] = [appdir]
+        if "resource://gre/" not in url_prefixes:
+            url_prefixes["resource://gre/"] = [gredir]
+
+        self._url_prefixes = url_prefixes
+        self._url_overrides = overrides
+
+        self._respath = None
+        if ('MOZ_MACBUNDLE_NAME' in buildconfig.substs and
+            buildconfig.substs['MOZ_MACBUNDLE_NAME']):
+            self._respath = mozpath.join('dist',
+                                         buildconfig.substs['MOZ_MACBUNDLE_NAME'],
+                                         'Contents',
+                                         'Resources')
+
+        self._install_mapping = install_info
+        self._populate_install_manifest()
+
+        self._rewrites = [
+            (lambda url: ' -> ' in url,
+             lambda url: url.split(' -> ')[1].rstrip()),
+            (lambda url: url.startswith('file://'),
+             lambda url: url[len('file://'):]),
+        ]
+
+    def _load_manifest(self, path, root):
+        install_manifest = InstallManifest(path)
+        reg = FileRegistry()
+        install_manifest.populate_registry(reg)
+
+        for dest, src in reg:
+            if hasattr(src, 'path'):
+                if not os.path.isabs(dest):
+                    dest = root + dest
+                self._install_mapping[dest] = (src.path,
+                                               isinstance(src, PreprocessedFile))
+
+    def _populate_install_manifest(self):
+        # Test files are mentioned in _build_manifests/install/tests.
+        # TODO: Add these to final target files and handle everything
+        # uniformly.
+        mp = os.path.join(self.topobjdir, '_build_manifests', 'install',
+                          '_tests')
+        self._load_manifest(mp, root='_tests/')
+
+    def _find_install_prefix(self, objdir_path):
+        path_parts = objdir_path.split('/')
+        offset = 0
+
+        def _prefix(s):
+            for p in mozpath.split(s):
+                if '*' not in p:
+                    yield p + '/'
+
+        for leaf in reversed(path_parts):
+            offset += len(leaf)
+            if objdir_path[:-offset] in self._install_mapping:
+                pattern_prefix, is_pp = self._install_mapping[objdir_path[:-offset]]
+                full_leaf = objdir_path[len(objdir_path) - offset:]
+                src_prefix = ''.join(_prefix(pattern_prefix))
+                self._install_mapping[objdir_path] = (mozpath.join(src_prefix, full_leaf),
+                                                      is_pp)
+            offset += 1
+
+    def _install_info(self, objdir_path):
+        if objdir_path not in self._install_mapping:
+            # If our path is missing, some prefix of it may be in the install
+            # mapping mapped to a wildcard.
+            self._find_install_prefix(objdir_path)
+        if objdir_path not in self._install_mapping:
+            raise UrlFinderError("Couldn't find entry in manifest for %s" %
+                                 objdir_path)
+        return self._install_mapping[objdir_path]
+
+    def _abs_objdir_install_info(self, term):
+        obj_relpath = term[len(self.topobjdir) + 1:]
+        res = self._install_info(obj_relpath)
+
+        # Some urls on osx will refer to paths in the mac bundle, so we
+        # re-interpret them as being their original location in dist/bin.
+        if (not res and self._respath and
+            obj_relpath.startswith(self._respath)):
+            obj_relpath = obj_relpath.replace(self._respath, 'dist/bin')
+            res = self._install_info(obj_relpath)
+
+        if not res:
+            raise UrlFinderError("Couldn't find entry in manifest for %s" %
+                                 obj_relpath)
+        return res
+
+    def find_files(self, url):
+        # Returns a tuple of (source file, objdir file, preprocessed)
+        # for the given "resource:", "chrome:", or "file:" uri.
+        term = url
+        if term in self._url_overrides:
+            term = self._url_overrides[term]
+
+        if os.path.isabs(term) and term.startswith(self.topobjdir):
+            source_path, preprocessed = self._abs_objdir_install_info(term)
+            return source_path, term, preprocessed
+
+        objdir_path = None
+        for prefix, dests in self._url_prefixes.iteritems():
+            if term.startswith(prefix):
+                for dest in dests:
+                    if not dest.endswith('/'):
+                        dest += '/'
+                    replaced = url.replace(prefix, dest)
+
+                    if os.path.isfile(mozpath.join(self.topobjdir, replaced)):
+                        objdir_path = replaced
+                        break
+
+                    if (dest.startswith('resource://') or
+                        dest.startswith('chrome://')):
+                        result = self.find_files(term.replace(prefix, dest))
+                        if result:
+                            return result
+
+        if not objdir_path:
+            raise UrlFinderError("No objdir path for %s" % term)
+        while objdir_path.startswith('//'):
+            # The mochitest harness produces some wonky file:// uris
+            # that need to be fixed.
+            objdir_path = objdir_path[1:]
+
+        if os.path.isabs(objdir_path) and objdir_path.startswith(self.topobjdir):
+            source_path, preprocessed = self._abs_objdir_install_info(objdir_path)
+            return source_path, term, preprocessed
+
+        src_path, preprocessed = self._install_info(objdir_path)
+        return mozpath.normpath(src_path), objdir_path, preprocessed
+
+    def rewrite_url(self, url):
+        # This applies one-off rules and returns None for urls that we aren't
+        # going to be able to resolve to a source file ("about:" urls, for
+        # instance).
+        if url in self._final_mapping:
+            return self._final_mapping[url]
+        if url.endswith('> eval'):
+            return None
+        if url.endswith('> Function'):
+            return None
+        if ' -> ' in url:
+             url = url.split(' -> ')[1].rstrip()
+
+        url_obj = urlparse.urlparse(url)
+        if url_obj.scheme == 'file' and os.path.isabs(url_obj.path):
+            path = url_obj.path
+            if not os.path.isfile(path):
+                # This may have been in a profile directory that no
+                # longer exists.
+                return None
+            if not path.startswith(self.topobjdir):
+                return path, None, False
+            url = url_obj.path
+        if url_obj.scheme in ('http', 'https', 'javascript', 'data', 'about'):
+            return None
+
+        result = self.find_files(url)
+        self._final_mapping[url] = result
+        return result
\ No newline at end of file