Bug 1380118 - aom: Support vendoring from github. r=froydnj draft
authorRalph Giles <giles@mozilla.com>
Mon, 10 Jul 2017 16:17:49 -0700
changeset 651854 c63a5619a2c1ab281b0614cc64d0b4d6b6bb8244
parent 651853 3a56cd81c25af572f8bb293289216b3f922549f2
child 651855 d208731c4bb924b0e6ad36ff46f6e08b161413b3
push id75832
push userbmo:giles@thaumas.net
push dateThu, 24 Aug 2017 03:49:46 +0000
reviewersfroydnj
bugs1380118
milestone57.0a1
Bug 1380118 - aom: Support vendoring from github. r=froydnj Add a --repo switch to `mach vendor aom` to allow specifying an alternate repository url. Update our vendor script to support commit query and snapshot download from github as well as upstream's gitiles instance. This lets us work with experimental branches for testing. Also cleans up some naming and checks the passed url for one of the two supported sites. We could fall back to doing a complete clone and query the local repository, but this covers most use cases.
python/mozbuild/mozbuild/mach_commands.py
python/mozbuild/mozbuild/vendor_aom.py
--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -2063,16 +2063,18 @@ class Vendor(MachCommandBase):
         from mozbuild.vendor_rust import VendorRust
         vendor_command = self._spawn(VendorRust)
         vendor_command.vendor(**kwargs)
 
     @SubCommand('vendor', 'aom',
                 description='Vendor av1 video codec reference implementation into the source repository.')
     @CommandArgument('-r', '--revision',
         help='Repository tag or commit to update to.')
+    @CommandArgument('--repo',
+        help='Repository url to pull a snapshot from. Supports github and googlesource.')
     @CommandArgument('--ignore-modified', action='store_true',
         help='Ignore modified files in current checkout',
         default=False)
     def vendor_aom(self, **kwargs):
         from mozbuild.vendor_aom import VendorAOM
         vendor_command = self._spawn(VendorAOM)
         vendor_command.vendor(**kwargs)
 
--- a/python/mozbuild/mozbuild/vendor_aom.py
+++ b/python/mozbuild/mozbuild/vendor_aom.py
@@ -7,35 +7,60 @@ from __future__ import absolute_import, 
 from distutils.version import LooseVersion
 import logging
 from mozbuild.base import (
     BuildEnvironmentNotFoundException,
     MozbuildObject,
 )
 import mozfile
 import mozpack.path as mozpath
+import os
 import requests
 import re
 import sys
 import tarfile
+from urlparse import urlparse
 
 class VendorAOM(MozbuildObject):
-    base_url = 'https://aomedia.googlesource.com/aom/'
-
-    def upstream_url(self, revision):
+    def upstream_snapshot(self, revision):
         '''Construct a url for a tarball snapshot of the given revision.'''
-        return mozpath.join(self.base_url, '+archive', revision + '.tar.gz')
+        if 'googlesource' in self.repo_url:
+            return mozpath.join(self.repo_url, '+archive', revision + '.tar.gz')
+        elif 'github' in self.repo_url:
+            return mozpath.join(self.repo_url, 'archive', revision + '.tar.gz')
+        else:
+            raise ValueError('Unknown git host, no snapshot lookup method')
 
     def upstream_commit(self, revision):
         '''Convert a revision to a git commit and timestamp.
 
         Ask the upstream repo to convert the requested revision to
         a git commit id and timestamp, so we can be precise in
         what we're vendoring.'''
-        url = mozpath.join(self.base_url, '+', revision + '?format=JSON')
+        if 'googlesource' in self.repo_url:
+            return self.upstream_googlesource_commit(revision)
+        elif 'github' in self.repo_url:
+            return self.upstream_github_commit(revision)
+        else:
+            raise ValueError('Unknown git host, no commit lookup method')
+
+    def upstream_validate(self, url):
+        '''Validate repository urls to make sure we can handle them.'''
+        host = urlparse(url).netloc
+        valid_domains = ('googlesource.com', 'github.com')
+        if not any(filter(lambda domain: domain in host, valid_domains)):
+            self.log(logging.ERROR, 'upstream_url', {},
+                     '''Unsupported git host %s; cannot fetch snapshots.
+
+Please set a repository url with --repo on either googlesource or github.''' % host)
+            sys.exit(1)
+
+    def upstream_googlesource_commit(self, revision):
+        '''Query gitiles for a git commit and timestamp.'''
+        url = mozpath.join(self.repo_url, '+', revision + '?format=JSON')
         self.log(logging.INFO, 'fetch', {'url': url},
                  'Fetching commit id from {url}')
         req = requests.get(url)
         req.raise_for_status()
         try:
             info = req.json()
         except ValueError as e:
             if 'No JSON object' in e.message:
@@ -43,35 +68,54 @@ class VendorAOM(MozbuildObject):
                 # at the beginning of the json response. Work around this.
                 # https://bugs.chromium.org/p/chromium/issues/detail?id=718550
                 import json
                 info = json.loads(req.text[4:])
             else:
                 raise
         return (info['commit'], info['committer']['time'])
 
+    def upstream_github_commit(self, revision):
+        '''Query the github api for a git commit id and timestamp.'''
+        github_api = 'https://api.github.com/'
+        repo = urlparse(self.repo_url).path[1:]
+        url = mozpath.join(github_api, 'repos', repo, 'commits', revision)
+        self.log(logging.INFO, 'fetch', {'url': url},
+                 'Fetching commit id from {url}')
+        req = requests.get(url)
+        req.raise_for_status()
+        info = req.json()
+        return (info['sha'], info['commit']['committer']['date'])
+
     def fetch_and_unpack(self, revision, target):
         '''Fetch and unpack upstream source'''
-        url = self.upstream_url(revision)
+        url = self.upstream_snapshot(revision)
         self.log(logging.INFO, 'fetch', {'url': url}, 'Fetching {url}')
-        filename = 'libaom-' + revision + '.tar.gz'
+        prefix = 'aom-' + revision
+        filename = prefix + '.tar.gz'
         with open(filename, 'wb') as f:
             req = requests.get(url, stream=True)
             for data in req.iter_content(4096):
                 f.write(data)
         tar = tarfile.open(filename)
         bad_paths = filter(lambda name: name.startswith('/') or '..' in name,
                            tar.getnames())
         if any(bad_paths):
             raise Exception("Tar archive contains non-local paths,"
                             "e.g. '%s'" % bad_paths[0])
         self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % target)
         mozfile.remove(target)
         self.log(logging.INFO, 'unpack', {}, 'Unpacking upstream files.')
         tar.extractall(target)
+        # Github puts everything properly down a directory; move it up.
+        if all(map(lambda name: name.startswith(prefix), tar.getnames())):
+            tardir = mozpath.join(target, prefix)
+            os.system('mv %s/* %s/.* %s' % (tardir, tardir, target))
+            os.rmdir(tardir)
+        # Remove the tarball.
         mozfile.remove(filename)
 
     def update_readme(self, revision, timestamp, target):
         filename = mozpath.join(target, 'README_MOZILLA')
         with open(filename) as f:
             readme = f.read()
 
         prefix = 'The git commit ID used was'
@@ -157,24 +201,30 @@ Please check manually and update the ven
                      '''You have uncommitted changes to the following files:
 
 {files}
 
 Please commit or stash these changes before vendoring, or re-run with `--ignore-modified`.
 '''.format(files='\n'.join(sorted(modified))))
             sys.exit(1)
 
-    def vendor(self, revision, ignore_modified=False):
+    def vendor(self, revision, repo, ignore_modified=False):
         self.populate_logger()
         self.log_manager.enable_unstructured()
 
         if not ignore_modified:
             self.check_modified_files()
         if not revision:
             revision = 'master'
+        if repo:
+            self.repo_url = repo
+        else:
+            self.repo_url = 'https://aomedia.googlesource.com/aom/'
+        self.upstream_validate(self.repo_url)
+
         commit, timestamp = self.upstream_commit(revision)
 
         vendor_dir = mozpath.join(self.topsrcdir, 'third_party/aom')
         self.fetch_and_unpack(commit, vendor_dir)
         self.log(logging.INFO, 'clean_upstream', {},
                  '''Removing unnecessary files.''')
         self.clean_upstream(vendor_dir)
         glue_dir = mozpath.join(self.topsrcdir, 'media/libaom')