--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -2063,16 +2063,18 @@ class Vendor(MachCommandBase):
from mozbuild.vendor_rust import VendorRust
vendor_command = self._spawn(VendorRust)
vendor_command.vendor(**kwargs)
@SubCommand('vendor', 'aom',
description='Vendor av1 video codec reference implementation into the source repository.')
@CommandArgument('-r', '--revision',
help='Repository tag or commit to update to.')
+ @CommandArgument('--repo',
+ help='Repository url to pull a snapshot from. Supports github and googlesource.')
@CommandArgument('--ignore-modified', action='store_true',
help='Ignore modified files in current checkout',
default=False)
def vendor_aom(self, **kwargs):
from mozbuild.vendor_aom import VendorAOM
vendor_command = self._spawn(VendorAOM)
vendor_command.vendor(**kwargs)
--- a/python/mozbuild/mozbuild/vendor_aom.py
+++ b/python/mozbuild/mozbuild/vendor_aom.py
@@ -7,35 +7,60 @@ from __future__ import absolute_import,
from distutils.version import LooseVersion
import logging
from mozbuild.base import (
BuildEnvironmentNotFoundException,
MozbuildObject,
)
import mozfile
import mozpack.path as mozpath
+import os
import requests
import re
import sys
import tarfile
+from urlparse import urlparse
class VendorAOM(MozbuildObject):
- base_url = 'https://aomedia.googlesource.com/aom/'
-
- def upstream_url(self, revision):
+ def upstream_snapshot(self, revision):
'''Construct a url for a tarball snapshot of the given revision.'''
- return mozpath.join(self.base_url, '+archive', revision + '.tar.gz')
+ if 'googlesource' in self.repo_url:
+ return mozpath.join(self.repo_url, '+archive', revision + '.tar.gz')
+ elif 'github' in self.repo_url:
+ return mozpath.join(self.repo_url, 'archive', revision + '.tar.gz')
+ else:
+ raise ValueError('Unknown git host, no snapshot lookup method')
def upstream_commit(self, revision):
'''Convert a revision to a git commit and timestamp.
Ask the upstream repo to convert the requested revision to
a git commit id and timestamp, so we can be precise in
what we're vendoring.'''
- url = mozpath.join(self.base_url, '+', revision + '?format=JSON')
+ if 'googlesource' in self.repo_url:
+ return self.upstream_googlesource_commit(revision)
+ elif 'github' in self.repo_url:
+ return self.upstream_github_commit(revision)
+ else:
+ raise ValueError('Unknown git host, no commit lookup method')
+
+ def upstream_validate(self, url):
+ '''Validate repository urls to make sure we can handle them.'''
+ host = urlparse(url).netloc
+ valid_domains = ('googlesource.com', 'github.com')
+ if not any(filter(lambda domain: domain in host, valid_domains)):
+ self.log(logging.ERROR, 'upstream_url', {},
+ '''Unsupported git host %s; cannot fetch snapshots.
+
+Please set a repository url with --repo on either googlesource or github.''' % host)
+ sys.exit(1)
+
+ def upstream_googlesource_commit(self, revision):
+ '''Query gitiles for a git commit and timestamp.'''
+ url = mozpath.join(self.repo_url, '+', revision + '?format=JSON')
self.log(logging.INFO, 'fetch', {'url': url},
'Fetching commit id from {url}')
req = requests.get(url)
req.raise_for_status()
try:
info = req.json()
except ValueError as e:
if 'No JSON object' in e.message:
@@ -43,35 +68,54 @@ class VendorAOM(MozbuildObject):
# at the beginning of the json response. Work around this.
# https://bugs.chromium.org/p/chromium/issues/detail?id=718550
import json
info = json.loads(req.text[4:])
else:
raise
return (info['commit'], info['committer']['time'])
+ def upstream_github_commit(self, revision):
+ '''Query the github api for a git commit id and timestamp.'''
+ github_api = 'https://api.github.com/'
+ repo = urlparse(self.repo_url).path[1:]
+ url = mozpath.join(github_api, 'repos', repo, 'commits', revision)
+ self.log(logging.INFO, 'fetch', {'url': url},
+ 'Fetching commit id from {url}')
+ req = requests.get(url)
+ req.raise_for_status()
+ info = req.json()
+ return (info['sha'], info['commit']['committer']['date'])
+
def fetch_and_unpack(self, revision, target):
'''Fetch and unpack upstream source'''
- url = self.upstream_url(revision)
+ url = self.upstream_snapshot(revision)
self.log(logging.INFO, 'fetch', {'url': url}, 'Fetching {url}')
- filename = 'libaom-' + revision + '.tar.gz'
+ prefix = 'aom-' + revision
+ filename = prefix + '.tar.gz'
with open(filename, 'wb') as f:
req = requests.get(url, stream=True)
for data in req.iter_content(4096):
f.write(data)
tar = tarfile.open(filename)
bad_paths = filter(lambda name: name.startswith('/') or '..' in name,
tar.getnames())
if any(bad_paths):
raise Exception("Tar archive contains non-local paths,"
"e.g. '%s'" % bad_paths[0])
self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % target)
mozfile.remove(target)
self.log(logging.INFO, 'unpack', {}, 'Unpacking upstream files.')
tar.extractall(target)
+ # Github puts everything properly down a directory; move it up.
+ if all(map(lambda name: name.startswith(prefix), tar.getnames())):
+ tardir = mozpath.join(target, prefix)
+ os.system('mv %s/* %s/.* %s' % (tardir, tardir, target))
+ os.rmdir(tardir)
+ # Remove the tarball.
mozfile.remove(filename)
def update_readme(self, revision, timestamp, target):
filename = mozpath.join(target, 'README_MOZILLA')
with open(filename) as f:
readme = f.read()
prefix = 'The git commit ID used was'
@@ -157,24 +201,30 @@ Please check manually and update the ven
'''You have uncommitted changes to the following files:
{files}
Please commit or stash these changes before vendoring, or re-run with `--ignore-modified`.
'''.format(files='\n'.join(sorted(modified))))
sys.exit(1)
- def vendor(self, revision, ignore_modified=False):
+ def vendor(self, revision, repo, ignore_modified=False):
self.populate_logger()
self.log_manager.enable_unstructured()
if not ignore_modified:
self.check_modified_files()
if not revision:
revision = 'master'
+ if repo:
+ self.repo_url = repo
+ else:
+ self.repo_url = 'https://aomedia.googlesource.com/aom/'
+ self.upstream_validate(self.repo_url)
+
commit, timestamp = self.upstream_commit(revision)
vendor_dir = mozpath.join(self.topsrcdir, 'third_party/aom')
self.fetch_and_unpack(commit, vendor_dir)
self.log(logging.INFO, 'clean_upstream', {},
'''Removing unnecessary files.''')
self.clean_upstream(vendor_dir)
glue_dir = mozpath.join(self.topsrcdir, 'media/libaom')