--- a/python/mozbuild/mozbuild/mach_commands.py
+++ b/python/mozbuild/mozbuild/mach_commands.py
@@ -1831,16 +1831,29 @@ class Vendor(MachCommandBase):
@CommandArgument('--build-peers-said-large-imports-were-ok', action='store_true',
help='Permit overly-large files to be added to the repository',
default=False)
def vendor_rust(self, **kwargs):
from mozbuild.vendor_rust import VendorRust
vendor_command = self._spawn(VendorRust)
vendor_command.vendor(**kwargs)
+ @SubCommand('vendor', 'aom',
+ description='Vendor av1 video codec reference implementation into the source repository.')
+ @CommandArgument('-r', '--revision',
+ help='Repository tag or commit to update to.')
+ @CommandArgument('--ignore-modified', action='store_true',
+ help='Ignore modified files in current checkout',
+ default=False)
+ def vendor_aom(self, **kwargs):
+ from mozbuild.vendor_aom import VendorAOM
+ vendor_command = self._spawn(VendorAOM)
+ vendor_command.vendor(**kwargs)
+
+
@CommandProvider
class WebRTCGTestCommands(GTestCommands):
@Command('webrtc-gtest', category='testing',
description='Run WebRTC.org GTest unit tests.')
@CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter',
help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns),"
"optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).")
@CommandArgumentGroup('debugging')
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/vendor_aom.py
@@ -0,0 +1,157 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from distutils.version import LooseVersion
+import logging
+from mozbuild.base import (
+ BuildEnvironmentNotFoundException,
+ MozbuildObject,
+)
+import mozfile
+import mozpack.path as mozpath
+import requests
+import re
+import sys
+import tarfile
+
+class VendorAOM(MozbuildObject):
+ base_url = 'https://aomedia.googlesource.com/aom/'
+
+ def upstream_url(self, revision):
+ '''Construct a url for a tarball snapshot of the given revision.'''
+ return mozpath.join(self.base_url, '+archive', revision + '.tar.gz')
+
+ def upstream_commit(self, revision):
+ '''Convert a revision to a git commit and timestamp.
+
+ Ask the upstream repo to convert the requested revision to
+ a git commit id and timestamp, so we can be precise in
+ what we're vendoring.'''
+ url = mozpath.join(self.base_url, '+', revision + '?format=JSON')
+ self.log(logging.INFO, 'fetch', {'url': url},
+ 'Fetching commit id from {url}')
+ req = requests.get(url)
+ req.raise_for_status()
+ try:
+ info = req.json()
+ except ValueError as e:
+ if 'No JSON object' in e.message:
+ # As of 2017 May, googlesource sends 4 garbage characters
+ # at the beginning of the json response. Work around this.
+ # https://bugs.chromium.org/p/chromium/issues/detail?id=718550
+ import json
+ info = json.loads(req.text[4:])
+ else:
+ raise
+ return (info['commit'], info['committer']['time'])
+
+ def fetch_and_unpack(self, revision, target):
+ '''Fetch and unpack upstream source'''
+ url = self.upstream_url(revision)
+ self.log(logging.INFO, 'fetch', {'url': url}, 'Fetching {url}')
+ filename = 'libaom-' + revision + '.tar.gz'
+ with open(filename, 'wb') as f:
+ req = requests.get(url, stream=True)
+ for data in req.iter_content(4096):
+ f.write(data)
+ tar = tarfile.open(filename)
+ bad_paths = filter(lambda name: name.startswith('/') or '..' in name,
+ tar.getnames())
+ if any(bad_paths):
+ raise Exception("Tar archive contains non-local paths,"
+ "e.g. '%s'" % bad_paths[0])
+ self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % target)
+ mozfile.remove(target)
+ self.log(logging.INFO, 'unpack', {}, 'Unpacking upstream files.')
+ tar.extractall(target)
+ mozfile.remove(filename)
+
+ def update_readme(self, revision, timestamp, target):
+ filename = mozpath.join(target, 'README_MOZILLA')
+ with open(filename) as f:
+ readme = f.read()
+
+ prefix = 'The git commit ID used was'
+ if prefix in readme:
+ new_readme = re.sub(prefix + ' [v\.a-f0-9]+.*$',
+ prefix + ' %s (%s).' % (revision, timestamp),
+ readme)
+ else:
+ new_readme = '%s\n\n%s %s.' % (readme, prefix, revision)
+
+ if readme != new_readme:
+ with open(filename, 'w') as f:
+ f.write(new_readme)
+
+ def clean_upstream(self, target):
+ '''Remove files we don't want to import.'''
+ mozfile.remove(mozpath.join(target, '.gitattributes'))
+ mozfile.remove(mozpath.join(target, '.gitignore'))
+ mozfile.remove(mozpath.join(target, 'build', '.gitattributes'))
+ mozfile.remove(mozpath.join(target, 'build' ,'.gitignore'))
+
+ def generate_sources(self, target):
+ '''
+ Run the library's native build system to update ours.
+
+ Invoke configure for each supported platform to generate
+ appropriate config and header files, then invoke the
+ makefile to obtain a list of source files, writing
+ these out in the appropriate format for our build
+ system to use.
+ '''
+ config_dir = mozpath.join(target, 'config')
+ self.log(logging.INFO, 'rm_confg_dir', {}, 'rm -rf %s' % config_dir)
+ mozfile.remove(config_dir)
+ self.run_process(args=['./generate_sources_mozbuild.sh'],
+ cwd=target)
+
+ def check_modified_files(self):
+ '''
+ Ensure that there aren't any uncommitted changes to files
+ in the working copy, since we're going to change some state
+ on the user.
+ '''
+ modified = self.repository.get_modified_files()
+ if modified:
+ self.log(logging.ERROR, 'modified_files', {},
+ '''You have uncommitted changes to the following files:
+
+{files}
+
+Please commit or stash these changes before vendoring, or re-run with `--ignore-modified`.
+'''.format(files='\n'.join(sorted(modified))))
+ sys.exit(1)
+
+ def vendor(self, revision, ignore_modified=False):
+ self.populate_logger()
+ self.log_manager.enable_unstructured()
+
+ if not ignore_modified:
+ self.check_modified_files()
+ if not revision:
+ revision = 'master'
+ commit, timestamp = self.upstream_commit(revision)
+
+ vendor_dir = mozpath.join(self.topsrcdir, 'third_party/aom')
+ self.fetch_and_unpack(commit, vendor_dir)
+ self.log(logging.INFO, 'clean_upstream', {},
+ '''Removing unnecessary files.''')
+ self.clean_upstream(vendor_dir)
+ glue_dir = mozpath.join(self.topsrcdir, 'media/libaom')
+ self.log(logging.INFO, 'generate_sources', {},
+ '''Generating build files...''')
+ self.generate_sources(glue_dir)
+ self.log(logging.INFO, 'update_readme', {},
+ '''Updating README_MOZILLA.''')
+ self.update_readme(commit, timestamp, glue_dir)
+ self.repository.add_remove_files(vendor_dir)
+ self.log(logging.INFO, 'add_remove_files', {},
+ '''Registering changes with version control.''')
+ self.repository.add_remove_files(vendor_dir)
+ self.repository.add_remove_files(glue_dir)
+ self.log(logging.INFO, 'done', {'revision': revision},
+ '''Update to aom version '{revision}' ready to commit.''')