new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/android-arm.py
@@ -0,0 +1,39 @@
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+ os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+ 'external_tools',
+)
+
+config = {
+ 'default_actions': [
+ 'get-tooltool',
+ 'checkout-sources',
+ 'build',
+ # 'test', # can't run android tests on linux hosts
+ 'package',
+ ],
+
+ 'tooltool_manifest_file': "android.manifest",
+ 'tooltool_cache': "/builds/tooltool_cache",
+ 'exes': {
+ 'gittool.py': [os.path.join(external_tools_path, 'gittool.py')],
+ 'tooltool.py': "/builds/tooltool.py",
+ 'python2.7': "/tools/python27/bin/python2.7",
+ },
+ 'dump_syms_binary': 'dump_syms',
+ 'arch': 'arm',
+ 'use_mock': True,
+ 'mock_target': 'mozilla-centos6-x86_64',
+ 'mock_packages': ['make', 'git', 'nasm', 'glibc-devel.i686',
+ 'libstdc++-devel.i686', 'zip', 'yasm',
+ 'mozilla-python27'],
+ 'mock_files': [
+ ('/home/cltbld/.ssh', '/home/mock_mozilla/.ssh'),
+ ('/builds/relengapi.tok', '/builds/relengapi.tok'),
+ ('/tools/tooltool.py', '/builds/tooltool.py'),
+ ],
+ 'operating_system': 'android',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/android-x86.py
@@ -0,0 +1,40 @@
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+ os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+ 'external_tools',
+)
+
+config = {
+ 'default_actions': [
+ 'get-tooltool',
+ 'checkout-sources',
+ 'build',
+ # 'test', # can't run android tests on linux hosts
+ 'package',
+ ],
+
+ 'tooltool_manifest_file': "android.manifest",
+ 'tooltool_cache': "/builds/tooltool_cache",
+ 'exes': {
+ 'gittool.py': [os.path.join(external_tools_path, 'gittool.py')],
+ 'tooltool.py': "/builds/tooltool.py",
+ 'python2.7': "/tools/python27/bin/python2.7",
+ },
+ 'avoid_avx2': True,
+ 'dump_syms_binary': 'dump_syms',
+ 'arch': 'x86',
+ 'use_mock': True,
+ 'mock_target': 'mozilla-centos6-x86_64',
+ 'mock_packages': ['make', 'git', 'nasm', 'glibc-devel.i686',
+ 'libstdc++-devel.i686', 'zip', 'yasm',
+ 'mozilla-python27'],
+ 'mock_files': [
+ ('/home/cltbld/.ssh', '/home/mock_mozilla/.ssh'),
+ ('/builds/relengapi.tok', '/builds/relengapi.tok'),
+ ('/tools/tooltool.py', '/builds/tooltool.py'),
+ ],
+ 'operating_system': 'android',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/linux32.py
@@ -0,0 +1,31 @@
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+ os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+ 'external_tools',
+)
+
+config = {
+ 'tooltool_manifest_file': "linux.manifest",
+ 'tooltool_cache': "/builds/tooltool_cache",
+ 'exes': {
+ 'gittool.py': [os.path.join(external_tools_path, 'gittool.py')],
+ 'tooltool.py': "/builds/tooltool.py",
+ 'python2.7': "/tools/python27/bin/python2.7",
+ },
+ 'dump_syms_binary': 'dump_syms',
+ 'arch': 'x86',
+ 'use_mock': True,
+ 'avoid_avx2': True,
+ 'mock_target': 'mozilla-centos6-x86_64',
+ 'mock_packages': ['make', 'git', 'nasm', 'glibc-devel.i686',
+ 'libstdc++-devel.i686', 'zip', 'yasm',
+ 'mozilla-python27'],
+ 'mock_files': [
+ ('/home/cltbld/.ssh', '/home/mock_mozilla/.ssh'),
+ ('/builds/relengapi.tok', '/builds/relengapi.tok'),
+ ],
+ 'operating_system': 'linux',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/linux64.py
@@ -0,0 +1,31 @@
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+ os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+ 'external_tools',
+)
+
+config = {
+ 'tooltool_manifest_file': "linux.manifest",
+ 'tooltool_cache': "/builds/tooltool_cache",
+ 'exes': {
+ 'gittool.py': [os.path.join(external_tools_path, 'gittool.py')],
+ 'tooltool.py': "/builds/tooltool.py",
+ 'python2.7': "/tools/python27/bin/python2.7",
+ },
+ 'dump_syms_binary': 'dump_syms',
+ 'arch': 'x64',
+ 'use_mock': True,
+ 'avoid_avx2': True,
+ 'mock_target': 'mozilla-centos6-x86_64',
+ 'mock_packages': ['make', 'git', 'nasm', 'glibc-devel.i686',
+ 'libstdc++-devel.i686', 'zip', 'yasm',
+ 'mozilla-python27'],
+ 'mock_files': [
+ ('/home/cltbld/.ssh', '/home/mock_mozilla/.ssh'),
+ ('/builds/relengapi.tok', '/builds/relengapi.tok'),
+ ],
+ 'operating_system': 'linux',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/macosx32.py
@@ -0,0 +1,21 @@
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+ os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+ 'external_tools',
+)
+
+config = {
+ 'tooltool_manifest_file': "osx.manifest",
+ 'tooltool_cache': "/builds/tooltool_cache",
+ 'exes': {
+ 'gittool.py': [os.path.join(external_tools_path, 'gittool.py')],
+ 'tooltool.py': "/builds/tooltool.py",
+ 'python2.7': "/tools/python27/bin/python2.7",
+ },
+ 'dump_syms_binary': 'dump_syms',
+ 'arch': 'x86',
+ 'use_yasm': True,
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/macosx64.py
@@ -0,0 +1,21 @@
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+ os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+ 'external_tools',
+)
+
+config = {
+ 'tooltool_manifest_file': "osx.manifest",
+ 'tooltool_cache': "/builds/tooltool_cache",
+ 'exes': {
+ 'gittool.py': [os.path.join(external_tools_path, 'gittool.py')],
+ 'tooltool.py': "/builds/tooltool.py",
+ 'python2.7': "/tools/python27/bin/python2.7",
+ },
+ 'dump_syms_binary': 'dump_syms',
+ 'arch': 'x64',
+ 'use_yasm': True,
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/tooltool-manifests/android.manifest
@@ -0,0 +1,17 @@
+[
+{
+"size": 357533004,
+"visibility": "internal",
+"digest": "9b24b6db45ca4f6418d18119e3d2dd48e2a352e977ebf4fd0b2a8c4103fd0735eacb0f7360ff3a2eda6777fdd1029325ca23b8a4c34542b3b1fa487ff633647c",
+"algorithm": "sha512",
+"filename": "android-ndk.tar.bz2",
+"unpack": true
+},
+{
+"size": 4086983,
+"visibility": "public",
+"digest": "f9b8922384b489cf4969b1d26d4fef93db5882dacadd9e679e728164220b237ce170e94f45e6a6ede1055f4d06097f994e5d0185f42cf90b9e3dc57dfc2d0005",
+"algorithm": "sha512",
+"filename": "dump_syms"
+}
+]
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/tooltool-manifests/linux.manifest
@@ -0,0 +1,9 @@
+[
+{
+"size": 4086983,
+"visibility": "public",
+"digest": "f9b8922384b489cf4969b1d26d4fef93db5882dacadd9e679e728164220b237ce170e94f45e6a6ede1055f4d06097f994e5d0185f42cf90b9e3dc57dfc2d0005",
+"algorithm": "sha512",
+"filename": "dump_syms"
+}
+]
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/tooltool-manifests/osx.manifest
@@ -0,0 +1,10 @@
+[
+{
+"size": 393368,
+"visibility": "public",
+"digest": "dcbd679a57ac3a1bc20df45cd8770b462d2c430d1c5fb85366c02904422824ed4193a66d936fa9d21fb49190f5915f9b24aad16e3eb8b7c9a90d09c93067982f",
+"algorithm": "sha512",
+"filename": "dump_syms"
+}
+]
+
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/tooltool-manifests/win.manifest
@@ -0,0 +1,10 @@
+[
+{
+"size": 51200,
+"visibility": "public",
+"digest": "3071d2c22325c6f952d99491005dce85887ebd6a7ec52a91af157cf6f1ef4453234489d1269f7f92c739e08fd8a747d1ea2d6a6cf13cc9818a2f99302b4eb970",
+"algorithm": "sha512",
+"filename": "dump_syms_vc1800.exe"
+}
+]
+
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/win32.py
@@ -0,0 +1,37 @@
+import sys
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+ os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+ 'external_tools',
+)
+
+VSPATH = 'C:/tools/vs2013'
+config = {
+ 'tooltool_manifest_file': "win.manifest",
+ 'exes': {
+ 'gittool.py': [sys.executable, os.path.join(external_tools_path, 'gittool.py')],
+ 'python2.7': 'c:\\mozilla-build\\python27\\python2.7.exe',
+ 'tooltool.py': [sys.executable, "c:\\mozilla-build\\tooltool.py"],
+ },
+ 'dump_syms_binary': 'dump_syms_vc1800.exe',
+ 'arch': 'x86',
+ 'use_yasm': True,
+ 'operating_system': 'msvc',
+ 'partial_env': {
+ 'PATH': '%s;%s;%s;%s;%s' % (
+ 'c:/Program Files (x86)/Windows Kits/8.1/bin/x86;{_VSPATH}/Common7/IDE;{_VSPATH}/VC/BIN/amd64_x86;{_VSPATH}/VC/BIN/amd64;{_VSPATH}/Common7/Tools;{_VSPATH}/VC/VCPackages;c:/mozilla-build/moztools'.format(_VSPATH=VSPATH),
+ 'c:/windows/Microsoft.NET/Framework/v3.5;c:/windows/Microsoft.NET/Framework/v4.0.30319',
+ os.environ['PATH'],
+ 'C:\\mozilla-build\\Git\\bin',
+ 'C:\\mozilla-build\\svn-win32-1.6.3\\bin',
+ ),
+ 'WIN32_REDIST_DIR': '{_VSPATH}/VC/redist/x86/Microsoft.VC120.CRT'.format(_VSPATH=VSPATH),
+ 'INCLUDE': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\shared;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\um;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\winrt;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\winrt\\wrl;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\winrt\\wrl\\wrappers;{_VSPATH}\\vc\\include;{_VSPATH}\\vc\\atlmfc\\include;c:\\tools\\sdks\\dx10\\include'.format(_VSPATH=VSPATH),
+ 'LIB': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\Lib\\winv6.3\\um\\x86;{_VSPATH}\\vc\\lib;{_VSPATH}\\vc\\atlmfc\\lib;c:\\tools\\sdks\\dx10\\lib'.format(_VSPATH=VSPATH),
+ 'LIBPATH': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\Lib\\winv6.3\\um\\x86;{_VSPATH}\\vc\\lib;{_VSPATH}\\vc\\atlmfc\\lib;c:\\tools\\sdks\\dx10\\lib'.format(_VSPATH=VSPATH),
+ 'WINDOWSSDKDIR': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\',
+ },
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/openh264/win64.py
@@ -0,0 +1,37 @@
+import sys
+import os
+
+import mozharness
+
+external_tools_path = os.path.join(
+ os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+ 'external_tools',
+)
+
+VSPATH = 'C:/tools/vs2013'
+config = {
+ 'tooltool_manifest_file': "win.manifest",
+ 'exes': {
+ 'gittool.py': [sys.executable, os.path.join(external_tools_path, 'gittool.py')],
+ 'python2.7': 'c:\\mozilla-build\\python27\\python2.7.exe',
+ 'tooltool.py': [sys.executable, "c:\\mozilla-build\\tooltool.py"],
+ },
+ 'dump_syms_binary': 'dump_syms_vc1800.exe',
+ 'arch': 'x64',
+ 'use_yasm': True,
+ 'operating_system': 'msvc',
+ 'partial_env': {
+ 'PATH': '%s;%s;%s;%s;%s' % (
+ 'c:/Program Files (x86)/Windows Kits/8.1/bin/x64;{_VSPATH}/Common7/IDE;{_VSPATH}/VC/BIN/amd64;{_VSPATH}/VC/BIN/x86_amd64;{_VSPATH}/VC/BIN;{_VSPATH}/Common7/Tools;{_VSPATH}/VC/VCPackages;c:/mozilla-build/moztools-x64'.format(_VSPATH=VSPATH),
+ 'c:/windows/Microsoft.NET/Framework64/v3.5;c:/windows/Microsoft.NET/Framework64/v4.0.30319',
+ os.environ['PATH'],
+ 'C:\\mozilla-build\\Git\\bin',
+ 'C:\\mozilla-build\\svn-win32-1.6.3\\bin',
+ ),
+ 'WIN32_REDIST_DIR': '{_VSPATH}/VC/redist/x64/Microsoft.VC120.CRT'.format(_VSPATH=VSPATH),
+ 'INCLUDE': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\shared;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\um;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\winrt;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\winrt\\wrl;c:\\Program Files (x86)\\Windows Kits\\8.1\\include\\winrt\\wrl\\wrappers;{_VSPATH}\\vc\\include;{_VSPATH}\\vc\\atlmfc\\include;c:\\tools\\sdks\\dx10\\include'.format(_VSPATH=VSPATH),
+ 'LIB': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\Lib\\winv6.3\\um\\x64;{_VSPATH}\\vc\\lib\\amd64;{_VSPATH}\\vc\\atlmfc\\lib\\amd64;c:\\tools\\sdks\\dx10\\lib\\x64'.format(_VSPATH=VSPATH),
+ 'LIBPATH': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\Lib\\winv6.3\\um\\x64;{_VSPATH}\\vc\\lib\\amd64;{_VSPATH}\\vc\\atlmfc\\lib\\amd64;c:\\tools\\sdks\\dx10\\lib\\x64'.format(_VSPATH=VSPATH),
+ 'WINDOWSSDKDIR': 'c:\\Program Files (x86)\\Windows Kits\\8.1\\',
+ },
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/external_tools/packagesymbols.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import argparse
+import os
+import subprocess
+import sys
+import zipfile
+
+
+class ProcError(Exception):
+ def __init__(self, returncode, stderr):
+ self.returncode = returncode
+ self.stderr = stderr
+
+
+def check_output(command):
+ proc = subprocess.Popen(command,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ stdout, stderr = proc.communicate()
+ if proc.returncode != 0:
+ raise ProcError(proc.returncode, stderr)
+ return stdout
+
+
+def process_file(dump_syms, path):
+ try:
+ stdout = check_output([dump_syms, path])
+ except ProcError as e:
+ print('Error: running "%s %s": %s' % (dump_syms, path, e.stderr))
+ return None, None, None
+ bits = stdout.splitlines()[0].split(' ', 4)
+ if len(bits) != 5:
+ return None, None, None
+ _, platform, cpu_arch, debug_id, debug_file = bits
+ if debug_file.lower().endswith('.pdb'):
+ sym_file = debug_file[:-4] + '.sym'
+ else:
+ sym_file = debug_file + '.sym'
+ filename = os.path.join(debug_file, debug_id, sym_file)
+ debug_filename = os.path.join(debug_file, debug_id, debug_file)
+ return filename, stdout, debug_filename
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('dump_syms', help='Path to dump_syms binary')
+ parser.add_argument('files', nargs='+',
+ help='Path to files to dump symbols from')
+ parser.add_argument('--symbol-zip', default='symbols.zip',
+ help='Name of zip file to put dumped symbols in')
+ parser.add_argument('--no-binaries',
+ action='store_true',
+ default=False,
+ help='Don\'t store binaries in zip file')
+ args = parser.parse_args()
+ count = 0
+ with zipfile.ZipFile(args.symbol_zip, 'w', zipfile.ZIP_DEFLATED) as zf:
+ for f in args.files:
+ filename, contents, debug_filename = process_file(args.dump_syms, f)
+ if not (filename and contents):
+ print('Error dumping symbols')
+ sys.exit(1)
+ zf.writestr(filename, contents)
+ count += 1
+ if not args.no_binaries:
+ zf.write(f, debug_filename)
+ count += 1
+ print('Added %d files to %s' % (count, args.symbol_zip))
+
+if __name__ == '__main__':
+ main()
--- a/testing/mozharness/mozharness/base/script.py
+++ b/testing/mozharness/mozharness/base/script.py
@@ -1453,17 +1453,17 @@ class ScriptMixin(PlatformMixin):
self.return_code = fatal_exit_code
self.fatal("Halting on failure while running %s" % command,
exit_code=fatal_exit_code)
if return_type == 'num_errors':
return parser.num_errors
return returncode
def get_output_from_command(self, command, cwd=None,
- halt_on_failure=False, env=None,
+ halt_on_failure=False, env=None, partial_env=None,
silent=False, log_level=INFO,
tmpfile_base_path='tmpfile',
return_type='output', save_tmpfiles=False,
throw_exception=False, fatal_exit_code=2,
ignore_errors=False, success_codes=None):
"""Similar to run_command, but where run_command is an
os.system(command) analog, get_output_from_command is a `command`
analog.
@@ -1485,16 +1485,18 @@ class ScriptMixin(PlatformMixin):
command (str | list): command or list of commands to
execute and log.
cwd (str, optional): directory path from where to execute the
command. Defaults to `None`.
halt_on_failure (bool, optional): whether or not to redefine the
log level as `FATAL` on error. Defaults to False.
env (dict, optional): key-value of environment values to use to
run the command. Defaults to None.
+ partial_env (dict, optional): key-value of environment values to
+ replace from the current environment values. Defaults to None.
silent (bool, optional): whether or not to output the stdout of
executing the command. Defaults to False.
log_level (str, optional): log level name to use on normal execution.
Defaults to `INFO`.
tmpfile_base_path (str, optional): base path of the file to which
the output will be writen to. Defaults to 'tmpfile'.
return_type (str, optional): if equal to 'output' then the complete
output of the executed command is returned, otherwise the written
@@ -1553,18 +1555,26 @@ class ScriptMixin(PlatformMixin):
except IOError:
level = ERROR
if halt_on_failure:
level = FATAL
self.log("Can't open %s for writing!" % tmp_stderr_filename +
self.exception(), level=level)
return None
shell = True
- if isinstance(command, list):
+ if isinstance(command, list) or isinstance(command, tuple):
shell = False
+
+ if env is None:
+ if partial_env:
+ self.info("Using partial env: %s" % pprint.pformat(partial_env))
+ env = self.query_env(partial_env=partial_env)
+ else:
+ self.info("Using env: %s" % pprint.pformat(env))
+
p = subprocess.Popen(command, shell=shell, stdout=tmp_stdout,
cwd=cwd, stderr=tmp_stderr, env=env)
# XXX: changed from self.debug to self.log due to this error:
# TypeError: debug() takes exactly 1 argument (2 given)
self.log("Temporary files: %s and %s" % (tmp_stdout_filename, tmp_stderr_filename), level=DEBUG)
p.wait()
tmp_stdout.close()
tmp_stderr.close()
--- a/testing/mozharness/scripts/openh264_build.py
+++ b/testing/mozharness/scripts/openh264_build.py
@@ -2,42 +2,55 @@
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
import sys
import os
import glob
+import re
# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))
# import the guts
+import mozharness
from mozharness.base.vcs.vcsbase import VCSScript
from mozharness.base.log import ERROR
from mozharness.base.transfer import TransferMixin
from mozharness.mozilla.mock import MockMixin
+from mozharness.mozilla.tooltool import TooltoolMixin
-class OpenH264Build(MockMixin, TransferMixin, VCSScript):
+external_tools_path = os.path.join(
+ os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
+ 'external_tools',
+)
+
+
+class OpenH264Build(MockMixin, TransferMixin, VCSScript, TooltoolMixin):
all_actions = [
'clobber',
+ 'get-tooltool',
'checkout-sources',
'build',
'test',
'package',
+ 'dump-symbols',
'upload',
]
default_actions = [
+ 'get-tooltool',
'checkout-sources',
'build',
'test',
'package',
+ 'dump-symbols',
]
config_options = [
[["--repo"], {
"dest": "repo",
"help": "OpenH264 repository to use",
"default": "https://github.com/cisco/openh264.git"
}],
@@ -46,26 +59,19 @@ class OpenH264Build(MockMixin, TransferM
"help": "revision to checkout",
"default": "master"
}],
[["--debug"], {
"dest": "debug_build",
"action": "store_true",
"help": "Do a debug build",
}],
- [["--64"], {
- "dest": "64bit",
- "action": "store_true",
- "help": "Do a 64-bit build",
- "default": True,
- }],
- [["--32"], {
- "dest": "64bit",
- "action": "store_false",
- "help": "Do a 32-bit build",
+ [["--arch"], {
+ "dest": "arch",
+ "help": "Arch type to use (x64, x86, or arm)",
}],
[["--os"], {
"dest": "operating_system",
"help": "Specify the operating system to build for",
}],
[["--use-mock"], {
"dest": "use_mock",
"help": "use mock to set up build environment",
@@ -73,28 +79,31 @@ class OpenH264Build(MockMixin, TransferM
"default": False,
}],
[["--use-yasm"], {
"dest": "use_yasm",
"help": "use yasm instead of nasm",
"action": "store_true",
"default": False,
}],
+ [["--avoid-avx2"], {
+ "dest": "avoid_avx2",
+ "help": "Pass HAVE_AVX2='false' through to Make to support older nasm",
+ "action": "store_true",
+ "default": False,
+ }]
]
def __init__(self, require_config_file=False, config={},
all_actions=all_actions,
default_actions=default_actions):
# Default configuration
default_config = {
'debug_build': False,
- 'mock_target': 'mozilla-centos6-x86_64',
- 'mock_packages': ['make', 'git', 'nasm', 'glibc-devel.i686', 'libstdc++-devel.i686', 'zip', 'yasm'],
- 'mock_files': [],
'upload_ssh_key': os.path.expanduser("~/.ssh/ffxbld_rsa"),
'upload_ssh_user': 'ffxbld',
'upload_ssh_host': 'stage.mozilla.org',
'upload_path_base': '/home/ffxbld/openh264',
'use_yasm': False,
}
default_config.update(config)
@@ -106,47 +115,85 @@ class OpenH264Build(MockMixin, TransferM
all_actions=all_actions,
default_actions=default_actions,
)
if self.config['use_mock']:
self.setup_mock()
self.enable_mock()
+ def get_tooltool(self):
+ c = self.config
+ if not c.get('tooltool_manifest_file'):
+ self.info("Skipping tooltool fetching since no tooltool manifest")
+ return
+ dirs = self.query_abs_dirs()
+ self.mkdir_p(dirs['abs_work_dir'])
+ manifest = os.path.join(dirs['base_work_dir'],
+ 'scripts', 'configs',
+ 'openh264', 'tooltool-manifests',
+ c['tooltool_manifest_file'])
+ self.info("Getting tooltool files from manifest (%s)" % manifest)
+ try:
+ self.tooltool_fetch(
+ manifest=manifest,
+ output_dir=dirs['abs_work_dir'],
+ cache=c.get('tooltool_cache')
+ )
+ except KeyError:
+ self.error('missing a required key.')
+
def query_package_name(self):
- if self.config['64bit']:
+ if self.config['arch'] == "x64":
bits = '64'
else:
bits = '32'
version = self.config['revision']
if sys.platform == 'linux2':
if self.config.get('operating_system') == 'android':
- return 'openh264-android-{version}.zip'.format(version=version, bits=bits)
+ if self.config.get('arch') == 'x86':
+ return 'openh264-android-{arch}-{version}.zip'.format(
+ version=version, bits=bits, arch=self.config['arch']
+ )
+ else:
+ return 'openh264-android-{arch}-{version}.zip'.format(
+ version=version, bits=bits, arch='arm'
+ )
else:
return 'openh264-linux{bits}-{version}.zip'.format(version=version, bits=bits)
elif sys.platform == 'darwin':
return 'openh264-macosx{bits}-{version}.zip'.format(version=version, bits=bits)
elif sys.platform == 'win32':
return 'openh264-win{bits}-{version}.zip'.format(version=version, bits=bits)
self.fatal("can't determine platform")
def query_make_params(self):
+ dirs = self.query_abs_dirs()
retval = []
if self.config['debug_build']:
retval.append('BUILDTYPE=Debug')
- if self.config['64bit']:
+ if self.config['avoid_avx2']:
+ retval.append('HAVE_AVX2=false')
+
+ if self.config['arch'] == 'x64':
retval.append('ENABLE64BIT=Yes')
else:
retval.append('ENABLE64BIT=No')
if "operating_system" in self.config:
retval.append("OS=%s" % self.config['operating_system'])
+ if self.config["operating_system"] == "android":
+ if self.config['arch'] == 'x86':
+ retval.append("ARCH=x86")
+ retval.append('TARGET=invalid')
+ retval.append('NDKLEVEL=9')
+ retval.append('NDKROOT=%s/android-ndk' % dirs['abs_work_dir'])
if self.config['use_yasm']:
retval.append('ASM=yasm')
return retval
def query_upload_ssh_key(self):
return self.config['upload_ssh_key']
@@ -155,21 +202,27 @@ class OpenH264Build(MockMixin, TransferM
return self.config['upload_ssh_host']
def query_upload_ssh_user(self):
return self.config['upload_ssh_user']
def query_upload_ssh_path(self):
return "%s/%s" % (self.config['upload_path_base'], self.config['revision'])
- def run_make(self, target):
+ def run_make(self, target, capture_output=False):
cmd = ['make', target] + self.query_make_params()
dirs = self.query_abs_dirs()
repo_dir = os.path.join(dirs['abs_work_dir'], 'src')
- return self.run_command(cmd, cwd=repo_dir)
+ env = self.config.get('env')
+ partial_env = self.config.get('partial_env')
+ kwargs = dict(cwd=repo_dir, env=env, partial_env=partial_env)
+ if capture_output:
+ return self.get_output_from_command(cmd, **kwargs)
+ else:
+ return self.run_command(cmd, **kwargs)
def checkout_sources(self):
repo = self.config['repo']
rev = self.config['revision']
dirs = self.query_abs_dirs()
repo_dir = os.path.join(dirs['abs_work_dir'], 'src')
@@ -212,23 +265,61 @@ class OpenH264Build(MockMixin, TransferM
def package(self):
dirs = self.query_abs_dirs()
srcdir = os.path.join(dirs['abs_work_dir'], 'src')
package_name = self.query_package_name()
package_file = os.path.join(dirs['abs_work_dir'], package_name)
if os.path.exists(package_file):
os.unlink(package_file)
- to_package = [os.path.basename(f) for f in glob.glob(os.path.join(srcdir, "*gmpopenh264*"))]
+ to_package = []
+ for f in glob.glob(os.path.join(srcdir, "*gmpopenh264*")):
+ if not re.search(
+ "(?:lib)?gmpopenh264(?!\.\d)\.(?:dylib|so|dll|info)(?!\.\d)",
+ f):
+ # Don't package unnecessary zip bloat
+ # Blocks things like libgmpopenh264.2.dylib and libgmpopenh264.so.1
+ self.log("Skipping packaging of {package}".format(package=f))
+ continue
+ to_package.append(os.path.basename(f))
+ self.log("Packaging files %s" % to_package)
cmd = ['zip', package_file] + to_package
retval = self.run_command(cmd, cwd=srcdir)
if retval != 0:
self.fatal("couldn't make package")
self.copy_to_upload_dir(package_file)
+ def dump_symbols(self):
+ dirs = self.query_abs_dirs()
+ c = self.config
+ srcdir = os.path.join(dirs['abs_work_dir'], 'src')
+ package_name = self.run_make('echo-plugin-name', capture_output=True)
+ if not package_name:
+ self.fatal("failure running make")
+ zip_package_name = self.query_package_name()
+ if not zip_package_name[-4:] == ".zip":
+ self.fatal("Unexpected zip_package_name")
+ symbol_package_name = "{base}.symbols.zip".format(base=zip_package_name[:-4])
+ symbol_zip_path = os.path.join(dirs['abs_upload_dir'], symbol_package_name)
+ repo_dir = os.path.join(dirs['abs_work_dir'], 'src')
+ env = self.config.get('env')
+ partial_env = self.config.get('partial_env')
+ kwargs = dict(cwd=repo_dir, env=env, partial_env=partial_env)
+ dump_syms = os.path.join(dirs['abs_work_dir'], c['dump_syms_binary'])
+ self.chmod(dump_syms, 0755)
+ python = self.query_exe('python2.7')
+ cmd = [python, os.path.join(external_tools_path, 'packagesymbols.py'),
+ '--symbol-zip', symbol_zip_path,
+ dump_syms, os.path.join(srcdir, package_name)]
+ if self.config['use_mock']:
+ self.disable_mock()
+ self.run_command(cmd, **kwargs)
+ if self.config['use_mock']:
+ self.enable_mock()
+
def upload(self):
if self.config['use_mock']:
self.disable_mock()
dirs = self.query_abs_dirs()
self.rsync_upload_directory(
dirs['abs_upload_dir'],
self.query_upload_ssh_key(),
self.query_upload_ssh_user(),