Bug 1380893 - Add CI for plain builds; r?Build
This commit adds CI tasks to perform "plain" builds. These builds use
the same toolchains used by official builds. But that's about it. The
mozconfig changes are minimal and only set up paths to toolchain
artifacts. sccache is not used.
The main goal of these builds is to have a "reference" build that
matches an out-of-the-box build environment as much as possible. We want
this mainly so we have timing and behavior information that matches what
developers are doing.
The Windows/generic Taskcluster worker doesn't like it when you specify
an artifact directory that doesn't exist. So we needed to add a
mozharness step to ensure UPLOAD_PATH exists to prevent those tasks from
erroring.
I'm not super thrilled about using mozharness here. We definitely don't
really need mozharness. But the main thing it is providing that we care
about is the Perfherder metrics data. I don't feel like scope bloating
to move that out of mozharness at this time.
I only implemented Linux64 and Windows64 because I'm not convinced
coverage on 32-bit build variations is useful. Tasks only run on trunk
because they are informational only and we don't need to waste resources
running these on release repos and on Try. They are tier 2 because they
aren't critical to shipping Firefox.
MozReview-Commit-ID: Gl6hGYbFX9b
new file mode 100644
--- /dev/null
+++ b/browser/config/mozconfigs/linux64/plain-debug
@@ -0,0 +1,3 @@
+. "$topsrcdir/browser/config/mozconfigs/linux64/plain-opt"
+
+ac_add_options --enable-debug
new file mode 100644
--- /dev/null
+++ b/browser/config/mozconfigs/linux64/plain-opt
@@ -0,0 +1,11 @@
+export TOOLTOOL_DIR="$topsrcdir"
+export LLVM_CONFIG="${TOOLTOOL_DIR}/clang/bin/llvm-config"
+
+CARGO="${TOOLTOOL_DIR}/rustc/bin/cargo"
+RUSTC="${TOOLTOOL_DIR}/rustc/bin/rustc"
+
+CC="${TOOLTOOL_DIR}/gcc/bin/gcc"
+CXX="${TOOLTOOL_DIR}/gcc/bin/g++"
+
+mk_add_options "export PATH=${TOOLTOOL_DIR}/gcc/bin:${PATH}"
+mk_add_options "export LD_LIBRARY_PATH=${TOOLTOOL_DIR}/gcc/lib64:${TOOLTOOL_DIR}/gcc/lib32:${TOOLTOOL_DIR}/gcc/lib"
new file mode 100644
--- /dev/null
+++ b/browser/config/mozconfigs/win64/plain-debug
@@ -0,0 +1,3 @@
+. $topsrcdir/browser/config/mozconfigs/win64/plain-opt
+
+ac_add_options --enable-debug
new file mode 100644
--- /dev/null
+++ b/browser/config/mozconfigs/win64/plain-opt
@@ -0,0 +1,11 @@
+TOOLTOOL_DIR=${topsrcdir}
+export MAKECAB=$TOOLTOOL_DIR/makecab.exe
+export LLVM_CONFIG="${TOOLTOOL_DIR}/clang/bin/llvm-config"
+
+RUSTC="${TOOLTOOL_DIR}/rustc/bin/rustc"
+CARGO="${TOOLTOOL_DIR}/rustc/bin/cargo"
+
+ac_add_options --target=x86_64-pc-mingw32
+ac_add_options --host=x86_64-pc-mingw32
+
+. $topsrcdir/build/win64/mozconfig.vs-latest
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -21,16 +21,44 @@ linux64/opt:
tooltool-downloads: public
need-xvfb: true
toolchains:
- linux64-clang
- linux64-gcc
- linux64-rust
- linux64-sccache
+linux64-plain/opt:
+ description: "Linux64 Opt Plain"
+ index:
+ product: firefox
+ job-name: linux64-plain-opt
+ treeherder:
+ platform: linux64/opt
+ symbol: Bp
+ tier: 2
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ max-run-time: 3600
+ env:
+ PERFHERDER_EXTRA_OPTIONS: plain
+ run:
+ using: mozharness
+ config:
+ - builds/releng_base_firefox.py
+ - builds/releng_base_linux_64_builds.py
+ - builds/releng_sub_linux_configs/64_plain_opt.py
+ script: mozharness/scripts/fx_desktop_build.py
+ tooltool-downloads: public
+ run-on-projects: [trunk]
+ toolchains:
+ - linux64-clang
+ - linux64-gcc
+ - linux64-rust
+
linux64-dmd/opt:
description: "Linux64 DMD Opt"
index:
product: firefox
job-name: linux64-dmd-opt
treeherder:
platform: linux64-dmd/opt
symbol: Bdmd
@@ -139,16 +167,44 @@ linux64/debug:
tooltool-downloads: public
need-xvfb: true
toolchains:
- linux64-clang
- linux64-gcc
- linux64-sccache
- linux64-rust
+linux64-plain/debug:
+ description: "Linux64 Debug Plain"
+ index:
+ product: firefox
+ job-name: linux64-plain-debug
+ treeherder:
+ platform: linux64/debug
+ symbol: Bp
+ tier: 2
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker:
+ max-run-time: 3600
+ env:
+ PERFHERDER_EXTRA_OPTIONS: plain
+ run:
+ using: mozharness
+ config:
+ - builds/releng_base_firefox.py
+ - builds/releng_base_linux_64_builds.py
+ - builds/releng_sub_linux_configs/64_plain_debug.py
+ script: mozharness/scripts/fx_desktop_build.py
+ tooltool-downloads: public
+ run-on-projects: [trunk]
+ toolchains:
+ - linux64-clang
+ - linux64-gcc
+ - linux64-rust
+
linux64-devedition-nightly/opt:
description: "Linux64 devedition Nightly"
attributes:
nightly: true
shipping-phase: build
shipping-product: devedition
index:
product: devedition
--- a/taskcluster/ci/build/windows.yml
+++ b/taskcluster/ci/build/windows.yml
@@ -136,16 +136,45 @@ win64/debug:
- builds/taskcluster_base_windows.py
- builds/taskcluster_base_win64.py
- builds/taskcluster_sub_win64/debug.py
toolchains:
- win64-clang-cl
- win64-rust
- win64-sccache
+win64-plain/debug:
+ description: "Win64 Debug Plain"
+ index:
+ product: firefox
+ job-name: win64-plain-debug
+ treeherder:
+ platform: windows2012-64/debug
+ symbol: Bp
+ tier: 2
+ worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+ worker:
+ max-run-time: 7200
+ env:
+ PERFHERDER_EXTRA_OPTIONS: plain
+ TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
+ run:
+ using: mozharness
+ options: [append-env-variables-from-configs]
+ script: mozharness/scripts/fx_desktop_build.py
+ config:
+ - builds/releng_base_firefox.py
+ - builds/taskcluster_base_windows.py
+ - builds/taskcluster_base_win64.py
+ - builds/taskcluster_sub_win64/plain_debug.py
+ run-on-projects: [trunk]
+ toolchains:
+ - win64-clang-cl
+ - win64-rust
+
win64/opt:
description: "Win64 Opt"
index:
product: firefox
job-name: win64-opt
treeherder:
platform: windows2012-64/opt
symbol: B
@@ -164,16 +193,45 @@ win64/opt:
- builds/taskcluster_base_windows.py
- builds/taskcluster_base_win64.py
- builds/taskcluster_sub_win64/opt.py
toolchains:
- win64-clang-cl
- win64-rust
- win64-sccache
+win64-plain/opt:
+ description: "Win64 Opt Plain"
+ index:
+ product: firefox
+ job-name: win64-plain-opt
+ treeherder:
+ platform: windows2012-64/opt
+ symbol: Bp
+ tier: 2
+ worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+ worker:
+ max-run-time: 7200
+ env:
+ PERFHERDER_EXTRA_OPTIONS: plain
+ TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
+ run:
+ using: mozharness
+ options: [append-env-variables-from-configs]
+ script: mozharness/scripts/fx_desktop_build.py
+ config:
+ - builds/releng_base_firefox.py
+ - builds/taskcluster_base_windows.py
+ - builds/taskcluster_base_win64.py
+ - builds/taskcluster_sub_win64/plain_opt.py
+ run-on-projects: [trunk]
+ toolchains:
+ - win64-clang-cl
+ - win64-rust
+
win64-dmd/opt:
description: "Win64 DMD Opt"
index:
product: firefox
job-name: win64-dmd-opt
treeherder:
platform: windows2012-64-dmd/opt
symbol: Bdmd
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_plain_debug.py
@@ -0,0 +1,7 @@
+config = {
+ 'default_actions': [
+ 'build',
+ ],
+ 'disable_package_metrics': True,
+ 'mozconfig_variant': 'plain-debug',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_plain_opt.py
@@ -0,0 +1,7 @@
+config = {
+ 'default_actions': [
+ 'build',
+ ],
+ 'disable_package_metrics': True,
+ 'mozconfig_variant': 'plain-opt',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/taskcluster_sub_win64/plain_debug.py
@@ -0,0 +1,9 @@
+config = {
+ 'default_actions': [
+ 'build',
+ 'ensure-upload-path',
+ ],
+ 'disable_package_metrics': True,
+ 'mozconfig_variant': 'plain-debug',
+ 'stage_platform': 'win64',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/taskcluster_sub_win64/plain_opt.py
@@ -0,0 +1,9 @@
+config = {
+ 'default_actions': [
+ 'build',
+ 'ensure-upload-path',
+ ],
+ 'disable_package_metrics': True,
+ 'mozconfig_variant': 'plain-opt',
+ 'stage_platform': 'win64',
+}
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -1775,17 +1775,24 @@ or run without that action (ie: --no-{ac
if return_code:
self.return_code = self.worst_level(
EXIT_STATUS_DICT[TBPL_FAILURE], self.return_code,
AUTOMATION_EXIT_CODES[::-1]
)
self.fatal("'mach valgrind-test' did not run successfully. Please check "
"log for errors.")
+ def ensure_upload_path(self):
+ env = self.query_mach_build_env()
+ # Some Taskcluster workers don't like it if an artifacts directory
+ # is defined but no artifacts are uploaded. Guard against this by always
+ # ensuring the artifacts directory exists.
+ if 'UPLOAD_PATH' in env and not os.path.exists(env['UPLOAD_PATH']):
+ os.makedirs(env['UPLOAD_PATH'])
def _post_fatal(self, message=None, exit_code=None):
if not self.return_code: # only overwrite return_code if it's 0
self.error('setting return code to 2 because fatal was called')
self.return_code = 2
@PostScriptRun
def _summarize(self):
--- a/testing/mozharness/scripts/fx_desktop_build.py
+++ b/testing/mozharness/scripts/fx_desktop_build.py
@@ -36,16 +36,17 @@ class FxDesktopBuild(BuildScript, TryToo
'get-secrets',
'clobber',
'build',
'check-test',
'valgrind-test',
'multi-l10n',
'package-source',
'update',
+ 'ensure-upload-path',
],
'require_config_file': True,
# Default configuration
'config': {
'is_automation': True,
"pgo_build": False,
"debug_build": False,
"pgo_platforms": ['linux', 'linux64', 'win32', 'win64'],