Bug 1430878 - Use larger EC2 instances for Clang toolchain tasks; r?glandium
We now have access to workers running on EC2 instances with dozens of
vCPUs. gecko-<L>-b-linux-large is m4.10xlarge, m5.12xlarge, c5.9xlarge,
or c4.8xlarge. gecko-<L>-b-linux-xlarge is m5.24xlarge, m4.16xlarge,
or c5.18xlarge.
Experimentation reveals that Clang tasks are the only tasks that
are CPU efficient enough (read: cost effective) to run on these
larger worker types.
This commit defines the new worker types and switches Clang toolchain
tasks to run on the new workers. clang5 and clang6 tasks take ~30 minutes
on the -large variant but ~17 minutes on the -xlarge variant. All other
tasks don't show as linear of a speedup. So running them on the
-xlarge variant isn't justified.
As part of this change, Mac toolchain tasks have been converted
to run on gecko-<L>-b-linux* workers. The gecko-<L>-b-macosx64 workers
are actually Linux. IMO the b-macosx64 worker type is no longer needed.
Moving the toolchain tasks off the worker should hopefully not be very
controversial.
MozReview-Commit-ID: HynQPMWiWHo
--- a/taskcluster/ci/toolchain/linux.yml
+++ b/taskcluster/ci/toolchain/linux.yml
@@ -8,17 +8,17 @@ job-defaults:
linux64-clang-3.9:
description: "Clang 3.9 toolchain build"
treeherder:
kind: build
platform: toolchains/opt
symbol: TL(clang3.9)
tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux-large
worker:
max-run-time: 7200
run:
using: toolchain-script
script: build-clang-3.9-linux.sh
resources:
- 'build/build-clang/build-clang.py'
- 'build/build-clang/clang-3.9-linux64.json'
@@ -29,17 +29,17 @@ linux64-clang-3.9:
linux64-clang-4:
description: "Clang 4 toolchain build"
treeherder:
kind: build
platform: toolchains/opt
symbol: TL(clang4)
tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux-large
worker:
max-run-time: 7200
run:
using: toolchain-script
script: build-clang-4-linux.sh
resources:
- 'build/build-clang/build-clang.py'
- 'build/build-clang/clang-4-linux64.json'
@@ -50,17 +50,17 @@ linux64-clang-4:
linux64-clang-5:
description: "Clang 5 toolchain build"
treeherder:
kind: build
platform: toolchains/opt
symbol: TL(clang5)
tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux-xlarge
worker:
max-run-time: 7200
run:
using: toolchain-script
script: build-clang-5-linux.sh
resources:
- 'build/build-clang/build-clang.py'
- 'build/build-clang/clang-5-linux64.json'
@@ -72,17 +72,17 @@ linux64-clang-5:
linux64-clang-6-pre:
description: "Clang 6 Pre toolchain build"
treeherder:
kind: build
platform: toolchains/opt
symbol: TL(clang6p)
tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux-xlarge
worker:
max-run-time: 7200
run:
using: toolchain-script
script: build-clang-6-pre-linux.sh
resources:
- 'build/build-clang/build-clang.py'
- 'build/build-clang/clang-6-pre-linux64.json'
@@ -122,17 +122,17 @@ linux64-clang-tidy:
index:
product: static-analysis
job-name: linux64-clang-tidy
treeherder:
kind: build
platform: toolchains/opt
symbol: TL(clang-tidy)
tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux-large
worker:
max-run-time: 36000
run:
using: toolchain-script
script: build-clang-tidy-linux.sh
resources:
- 'build/clang-plugin/**'
- 'build/build-clang/build-clang.py'--- a/taskcluster/ci/toolchain/macosx.yml
+++ b/taskcluster/ci/toolchain/macosx.yml
@@ -11,17 +11,17 @@ job-defaults:
macosx64-clang:
description: "Clang toolchain build"
treeherder:
kind: build
platform: toolchains/opt
symbol: TM(clang)
tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux-large
worker:
max-run-time: 36000
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
run:
using: toolchain-script
script: build-clang-macosx.sh
tooltool-downloads: internal
@@ -40,17 +40,17 @@ macosx64-clang-tidy:
index:
product: static-analysis
job-name: macosx64-clang-tidy
treeherder:
kind: build
platform: toolchains/opt
symbol: TM(clang-tidy)
tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux-large
worker:
max-run-time: 36000
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
run:
using: toolchain-script
script: build-clang-tidy-macosx.sh
tooltool-downloads: internal
@@ -67,17 +67,17 @@ macosx64-clang-tidy:
macosx64-cctools-port:
description: "cctools-port toolchain build"
treeherder:
kind: build
platform: toolchains/opt
symbol: TM(cctools)
tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
worker:
max-run-time: 36000
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
run:
using: toolchain-script
script: build-cctools-port-macosx.sh
tooltool-downloads: internal
@@ -90,17 +90,17 @@ macosx64-cctools-port:
macosx64-gn:
description: "gn toolchain build"
treeherder:
kind: build
platform: toolchains/opt
symbol: TM(gn)
tier: 1
- worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+ worker-type: aws-provisioner-v1/gecko-{level}-b-linux
worker:
max-run-time: 36000
env:
TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-clang.manifest"
run:
using: toolchain-script
script: build-gn-macosx.sh
tooltool-downloads: internal--- a/taskcluster/taskgraph/util/workertypes.py
+++ b/taskcluster/taskgraph/util/workertypes.py
@@ -2,26 +2,32 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
WORKER_TYPES = {
'aws-provisioner-v1/gecko-1-b-android': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-1-b-linux': ('docker-worker', 'linux'),
+ 'aws-provisioner-v1/gecko-1-b-linux-large': ('docker-worker', 'linux'),
+ 'aws-provisioner-v1/gecko-1-b-linux-xlarge': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-1-b-macosx64': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-1-b-win2012': ('generic-worker', 'windows'),
'aws-provisioner-v1/gecko-1-images': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-2-b-android': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-2-b-linux': ('docker-worker', 'linux'),
+ 'aws-provisioner-v1/gecko-2-b-linux-large': ('docker-worker', 'linux'),
+ 'aws-provisioner-v1/gecko-2-b-linux-xlarge': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-2-b-macosx64': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-2-b-win2012': ('generic-worker', 'windows'),
'aws-provisioner-v1/gecko-2-images': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-3-b-android': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-3-b-linux': ('docker-worker', 'linux'),
+ 'aws-provisioner-v1/gecko-3-b-linux-large': ('docker-worker', 'linux'),
+ 'aws-provisioner-v1/gecko-3-b-linux-xlarge': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-3-b-macosx64': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-3-b-win2012': ('generic-worker', 'windows'),
'aws-provisioner-v1/gecko-3-images': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-symbol-upload': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-t-linux-large': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-t-linux-medium': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-t-linux-xlarge': ('docker-worker', 'linux'),
'aws-provisioner-v1/gecko-t-win10-64': ('generic-worker', 'windows'),