Bug 1436499 - Spin up a dev treescriptworker. r=aki
MozReview-Commit-ID: 8RHK0M68goW
--- a/manifests/moco-nodes.pp
+++ b/manifests/moco-nodes.pp
@@ -1004,17 +1004,17 @@ node /^shipitworker-.*\.srv\.releng\..*\
# Treescript workers
node /^treescriptworker-dev\d*\.srv\.releng\..*\.mozilla\.com$/ {
$aspects = [ 'maximum-security' ]
$treescriptworker_env = 'dev'
$timezone = 'UTC'
$only_user_ssh = true
- include toplevel::server
+ include toplevel::server::treescriptworker
}
## Loaners
node 'dhouse-1330169.srv.releng.scl3.mozilla.com' {
$aspects = [ 'low-security' ]
include toplevel::server
--- a/modules/packages/manifests/mozilla/python27.pp
+++ b/modules/packages/manifests/mozilla/python27.pp
@@ -41,17 +41,18 @@ class packages::mozilla::python27 {
} -> Anchor['packages::mozilla::python27::end']
case $::operatingsystem {
CentOS: {
# Bug 1307757 - Deploy python 2.7.12 to releng servers
# install Python 2.7.12 for 'buildduty-tools' and 'cruncher-aws' servers in order
# to silence the alerts caused by using on older Python version (2.7.3) when
# performing HG operation.
- if $::hostname in [ 'buildduty-tools', 'cruncher-aws', 'aws-manager1', 'aws-manager2' ] {
+ if $::hostname in [ 'buildduty-tools', 'cruncher-aws', 'aws-manager1', 'aws-manager2',
+ 'treescriptworker-dev1', 'treescriptworker-1' ] {
realize(Packages::Yumrepo['python27-12'])
Anchor['packages::mozilla::python27::begin'] ->
package {
'mozilla-python27':
ensure => '2.7.12-1.el6';
} -> Anchor['packages::mozilla::python27::end']
}
else {
new file mode 100644
--- /dev/null
+++ b/modules/toplevel/manifests/server/treescriptworker.pp
@@ -0,0 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class toplevel::server::treescriptworker inherits toplevel::server {
+ include ::tree_scriptworker
+}
new file mode 100644
--- /dev/null
+++ b/modules/tree_scriptworker/manifests/init.pp
@@ -0,0 +1,96 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class tree_scriptworker {
+ include tree_scriptworker::settings
+ include dirs::builds
+ include dirs::builds::hg_shared
+ include packages::mozilla::python35
+ include packages::mozilla::python27
+ include packages::mozilla::py27_mercurial
+ include users::builder
+ include tweaks::swap_on_instance_storage
+ include packages::gcc
+ include packages::make
+ include packages::libffi
+ include tweaks::scriptworkerlogrotate
+
+ $env_config = $tree_scriptworker::settings::env_config[$treescriptworker_env]
+
+ python35::virtualenv {
+ $tree_scriptworker::settings::root:
+ python3 => $packages::mozilla::python35::python3,
+ require => Class['packages::mozilla::python35'],
+ user => $users::builder::username,
+ group => $users::builder::group,
+ mode => '0700',
+ packages => [
+ 'PyYAML==3.12',
+ 'aiohttp==2.3.9',
+ 'arrow==0.12.1',
+ 'async_timeout==1.4.0',
+ 'certifi==2018.1.18',
+ 'chardet==3.0.4',
+ 'defusedxml==0.5.0',
+ 'dictdiffer==0.7.0',
+ 'frozendict==1.2',
+ 'idna==2.6',
+ 'json-e==2.5.0',
+ 'jsonschema==2.6.0',
+ 'mohawk==0.3.4',
+ 'multidict==4.1.0',
+ 'pexpect==4.3.1',
+ 'ptyprocess==0.5.2',
+ 'python-dateutil==2.6.1',
+ 'python-gnupg==0.4.1',
+ 'requests==2.18.4',
+ 'scriptworker==8.0.0',
+ 'treescript==0.1',
+ 'six==1.10.0',
+ 'slugid==1.0.7',
+ 'taskcluster==2.1.3',
+ 'urllib3==1.22',
+ 'virtualenv==15.1.0',
+ 'yarl==1.1.0',
+ ];
+ }
+
+ scriptworker::instance {
+ $tree_scriptworker::settings::root:
+ instance_name => $module_name,
+ basedir => $tree_scriptworker::settings::root,
+
+ task_script_executable => $tree_scriptworker::settings::task_script_executable,
+ task_script => $tree_scriptworker::settings::task_script,
+ task_script_config => $tree_scriptworker::settings::task_script_config,
+
+ username => $users::builder::username,
+ group => $users::builder::group,
+
+ taskcluster_client_id => $env_config["taskcluster_client_id"],
+ taskcluster_access_token => $env_config["taskcluster_access_token"],
+ worker_group => $tree_scriptworker::settings::worker_group,
+ worker_type => $env_config["worker_type"],
+
+ task_max_timeout => $tree_scriptworker::settings::task_max_timeout,
+
+ cot_job_type => 'scriptworker',
+
+ sign_chain_of_trust => $env_config["sign_chain_of_trust"],
+ verify_chain_of_trust => $env_config["verify_chain_of_trust"],
+ verify_cot_signature => $env_config["verify_cot_signature"],
+
+ verbose_logging => $tree_scriptworker::settings::verbose_logging,
+ }
+
+ file {
+ "${tree_scriptworker::settings::root}/script_config.json":
+ require => Python35::Virtualenv[$tree_scriptworker::settings::root],
+ mode => '0600',
+ owner => $users::builder::username,
+ group => $users::builder::group,
+ content => template("${module_name}/script_config.json.erb"),
+ show_diff => false;
+ }
+}
new file mode 100644
--- /dev/null
+++ b/modules/tree_scriptworker/manifests/settings.pp
@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class tree_scriptworker::settings {
+ $root = '/builds/scriptworker'
+ $task_script = "${root}/bin/treescript"
+ $task_script_config = "${root}/script_config.json"
+ $task_max_timeout = 1800
+ $upstream_repo = 'https://hg.mozilla.org/mozilla-unified'
+ $share_base = '/builds/hg-shared'
+ $worker_group = 'treescriptworker-v1'
+ $verbose_logging = true
+
+ $env_config = {
+ 'dev' => {
+ taskcluster_client_id => 'project/releng/scriptworker/treescriptworker-dev',
+ taskcluster_access_token => secret('treescriptworker_dev_taskcluster_access_token'),
+ worker_type => 'treescript-dev',
+ sign_chain_of_trust => false,
+ verify_chain_of_trust => false,
+ verify_cot_signature => false,
+ },
+ 'prod' => {
+ taskcluster_client_id => 'project/releng/scriptworker/treescriptworker',
+ # No prod secret yet, this line would break puppet if not commented out
+ # taskcluster_access_token => secret('treescriptworker_prod_taskcluster_access_token'),
+ worker_type => 'balrogworker-v1',
+ sign_chain_of_trust => true,
+ verify_chain_of_trust => true,
+ verify_cot_signature => true,
+ }
+ }
+}
new file mode 100644
--- /dev/null
+++ b/modules/tree_scriptworker/templates/script_config.json.erb
@@ -0,0 +1,8 @@
+{
+ "work_dir": "<%= scope.lookupvar("tree_scriptworker::settings::root") %>/work",
+ "artifact_dir": "<%= scope.lookupvar("tree_scriptworker::settings::root") %>/artifacts",
+
+ "hg": "<%= scope.lookupvar("packages::mozilla::py27_mercurial::mercurial") %>",
+ "hg_share_base_dir": "<%= scope.lookupvar("tree_scriptworker::settings::share_base") %>",
+ "upstream_repo": "<%= scope.lookupvar("tree_scriptworker::settings::upstream_repo") %>"
+}