Bug 1441353 - Add addon_scriptworker instances r=Callek
MozReview-Commit-ID: 8Xa9QVLAzr0
--- a/manifests/moco-nodes.pp
+++ b/manifests/moco-nodes.pp
@@ -911,16 +911,33 @@ node /^signing-linux-dev.*\.releng\..*\.
node /^tb-depsigning-worker.*\.srv\.releng\..*\.mozilla\.com$/ {
$aspects = [ 'maximum-security' ]
$signing_scriptworker_env = 'comm-thunderbird-dep'
$timezone = 'UTC'
$only_user_ssh = true
include toplevel::server::signingscriptworker
}
+# Addon scriptworkers
+node /^addonworker-\d*\.srv\.releng\..*\.mozilla\.com$/ {
+ $aspects = [ 'maximum-security' ]
+ $addon_scriptworker_env = 'prod'
+ $timezone = 'UTC'
+ $only_user_ssh = true
+ include toplevel::server::addonscriptworker
+}
+
+node /^addonworker-dev-\d*\.srv\.releng\..*\.mozilla\.com$/ {
+ $aspects = [ 'maximum-security' ]
+ $addon_scriptworker_env = 'dev'
+ $timezone = 'UTC'
+ $only_user_ssh = true
+ include toplevel::server::addonscriptworker
+}
+
# Balrog scriptworkers
node /^balrogworker-\d*\.srv\.releng\..*\.mozilla\.com$/ {
$aspects = [ 'maximum-security' ]
$balrogworker_env = 'prod'
$timezone = 'UTC'
$only_user_ssh = true
include toplevel::server::balrogscriptworker
}
@@ -1092,9 +1109,8 @@ node 't-linux64-ms-280.test.releng.mdc1.
include toplevel::server
}
# Loaner for dividehex; bug 1445842 and 1447766
node 'ds-test1.srv.releng.mdc2.mozilla.com' {
$aspects = [ 'low-security' ]
include toplevel::server
}
-
new file mode 100644
--- /dev/null
+++ b/modules/addon_scriptworker/manifests/init.pp
@@ -0,0 +1,96 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class addon_scriptworker {
+ include addon_scriptworker::settings
+ include dirs::builds
+ include packages::mozilla::python35
+ include tweaks::swap_on_instance_storage
+ include packages::gcc
+ include packages::make
+ include tweaks::scriptworkerlogrotate
+
+ python35::virtualenv {
+ $addon_scriptworker::settings::root:
+ python3 => $packages::mozilla::python35::python3,
+ require => Class['packages::mozilla::python35'],
+ user => $addon_scriptworker::settings::user,
+ group => $addon_scriptworker::settings::group,
+ mode => 700,
+ packages => [
+ 'PyYAML==3.12',
+ 'addonscript==0.2.1',
+ 'aiohttp==2.3.9',
+ 'arrow==0.12.1',
+ 'async_timeout==1.4.0',
+ 'certifi==2018.1.18',
+ 'chardet==3.0.4',
+ 'defusedxml==0.5.0',
+ 'dictdiffer==0.7.0',
+ 'ecdsa==0.13',
+ 'frozendict==1.2',
+ 'future==0.16.0',
+ 'idna==2.6',
+ 'json-e==2.5.0',
+ 'jsonschema==2.6.0',
+ 'mohawk==0.3.4',
+ 'multidict==4.0.0',
+ 'pexpect==4.3.1',
+ 'ptyprocess==0.5.2',
+ 'pycryptodome==3.5.1',
+ 'python-dateutil==2.6.1',
+ 'python-gnupg==0.4.1',
+ 'python-jose==2.0.2',
+ 'redo==1.6',
+ 'requests==2.18.4',
+ 'scriptworker==10.2.0',
+ 'six==1.10.0',
+ 'slugid==1.0.7',
+ 'taskcluster==2.1.3',
+ 'urllib3==1.22',
+ 'virtualenv==15.1.0',
+ 'yarl==1.0.0',
+ ];
+ }
+
+ scriptworker::instance {
+ $addon_scriptworker::settings::root:
+ instance_name => $module_name,
+ basedir => $addon_scriptworker::settings::root,
+ work_dir => $addon_scriptworker::settings::work_dir,
+
+ task_script => $addon_scriptworker::settings::task_script,
+
+ username => $addon_scriptworker::settings::user,
+ group => $addon_scriptworker::settings::group,
+
+ taskcluster_client_id => $addon_scriptworker::settings::taskcluster_client_id,
+ taskcluster_access_token => $addon_scriptworker::settings::taskcluster_access_token,
+ worker_group => $addon_scriptworker::settings::worker_group,
+ worker_type => $addon_scriptworker::settings::worker_type,
+
+ cot_job_type => 'shipit',
+
+ sign_chain_of_trust => $addon_scriptworker::settings::sign_chain_of_trust,
+ verify_chain_of_trust => $addon_scriptworker::settings::verify_chain_of_trust,
+ verify_cot_signature => $addon_scriptworker::settings::verify_cot_signature,
+
+ verbose_logging => $addon_scriptworker::settings::verbose_logging,
+ }
+
+ File {
+ ensure => present,
+ mode => '0600',
+ owner => $addon_scriptworker::settings::user,
+ group => $addon_scriptworker::settings::group,
+ show_diff => false,
+ }
+
+ $config_content = $addon_scriptworker::settings::script_config_content
+ file {
+ $addon_scriptworker::settings::script_config:
+ require => Python35::Virtualenv[$addon_scriptworker::settings::root],
+ content => inline_template("<%- require 'json' -%><%= JSON.pretty_generate(@config_content) %>");
+ }
+}
new file mode 100644
--- /dev/null
+++ b/modules/addon_scriptworker/manifests/settings.pp
@@ -0,0 +1,83 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class addon_scriptworker::settings {
+ include ::config
+ include users::builder
+
+ $root = $config::scriptworker_root
+
+ $amo_stage_instance_scope = 'project:releng:addons.mozilla.org:server:staging'
+ $amo_stage_instance_config = {
+ amo_server => 'https://addons.allizom.org',
+ jwt_user => 'user:11686445:783',
+ jwt_secret => secret('addon_scriptworker_amo_password_staging'),
+ }
+
+ $_env_configs = {
+ 'dev' => {
+ worker_group => 'addon-dev',
+ worker_type => 'addon-dev',
+ verbose_logging => true,
+ taskcluster_client_id => 'project/releng/scriptworker/addon/dev',
+ taskcluster_access_token => secret('addon_scriptworker_taskcluster_access_token_dev'),
+
+ sign_chain_of_trust => false,
+ verify_chain_of_trust => true,
+ verify_cot_signature => false,
+
+
+ amo_instances => {
+ "${amo_stage_instance_scope}" => $amo_stage_instance_config,
+ },
+ },
+ 'prod' => {
+ worker_group => 'addon-v1',
+ worker_type => 'addon-v1',
+ verbose_logging => true,
+ taskcluster_client_id => 'project/releng/scriptworker/addon/production',
+ taskcluster_access_token => secret('addon_scriptworker_taskcluster_access_token_prod'),
+
+ sign_chain_of_trust => true,
+ verify_chain_of_trust => true,
+ verify_cot_signature => true,
+
+ amo_instances => {
+ "${amo_stage_instance_scope}" => $amo_stage_instance_config,
+ 'project:releng:addons.mozilla.org:server:production' => {
+ amo_server => 'https://addons.mozilla.org',
+ jwt_user => 'user:13856839:824',
+ jwt_secret => secret('addon_scriptworker_amo_password_prod'),
+ },
+ },
+ },
+ }
+
+ $_env_config = $_env_configs[$addon_scriptworker_env]
+ $work_dir = "${root}/work"
+ $artifact_dir = "${root}/artifacts"
+ $task_script = "${root}/bin/addonscript"
+
+ $user = $users::builder::username
+ $group = $users::builder::group
+
+ $taskcluster_client_id = $_env_config['taskcluster_client_id']
+ $taskcluster_access_token = $_env_config['taskcluster_access_token']
+ $worker_group = $_env_config['worker_group']
+ $worker_type = $_env_config['worker_type']
+
+ $sign_chain_of_trust = $_env_config['sign_chain_of_trust']
+ $verify_chain_of_trust = $_env_config['verify_chain_of_trust']
+ $verify_cot_signature = $_env_config['verify_cot_signature']
+
+ $verbose_logging = $_env_config['verbose_logging']
+
+ $script_config = "${root}/script_config.json"
+ $script_config_content = {
+ work_dir => $work_dir,
+ artifact_dir => $artifact_dir,
+ verbose => $verbose_logging,
+ amo_instances => $_env_config['amo_instances'],
+ }
+}
new file mode 100644
--- /dev/null
+++ b/modules/toplevel/manifests/server/addonscriptworker.pp
@@ -0,0 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class toplevel::server::addonscriptworker inherits toplevel::server {
+ include ::addon_scriptworker
+}