Bug 1441353 - Add addon_scriptworker instances r=Callek
authorJohan Lorenzo <jlorenzo@mozilla.com>
Wed, 04 Apr 2018 11:52:16 +0200
changeset 6574 a6db788c22109b564fcb0f5c9c897a2177c88210
parent 6573 7c6dd4898e99fac911965dbd8227a822613e04eb
child 6575 18ae4183e75da1aff43cb56fb891929c3bb35dc1
child 6611 29d4bdb13bb2c491614c5a5c5d41f4ba82b7d952
child 6612 b213b2930f3533d2362485aac06ac263b1cec444
push id2746
push userbmo:jlorenzo@mozilla.com
push dateWed, 11 Apr 2018 07:34:57 +0000
reviewersCallek
bugs1441353
Bug 1441353 - Add addon_scriptworker instances r=Callek MozReview-Commit-ID: 8Xa9QVLAzr0
manifests/moco-nodes.pp
modules/addon_scriptworker/manifests/init.pp
modules/addon_scriptworker/manifests/settings.pp
modules/toplevel/manifests/server/addonscriptworker.pp
--- a/manifests/moco-nodes.pp
+++ b/manifests/moco-nodes.pp
@@ -911,16 +911,33 @@ node /^signing-linux-dev.*\.releng\..*\.
 node /^tb-depsigning-worker.*\.srv\.releng\..*\.mozilla\.com$/ {
     $aspects                  = [ 'maximum-security' ]
     $signing_scriptworker_env = 'comm-thunderbird-dep'
     $timezone                 = 'UTC'
     $only_user_ssh            = true
     include toplevel::server::signingscriptworker
 }
 
+# Addon scriptworkers
+node /^addonworker-\d*\.srv\.releng\..*\.mozilla\.com$/ {
+    $aspects          = [ 'maximum-security' ]
+    $addon_scriptworker_env = 'prod'
+    $timezone         = 'UTC'
+    $only_user_ssh    = true
+    include toplevel::server::addonscriptworker
+}
+
+node /^addonworker-dev-\d*\.srv\.releng\..*\.mozilla\.com$/ {
+    $aspects          = [ 'maximum-security' ]
+    $addon_scriptworker_env = 'dev'
+    $timezone         = 'UTC'
+    $only_user_ssh    = true
+    include toplevel::server::addonscriptworker
+}
+
 # Balrog scriptworkers
 node /^balrogworker-\d*\.srv\.releng\..*\.mozilla\.com$/ {
     $aspects          = [ 'maximum-security' ]
     $balrogworker_env = 'prod'
     $timezone         = 'UTC'
     $only_user_ssh    = true
     include toplevel::server::balrogscriptworker
 }
@@ -1092,9 +1109,8 @@ node 't-linux64-ms-280.test.releng.mdc1.
     include toplevel::server
 }
 
 # Loaner for dividehex; bug 1445842 and 1447766
 node 'ds-test1.srv.releng.mdc2.mozilla.com' {
     $aspects = [ 'low-security' ]
     include toplevel::server
 }
-
new file mode 100644
--- /dev/null
+++ b/modules/addon_scriptworker/manifests/init.pp
@@ -0,0 +1,96 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class addon_scriptworker {
+    include addon_scriptworker::settings
+    include dirs::builds
+    include packages::mozilla::python35
+    include tweaks::swap_on_instance_storage
+    include packages::gcc
+    include packages::make
+    include tweaks::scriptworkerlogrotate
+
+    python35::virtualenv {
+        $addon_scriptworker::settings::root:
+            python3  => $packages::mozilla::python35::python3,
+            require  => Class['packages::mozilla::python35'],
+            user     => $addon_scriptworker::settings::user,
+            group    => $addon_scriptworker::settings::group,
+            mode     => 700,
+            packages => [
+                'PyYAML==3.12',
+                'addonscript==0.2.1',
+                'aiohttp==2.3.9',
+                'arrow==0.12.1',
+                'async_timeout==1.4.0',
+                'certifi==2018.1.18',
+                'chardet==3.0.4',
+                'defusedxml==0.5.0',
+                'dictdiffer==0.7.0',
+                'ecdsa==0.13',
+                'frozendict==1.2',
+                'future==0.16.0',
+                'idna==2.6',
+                'json-e==2.5.0',
+                'jsonschema==2.6.0',
+                'mohawk==0.3.4',
+                'multidict==4.0.0',
+                'pexpect==4.3.1',
+                'ptyprocess==0.5.2',
+                'pycryptodome==3.5.1',
+                'python-dateutil==2.6.1',
+                'python-gnupg==0.4.1',
+                'python-jose==2.0.2',
+                'redo==1.6',
+                'requests==2.18.4',
+                'scriptworker==10.2.0',
+                'six==1.10.0',
+                'slugid==1.0.7',
+                'taskcluster==2.1.3',
+                'urllib3==1.22',
+                'virtualenv==15.1.0',
+                'yarl==1.0.0',
+            ];
+    }
+
+    scriptworker::instance {
+        $addon_scriptworker::settings::root:
+            instance_name            => $module_name,
+            basedir                  => $addon_scriptworker::settings::root,
+            work_dir                 => $addon_scriptworker::settings::work_dir,
+
+            task_script              => $addon_scriptworker::settings::task_script,
+
+            username                 => $addon_scriptworker::settings::user,
+            group                    => $addon_scriptworker::settings::group,
+
+            taskcluster_client_id    => $addon_scriptworker::settings::taskcluster_client_id,
+            taskcluster_access_token => $addon_scriptworker::settings::taskcluster_access_token,
+            worker_group             => $addon_scriptworker::settings::worker_group,
+            worker_type              => $addon_scriptworker::settings::worker_type,
+
+            cot_job_type             => 'shipit',
+
+            sign_chain_of_trust      => $addon_scriptworker::settings::sign_chain_of_trust,
+            verify_chain_of_trust    => $addon_scriptworker::settings::verify_chain_of_trust,
+            verify_cot_signature     => $addon_scriptworker::settings::verify_cot_signature,
+
+            verbose_logging          => $addon_scriptworker::settings::verbose_logging,
+    }
+
+    File {
+        ensure      => present,
+        mode        => '0600',
+        owner       => $addon_scriptworker::settings::user,
+        group       => $addon_scriptworker::settings::group,
+        show_diff   => false,
+    }
+
+    $config_content = $addon_scriptworker::settings::script_config_content
+    file {
+        $addon_scriptworker::settings::script_config:
+            require => Python35::Virtualenv[$addon_scriptworker::settings::root],
+            content => inline_template("<%- require 'json' -%><%= JSON.pretty_generate(@config_content) %>");
+    }
+}
new file mode 100644
--- /dev/null
+++ b/modules/addon_scriptworker/manifests/settings.pp
@@ -0,0 +1,83 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class addon_scriptworker::settings {
+    include ::config
+    include users::builder
+
+    $root                     = $config::scriptworker_root
+
+    $amo_stage_instance_scope = 'project:releng:addons.mozilla.org:server:staging'
+    $amo_stage_instance_config = {
+        amo_server                => 'https://addons.allizom.org',
+        jwt_user                  => 'user:11686445:783',
+        jwt_secret                => secret('addon_scriptworker_amo_password_staging'),
+    }
+
+    $_env_configs             = {
+      'dev'  => {
+        worker_group             => 'addon-dev',
+        worker_type              => 'addon-dev',
+        verbose_logging          => true,
+        taskcluster_client_id    => 'project/releng/scriptworker/addon/dev',
+        taskcluster_access_token => secret('addon_scriptworker_taskcluster_access_token_dev'),
+
+        sign_chain_of_trust      => false,
+        verify_chain_of_trust    => true,
+        verify_cot_signature     => false,
+
+
+        amo_instances        => {
+            "${amo_stage_instance_scope}" => $amo_stage_instance_config,
+        },
+      },
+      'prod' => {
+        worker_group             => 'addon-v1',
+        worker_type              => 'addon-v1',
+        verbose_logging          => true,
+        taskcluster_client_id    => 'project/releng/scriptworker/addon/production',
+        taskcluster_access_token => secret('addon_scriptworker_taskcluster_access_token_prod'),
+
+        sign_chain_of_trust      => true,
+        verify_chain_of_trust    => true,
+        verify_cot_signature     => true,
+
+        amo_instances        => {
+            "${amo_stage_instance_scope}"   => $amo_stage_instance_config,
+            'project:releng:addons.mozilla.org:server:production' => {
+                amo_server     => 'https://addons.mozilla.org',
+                jwt_user       => 'user:13856839:824',
+                jwt_secret     => secret('addon_scriptworker_amo_password_prod'),
+            },
+        },
+      },
+    }
+
+    $_env_config                = $_env_configs[$addon_scriptworker_env]
+    $work_dir                   = "${root}/work"
+    $artifact_dir               = "${root}/artifacts"
+    $task_script                = "${root}/bin/addonscript"
+
+    $user                       = $users::builder::username
+    $group                      = $users::builder::group
+
+    $taskcluster_client_id      = $_env_config['taskcluster_client_id']
+    $taskcluster_access_token   = $_env_config['taskcluster_access_token']
+    $worker_group               = $_env_config['worker_group']
+    $worker_type                = $_env_config['worker_type']
+
+    $sign_chain_of_trust        = $_env_config['sign_chain_of_trust']
+    $verify_chain_of_trust      = $_env_config['verify_chain_of_trust']
+    $verify_cot_signature       = $_env_config['verify_cot_signature']
+
+    $verbose_logging            = $_env_config['verbose_logging']
+
+    $script_config              = "${root}/script_config.json"
+    $script_config_content      = {
+        work_dir           => $work_dir,
+        artifact_dir       => $artifact_dir,
+        verbose            => $verbose_logging,
+        amo_instances      => $_env_config['amo_instances'],
+    }
+}
new file mode 100644
--- /dev/null
+++ b/modules/toplevel/manifests/server/addonscriptworker.pp
@@ -0,0 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class toplevel::server::addonscriptworker inherits toplevel::server {
+    include ::addon_scriptworker
+}