Bug 1433459 - Add bouncer_scriptworker instances r=mtabara draft
authorJohan Lorenzo <jlorenzo@mozilla.com>
Thu, 22 Feb 2018 10:15:28 +0000
changeset 6428 403d619480905c4574bdba96cf790511787bab87
parent 6423 a5c22656b3c8d1664c51ec6f663f71a32fb176c3
push id2722
push userbmo:jlorenzo@mozilla.com
push dateTue, 27 Feb 2018 14:36:23 +0000
reviewersmtabara
bugs1433459
Bug 1433459 - Add bouncer_scriptworker instances r=mtabara MozReview-Commit-ID: 4qPk6xjSrix
manifests/moco-nodes.pp
modules/bouncer_scriptworker/manifests/init.pp
modules/bouncer_scriptworker/manifests/settings.pp
modules/toplevel/manifests/server/bouncerscriptworker.pp
--- a/manifests/moco-nodes.pp
+++ b/manifests/moco-nodes.pp
@@ -954,16 +954,33 @@ node /^beetmoverworker-.*\.srv\.releng\.
 node /^beetmover-dev.*\.srv\.releng\..*\.mozilla\.com$/ {
     $aspects             = [ 'maximum-security' ]
     $beetmoverworker_env = 'dev'
     $timezone            = 'UTC'
     $only_user_ssh       = true
     include toplevel::server::beetmoverscriptworker
 }
 
+# Bouncer scriptworkers
+node /^bouncerworker-dev.*\.srv\.releng\..*\.mozilla\.com$/ {
+    $aspects             = [ 'maximum-security' ]
+    $bouncer_scriptworker_env = 'dev'
+    $timezone            = 'UTC'
+    $only_user_ssh       = true
+    include toplevel::server::bouncerscriptworker
+}
+
+node /^bouncerworker-.*\.srv\.releng\..*\.mozilla\.com$/ {
+    $aspects             = [ 'maximum-security' ]
+    $bouncer_scriptworker_env = 'prod'
+    $timezone            = 'UTC'
+    $only_user_ssh       = true
+    include toplevel::server::bouncerscriptworker
+}
+
 # Pushapk scriptworkers
 node /^dep-pushapkworker-.*\.srv\.releng\..*\.mozilla\.com$/ {
     $aspects                  = [ 'maximum-security' ]
     $pushapk_scriptworker_env = 'dep'
     $timezone                 = 'UTC'
     $only_user_ssh            = true
     include toplevel::server::pushapkscriptworker
 }
new file mode 100644
--- /dev/null
+++ b/modules/bouncer_scriptworker/manifests/init.pp
@@ -0,0 +1,97 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class bouncer_scriptworker {
+    include bouncer_scriptworker::settings
+    include dirs::builds
+    include packages::mozilla::python35
+    include users::builder
+    include tweaks::swap_on_instance_storage
+    include packages::gcc
+    include packages::make
+    include packages::libffi
+    include tweaks::scriptworkerlogrotate
+    include tweaks::scriptworkerlogrotate
+
+    python35::virtualenv {
+        $bouncer_scriptworker::settings::root:
+            python3  => $packages::mozilla::python35::python3,
+            require  => Class['packages::mozilla::python35'],
+            user     => $users::builder::username,
+            group    => $users::builder::group,
+            mode     => '0700',
+            packages => [
+                'PyYAML==3.12',
+                'aiohttp==2.3.9',
+                'arrow==0.12.1',
+                'async_timeout==1.4.0',
+                'bouncerscript==0.1.0',
+                'certifi==2018.1.18',
+                'chardet==3.0.4',
+                'defusedxml==0.5.0',
+                'dictdiffer==0.7.0',
+                'frozendict==1.2',
+                'idna==2.6',
+                'json-e==2.5.0',
+                'jsonschema==2.6.0',
+                'mohawk==0.3.4',
+                'multidict==4.0.0',
+                'pexpect==4.3.1',
+                'ptyprocess==0.5.2',
+                'python-dateutil==2.6.1',
+                'python-gnupg==0.4.1',
+                'redo==1.6',
+                'requests==2.18.4',
+                'scriptworker==8.2.0',
+                'six==1.10.0',
+                'slugid==1.0.7',
+                'taskcluster==2.1.3',
+                'urllib3==1.22',
+                'virtualenv==15.1.0',
+                'yarl==1.0.0',
+            ];
+    }
+
+    scriptworker::instance {
+        $bouncer_scriptworker::settings::root:
+            instance_name            => $module_name,
+            basedir                  => $bouncer_scriptworker::settings::root,
+
+            task_script              => $bouncer_scriptworker::settings::task_script,
+            task_script_config       => $bouncer_scriptworker::settings::task_script_config,
+
+            username                 => $users::builder::username,
+            group                    => $users::builder::group,
+
+            taskcluster_client_id    => $bouncer_scriptworker::settings::taskcluster_client_id,
+            taskcluster_access_token => $bouncer_scriptworker::settings::taskcluster_access_token,
+            worker_group             => $bouncer_scriptworker::settings::worker_group,
+            worker_type              => $bouncer_scriptworker::settings::worker_type,
+
+            task_max_timeout         => $bouncer_scriptworker::settings::task_max_timeout,
+
+            cot_job_type             => 'bouncer',
+
+            sign_chain_of_trust      => $bouncer_scriptworker::settings::sign_chain_of_trust,
+            verify_chain_of_trust    => $bouncer_scriptworker::settings::verify_chain_of_trust,
+            verify_cot_signature     => $bouncer_scriptworker::settings::verify_cot_signature,
+
+            verbose_logging          => $bouncer_scriptworker::settings::verbose_logging,
+    }
+
+    File {
+        ensure      => present,
+        mode        => '0600',
+        owner       => $bouncer_scriptworker::settings::user,
+        group       => $bouncer_scriptworker::settings::group,
+        show_diff   => false,
+    }
+
+    $config_content = $bouncer_scriptworker::settings::script_config_content
+    file {
+        $bouncer_scriptworker::settings::script_config:
+            require => Python35::Virtualenv[$bouncer_scriptworker::settings::root],
+            content => inline_template("<%- require 'json' -%><%= JSON.pretty_generate(@config_content) %>");
+    }
+}
new file mode 100644
--- /dev/null
+++ b/modules/bouncer_scriptworker/manifests/settings.pp
@@ -0,0 +1,88 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class bouncer_scriptworker::settings {
+    include ::config
+    include users::builder
+
+    $root                     = $config::scriptworker_root
+
+    $bouncer_stage_instance_scope = 'project:releng:bouncer:server:staging'
+    $bouncer_stage_instance_config = {
+        api_root                  => 'https://admin-bouncer-releng.stage.mozaws.net/',
+        timeout_in_seconds        => 60,
+        username                  => 'ffxbld',
+        # TODO Split credentials
+        password                  => secret('tuxedo_password'),
+    }
+
+    $_env_configs             = {
+      'dev'  => {
+        worker_group             => 'bouncer-dev',
+        worker_type              => 'bouncer-dev',
+        verbose_logging          => true,
+        taskcluster_client_id    => 'project/releng/scriptworker/bouncer/dev',
+        taskcluster_access_token => secret('bouncer_scriptworker_taskcluster_access_token_dev'),
+
+        sign_chain_of_trust      => false,
+        verify_chain_of_trust    => true,
+        verify_cot_signature     => false,
+
+        bouncer_instances        => {
+            "${bouncer_stage_instance_scope}" => $bouncer_stage_instance_config,
+        },
+      },
+      'prod' => {
+        worker_group             => 'bouncer-v1',
+        worker_type              => 'bouncer-v1',
+        verbose_logging          => true,
+        taskcluster_client_id    => 'project/releng/scriptworker/bouncer/production',
+        taskcluster_access_token => secret('bouncer_scriptworker_taskcluster_access_token_prod'),
+
+        sign_chain_of_trust      => true,
+        verify_chain_of_trust    => true,
+        verify_cot_signature     => true,
+
+        bouncer_instances        => {
+            "${bouncer_stage_instance_scope}"   => $bouncer_stage_instance_config,
+            'project:releng:bouncer:server:production' => {
+                api_root                  => 'https://bounceradmin.mozilla.com/',
+                timeout_in_seconds        => 60,
+                username                  => 'ffxbld',
+                # TODO Split credentials
+                password                  => secret('tuxedo_password'),
+            },
+        },
+      },
+    }
+
+    $_env_config                = $_env_configs[$bouncer_scriptworker_env]
+    $work_dir                   = "${root}/work"
+    $task_script                = "${root}/bin/bouncerscript"
+
+    $user                       = $users::builder::username
+    $group                      = $users::builder::group
+
+    $taskcluster_client_id      = $_env_config['taskcluster_client_id']
+    $taskcluster_access_token   = $_env_config['taskcluster_access_token']
+    $worker_group               = $_env_config['worker_group']
+    $worker_type                = $_env_config['worker_type']
+
+    $sign_chain_of_trust        = $_env_config['sign_chain_of_trust']
+    $verify_chain_of_trust      = $_env_config['verify_chain_of_trust']
+    $verify_cot_signature       = $_env_config['verify_cot_signature']
+
+    $verbose_logging            = $_env_config['verbose_logging']
+
+    $script_config              = "${root}/script_config.json"
+    $script_config_content      = {
+        work_dir           => $work_dir,
+        schema_files       => {
+            submission => "${root}/lib/python3.5/site-packages/bouncerscript/data/bouncer_submission_task_schema.json",
+            aliases    => "${root}/lib/python3.5/site-packages/bouncerscript/data/bouncer_aliases_task_schema.json",
+        },
+        verbose            => $verbose_logging,
+        bouncer_config     => $_env_config['bouncer_instances'],
+    }
+}
new file mode 100644
--- /dev/null
+++ b/modules/toplevel/manifests/server/bouncerscriptworker.pp
@@ -0,0 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class toplevel::server::bouncerscriptworker inherits toplevel::server {
+    include ::bouncer_scriptworker
+}