Bug 1431764 - Add shipit_scriptworkers r=aki draft
authorJohan Lorenzo <jlorenzo@mozilla.com>
Wed, 24 Jan 2018 18:32:48 +0100
changeset 5819 4ff28d124d0c1bcbdb291359f08b9aa196294c76
parent 5812 e33efd78fdf8359a49131d4a92fb434d9d0d18a6
push id2708
push userbmo:jlorenzo@mozilla.com
push dateWed, 31 Jan 2018 13:47:15 +0000
reviewersaki
bugs1431764
Bug 1431764 - Add shipit_scriptworkers r=aki MozReview-Commit-ID: CYml5CGwp2g
manifests/moco-nodes.pp
modules/scriptworker/files/git_pubkeys/jlorenzo@mozilla.com.asc
modules/shipit_scriptworker/manifests/init.pp
modules/shipit_scriptworker/manifests/settings.pp
modules/toplevel/manifests/server/shipitscriptworker.pp
--- a/manifests/moco-nodes.pp
+++ b/manifests/moco-nodes.pp
@@ -961,16 +961,34 @@ node /pushapkworker-.*\.srv\.releng\..*\
 node /binarytransparencyworker-.*\.srv\.releng\..*\.mozilla\.com/ {
     $aspects = [ 'maximum-security' ]
     $transparencyworker_env = "dev"
     $timezone = "UTC"
     $only_user_ssh = true
     include toplevel::server::transparencyscriptworker
 }
 
+# shipit scriptworkers
+node /^shipitworker-dev-.*\.srv\.releng\..*\.mozilla\.com$/ {
+    $aspects                  = [ 'maximum-security' ]
+    $shipit_scriptworker_env  = 'dev'
+    $timezone                 = 'UTC'
+    $only_user_ssh            = true
+    include toplevel::server::shipitscriptworker
+}
+
+node /^shipitworker-.*\.srv\.releng\..*\.mozilla\.com$/ {
+    $aspects                  = [ 'maximum-security' ]
+    $shipit_scriptworker_env  = 'prod'
+    $timezone                 = 'UTC'
+    $only_user_ssh            = true
+    include toplevel::server::shipitscriptworker
+}
+
+
 ## Loaners
 
 node 'dhouse-1330169.srv.releng.scl3.mozilla.com' {
     $aspects = [ 'low-security' ]
     include toplevel::server
 }
 
 # See Bug 1343963
@@ -1015,9 +1033,8 @@ node 't-linux64-xe-264.test.releng.mdc1.
     include toplevel::worker::releng::taskcluster_worker::test::gpu
 }
 
 node 'ms1-1.test.releng.mdc1.mozilla.com' {
     $aspects          = [ 'low-security' ]
     $slave_trustlevel = 'try'
     include toplevel::base
 }
-
new file mode 100644
--- /dev/null
+++ b/modules/scriptworker/files/git_pubkeys/jlorenzo@mozilla.com.asc
@@ -0,0 +1,96 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFpxp+QBEACqnYQN+XwCB0PeRLKdYMtnENTUKfx2i69Vp86KALNuZFFiJVOX
+84mt869ZDsZeNpXRRmW0CoQjx7k3LTIruDKiXcNyCd7rxX5B4k059dwIEkY9OWNX
+/1KQNZt9vbUoQjiUtzhqDhIS+NZau/RydQxtsFVCyKMereXwxWBX2wwMUpGIN0bq
+6BMTz/D++KO5JsLzsgYbn57SGX6jBCLnmNAJfN8AIUZC6iGWTtXgEG382UmI+RT/
+NplUGSYJJIF4QxBoYEyapp7BFihdHNjP+4SoeCrjB0LG7KCP9k2RNEYVj38a4Oea
+Vq3CYtMjFz+iuKvuxv+ehp4NFAVw9nzhfwEVNWo1GWaNmO4UcMp8yc793FpvFKV3
+B9JJR8m6emzv3DW8AIWhcyGFwFoj3y8VbSv/UZz1ETeITuGKwD6ANwopV+bKVYp5
+KMp8D2V/x7WybwZzxXJVzVVa5kexIUlK3AFIayDATUaFv+jZoaaW+kMW4Gl8JZHq
+gZOpsuej40e+zNcbjzBp4vN+I5u9J6KcXEZz9qipJofpzWj9Jlz9p5JiD3mXlqFM
+ojDlXG9oNm00hC3MqoCOXTU9W0Oowg3QCJQodU/Mhk/lsxTZ5aFo+UrniaDsv9j8
+SEgKdP+MMT6bJulL1F9oCcGl88LUIib4t7mV1MVQY4nQWJzhWDZM1Rd8PwARAQAB
+tC9Kb2hhbiBMb3JlbnpvICgyMDE4IGtleSkgPGpsb3JlbnpvQG1vemlsbGEuY29t
+PokCPQQTAQgAJwUCWnGn5AIbAQUJA8JnAAULCQgHAgYVCAkKCwIEFgIDAQIeAQIX
+gAAKCRBkXVVgLUhR+RlaEACWuAKzpL2XC2deWKdDiKSdb42P3dr+EjpnYIIK4tz3
+FCiko41fzvbHBTIo2TPpC5wiVvXI+ClMlA8Ah9nbUiZ/Rsyo4hBH/vBGzue6Gwmw
+DTQSq5hDID4NfKLitMz8MX3lI/KD5Uh0ggO3p4vHowdQOCDMPVOZgCACzBp21kXI
+xGCs3ra7wJjLecfmxAhlsvEEK9xl36f4EUU3Y6nVI7tPbK8j+gKUDgCXe8bYN5qQ
+n7BNVej36SFIdRr3D1B/IwradvWmBH2VfQXzQvDza/3fMv6rtwxC1l8QfvLbKMai
+GV12T76b4Nr6p1sY4snz3BSlbNe/4inS847IkH+Oin3BNuR5PYQ9oU+GkRs4T0QD
+Cdlo4fjkL+X5+EC0lU1OMC2LXqDG3teQPSMunfbZhN53BLtuxmM60+HECxTWSEfs
+dOJrRJBLk/DsBNuJqVgzC8iQasxpRbne4/DSnXtIsgRMjm8AlIQLJ4WodRYZRlX3
+MBdsbcM/8NiBOA5jxsmQXnbYlRSFM4zz6n/oF/pYLUj1J54UUW9/4CduBNIA7R/N
+VEtAHwM4tXp2e/TKzt86GEldrft5ROk3DPZhXC2o3Pi53p4hmcm+ZfQUlLjoH6v0
+P5jebr5W1uy270oxCTlblZnoS7Mm1FKmDAF5faC20JXLc5qMtm+38mrMEcNYDATp
+HYkCHAQQAQoABgUCWnGtjgAKCRDVREAOtiM0Ik9kD/4jH95KEnonoKqJsGZZEcua
+9NWNppNTkmGnxOs7OjDDAPIG1/jtuht02NE7NU+KEUIQJhlcDVoLnZQnNn8FAwbD
+cRc8SaGqFBkaInug+5KyIC5VQIQ4XHCuUj7w1JgVQKf2HqiHa9/tqFgXKQLO67PQ
+VbyDvkTmNEGOJ8Jr9bI53X/DIGt9d1G8YC5G7DSDjCJM5SGkB6UBxDnVMDs8FtcA
+aa+cLC28n1kdkP1ORHzMX6uKK6g+asF6jkRk9GKeH+H01EgvTuMJaLEmZSLLBDds
+lRX7jshLiuwyhW0kgqfhhxbKDr0uNShNyj3QBB9N0uhQYotGkQhF996hUm0q3Iv+
+VS4tbhkT7fxGeJjEDBzljDpgy239oLxfX8t1O2D/ySlD5IDIZc/1vCB3vGT/XS4u
+hW5Ke5h2JzAq8EXjphDHoJGElqsPtsQZaR4JdK1Huod2PElnN2DIBwV7/7gbaCsZ
+6/a55tCkZdPB7Et12syfRMU6WuKPV+KT+LY7of8LlBLVC62yOJYNYev6iQKFYMxJ
+W61ymrI6ZutXHZRMhOVW0gxOd9ZO1QQ5Fp69cSwTlmLZVqoU3xEn3ymSqvsF4Mee
+oldYfsJHJBvDcWJzU+b3nw9w3frSf8xZXoyNmg+IYmRhTOjqSOHzb6PCdVuCYAAY
+x/JtHRWIZJSCeqSvIoGA7LkCDQRacauDARAAkrZMEAvzopHkUouCJwzTFV+9cHA8
+yNS9/M7q+y54YlVc5Y2uh4h/IuNpgioSe5RQJo+SDknHqgd41AvwF+GTaz7ApMuk
+j/Y2RHfgZ5Hdp0t2qhBJJk/vyc8p776/vnLnT5HdIH46E5eHi9gTfFpgkZVuzi6E
+bVorSqUnhNRX/0beQUvuldrgTacbtq8o9dJfLFBAT5OK01e3VEFZm3+D7WWc8vG3
+Zdb46ppAbd8WRsatWEAPIT3y7vaKZwxAqqfhsz9DqAmLJ5JZDBHa4hFuYynCg6xO
+08kchqqqywAKy5S3nemBr7da9L2wZyMSqxT9a+pfiDJy5M76yS0RuBUckOk4NJ2B
+DLeYFZilug2cGRh9q84dpXVWgbaPJnJRHuZ7ennWWlbYavyUFkJLfWpsIPK/f2Rq
+8bZwj+CMhbYou9wL67FhDRlsO5ccbAfaqSKZxFJI9bZo8KBIRsPKkCIzJ+DKY7fg
+gdjNmlX6MyWDCUCe/uiXVR7P2hF1ErUNN+cDo1MGQcrtXlobvvw2FFVM+o4TqCaz
+gDIihq8evoqxVziUBmWgTkjOorhESWvOnD1+gGUiHJ+6kp4A8wJORVJfnBYjANj6
+iBoP4y01yoixIMFE06CpTi2nkF5ohDEwj1+vwN2LIzqfoB4SILBTj6vPw60AI17y
+6u0rHkjy5aQdQyMAEQEAAYkERAQYAQgADwUCWnGrgwIbAgUJA8JnAAIpCRBkXVVg
+LUhR+cFdIAQZAQgABgUCWnGrgwAKCRACXe1IhvSglkIrEACF07S1GhuM5dSH6E+O
+Rihwi8czfOnAK96OAj4lMrGNoJdBDdZM3RhlRsTSQv64kdtYFdhoih0HJ0mkycfl
+iaexzcmRK/erOjD8UyN1MZqPLjVNt+ENJ5aUbkfqBWOrpxiKvqbPe9oZHmajSrQz
+9SFleARGQYRWR4jqZjHbNiP+4CpYypZcEoZmqMM7tG5RsnAz/ePRdQHKZTK9uacC
+PvyuMD6uJsnsRCrMyiJYvMlITUW+547sivJdi7zdA8ZHfamgl5NdXRjGdQGBPKXd
+7tUiIRwSYnLWOnRrNiD351e/sswlU1mKtjwlKdJL0kuDL1mSddFE953yP2IjOOYU
+p1CfCpRGeUaOl0KImMDinNwBvz0BEx62pvJyqvwt7QFiw8qlCdFuHakmMe30lsVK
+eXtVrZOvsg6W3RvjmWIs1Xf5Kq/LuQxvhp2NiE5OgAPEyEPqxQJzfaCco01r2O15
+ZGV9876as278Dttvsghfw9JCKSY08VnNkKCQLHv1a0rePqbrc6Cdsq/5NMc/8HpM
+9ojmQ/XobJ19g3DkQzqULc/WNo2aLzCJ2wvGGWB5U8yx+dEADC53igmhgKadLVsp
+Sfj6Arvs458qiP3GYaT1dXTcSQYhQpzQlNGt6AV3kI9ppmMssn7Le3EkhbgXiwgf
+Wf64EOfWjkwgTObqM85us2A3hoJ/D/9G8eBVs9TuldDsl3soHG6lQHImwQlw960j
+1RDeA2k9znGjWZKq+iQlKt7b3YeQx05Q02d8H5hgua6smGW8eQxlDXBwtMbJWXnH
+5PB7G/wan65Ve3nU3L1HqReb1G3PUxIkPsHWPvLwgYah/chZFm1B9z/52BtyMqxF
+kNSYW4LDOnLiFIOOsGGRm++ikSOR5ezpXWRoGRXQSlakBClT7qzmX8I3B1EpKq5u
+jOTHGydNru7LEtcSu+XeJAFGwUfw9OMOWqUwyRfEZFLUaq3LYxFZlu8XWaXu5K4D
+w6tQCYv4w1DzjdCnCYHUAa3QiOVSTSIMx37HDoBOn93XMoBwYZ7tJ4dtrVujnpvr
+fZePp2/wB/q53G9w72SIp7iq20anWAesQNMFuxYO8fOqvdrRv5zQP39I6PTJz4no
+JBa85+tANFWoqqE05Onxk0PrUXHZdRG4Jox2NuRCgNFzeEcYfIxufABREmNA+kdf
+uzDT7jResJl4iCq6tyWIjwlEHX332ffnFa6j1N7iBm6RE/XkaWpsL8xIU8q9sn6N
+zeXV4+gO9oiKtUDqKp6pT5Q8j5wAmDaFzatZXAh6Vf70XQxOrCINJ0F7ls8qtBf5
+cThUz+rS8+q35CurTnB9DBZK5MOhSu2VGXpVckvzwVId75nVCHzlNTFzfTWrfDKk
+OJ8yKwHmlrkCDQRacawYARAA5ts3F8Xjq5qyLLqBtAMUrU29rtNgfx/Zb+1p6P3H
+zC8RYr8PuYr/UMNukCjl866nb4ZgOtu8ijWm7jDhpjdWwTE/Tt6JUWL2prCe7vf5
+mUZQ27r+i09ZGtJ+iHL3TCswAxJlLg6fCyksJiItuN0lSmo3ROL1/m3ra/t5Poqf
+vZoxU3yXlWtANE1Y1N7hwxPp9qnkQYca6kvDVMc64RbsJJXt5/Bdr2MRxuwh5UE4
+T/7yZ4tUEthbZVXaTzTZ660DC96W65QNnyGFvhwJbI3f6k0s6ND0hFePoxoBhq4M
+3010WBrirQBI6aP9xleAaokmt7f+LMksYaGQN2XgbdXFTpTK3bhTPCT1Buc7Euw1
+/79eaCsrN56za5tIzIP4jSmBSc4x6/5jxmKdLk169B2zc5+aQYcuGBLEJFo+l6sg
+JJC+yRtqAR+5fHWaxCrCQEQQ4tnJduO4QbMUyxRMNEKoXAZQIb3PHGUJglURvANi
+3gqIeCQc2xLQZy7P4IWP+dhW6HKCFwDsr3ERaYwL+Q29kY2WZ9w3Cas75wM2MrKE
+0sfYXi6fU2b8mtbPSp0PlHiAwO6AkGhZ7Pd34tZsQ9lcR+n3/9Z+Hai8jrOlpLEI
+Wlr+rMYXJ7+ICIrJT5bUH8/1PBKUzzzxu4CPPpcnWHs8lcZR2ZUmrH2x/ATeWLJ7
+grUAEQEAAYkCJQQYAQgADwUCWnGsGAIbDAUJA8JnAAAKCRBkXVVgLUhR+YuMD/0R
+0Gck15pqR6OFjqsc7KKxjmwdUHDehTvqHLk3lTUqoKQL3D0mGfs0rK+MWYWY/xFW
+eEjmHZ2OrBJhaLwvi60tDjCBaXCL5WHojPoQ4CEVd/pE56fb+m8htN6Kk3hgjIVi
+OxqxbA6bTFUp16b6ik64PnCBaCx+wcaW+dzoEbtpVTW8ZfFtmtBkuGq6WxWDAMXJ
+epWS/FG8wV2b6GclPT2nsoi9G9HMo/DdrdoEcEArxTbZclKHQI+L5GW1jZSvq+d2
+A/bDhiXp5Rmh7jisURqIidvAbUri51svtYaV7Rz/M0qeG3kJN9KnFYFZulEgifaz
+5Oz9D4g5+eBxmhSIql+KCX3WBju9Tp1dhLhIcJQcC0Vht4cZ394WjK9BBFaLVaz2
+O63FlzumRkRuZrwGdXPDoFfDwuw57iFNEN5ZWUd1UBztUBtD4p84b0IqT11bexMh
+JTZovPg0W1QjzJY6tHyqD2MiKQDinyheMJaZyM+Gwr3gjfmbus7YgEIE9XPExLe6
+32zcR5YIbtR1QCy1GDWgZJBl8/UmhkWl1dtfOY1Xg+XU31Yh+P+VMf4r+Rz6jvoA
+3SluJ9cZu1K/W/KIFjw1860NL7bWItDzhymbwp9L87qxB+V0ibQixvBO/VgWBymU
+PEtY0KRawbNenVyt78e1ujqyiHhS3/Mi+2GAOrs+Rg==
+=/8cb
+-----END PGP PUBLIC KEY BLOCK-----
new file mode 100644
--- /dev/null
+++ b/modules/shipit_scriptworker/manifests/init.pp
@@ -0,0 +1,93 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class shipit_scriptworker {
+    include shipit_scriptworker::settings
+    include dirs::builds
+    include packages::mozilla::python35
+    include tweaks::swap_on_instance_storage
+    include packages::gcc
+    include packages::make
+    include tweaks::scriptworkerlogrotate
+
+    python35::virtualenv {
+        $shipit_scriptworker::settings::root:
+            python3  => $packages::mozilla::python35::python3,
+            require  => Class['packages::mozilla::python35'],
+            user     => $shipit_scriptworker::settings::user,
+            group    => $shipit_scriptworker::settings::group,
+            mode     => 700,
+            packages => [
+                'PyYAML==3.12',
+                'aiohttp==2.3.9',
+                'arrow==0.12.1',
+                'async_timeout==1.4.0',
+                'certifi==2018.1.18',
+                'chardet==3.0.4',
+                'defusedxml==0.5.0',
+                'dictdiffer==0.7.0',
+                'frozendict==1.2',
+                'idna==2.6',
+                'json-e==2.5.0',
+                'jsonschema==2.6.0',
+                'mohawk==0.3.4',
+                'multidict==4.0.0',
+                'pexpect==4.3.1',
+                'ptyprocess==0.5.2',
+                'python-dateutil==2.6.1',
+                'python-gnupg==0.4.1',
+                'redo==1.6',
+                'requests==2.18.4',
+                'scriptworker==8.0.1',
+                'shipitapi==0.1.0',
+                'shipitscript==0.1.0',
+                'six==1.10.0',
+                'slugid==1.0.7',
+                'taskcluster==2.1.3',
+                'urllib3==1.22',
+                'virtualenv==15.1.0',
+                'yarl==1.0.0',
+            ];
+    }
+
+    scriptworker::instance {
+        $shipit_scriptworker::settings::root:
+            instance_name            => $module_name,
+            basedir                  => $shipit_scriptworker::settings::root,
+            work_dir                 => $shipit_scriptworker::settings::work_dir,
+
+            task_script              => $shipit_scriptworker::settings::task_script,
+
+            username                 => $shipit_scriptworker::settings::user,
+            group                    => $shipit_scriptworker::settings::group,
+
+            taskcluster_client_id    => $shipit_scriptworker::settings::taskcluster_client_id,
+            taskcluster_access_token => $shipit_scriptworker::settings::taskcluster_access_token,
+            worker_group             => $shipit_scriptworker::settings::worker_group,
+            worker_type              => $shipit_scriptworker::settings::worker_type,
+
+            cot_job_type             => 'shipit',
+
+            sign_chain_of_trust      => $shipit_scriptworker::settings::sign_chain_of_trust,
+            verify_chain_of_trust    => $shipit_scriptworker::settings::verify_chain_of_trust,
+            verify_cot_signature     => $shipit_scriptworker::settings::verify_cot_signature,
+
+            verbose_logging          => $shipit_scriptworker::settings::verbose_logging,
+    }
+
+    File {
+        ensure      => present,
+        mode        => '0600',
+        owner       => $shipit_scriptworker::settings::user,
+        group       => $shipit_scriptworker::settings::group,
+        show_diff   => false,
+    }
+
+    $config_content = $shipit_scriptworker::settings::script_config_content
+    file {
+        $shipit_scriptworker::settings::script_config:
+            require => Python35::Virtualenv[$shipit_scriptworker::settings::root],
+            content => inline_template("<%- require 'json' -%><%= JSON.pretty_generate(@config_content) %>");
+    }
+}
new file mode 100644
--- /dev/null
+++ b/modules/shipit_scriptworker/manifests/settings.pp
@@ -0,0 +1,84 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class shipit_scriptworker::settings {
+    include ::config
+    include users::builder
+
+    $root                     = $config::scriptworker_root
+
+    $ship_it_stage_instance_scope = 'project:releng:ship-it:staging'
+    $ship_it_stage_instance_config = {
+        api_root                  => 'https://ship-it-dev.allizom.org',
+        timeout_in_seconds        => 60,
+        username                  => 'shipit-scriptworker-stage',
+        password                  => secret('shipit_scriptworker_ship_it_password_dev'),
+    }
+
+    $_env_configs             = {
+      'dev'  => {
+        worker_group             => 'shipit-dev',
+        worker_type              => 'shipit-dev',
+        verbose_logging          => true,
+        taskcluster_client_id    => 'project/releng/scriptworker/shipit/dev',
+        taskcluster_access_token => secret('shipit_scriptworker_taskcluster_access_token_dev'),
+
+        sign_chain_of_trust      => false,
+        verify_chain_of_trust    => true,
+        verify_cot_signature     => false,
+
+        ship_it_instances        => {
+            "$ship_it_stage_instance_scope" => $ship_it_stage_instance_config,
+        },
+      },
+      'prod' => {
+        worker_group             => 'shipit-v1',
+        worker_type              => 'shipit-v1',
+        verbose_logging          => true,
+        taskcluster_client_id    => 'project/releng/scriptworker/shipit/production',
+        taskcluster_access_token => secret('shipit_scriptworker_taskcluster_access_token_prod'),
+
+        sign_chain_of_trust      => true,
+        verify_chain_of_trust    => true,
+        verify_cot_signature     => true,
+
+        ship_it_instances        => {
+            "$ship_it_stage_instance_scope"     => $ship_it_stage_instance_config,
+            'project:releng:ship-it:production' => {
+                api_root                  => 'https://ship-it.mozilla.org',
+                timeout_in_seconds        => 60,
+                username                  => 'shipit-scriptworker',
+                password                  => secret('shipit_scriptworker_ship_it_password_prod'),
+            },
+        },
+      },
+    }
+
+    $_env_config                = $_env_configs[$shipit_scriptworker_env]
+    $schema_file                = "${root}/lib/python3.5/site-packages/shipitscript/data/shipit_task_schema.json"
+    $work_dir                   = "${root}/work"
+    $task_script                = "${root}/bin/shipitscript"
+
+    $user                       = $users::builder::username
+    $group                      = $users::builder::group
+
+    $taskcluster_client_id      = $_env_config['taskcluster_client_id']
+    $taskcluster_access_token   = $_env_config['taskcluster_access_token']
+    $worker_group               = $_env_config['worker_group']
+    $worker_type                = $_env_config['worker_type']
+
+    $sign_chain_of_trust        = $_env_config['sign_chain_of_trust']
+    $verify_chain_of_trust      = $_env_config['verify_chain_of_trust']
+    $verify_cot_signature       = $_env_config['verify_cot_signature']
+
+    $verbose_logging            = $_env_config['verbose_logging']
+
+    $script_config              = "${root}/script_config.json"
+    $script_config_content      = {
+        work_dir           => $work_dir,
+        schema_file        => $schema_file,
+        verbose            => $verbose_logging,
+        ship_it_instances  => $_env_config['ship_it_instances'],
+    }
+}
new file mode 100644
--- /dev/null
+++ b/modules/toplevel/manifests/server/shipitscriptworker.pp
@@ -0,0 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class toplevel::server::shipitscriptworker inherits toplevel::server {
+    include ::shipit_scriptworker
+}