Bug 1431764 - Add shipit_scriptworkers r=aki
MozReview-Commit-ID: CYml5CGwp2g
--- a/manifests/moco-nodes.pp
+++ b/manifests/moco-nodes.pp
@@ -961,16 +961,34 @@ node /pushapkworker-.*\.srv\.releng\..*\
node /binarytransparencyworker-.*\.srv\.releng\..*\.mozilla\.com/ {
$aspects = [ 'maximum-security' ]
$transparencyworker_env = "dev"
$timezone = "UTC"
$only_user_ssh = true
include toplevel::server::transparencyscriptworker
}
+# shipit scriptworkers
+node /^shipitworker-dev-.*\.srv\.releng\..*\.mozilla\.com$/ {
+ $aspects = [ 'maximum-security' ]
+ $shipit_scriptworker_env = 'dev'
+ $timezone = 'UTC'
+ $only_user_ssh = true
+ include toplevel::server::shipitscriptworker
+}
+
+node /^shipitworker-.*\.srv\.releng\..*\.mozilla\.com$/ {
+ $aspects = [ 'maximum-security' ]
+ $shipit_scriptworker_env = 'prod'
+ $timezone = 'UTC'
+ $only_user_ssh = true
+ include toplevel::server::shipitscriptworker
+}
+
+
## Loaners
node 'dhouse-1330169.srv.releng.scl3.mozilla.com' {
$aspects = [ 'low-security' ]
include toplevel::server
}
# See Bug 1343963
@@ -1015,9 +1033,8 @@ node 't-linux64-xe-264.test.releng.mdc1.
include toplevel::worker::releng::taskcluster_worker::test::gpu
}
node 'ms1-1.test.releng.mdc1.mozilla.com' {
$aspects = [ 'low-security' ]
$slave_trustlevel = 'try'
include toplevel::base
}
-
new file mode 100644
--- /dev/null
+++ b/modules/scriptworker/files/git_pubkeys/jlorenzo@mozilla.com.asc
@@ -0,0 +1,96 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFpxp+QBEACqnYQN+XwCB0PeRLKdYMtnENTUKfx2i69Vp86KALNuZFFiJVOX
+84mt869ZDsZeNpXRRmW0CoQjx7k3LTIruDKiXcNyCd7rxX5B4k059dwIEkY9OWNX
+/1KQNZt9vbUoQjiUtzhqDhIS+NZau/RydQxtsFVCyKMereXwxWBX2wwMUpGIN0bq
+6BMTz/D++KO5JsLzsgYbn57SGX6jBCLnmNAJfN8AIUZC6iGWTtXgEG382UmI+RT/
+NplUGSYJJIF4QxBoYEyapp7BFihdHNjP+4SoeCrjB0LG7KCP9k2RNEYVj38a4Oea
+Vq3CYtMjFz+iuKvuxv+ehp4NFAVw9nzhfwEVNWo1GWaNmO4UcMp8yc793FpvFKV3
+B9JJR8m6emzv3DW8AIWhcyGFwFoj3y8VbSv/UZz1ETeITuGKwD6ANwopV+bKVYp5
+KMp8D2V/x7WybwZzxXJVzVVa5kexIUlK3AFIayDATUaFv+jZoaaW+kMW4Gl8JZHq
+gZOpsuej40e+zNcbjzBp4vN+I5u9J6KcXEZz9qipJofpzWj9Jlz9p5JiD3mXlqFM
+ojDlXG9oNm00hC3MqoCOXTU9W0Oowg3QCJQodU/Mhk/lsxTZ5aFo+UrniaDsv9j8
+SEgKdP+MMT6bJulL1F9oCcGl88LUIib4t7mV1MVQY4nQWJzhWDZM1Rd8PwARAQAB
+tC9Kb2hhbiBMb3JlbnpvICgyMDE4IGtleSkgPGpsb3JlbnpvQG1vemlsbGEuY29t
+PokCPQQTAQgAJwUCWnGn5AIbAQUJA8JnAAULCQgHAgYVCAkKCwIEFgIDAQIeAQIX
+gAAKCRBkXVVgLUhR+RlaEACWuAKzpL2XC2deWKdDiKSdb42P3dr+EjpnYIIK4tz3
+FCiko41fzvbHBTIo2TPpC5wiVvXI+ClMlA8Ah9nbUiZ/Rsyo4hBH/vBGzue6Gwmw
+DTQSq5hDID4NfKLitMz8MX3lI/KD5Uh0ggO3p4vHowdQOCDMPVOZgCACzBp21kXI
+xGCs3ra7wJjLecfmxAhlsvEEK9xl36f4EUU3Y6nVI7tPbK8j+gKUDgCXe8bYN5qQ
+n7BNVej36SFIdRr3D1B/IwradvWmBH2VfQXzQvDza/3fMv6rtwxC1l8QfvLbKMai
+GV12T76b4Nr6p1sY4snz3BSlbNe/4inS847IkH+Oin3BNuR5PYQ9oU+GkRs4T0QD
+Cdlo4fjkL+X5+EC0lU1OMC2LXqDG3teQPSMunfbZhN53BLtuxmM60+HECxTWSEfs
+dOJrRJBLk/DsBNuJqVgzC8iQasxpRbne4/DSnXtIsgRMjm8AlIQLJ4WodRYZRlX3
+MBdsbcM/8NiBOA5jxsmQXnbYlRSFM4zz6n/oF/pYLUj1J54UUW9/4CduBNIA7R/N
+VEtAHwM4tXp2e/TKzt86GEldrft5ROk3DPZhXC2o3Pi53p4hmcm+ZfQUlLjoH6v0
+P5jebr5W1uy270oxCTlblZnoS7Mm1FKmDAF5faC20JXLc5qMtm+38mrMEcNYDATp
+HYkCHAQQAQoABgUCWnGtjgAKCRDVREAOtiM0Ik9kD/4jH95KEnonoKqJsGZZEcua
+9NWNppNTkmGnxOs7OjDDAPIG1/jtuht02NE7NU+KEUIQJhlcDVoLnZQnNn8FAwbD
+cRc8SaGqFBkaInug+5KyIC5VQIQ4XHCuUj7w1JgVQKf2HqiHa9/tqFgXKQLO67PQ
+VbyDvkTmNEGOJ8Jr9bI53X/DIGt9d1G8YC5G7DSDjCJM5SGkB6UBxDnVMDs8FtcA
+aa+cLC28n1kdkP1ORHzMX6uKK6g+asF6jkRk9GKeH+H01EgvTuMJaLEmZSLLBDds
+lRX7jshLiuwyhW0kgqfhhxbKDr0uNShNyj3QBB9N0uhQYotGkQhF996hUm0q3Iv+
+VS4tbhkT7fxGeJjEDBzljDpgy239oLxfX8t1O2D/ySlD5IDIZc/1vCB3vGT/XS4u
+hW5Ke5h2JzAq8EXjphDHoJGElqsPtsQZaR4JdK1Huod2PElnN2DIBwV7/7gbaCsZ
+6/a55tCkZdPB7Et12syfRMU6WuKPV+KT+LY7of8LlBLVC62yOJYNYev6iQKFYMxJ
+W61ymrI6ZutXHZRMhOVW0gxOd9ZO1QQ5Fp69cSwTlmLZVqoU3xEn3ymSqvsF4Mee
+oldYfsJHJBvDcWJzU+b3nw9w3frSf8xZXoyNmg+IYmRhTOjqSOHzb6PCdVuCYAAY
+x/JtHRWIZJSCeqSvIoGA7LkCDQRacauDARAAkrZMEAvzopHkUouCJwzTFV+9cHA8
+yNS9/M7q+y54YlVc5Y2uh4h/IuNpgioSe5RQJo+SDknHqgd41AvwF+GTaz7ApMuk
+j/Y2RHfgZ5Hdp0t2qhBJJk/vyc8p776/vnLnT5HdIH46E5eHi9gTfFpgkZVuzi6E
+bVorSqUnhNRX/0beQUvuldrgTacbtq8o9dJfLFBAT5OK01e3VEFZm3+D7WWc8vG3
+Zdb46ppAbd8WRsatWEAPIT3y7vaKZwxAqqfhsz9DqAmLJ5JZDBHa4hFuYynCg6xO
+08kchqqqywAKy5S3nemBr7da9L2wZyMSqxT9a+pfiDJy5M76yS0RuBUckOk4NJ2B
+DLeYFZilug2cGRh9q84dpXVWgbaPJnJRHuZ7ennWWlbYavyUFkJLfWpsIPK/f2Rq
+8bZwj+CMhbYou9wL67FhDRlsO5ccbAfaqSKZxFJI9bZo8KBIRsPKkCIzJ+DKY7fg
+gdjNmlX6MyWDCUCe/uiXVR7P2hF1ErUNN+cDo1MGQcrtXlobvvw2FFVM+o4TqCaz
+gDIihq8evoqxVziUBmWgTkjOorhESWvOnD1+gGUiHJ+6kp4A8wJORVJfnBYjANj6
+iBoP4y01yoixIMFE06CpTi2nkF5ohDEwj1+vwN2LIzqfoB4SILBTj6vPw60AI17y
+6u0rHkjy5aQdQyMAEQEAAYkERAQYAQgADwUCWnGrgwIbAgUJA8JnAAIpCRBkXVVg
+LUhR+cFdIAQZAQgABgUCWnGrgwAKCRACXe1IhvSglkIrEACF07S1GhuM5dSH6E+O
+Rihwi8czfOnAK96OAj4lMrGNoJdBDdZM3RhlRsTSQv64kdtYFdhoih0HJ0mkycfl
+iaexzcmRK/erOjD8UyN1MZqPLjVNt+ENJ5aUbkfqBWOrpxiKvqbPe9oZHmajSrQz
+9SFleARGQYRWR4jqZjHbNiP+4CpYypZcEoZmqMM7tG5RsnAz/ePRdQHKZTK9uacC
+PvyuMD6uJsnsRCrMyiJYvMlITUW+547sivJdi7zdA8ZHfamgl5NdXRjGdQGBPKXd
+7tUiIRwSYnLWOnRrNiD351e/sswlU1mKtjwlKdJL0kuDL1mSddFE953yP2IjOOYU
+p1CfCpRGeUaOl0KImMDinNwBvz0BEx62pvJyqvwt7QFiw8qlCdFuHakmMe30lsVK
+eXtVrZOvsg6W3RvjmWIs1Xf5Kq/LuQxvhp2NiE5OgAPEyEPqxQJzfaCco01r2O15
+ZGV9876as278Dttvsghfw9JCKSY08VnNkKCQLHv1a0rePqbrc6Cdsq/5NMc/8HpM
+9ojmQ/XobJ19g3DkQzqULc/WNo2aLzCJ2wvGGWB5U8yx+dEADC53igmhgKadLVsp
+Sfj6Arvs458qiP3GYaT1dXTcSQYhQpzQlNGt6AV3kI9ppmMssn7Le3EkhbgXiwgf
+Wf64EOfWjkwgTObqM85us2A3hoJ/D/9G8eBVs9TuldDsl3soHG6lQHImwQlw960j
+1RDeA2k9znGjWZKq+iQlKt7b3YeQx05Q02d8H5hgua6smGW8eQxlDXBwtMbJWXnH
+5PB7G/wan65Ve3nU3L1HqReb1G3PUxIkPsHWPvLwgYah/chZFm1B9z/52BtyMqxF
+kNSYW4LDOnLiFIOOsGGRm++ikSOR5ezpXWRoGRXQSlakBClT7qzmX8I3B1EpKq5u
+jOTHGydNru7LEtcSu+XeJAFGwUfw9OMOWqUwyRfEZFLUaq3LYxFZlu8XWaXu5K4D
+w6tQCYv4w1DzjdCnCYHUAa3QiOVSTSIMx37HDoBOn93XMoBwYZ7tJ4dtrVujnpvr
+fZePp2/wB/q53G9w72SIp7iq20anWAesQNMFuxYO8fOqvdrRv5zQP39I6PTJz4no
+JBa85+tANFWoqqE05Onxk0PrUXHZdRG4Jox2NuRCgNFzeEcYfIxufABREmNA+kdf
+uzDT7jResJl4iCq6tyWIjwlEHX332ffnFa6j1N7iBm6RE/XkaWpsL8xIU8q9sn6N
+zeXV4+gO9oiKtUDqKp6pT5Q8j5wAmDaFzatZXAh6Vf70XQxOrCINJ0F7ls8qtBf5
+cThUz+rS8+q35CurTnB9DBZK5MOhSu2VGXpVckvzwVId75nVCHzlNTFzfTWrfDKk
+OJ8yKwHmlrkCDQRacawYARAA5ts3F8Xjq5qyLLqBtAMUrU29rtNgfx/Zb+1p6P3H
+zC8RYr8PuYr/UMNukCjl866nb4ZgOtu8ijWm7jDhpjdWwTE/Tt6JUWL2prCe7vf5
+mUZQ27r+i09ZGtJ+iHL3TCswAxJlLg6fCyksJiItuN0lSmo3ROL1/m3ra/t5Poqf
+vZoxU3yXlWtANE1Y1N7hwxPp9qnkQYca6kvDVMc64RbsJJXt5/Bdr2MRxuwh5UE4
+T/7yZ4tUEthbZVXaTzTZ660DC96W65QNnyGFvhwJbI3f6k0s6ND0hFePoxoBhq4M
+3010WBrirQBI6aP9xleAaokmt7f+LMksYaGQN2XgbdXFTpTK3bhTPCT1Buc7Euw1
+/79eaCsrN56za5tIzIP4jSmBSc4x6/5jxmKdLk169B2zc5+aQYcuGBLEJFo+l6sg
+JJC+yRtqAR+5fHWaxCrCQEQQ4tnJduO4QbMUyxRMNEKoXAZQIb3PHGUJglURvANi
+3gqIeCQc2xLQZy7P4IWP+dhW6HKCFwDsr3ERaYwL+Q29kY2WZ9w3Cas75wM2MrKE
+0sfYXi6fU2b8mtbPSp0PlHiAwO6AkGhZ7Pd34tZsQ9lcR+n3/9Z+Hai8jrOlpLEI
+Wlr+rMYXJ7+ICIrJT5bUH8/1PBKUzzzxu4CPPpcnWHs8lcZR2ZUmrH2x/ATeWLJ7
+grUAEQEAAYkCJQQYAQgADwUCWnGsGAIbDAUJA8JnAAAKCRBkXVVgLUhR+YuMD/0R
+0Gck15pqR6OFjqsc7KKxjmwdUHDehTvqHLk3lTUqoKQL3D0mGfs0rK+MWYWY/xFW
+eEjmHZ2OrBJhaLwvi60tDjCBaXCL5WHojPoQ4CEVd/pE56fb+m8htN6Kk3hgjIVi
+OxqxbA6bTFUp16b6ik64PnCBaCx+wcaW+dzoEbtpVTW8ZfFtmtBkuGq6WxWDAMXJ
+epWS/FG8wV2b6GclPT2nsoi9G9HMo/DdrdoEcEArxTbZclKHQI+L5GW1jZSvq+d2
+A/bDhiXp5Rmh7jisURqIidvAbUri51svtYaV7Rz/M0qeG3kJN9KnFYFZulEgifaz
+5Oz9D4g5+eBxmhSIql+KCX3WBju9Tp1dhLhIcJQcC0Vht4cZ394WjK9BBFaLVaz2
+O63FlzumRkRuZrwGdXPDoFfDwuw57iFNEN5ZWUd1UBztUBtD4p84b0IqT11bexMh
+JTZovPg0W1QjzJY6tHyqD2MiKQDinyheMJaZyM+Gwr3gjfmbus7YgEIE9XPExLe6
+32zcR5YIbtR1QCy1GDWgZJBl8/UmhkWl1dtfOY1Xg+XU31Yh+P+VMf4r+Rz6jvoA
+3SluJ9cZu1K/W/KIFjw1860NL7bWItDzhymbwp9L87qxB+V0ibQixvBO/VgWBymU
+PEtY0KRawbNenVyt78e1ujqyiHhS3/Mi+2GAOrs+Rg==
+=/8cb
+-----END PGP PUBLIC KEY BLOCK-----
new file mode 100644
--- /dev/null
+++ b/modules/shipit_scriptworker/manifests/init.pp
@@ -0,0 +1,93 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class shipit_scriptworker {
+ include shipit_scriptworker::settings
+ include dirs::builds
+ include packages::mozilla::python35
+ include tweaks::swap_on_instance_storage
+ include packages::gcc
+ include packages::make
+ include tweaks::scriptworkerlogrotate
+
+ python35::virtualenv {
+ $shipit_scriptworker::settings::root:
+ python3 => $packages::mozilla::python35::python3,
+ require => Class['packages::mozilla::python35'],
+ user => $shipit_scriptworker::settings::user,
+ group => $shipit_scriptworker::settings::group,
+ mode => 700,
+ packages => [
+ 'PyYAML==3.12',
+ 'aiohttp==2.3.9',
+ 'arrow==0.12.1',
+ 'async_timeout==1.4.0',
+ 'certifi==2018.1.18',
+ 'chardet==3.0.4',
+ 'defusedxml==0.5.0',
+ 'dictdiffer==0.7.0',
+ 'frozendict==1.2',
+ 'idna==2.6',
+ 'json-e==2.5.0',
+ 'jsonschema==2.6.0',
+ 'mohawk==0.3.4',
+ 'multidict==4.0.0',
+ 'pexpect==4.3.1',
+ 'ptyprocess==0.5.2',
+ 'python-dateutil==2.6.1',
+ 'python-gnupg==0.4.1',
+ 'redo==1.6',
+ 'requests==2.18.4',
+ 'scriptworker==8.0.1',
+ 'shipitapi==0.1.0',
+ 'shipitscript==0.1.0',
+ 'six==1.10.0',
+ 'slugid==1.0.7',
+ 'taskcluster==2.1.3',
+ 'urllib3==1.22',
+ 'virtualenv==15.1.0',
+ 'yarl==1.0.0',
+ ];
+ }
+
+ scriptworker::instance {
+ $shipit_scriptworker::settings::root:
+ instance_name => $module_name,
+ basedir => $shipit_scriptworker::settings::root,
+ work_dir => $shipit_scriptworker::settings::work_dir,
+
+ task_script => $shipit_scriptworker::settings::task_script,
+
+ username => $shipit_scriptworker::settings::user,
+ group => $shipit_scriptworker::settings::group,
+
+ taskcluster_client_id => $shipit_scriptworker::settings::taskcluster_client_id,
+ taskcluster_access_token => $shipit_scriptworker::settings::taskcluster_access_token,
+ worker_group => $shipit_scriptworker::settings::worker_group,
+ worker_type => $shipit_scriptworker::settings::worker_type,
+
+ cot_job_type => 'shipit',
+
+ sign_chain_of_trust => $shipit_scriptworker::settings::sign_chain_of_trust,
+ verify_chain_of_trust => $shipit_scriptworker::settings::verify_chain_of_trust,
+ verify_cot_signature => $shipit_scriptworker::settings::verify_cot_signature,
+
+ verbose_logging => $shipit_scriptworker::settings::verbose_logging,
+ }
+
+ File {
+ ensure => present,
+ mode => '0600',
+ owner => $shipit_scriptworker::settings::user,
+ group => $shipit_scriptworker::settings::group,
+ show_diff => false,
+ }
+
+ $config_content = $shipit_scriptworker::settings::script_config_content
+ file {
+ $shipit_scriptworker::settings::script_config:
+ require => Python35::Virtualenv[$shipit_scriptworker::settings::root],
+ content => inline_template("<%- require 'json' -%><%= JSON.pretty_generate(@config_content) %>");
+ }
+}
new file mode 100644
--- /dev/null
+++ b/modules/shipit_scriptworker/manifests/settings.pp
@@ -0,0 +1,84 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class shipit_scriptworker::settings {
+ include ::config
+ include users::builder
+
+ $root = $config::scriptworker_root
+
+ $ship_it_stage_instance_scope = 'project:releng:ship-it:staging'
+ $ship_it_stage_instance_config = {
+ api_root => 'https://ship-it-dev.allizom.org',
+ timeout_in_seconds => 60,
+ username => 'shipit-scriptworker-stage',
+ password => secret('shipit_scriptworker_ship_it_password_dev'),
+ }
+
+ $_env_configs = {
+ 'dev' => {
+ worker_group => 'shipit-dev',
+ worker_type => 'shipit-dev',
+ verbose_logging => true,
+ taskcluster_client_id => 'project/releng/scriptworker/shipit/dev',
+ taskcluster_access_token => secret('shipit_scriptworker_taskcluster_access_token_dev'),
+
+ sign_chain_of_trust => false,
+ verify_chain_of_trust => true,
+ verify_cot_signature => false,
+
+ ship_it_instances => {
+ "$ship_it_stage_instance_scope" => $ship_it_stage_instance_config,
+ },
+ },
+ 'prod' => {
+ worker_group => 'shipit-v1',
+ worker_type => 'shipit-v1',
+ verbose_logging => true,
+ taskcluster_client_id => 'project/releng/scriptworker/shipit/production',
+ taskcluster_access_token => secret('shipit_scriptworker_taskcluster_access_token_prod'),
+
+ sign_chain_of_trust => true,
+ verify_chain_of_trust => true,
+ verify_cot_signature => true,
+
+ ship_it_instances => {
+ "$ship_it_stage_instance_scope" => $ship_it_stage_instance_config,
+ 'project:releng:ship-it:production' => {
+ api_root => 'https://ship-it.mozilla.org',
+ timeout_in_seconds => 60,
+ username => 'shipit-scriptworker',
+ password => secret('shipit_scriptworker_ship_it_password_prod'),
+ },
+ },
+ },
+ }
+
+ $_env_config = $_env_configs[$shipit_scriptworker_env]
+ $schema_file = "${root}/lib/python3.5/site-packages/shipitscript/data/shipit_task_schema.json"
+ $work_dir = "${root}/work"
+ $task_script = "${root}/bin/shipitscript"
+
+ $user = $users::builder::username
+ $group = $users::builder::group
+
+ $taskcluster_client_id = $_env_config['taskcluster_client_id']
+ $taskcluster_access_token = $_env_config['taskcluster_access_token']
+ $worker_group = $_env_config['worker_group']
+ $worker_type = $_env_config['worker_type']
+
+ $sign_chain_of_trust = $_env_config['sign_chain_of_trust']
+ $verify_chain_of_trust = $_env_config['verify_chain_of_trust']
+ $verify_cot_signature = $_env_config['verify_cot_signature']
+
+ $verbose_logging = $_env_config['verbose_logging']
+
+ $script_config = "${root}/script_config.json"
+ $script_config_content = {
+ work_dir => $work_dir,
+ schema_file => $schema_file,
+ verbose => $verbose_logging,
+ ship_it_instances => $_env_config['ship_it_instances'],
+ }
+}
new file mode 100644
--- /dev/null
+++ b/modules/toplevel/manifests/server/shipitscriptworker.pp
@@ -0,0 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class toplevel::server::shipitscriptworker inherits toplevel::server {
+ include ::shipit_scriptworker
+}