--- a/manifests/moco-config.pp
+++ b/manifests/moco-config.pp
@@ -401,88 +401,20 @@ class config inherits config::base {
}
}
# TC signing workers
$signingworker_exchange = "exchange/taskcluster-queue/v1/task-pending"
$signingworker_worker_type = "signing-worker-v1"
# scriptworker
- $scriptworker_root = "/builds/scriptworker"
+ $scriptworker_root = "/builds/scriptworker" # Used by scriptworker instances
$scriptworker_gpg_private_keys = hiera_hash('scriptworker_gpg_private_keys')
$scriptworker_gpg_public_keys = hiera_hash('scriptworker_gpg_public_keys')
- ## TC pushapk scriptworkers
- $pushapk_scriptworker_old_root = '/builds/pushapkworker' # TODO Remove this line once bug 1321513 reaches production
- $pushapk_scriptworker_root = $scriptworker_root
- $pushapk_scriptworker_worker_config = "${pushapk_scriptworker_root}/config.json"
- $pushapk_scriptworker_script_config = "${pushapk_scriptworker_root}/script_config.json"
-
- $pushapk_scriptworker_jarsigner_keystore = "${pushapk_scriptworker_root}/mozilla-android-keystore"
- $pushapk_scriptworker_jarsigner_nightly_certificate_alias = 'nightly'
- $pushapk_scriptworker_jarsigner_release_certificate_alias = 'release'
- $pushapk_scriptworker_taskcluster_artifact_expiration_hours = 336
- $pushapk_scriptworker_taskcluster_artifact_upload_timeout = 1200
- $pushapk_scriptworker_task_max_timeout = 1200
- $pushapk_scriptworker_artifact_expiration_hours = 336
- $pushapk_scriptworker_artifact_upload_timeout = 600
- $pushapk_scriptworker_env_config = {
- 'dev' => {
- provisioner_id => 'scriptworker-prov-v1',
- worker_group => 'pushapk-v1-dev',
- worker_type => 'pushapk-v1-dev',
- worker_id => 'jlorenzo-dev',
- verbose_logging => true,
- taskcluster_client_id => secret('pushapk_scriptworker_taskcluster_client_id_dev'),
- taskcluster_access_token => secret('pushapk_scriptworker_taskcluster_access_token_dev'),
- google_play_config => {
- 'aurora' => {
- service_account => secret('pushapk_scriptworker_aurora_google_play_service_account_dev'),
- certificate => secret('pushapk_scriptworker_aurora_google_play_certificate_dev'),
- certificate_target_location => "${pushapk_scriptworker_root}/aurora.p12",
- },
- 'beta' => {
- service_account => secret('pushapk_scriptworker_beta_google_play_service_account_dev'),
- certificate => secret('pushapk_scriptworker_beta_google_play_certificate_dev'),
- certificate_target_location => "${pushapk_scriptworker_root}/beta.p12",
- },
- 'release' => {
- service_account => secret('pushapk_scriptworker_release_google_play_service_account_dev'),
- certificate => secret('pushapk_scriptworker_release_google_play_certificate_dev'),
- certificate_target_location => "${pushapk_scriptworker_root}/release.p12",
- },
- },
- },
- 'prod' => {
- provisioner_id => 'scriptworker-prov-v1',
- worker_group => 'pushapk-v1',
- worker_type => 'pushapk-v1',
- verbose_logging => true,
- taskcluster_client_id => secret('pushapk_scriptworker_taskcluster_client_id_prod'),
- taskcluster_access_token => secret('pushapk_scriptworker_taskcluster_access_token_prod'),
- google_play_config => {
- 'aurora' => {
- service_account => secret('pushapk_scriptworker_aurora_google_play_service_account_prod'),
- certificate => secret('pushapk_scriptworker_aurora_google_play_certificate_prod'),
- certificate_target_location => "${pushapk_scriptworker_root}/aurora.p12",
- },
- 'beta' => {
- service_account => secret('pushapk_scriptworker_beta_google_play_service_account_prod'),
- certificate => secret('pushapk_scriptworker_beta_google_play_certificate_prod'),
- certificate_target_location => "${pushapk_scriptworker_root}/beta.p12",
- },
- 'release' => {
- service_account => secret('pushapk_scriptworker_release_google_play_service_account_prod'),
- certificate => secret('pushapk_scriptworker_release_google_play_certificate_prod'),
- certificate_target_location => "${pushapk_scriptworker_root}/release.p12",
- },
- },
- },
- }
-
# Funsize Scheduler configuration
$funsize_scheduler_root = "/builds/funsize"
$funsize_scheduler_balrog_username = "funsize"
$funsize_scheduler_pulse_username = "funsize"
$funsize_scheduler_pulse_queue = "scheduler"
$funsize_scheduler_bb_pulse_exchange = "exchange/build"
$funsize_scheduler_tc_pulse_exchange = "exchange/taskcluster-queue/v1/task-completed"
$funsize_scheduler_s3_bucket = "mozilla-nightly-updates"
--- a/modules/balrog_scriptworker/manifests/init.pp
+++ b/modules/balrog_scriptworker/manifests/init.pp
@@ -1,10 +1,9 @@
class balrog_scriptworker {
- include balrog_scriptworker::services
include balrog_scriptworker::settings
include dirs::builds
include packages::mozilla::python35
include packages::mozilla::python27
include packages::mozilla::py27_mercurial
include users::builder
include tweaks::swap_on_instance_storage
include packages::gcc
@@ -72,29 +71,36 @@ class balrog_scriptworker {
"pycparser==2.14",
"requests==2.8.1",
"six==1.10.0",
];
}
scriptworker::instance {
"${balrog_scriptworker::settings::root}":
- basedir => "${balrog_scriptworker::settings::root}",
- task_script_executable => "${balrog_scriptworker::settings::task_script_executable}",
- task_script => "${balrog_scriptworker::settings::task_script}",
- task_script_config => "${balrog_scriptworker::settings::task_script_config}",
+ instance_name => $module_name,
+ basedir => $balrog_scriptworker::settings::root,
+
+ task_script_executable => $balrog_scriptworker::settings::task_script_executable,
+ task_script => $balrog_scriptworker::settings::task_script,
+ task_script_config => $balrog_scriptworker::settings::task_script_config,
+
+ username => $users::builder::username,
+ group => $users::builder::group,
+
+ taskcluster_client_id => $balrog_scriptworker::settings::taskcluster_client_id,
+ taskcluster_access_token => $balrog_scriptworker::settings::taskcluster_access_token,
+ worker_group => $balrog_scriptworker::settings::worker_group,
+ worker_type => $balrog_scriptworker::settings::worker_type,
+
task_max_timeout => $balrog_scriptworker::settings::task_max_timeout,
- username => "${users::builder::username}",
- group => "${users::builder::group}",
- worker_group => "${balrog_scriptworker::settings::worker_group}",
- worker_type => "${balrog_scriptworker::settings::worker_type}",
+
cot_job_type => "balrog",
+
verbose_logging => $balrog_scriptworker::settings::verbose_logging,
- taskcluster_client_id => "${balrog_scriptworker::settings::taskcluster_client_id}",
- taskcluster_access_token => "${balrog_scriptworker::settings::taskcluster_access_token}",
}
mercurial::repo {
"tools":
hg_repo => "${balrog_scriptworker::settings::tools_repo}",
dst_dir => "${balrog_scriptworker::settings::root}/tools",
user => "${users::builder::username}",
branch => "${balrog_scriptworker::settings::tools_branch}",
deleted file mode 100644
--- a/modules/balrog_scriptworker/manifests/services.pp
+++ /dev/null
@@ -1,20 +0,0 @@
-class balrog_scriptworker::services {
- include ::config
- include balrog_scriptworker::settings
- include packages::mozilla::supervisor
-
- supervisord::supervise {
- "balrog_scriptworker":
- command => "${balrog_scriptworker::settings::root}/bin/scriptworker ${balrog_scriptworker::settings::root}/scriptworker.yaml",
- user => $::config::builder_username,
- require => [ File["${balrog_scriptworker::settings::root}/scriptworker.yaml"]],
- extra_config => template("${module_name}/supervisor_config.erb");
- }
- exec {
- "restart-balrogworker":
- command => "/usr/bin/supervisorctl restart balrog_scriptworker",
- refreshonly => true,
- subscribe => [Python35::Virtualenv["${balrog_scriptworker::settings::root}"],
- File["${balrog_scriptworker::settings::root}/scriptworker.yaml"]];
- }
-}
deleted file mode 100644
--- a/modules/balrog_scriptworker/templates/supervisor_config.erb
+++ /dev/null
@@ -1,6 +0,0 @@
-log_stderr=true
-log_stdout=true
-redirect_stderr=true
-stdout_logfile=syslog
-autorestart=true
-autostart=true
--- a/modules/beetmover_scriptworker/manifests/init.pp
+++ b/modules/beetmover_scriptworker/manifests/init.pp
@@ -1,10 +1,9 @@
class beetmover_scriptworker {
- include beetmover_scriptworker::services
include beetmover_scriptworker::settings
include dirs::builds
include packages::mozilla::python35
include users::builder
include tweaks::swap_on_instance_storage
include packages::gcc
include packages::make
include packages::libffi
@@ -54,29 +53,35 @@ class beetmover_scriptworker {
"s3transfer==0.1.8",
"docutils==0.12",
"yarl==0.7.0",
];
}
scriptworker::instance {
"${beetmover_scriptworker::settings::root}":
- basedir => "${beetmover_scriptworker::settings::root}",
- task_script_executable => "${beetmover_scriptworker::settings::task_script_executable}",
- task_script => "${beetmover_scriptworker::settings::task_script}",
- task_script_config => "${beetmover_scriptworker::settings::task_script_config}",
+ instance_name => $module_name,
+ basedir => $beetmover_scriptworker::settings::root,
+
+ task_script => $beetmover_scriptworker::settings::task_script,
+ task_script_config => $beetmover_scriptworker::settings::task_script_config,
+
+ username => $users::builder::username,
+ group => $users::builder::group,
+
+ taskcluster_client_id => $beetmover_scriptworker::settings::taskcluster_client_id,
+ taskcluster_access_token => $beetmover_scriptworker::settings::taskcluster_access_token,
+ worker_group => $beetmover_scriptworker::settings::worker_group,
+ worker_type => $beetmover_scriptworker::settings::worker_type,
+
task_max_timeout => $beetmover_scriptworker::settings::task_max_timeout,
- username => "${users::builder::username}",
- group => "${users::builder::group}",
- worker_group => "${beetmover_scriptworker::settings::worker_group}",
- worker_type => "${beetmover_scriptworker::settings::worker_type}",
+
cot_job_type => "beetmover",
+
verbose_logging => $beetmover_scriptworker::settings::verbose_logging,
- taskcluster_client_id => "${beetmover_scriptworker::settings::taskcluster_client_id}",
- taskcluster_access_token => "${beetmover_scriptworker::settings::taskcluster_access_token}",
}
file {
"${beetmover_scriptworker::settings::root}/script_config.json":
require => Python35::Virtualenv["${beetmover_scriptworker::settings::root}"],
mode => 600,
owner => "${users::builder::username}",
group => "${users::builder::group}",
deleted file mode 100644
--- a/modules/beetmover_scriptworker/manifests/services.pp
+++ /dev/null
@@ -1,20 +0,0 @@
-class beetmover_scriptworker::services {
- include ::config
- include beetmover_scriptworker::settings
- include packages::mozilla::supervisor
-
- supervisord::supervise {
- "beetmover_scriptworker":
- command => "${beetmover_scriptworker::settings::root}/bin/scriptworker ${beetmover_scriptworker::settings::root}/scriptworker.yaml",
- user => $::config::builder_username,
- require => [ File["${beetmover_scriptworker::settings::root}/scriptworker.yaml"]],
- extra_config => template("${module_name}/supervisor_config.erb");
- }
- exec {
- "restart-beetmoverworker":
- command => "/usr/bin/supervisorctl restart beetmover_scriptworker",
- refreshonly => true,
- subscribe => [Python35::Virtualenv["${beetmover_scriptworker::settings::root}"],
- File["${beetmover_scriptworker::settings::root}/scriptworker.yaml"]];
- }
-}
--- a/modules/beetmover_scriptworker/manifests/settings.pp
+++ b/modules/beetmover_scriptworker/manifests/settings.pp
@@ -1,11 +1,10 @@
class beetmover_scriptworker::settings {
$root = "/builds/scriptworker"
- $task_script_executable = "${root}/bin/python"
$task_script = "${root}/bin/beetmoverscript"
$task_script_config = "${root}/script_config.json"
$task_max_timeout = 1800
$worker_group = "beetmoverworker-v1"
$worker_type = "beetmoverworker-v1"
$taskcluster_client_id = secret("beetmoverworker_dev_taskcluster_client_id")
$taskcluster_access_token = secret("beetmoverworker_dev_taskcluster_access_token")
deleted file mode 100644
--- a/modules/beetmover_scriptworker/templates/supervisor_config.erb
+++ /dev/null
@@ -1,6 +0,0 @@
-log_stderr=true
-log_stdout=true
-redirect_stderr=true
-stdout_logfile=syslog
-autorestart=true
-autostart=true
--- a/modules/pushapkworker/manifests/init.pp
+++ b/modules/pushapkworker/manifests/init.pp
@@ -1,100 +1,109 @@
class pushapkworker {
- include ::config
- include pushapkworker::services
include pushapkworker::settings
include dirs::builds
include packages::mozilla::python35
- include users::builder
include tweaks::swap_on_instance_storage
include packages::gcc
include packages::make
include packages::libffi
include pushapkworker::jarsigner_init
include pushapkworker::mime_types
- $env_config = $config::pushapk_scriptworker_env_config[$pushapkworker_env]
- $google_play_config = $env_config['google_play_config']
-
python35::virtualenv {
$pushapkworker::settings::root:
python3 => $packages::mozilla::python35::python3,
require => Class['packages::mozilla::python35'],
- user => $users::builder::username,
- group => $users::builder::group,
+ user => $pushapkworker::settings::user,
+ group => $pushapkworker::settings::group,
mode => 700,
packages => [
- 'aiohttp==1.0.2',
+ 'aiohttp==1.1.2',
'arrow==0.8.0',
- 'async-timeout==1.0.0',
+ 'async-timeout==1.1.0',
'cffi==1.8.3',
'chardet==2.3.0',
'cryptography==1.5.2',
'defusedxml==0.4.1',
- 'frozendict==1.0',
+ 'frozendict==1.2',
'google-api-python-client==1.5.3',
'httplib2==0.9.2',
'idna==2.1',
'jsonschema==2.5.1',
'mohawk==0.3.3',
- 'mozapkpublisher==0.1.3',
+ 'mozapkpublisher==0.1.5',
'multidict==2.1.2',
'oauth2client==3.0.0',
'pexpect==4.2.1',
'ptyprocess==0.5.1',
- 'pushapkscript==0.1.4',
+ 'pushapkscript==0.2.0',
'pyasn1==0.1.9',
'pyasn1-modules==0.0.8',
'pycparser==2.14',
- 'pyOpenSSL==16.1.0',
+ 'pyOpenSSL==16.2.0',
'python-dateutil==2.5.3',
'python-gnupg==0.3.9',
- 'requests==2.11.1',
+ 'PyYAML==3.12',
+ 'requests==2.12.4',
'rsa==3.4.2',
- 'scriptworker==0.7.2',
+ 'scriptworker==1.0.0b7',
'simplejson==3.8.2',
'six==1.10.0',
'slugid==1.0.7',
'taskcluster==0.3.4',
'uritemplate==0.6',
- 'virtualenv==15.0.3'
+ 'virtualenv==15.0.3',
+ 'yarl==0.7.0',
];
}
- nrpe::custom {
- 'pushapkworker.cfg':
- content => template("${module_name}/nagios.cfg.erb");
+ scriptworker::instance {
+ "${pushapkworker::settings::root}":
+ instance_name => $module_name,
+ basedir => $pushapkworker::settings::root,
+ work_dir => $pushapkworker::settings::work_dir,
+
+ task_script => $pushapkworker::settings::task_script,
+
+ username => $pushapkworker::settings::user,
+ group => $pushapkworker::settings::group,
+
+ taskcluster_client_id => $pushapkworker::settings::taskcluster_client_id,
+ taskcluster_access_token => $pushapkworker::settings::taskcluster_access_token,
+ worker_group => $pushapkworker::settings::worker_group,
+ worker_type => $pushapkworker::settings::worker_type,
+
+ # TODO Enable one of the next 3 lines to turn on Chain of Trust (bug 1317783)
+ sign_chain_of_trust => false,
+ verify_chain_of_trust => false,
+ verify_cot_signature => false,
+ cot_job_type => 'pushapk',
+
+ verbose_logging => $pushapkworker::settings::verbose_logging,
}
File {
ensure => present,
mode => 600,
- owner => $users::builder::username,
- group => $users::builder::group,
+ owner => $pushapkworker::settings::user,
+ group => $pushapkworker::settings::group,
show_diff => false,
}
+ $google_play_config = $pushapkworker::settings::google_play_config
+
file {
- $config::pushapk_scriptworker_script_config:
+ $pushapkworker::settings::script_config:
require => Python35::Virtualenv[$pushapkworker::settings::root],
content => template("${module_name}/script_config.json.erb"),
show_diff => true;
- $config::pushapk_scriptworker_worker_config:
- require => Python35::Virtualenv[$pushapkworker::settings::root],
- content => template("${module_name}/config.json.erb");
-
$google_play_config['aurora']['certificate_target_location']:
content => $google_play_config['aurora']['certificate'];
$google_play_config['beta']['certificate_target_location']:
content => $google_play_config['beta']['certificate'];
$google_play_config['release']['certificate_target_location']:
content => $google_play_config['release']['certificate'];
-
- # TODO Remove the following statement line once bug 1321513 reaches production
- $config::pushapk_scriptworker_old_root:
- ensure => absent,
- force => true; # Needed to delete a folder
}
}
--- a/modules/pushapkworker/manifests/jarsigner_init.pp
+++ b/modules/pushapkworker/manifests/jarsigner_init.pp
@@ -1,35 +1,35 @@
class pushapkworker::jarsigner_init {
include ::config
include packages::jdk17
- $nightly = "${pushapkworker::settings::root}/nightly.cer"
- $release = "${pushapkworker::settings::root}/release.cer"
+ $nightly = $pushapkworker::settings::jarsigner_nightly_certificate
+ $release = $pushapkworker::settings::jarsigner_release_certificate
+
+ File {
+ ensure => 'present',
+ show_diff => false,
+ }
file {
$nightly:
- ensure => 'present',
- content => secret('pushapk_scriptworker_nightly_jarsigner_certificate'),
- show_diff => false;
+ content => secret('pushapk_scriptworker_nightly_jarsigner_certificate');
$release:
- ensure => 'present',
- content => secret('pushapk_scriptworker_release_jarsigner_certificate'),
- show_diff => false;
+ content => secret('pushapk_scriptworker_release_jarsigner_certificate');
+ }
+
+ Java_ks {
+ ensure => latest,
+ target => $pushapkworker::settings::jarsigner_keystore,
+ password => $pushapkworker::settings::jarsigner_keystore_password,
+ trustcacerts => true,
}
java_ks {
- $config::pushapk_scriptworker_jarsigner_nightly_certificate_alias:
- ensure => latest,
- certificate => $nightly,
- target => $config::pushapk_scriptworker_jarsigner_keystore,
- password => secret('pushapk_scriptworker_jarsigner_keystore_password'),
- trustcacerts => true;
+ $pushapkworker::settings::jarsigner_nightly_certificate_alias:
+ certificate => $nightly;
- $config::pushapk_scriptworker_jarsigner_release_certificate_alias:
- ensure => latest,
- certificate => $release,
- target => $config::pushapk_scriptworker_jarsigner_keystore,
- password => secret('pushapk_scriptworker_jarsigner_keystore_password'),
- trustcacerts => true;
+ $pushapkworker::settings::jarsigner_release_certificate_alias:
+ certificate => $release;
}
}
deleted file mode 100644
--- a/modules/pushapkworker/manifests/services.pp
+++ /dev/null
@@ -1,22 +0,0 @@
-class pushapkworker::services {
- include ::config
- include pushapkworker::settings
- include packages::mozilla::supervisor
-
- supervisord::supervise {
- 'pushapkworker':
- command => "${pushapkworker::settings::root}/bin/scriptworker ${config::pushapk_scriptworker_worker_config}",
- user => $::config::builder_username,
- require => [ File[$config::pushapk_scriptworker_worker_config],
- File[$config::pushapk_scriptworker_script_config]],
- extra_config => template("${module_name}/supervisor_config.erb");
- }
- exec {
- 'restart-pushapkworker':
- command => '/usr/bin/supervisorctl restart pushapkworker',
- refreshonly => true,
- subscribe => [Python35::Virtualenv[$pushapkworker::settings::root],
- File[$config::pushapk_scriptworker_worker_config],
- File[$config::pushapk_scriptworker_script_config]];
- }
-}
--- a/modules/pushapkworker/manifests/settings.pp
+++ b/modules/pushapkworker/manifests/settings.pp
@@ -1,5 +1,85 @@
class pushapkworker::settings {
include ::config
+ include users::builder
- $root = $config::pushapk_scriptworker_root
+ $root = $config::scriptworker_root
+
+ $_env_configs = {
+ 'dev' => {
+ worker_group => 'pushapk-v1-dev',
+ worker_type => 'pushapk-v1-dev',
+ verbose_logging => true,
+ taskcluster_client_id => secret('pushapk_scriptworker_taskcluster_client_id_dev'),
+ taskcluster_access_token => secret('pushapk_scriptworker_taskcluster_access_token_dev'),
+ google_play_config => {
+ 'aurora' => {
+ service_account => secret('pushapk_scriptworker_aurora_google_play_service_account_dev'),
+ certificate => secret('pushapk_scriptworker_aurora_google_play_certificate_dev'),
+ certificate_target_location => "${root}/aurora.p12",
+ },
+ 'beta' => {
+ service_account => secret('pushapk_scriptworker_beta_google_play_service_account_dev'),
+ certificate => secret('pushapk_scriptworker_beta_google_play_certificate_dev'),
+ certificate_target_location => "${root}/beta.p12",
+ },
+ 'release' => {
+ service_account => secret('pushapk_scriptworker_release_google_play_service_account_dev'),
+ certificate => secret('pushapk_scriptworker_release_google_play_certificate_dev'),
+ certificate_target_location => "${root}/release.p12",
+ },
+ },
+ },
+ 'prod' => {
+ worker_group => 'pushapk-v1',
+ worker_type => 'pushapk-v1',
+ verbose_logging => true,
+ taskcluster_client_id => secret('pushapk_scriptworker_taskcluster_client_id_prod'),
+ taskcluster_access_token => secret('pushapk_scriptworker_taskcluster_access_token_prod'),
+ google_play_config => {
+ 'aurora' => {
+ service_account => secret('pushapk_scriptworker_aurora_google_play_service_account_prod'),
+ certificate => secret('pushapk_scriptworker_aurora_google_play_certificate_prod'),
+ certificate_target_location => "${root}/aurora.p12",
+ },
+ 'beta' => {
+ service_account => secret('pushapk_scriptworker_beta_google_play_service_account_prod'),
+ certificate => secret('pushapk_scriptworker_beta_google_play_certificate_prod'),
+ certificate_target_location => "${root}/beta.p12",
+ },
+ 'release' => {
+ service_account => secret('pushapk_scriptworker_release_google_play_service_account_prod'),
+ certificate => secret('pushapk_scriptworker_release_google_play_certificate_prod'),
+ certificate_target_location => "${root}/release.p12",
+ },
+ },
+ },
+ }
+
+ $_env_config = $_env_configs[$pushapkworker_env]
+ $schema_file = "${root}/lib/python3.5/site-packages/pushapkscript/data/pushapk_task_schema.json"
+ $work_dir = "${root}/work"
+ $script_config = "${root}/script_config.json"
+ $task_script = "${root}/bin/pushapkscript"
+
+ $user = $users::builder::username
+ $group = $users::builder::group
+
+ $taskcluster_client_id = $_env_config['taskcluster_client_id']
+ $taskcluster_access_token = $_env_config['taskcluster_access_token']
+ $worker_group = $_env_config['worker_group']
+ $worker_type = $_env_config['worker_type']
+
+ $google_play_config = $_env_config['google_play_config']
+
+ $jarsigner_keystore = "${root}/mozilla-android-keystore"
+ $jarsigner_keystore_password = secret('pushapk_scriptworker_jarsigner_keystore_password')
+
+ $jarsigner_nightly_certificate = "${root}/nightly.cer"
+ $jarsigner_nightly_certificate_alias = 'nightly'
+
+ $jarsigner_release_certificate = "${root}/release.cer"
+ $jarsigner_release_certificate_alias = 'release'
+
+ $verbose_logging = $_env_config['verbose_logging']
+
}
deleted file mode 100644
--- a/modules/pushapkworker/templates/config.json.erb
+++ /dev/null
@@ -1,30 +0,0 @@
-{
- "provisioner_id": "<%= @env_config['provisioner_id'] %>",
- "worker_group": "<%= @env_config['worker_group'] %>",
- "worker_type": "<%= @env_config['worker_type'] %>",
-<% if @env_config['worker_id'] %>
- "worker_id": "<%= @env_config['worker_id'] %>",
-<% else %>
- "worker_id": "<%= @hostname %>",
-<% end %>
-
- "work_dir": "<%= scope.lookupvar('config::pushapk_scriptworker_root') %>/work",
- "log_dir": "<%= scope.lookupvar('config::pushapk_scriptworker_root') %>/logs",
- "artifact_dir": "<%= scope.lookupvar('config::pushapk_scriptworker_root') %>/artifacts",
- "task_log_dir": "<%= scope.lookupvar('config::pushapk_scriptworker_root') %>/artifacts/public/logs",
- "valid_artifact_path_regexes": ["^/v1/task/(?P<taskId>[^/]+)(/runs/\\d+)?/artifacts/(?P<filepath>.*)$"],
- "verify_chain_of_trust": false,
- "sign_chain_of_trust": false,
-
- "credentials": {
- "clientId": "<%= @env_config['taskcluster_client_id'] %>",
- "accessToken": "<%= @env_config['taskcluster_access_token'] %>"
- },
-
- "artifact_expiration_hours": <%= scope.lookupvar('config::pushapk_scriptworker_artifact_expiration_hours') %>,
- "artifact_upload_timeout": <%= scope.lookupvar('config::pushapk_scriptworker_artifact_upload_timeout') %>,
- "task_script": ["<%= scope.lookupvar('config::pushapk_scriptworker_root') %>/bin/pushapkscript", "<%= scope.lookupvar('config::pushapk_scriptworker_script_config') %>" ],
- "task_max_timeout": <%= scope.lookupvar('config::pushapk_scriptworker_task_max_timeout') %>,
-
- "verbose": <%= @env_config['verbose_logging'] %>
-}
deleted file mode 100644
--- a/modules/pushapkworker/templates/nagios.cfg.erb
+++ /dev/null
@@ -1,1 +0,0 @@
-command[check_pushapkworker]=<%= scope.lookupvar('nrpe::base::plugins_dir') %>/check_procs -c 1:1 -C pushapkworker
--- a/modules/pushapkworker/templates/script_config.json.erb
+++ b/modules/pushapkworker/templates/script_config.json.erb
@@ -1,27 +1,27 @@
{
- "work_dir": "<%= scope.lookupvar('config::pushapk_scriptworker_root') %>/work",
- "schema_file": "<%= scope.lookupvar('config::pushapk_scriptworker_root') %>/lib/python3.5/site-packages/pushapkscript/data/pushapk_task_schema.json",
- "verbose": <%= @env_config['verbose_logging'] %>,
+ "work_dir": "<%= scope['pushapkworker::settings::work_dir'] %>",
+ "schema_file": "<%= scope['pushapkworker::settings::schema_file'] %>",
+ "verbose": <%= scope['pushapkworker::settings::verbose_logging'] %>,
"google_play_accounts": {
"aurora": {
"service_account": "<%= @google_play_config['aurora']['service_account'] %>",
"certificate": "<%= @google_play_config['aurora']['certificate_target_location'] %>"
},
"beta": {
"service_account": "<%= @google_play_config['beta']['service_account'] %>",
"certificate": "<%= @google_play_config['beta']['certificate_target_location'] %>"
},
"release": {
"service_account": "<%= @google_play_config['release']['service_account'] %>",
"certificate": "<%= @google_play_config['release']['certificate_target_location'] %>"
}
},
- "jarsigner_key_store": "<%= scope.lookupvar('config::pushapk_scriptworker_jarsigner_keystore') %>",
+ "jarsigner_key_store": "<%= scope['pushapkworker::settings::jarsigner_keystore'] %>",
"jarsigner_certificate_aliases": {
- "aurora": "<%= scope.lookupvar('config::pushapk_scriptworker_jarsigner_nightly_certificate_alias') %>",
- "beta": "<%= scope.lookupvar('config::pushapk_scriptworker_jarsigner_nightly_certificate_alias') %>",
- "release": "<%= scope.lookupvar('config::pushapk_scriptworker_jarsigner_release_certificate_alias') %>"
+ "aurora": "<%= scope['pushapkworker::settings::jarsigner_nightly_certificate_alias'] %>",
+ "beta": "<%= scope['pushapkworker::settings::jarsigner_nightly_certificate_alias'] %>",
+ "release": "<%= scope['pushapkworker::settings::jarsigner_release_certificate_alias'] %>"
}
}
deleted file mode 100644
--- a/modules/pushapkworker/templates/supervisor_config.erb
+++ /dev/null
@@ -1,6 +0,0 @@
-log_stderr=true
-log_stdout=true
-redirect_stderr=true
-stdout_logfile=syslog
-autorestart=true
-autostart=true
new file mode 100644
--- /dev/null
+++ b/modules/scriptworker/lib/puppet/parser/functions/validate_taskcluster_identifier.rb
@@ -0,0 +1,10 @@
+module Puppet::Parser::Functions
+ newfunction(:validate_taskcluster_identifier) do |args|
+ # TaskCluster has a 22-character-limit on some fields. See
+ # https://github.com/taskcluster/taskcluster-queue/blob/d45c29675461ceff3a38c6881edfe700855f4517/schemas/constants.yml#L23
+ identifier_value = args[0]
+ raise ArgumentError, "TaskCluster identifier must be a String" unless identifier_value.kind_of? String
+ raise ArgumentError, "TaskCluster identifier must not be empty" unless !identifier_value.empty?
+ raise ArgumentError, "Invalid TaskCluster identifier: '#{identifier_value}' contains more than 22 characters." unless identifier_value.length <= 22
+ end
+end
new file mode 100644
--- /dev/null
+++ b/modules/scriptworker/manifests/chain_of_trust.pp
@@ -0,0 +1,72 @@
+define scriptworker::chain_of_trust(
+ $basedir,
+
+ $git_key_repo_dir,
+ $git_key_repo_url,
+ $git_pubkey_dir,
+
+ $pubkey,
+ $privkey,
+
+ $username,
+) {
+ # This git repo has the various worker pubkeys
+ git::repo {
+ "scriptworker-${git_key_repo_dir}":
+ repo => $git_key_repo_url,
+ dst_dir => $git_key_repo_dir,
+ user => $username,
+ require => Python35::Virtualenv[$basedir];
+ }
+
+ File {
+ ensure => present,
+ mode => 600,
+ owner => $username,
+ group => $group,
+ show_diff => false,
+ }
+
+ file {
+ # $username's gpg homedir: for git commit signature verification
+ "/home/${username}/.gnupg":
+ ensure => directory;
+ # these are the pubkeys that can sign git commits
+ $git_pubkey_dir:
+ ensure => directory,
+ source => 'puppet:///modules/scriptworker/git_pubkeys',
+ recurse => true,
+ recurselimit => 1,
+ purge => true,
+ require => Python35::Virtualenv[$basedir];
+ # cron jobs to poll git + rebuild gpg homedirs
+ "/etc/cron.d/scriptworker":
+ content => template("scriptworker/scriptworker.cron.erb");
+ # Notify rebuild_gpg_homedirs if the pubkey dir changes
+ "${basedir}/.git-pubkey-dir-checksum":
+ notify => Exec['rebuild_gpg_homedirs'];
+ "/home/${username}/pubkey":
+ mode => 644,
+ content => $pubkey,
+ show_diff => true;
+ "/home/${username}/privkey":
+ content => $privkey;
+ }
+
+ exec {
+ # create gpg homedirs on change
+ 'rebuild_gpg_homedirs':
+ require => [Python35::Virtualenv[$basedir],
+ Git::Repo["scriptworker-${git_key_repo_dir}"],
+ File["${basedir}/scriptworker.yaml"]],
+ command => "${basedir}/bin/rebuild_gpg_homedirs ${basedir}/scriptworker.yaml",
+ subscribe => File[$git_pubkey_dir],
+ user => $username;
+ # Create checksum file of git pubkeys
+ "${basedir}/.git-pubkey-dir-checksum":
+ require => File[$git_pubkey_dir],
+ path => "/usr/local/bin/:/bin:/usr/sbin:/usr/bin",
+ user => $username,
+ command => "find ${git_pubkey_dir} -type f | xargs md5sum | sort > ${basedir}/.git-pubkey-dir-checksum";
+ }
+}
--- a/modules/scriptworker/manifests/instance.pp
+++ b/modules/scriptworker/manifests/instance.pp
@@ -1,117 +1,115 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Handle installing Python virtualenvs containing Python packages.
# https://wiki.mozilla.org/ReleaseEngineering/Puppet/Modules/python
define scriptworker::instance(
- $basedir, $task_script_executable, $task_script, $task_script_config,
- $username, $group, $worker_group, $worker_type, $cot_job_type,
- $taskcluster_client_id, $taskcluster_access_token,
- $task_max_timeout=1200, $artifact_expiration_hours=336,
- $artifact_upload_timeout=1200, $verbose_logging=false,
- $sign_chain_of_trust=true, $verify_chain_of_trust=true,
- $verify_cot_signature=true
+ $instance_name,
+ $basedir,
+ $work_dir = "${basedir}/work",
+
+ $script_worker_config = "${basedir}/scriptworker.yaml",
+ $task_script_executable = "${basedir}/bin/python",
+ $task_script,
+ $task_script_config = "${basedir}/script_config.json",
+
+ $username,
+ $group,
+
+ $taskcluster_client_id,
+ $taskcluster_access_token,
+ $worker_group,
+ $worker_type,
+ $worker_id = $hostname,
+ $task_max_timeout = 1200,
+ $artifact_expiration_hours = 336,
+ $artifact_upload_timeout = 1200,
+
+ $cot_job_type,
+ $sign_chain_of_trust = true,
+ $verify_chain_of_trust = true,
+ $verify_cot_signature = true,
+
+ $verbose_logging = false,
+
+ $restart_process_when_changed = undef,
) {
include scriptworker::instance::settings
include packages::mozilla::git
include packages::mozilla::supervisor
- # some constants
+ # These constants need to be filled in $script_worker_config, even though Chain of Trust is not enabled.
$git_key_repo_dir = "${basedir}/gpg_key_repo/"
$git_pubkey_dir = "${basedir}/git_pubkeys/"
- # This git repo has the various worker pubkeys
- git::repo {
- "scriptworker-${git_key_repo_dir}":
- repo => "${scriptworker::instance::settings::git_key_repo_url}",
- dst_dir => $git_key_repo_dir,
- user => "${username}",
- require => Python35::Virtualenv["${basedir}"];
+ validate_taskcluster_identifier($worker_group)
+ validate_taskcluster_identifier($worker_type)
+ # Hostname may be longer than 22 characters. Getting an error is painful especially in dev environments.
+ # That's why we strip worker_id if the default value (aka hostname) is used.
+ if $worker_id == $hostname {
+ $sanitized_worker_id = regsubst($hostname, '^.*(.{22})$', '\1')
+ if $sanitized_worker_id != $worker_id {
+ notify {
+ "Hostname '${hostname}' too long! worker_id has been stripped to '${sanitized_worker_id}'":
+ loglevel => warning,
+ }
+ }
+ } else {
+ validate_taskcluster_identifier($worker_id)
+ $sanitized_worker_id = $worker_id
}
- nrpe::custom {
- "scriptworker.cfg":
- content => template("scriptworker/nagios.cfg.erb");
+ # XXX Workaround to have arrays as default values
+ if $restart_process_when_changed == undef {
+ $_restart_process_when_changed = [Python35::Virtualenv[$basedir], File[$task_script_config]]
+ } else {
+ $_restart_process_when_changed = $restart_process_when_changed
+ }
+
+
+ File {
+ ensure => present,
+ mode => 600,
+ owner => $username,
+ group => $group,
+ show_diff => false,
}
file {
- # scriptworker config
- "${basedir}/scriptworker.yaml":
- require => Python35::Virtualenv["${basedir}"],
- mode => 600,
- owner => "${username}",
- group => "${group}",
- content => template("scriptworker/scriptworker.yaml.erb"),
- show_diff => false;
+ $script_worker_config:
+ require => Python35::Virtualenv[$basedir],
+ content => template("scriptworker/scriptworker.yaml.erb");
# cleanup per bug 1298199
'/root/certs.sh':
ensure => absent;
- # $username's gpg homedir: for git commit signature verification
- "/home/${username}/.gnupg":
- ensure => directory,
- mode => 700,
- owner => "${username}",
- group => "${group}";
- # these are the pubkeys that can sign git commits
- "${git_pubkey_dir}":
- ensure => directory,
- mode => 700,
- owner => "${username}",
- group => "${group}",
- source => 'puppet:///modules/scriptworker/git_pubkeys',
- recurse => true,
- recurselimit => 1,
- purge => true,
- require => Python35::Virtualenv["${basedir}"];
- # cron jobs to poll git + rebuild gpg homedirs
- "/etc/cron.d/scriptworker":
- content => template("scriptworker/scriptworker.cron.erb");
- # Notify rebuild_gpg_homedirs if the pubkey dir changes
- "${basedir}/.git-pubkey-dir-checksum":
- owner => "${username}",
- group => "${group}",
- notify => Exec['rebuild_gpg_homedirs'];
- "/home/${username}/pubkey":
- mode => 644,
- content => $config::scriptworker_gpg_public_keys[$fqdn],
- owner => "${username}",
- group => "${group}";
- "/home/${username}/privkey":
- mode => 600,
- content => $config::scriptworker_gpg_private_keys[$fqdn],
- owner => "${username}",
- group => "${group}",
- show_diff => false;
- "${nrpe::base::plugins_dir}/nagios_file_age_check.py":
- require => Python35::Virtualenv["${basedir}"],
- mode => 750,
- owner => "${username}",
- group => "${group}",
- source => "puppet:///modules/scriptworker/nagios_file_age_check.py",
- show_diff => false;
- "${nrpe::base::plugins_dir}/nagios_pending_tasks.py":
- require => Python35::Virtualenv["${basedir}"],
- mode => 750,
- owner => "${username}",
- group => "${group}",
- content => template("scriptworker/nagios_pending_tasks.py.erb"),
- show_diff => false;
+ }
+
+ scriptworker::supervisord { $instance_name:
+ instance_name => $instance_name,
+ basedir => $basedir,
+ script_worker_config => $script_worker_config,
+ task_script_config => $task_script_config,
+ username => $username,
+ restart_process_when_changed => $_restart_process_when_changed,
+ }
+
+ scriptworker::nagios { $instance_name:
+ basedir => $basedir,
}
- exec {
- # create gpg homedirs on change
- 'rebuild_gpg_homedirs':
- require => [Python35::Virtualenv["${basedir}"],
- Git::Repo["scriptworker-${git_key_repo_dir}"],
- File["${basedir}/scriptworker.yaml"]],
- command => "${basedir}/bin/rebuild_gpg_homedirs ${basedir}/scriptworker.yaml",
- subscribe => File["${git_pubkey_dir}"],
- user => "${username}";
- # Create checksum file of git pubkeys
- "${basedir}/.git-pubkey-dir-checksum":
- require => File["${git_pubkey_dir}"],
- path => "/usr/local/bin/:/bin:/usr/sbin:/usr/bin",
- user => "${username}",
- command => "find ${git_pubkey_dir} -type f | xargs md5sum | sort > ${basedir}/.git-pubkey-dir-checksum";
+ # Activate Chain Of Trust
+ if $sign_chain_of_trust or $verify_chain_of_trust or $verify_cot_signature {
+ scriptworker::chain_of_trust { $instance_name:
+ basedir => $basedir,
+
+ git_key_repo_dir => $git_key_repo_dir,
+ git_key_repo_url => $scriptworker::instance::settings::git_key_repo_url,
+ git_pubkey_dir => $git_pubkey_dir,
+
+ pubkey => $config::scriptworker_gpg_public_keys[$fqdn],
+ privkey => $config::scriptworker_gpg_private_keys[$fqdn],
+
+ username => $username,
+ }
}
}
new file mode 100644
--- /dev/null
+++ b/modules/scriptworker/manifests/nagios.pp
@@ -0,0 +1,21 @@
+define scriptworker::nagios(
+ $basedir,
+) {
+
+ nrpe::custom {
+ "scriptworker.cfg":
+ content => template("scriptworker/nagios.cfg.erb");
+ }
+
+ File {
+ require => Python35::Virtualenv[$basedir],
+ mode => 750,
+ }
+
+ file {
+ "${nrpe::base::plugins_dir}/nagios_file_age_check.py":
+ source => "puppet:///modules/scriptworker/nagios_file_age_check.py";
+ "${nrpe::base::plugins_dir}/nagios_pending_tasks.py":
+ content => template("scriptworker/nagios_pending_tasks.py.erb");
+ }
+}
new file mode 100644
--- /dev/null
+++ b/modules/scriptworker/manifests/supervisord.pp
@@ -0,0 +1,23 @@
+define scriptworker::supervisord(
+ $instance_name,
+ $basedir,
+ $script_worker_config,
+ $task_script_config,
+ $username,
+ $restart_process_when_changed,
+) {
+ supervisord::supervise {
+ $instance_name:
+ command => "${basedir}/bin/scriptworker ${script_worker_config}",
+ user => $username,
+ require => $restart_process_when_changed,
+ extra_config => template("${module_name}/supervisor_config.erb");
+ }
+
+ exec {
+ "restart-${instance_name}":
+ command => "/usr/bin/supervisorctl restart ${instance_name}",
+ refreshonly => true,
+ subscribe => $restart_process_when_changed;
+ }
+}
--- a/modules/scriptworker/templates/scriptworker.yaml.erb
+++ b/modules/scriptworker/templates/scriptworker.yaml.erb
@@ -1,12 +1,12 @@
provisioner_id: <%= scope["scriptworker::instance::settings::provisioner_id"] %>
worker_group: <%= @worker_group %>
worker_type: <%= @worker_type %>
-worker_id: <%= @hostname %>
+worker_id: <%= @sanitized_worker_id %>
credentials:
clientId: <%= @taskcluster_client_id %>
accessToken: <%= @taskcluster_access_token %>
artifact_expiration_hours: <%= @artifact_expiration_hours %>
artifact_upload_timeout: <%= @artifact_upload_timeout %>
task_max_timeout: <%= @task_max_timeout %>
@@ -14,17 +14,17 @@ task_script: ["<%= @task_script_executab
verbose: <%= @verbose_logging %>
sign_chain_of_trust: <%= @sign_chain_of_trust %>
verify_chain_of_trust: <%= @verify_chain_of_trust %>
verify_cot_signature: <%= @verify_cot_signature %>
cot_job_type: <%= @cot_job_type %>
log_dir: <%= @basedir %>/logs
-work_dir: <%= @basedir %>/work
+work_dir: <%= @work_dir %>
artifact_dir: <%= @basedir %>/artifacts
task_log_dir: <%= @basedir %>/artifacts/public/logs
base_gpg_home_dir: <%= @basedir %>/gpg
gpg_lockfile: <%= @basedir %>/.gpg_homedirs.lock
git_key_repo_dir: <%= @git_key_repo_dir %>
git_key_repo_url: <%= scope["scriptworker::instance::settings::git_key_repo_url"] %>
git_commit_signing_pubkey_dir: <%= @git_pubkey_dir %>
last_good_git_revision_file: <%= @basedir %>/.git_revision
new file mode 100644
--- /dev/null
+++ b/modules/scriptworker/templates/supervisor_config.erb
@@ -0,0 +1,6 @@
+log_stderr=true
+log_stdout=true
+redirect_stderr=true
+stdout_logfile=syslog
+autorestart=true
+autostart=true
--- a/modules/signing_scriptworker/manifests/init.pp
+++ b/modules/signing_scriptworker/manifests/init.pp
@@ -1,11 +1,10 @@
class signing_scriptworker {
include ::config
- include signing_scriptworker::services
include signing_scriptworker::settings
include dirs::builds
include packages::mozilla::python35
include users::signer
include tweaks::swap_on_instance_storage
include packages::gcc
include packages::make
@@ -48,29 +47,34 @@ class signing_scriptworker {
"taskcluster==0.3.4",
"virtualenv==15.0.3",
"yarl==0.7.0",
];
}
scriptworker::instance {
"${signing_scriptworker::settings::root}":
- basedir => "${signing_scriptworker::settings::root}",
- task_script_executable => "${signing_scriptworker::settings::task_script_executable}",
- task_script => "${signing_scriptworker::settings::task_script}",
- task_script_config => "${signing_scriptworker::settings::task_script_config}",
- task_max_timeout => $signing_scriptworker::settings::task_max_timeout,
- username => "${users::signer::username}",
- group => "${users::signer::group}",
- worker_group => "${signing_scriptworker::settings::worker_group}",
- worker_type => "${signing_scriptworker::settings::worker_type}",
+ instance_name => $module_name,
+ basedir => $signing_scriptworker::settings::root,
+
+ task_script => $signing_scriptworker::settings::task_script,
+ task_script_config => $signing_scriptworker::settings::task_script_config,
+
+ username => $users::signer::username,
+ group => $users::signer::group,
+
+ taskcluster_client_id => secret("signing_scriptworker_taskcluster_client_id"),
+ taskcluster_access_token => secret("signing_scriptworker_taskcluster_access_token"),
+ worker_group => $signing_scriptworker::settings::worker_group,
+ worker_type => $signing_scriptworker::settings::worker_type,
+ task_max_timeout => 1800,
+
cot_job_type => "signing",
+
verbose_logging => $verbose_logging,
- taskcluster_client_id => secret("signing_scriptworker_taskcluster_client_id"),
- taskcluster_access_token => secret("signing_scriptworker_taskcluster_access_token");
}
nrpe::custom {
"signingworker.cfg":
content => template("${module_name}/nagios.cfg.erb");
}
file {
deleted file mode 100644
--- a/modules/signing_scriptworker/manifests/services.pp
+++ /dev/null
@@ -1,22 +0,0 @@
-class signing_scriptworker::services {
- include ::config
- include signing_scriptworker::settings
- include packages::mozilla::supervisor
-
- supervisord::supervise {
- "signing_scriptworker":
- command => "${signing_scriptworker::settings::root}/bin/scriptworker ${signing_scriptworker::settings::root}/scriptworker.yaml",
- user => $::config::signer_username,
- require => [ File["${signing_scriptworker::settings::root}/scriptworker.yaml"],
- File["${signing_scriptworker::settings::root}/passwords.json"]],
- extra_config => template("${module_name}/supervisor_config.erb");
- }
- exec {
- "restart-scriptworker":
- command => "/usr/bin/supervisorctl restart signing_scriptworker",
- refreshonly => true,
- subscribe => [Python35::Virtualenv["${signing_scriptworker::settings::root}"],
- File["${signing_scriptworker::settings::root}/scriptworker.yaml"],
- File["${signing_scriptworker::settings::root}/passwords.json"]];
- }
-}
--- a/modules/signing_scriptworker/manifests/settings.pp
+++ b/modules/signing_scriptworker/manifests/settings.pp
@@ -1,13 +1,12 @@
class signing_scriptworker::settings {
include ::config
$root = $config::scriptworker_root
$worker_group = "signing-linux-v1"
$worker_type = "signing-linux-v1"
$taskcluster_client_id = secret("signing_scriptworker_taskcluster_client_id")
$taskcluster_access_token = secret("signing_scriptworker_taskcluster_access_token")
- $task_script_executable = "${root}/bin/python"
$task_script = "${root}/bin/signingscript"
$task_script_config = "${root}/script_config.json"
$task_max_timeout = 1800
}
deleted file mode 100644
--- a/modules/signing_scriptworker/templates/supervisor_config.erb
+++ /dev/null
@@ -1,6 +0,0 @@
-log_stderr=true
-log_stdout=true
-redirect_stderr=true
-stdout_logfile=syslog
-autorestart=true
-autostart=true
