--- a/manifests/moco-config.pp
+++ b/manifests/moco-config.pp
@@ -405,58 +405,16 @@ class config inherits config::base {
$signingworker_exchange = "exchange/taskcluster-queue/v1/task-pending"
$signingworker_worker_type = "signing-worker-v1"
# scriptworker
$scriptworker_root = "/builds/scriptworker"
$scriptworker_gpg_private_keys = hiera_hash('scriptworker_gpg_private_keys')
$scriptworker_gpg_public_keys = hiera_hash('scriptworker_gpg_public_keys')
- # TC balrog scriptworkers
- $balrog_scriptworker_task_max_timeout = 1200
- $balrog_scriptworker_artifact_expiration_hours = 336
- $balrog_scriptworker_artifact_upload_timeout = 600
- $balrog_scriptworker_verbose_logging = false
- $balrog_scriptworker_root = "/builds/balrogworker"
- $balrog_scriptworker_git_balrogscript_repo = 'https://github.com/mozilla-releng/balrogscript.git'
- $balrog_scriptworker_hg_tools_repo = 'https://hg.mozilla.org/build/tools'
- $balrog_scriptworker_hg_tools_branch = 'default'
- $balrog_scriptworker_env_config = {
- "dev" => {
- provisioner_id => "scriptworker-prov-v1",
- worker_group => "balrogworker-v1",
- worker_type => "balrogworker-v1",
- taskcluster_client_id => secret("balrogworker_dev_taskcluster_client_id"),
- taskcluster_access_token => secret("balrogworker_dev_taskcluster_access_token"),
- balrog_username => "stage-ffxbld",
- balrog_password => secret("stage-ffxbld_ldap_password"),
- balrog_api_root => "https://balrog-admin.stage.mozaws.net/api",
- }
- }
-
- # TC beetmover scriptworkers
- $beetmover_scriptworker_task_max_timeout = 2400
- $beetmover_scriptworker_artifact_expiration_hours = 336
- $beetmover_scriptworker_artifact_upload_timeout = 600
- $beetmover_scriptworker_verbose_logging = false
- $beetmover_scriptworker_root = "/builds/beetmoverworker"
- $beetmover_scriptworker_env_config = {
- "dev" => {
- provisioner_id => "scriptworker-prov-v1",
- worker_group => "beetmoverworker-v1",
- worker_type => "beetmoverworker-v1",
- taskcluster_client_id => secret("beetmoverworker_dev_taskcluster_client_id"),
- taskcluster_access_token => secret("beetmoverworker_dev_taskcluster_access_token"),
- beetmover_aws_access_key_id => secret("nightly-beetmover-aws_access_key_id"),
- beetmover_aws_secret_access_key => secret("nightly-beetmover-aws_secret_access_key"),
- beetmover_aws_s3_firefox_bucket => "net-mozaws-prod-delivery-firefox",
- beetmover_aws_s3_fennec_bucket => "net-mozaws-prod-delivery-archive",
- }
- }
-
## TC pushapk scriptworkers
$pushapk_scriptworker_old_root = '/builds/pushapkworker' # TODO Remove this line once bug 1321513 reaches production
$pushapk_scriptworker_root = $scriptworker_root
$pushapk_scriptworker_worker_config = "${pushapk_scriptworker_root}/config.json"
$pushapk_scriptworker_script_config = "${pushapk_scriptworker_root}/script_config.json"
$pushapk_scriptworker_jarsigner_keystore = "${pushapk_scriptworker_root}/mozilla-android-keystore"
$pushapk_scriptworker_jarsigner_nightly_certificate_alias = 'nightly'
deleted file mode 100644
--- a/modules/balrog_scriptworker/files/dep.pubkey
+++ /dev/null
@@ -1,9 +0,0 @@
------BEGIN PUBLIC KEY-----
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzjHSobdeiQ3JHP/cCIOp
-WaX9y12rL5mIo9OR9bpqEZdD0yXJJJeZA887Mv8slqsM+qObMUpKvfEE6zyYPIZJ
-ANib31neI5BBYHhfhf2f5EnkilSYlmU3Gx+uRsmsdt58PpYe124tOAGgca/8bUy3
-eb6kUUTwvMI0oWQuPkGUaoHVQyj/bBMTrIkyF3UbfFtiX/SfOPvIoabNUe+pQHUe
-pqC2+RxzDGj+shTq/hYhtXlptFzsEEb2+0foLy0MY8C30dP2QqbM2iavvr/P8OcS
-Gm3H0TQcRzIEBzvPcIjiZi1nQj/r/3TlYRNCjuYT/HsNLXrB/U5Tc990jjAUJxdH
-0wIDAQAB
------END PUBLIC KEY-----
deleted file mode 100644
--- a/modules/balrog_scriptworker/files/nightly.pubkey
+++ /dev/null
@@ -1,9 +0,0 @@
------BEGIN PUBLIC KEY-----
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4j/IS3gWbyVVnWn4ZRnC
-Fuzb6VAaHa0I+4E504ekhVAhbKlSfBstkLbXajdjUVAJpn02zWnOaTl5KAdpDpIp
-SkdA4mK20ej3/Ij7gIt8IwaX+ArXL8mP84pxDn5BgaNADm3206Z6YQzc/TDYu529
-qkDFmLqNUVRJAhPO+qqhKHIcVGh8HUHXN6XV1qOFip+UU0M474jAGgurVmAv8Rh7
-VvM0v5KmB6V6WHwM5gwjg2yRY/o+xYIsNeSes9rpp+MOs/RnUA6LI4WZGY4YahvX
-VclIXBDgbWPYtojexIJkmYj8JIIRsh3eCsrRRe14fq7cBurp3CxBYMlDHf0RUoaq
-hQIDAQAB
------END PUBLIC KEY-----
deleted file mode 100644
--- a/modules/balrog_scriptworker/files/release.pubkey
+++ /dev/null
@@ -1,9 +0,0 @@
------BEGIN PUBLIC KEY-----
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvH4r94FpQ0gvr1hhTfV9
-NUeWPJ5CN6TZRq7v/Dc4nkJ1J4IP1B3UEii34tcNKpy1nKupiZuTT6T1zQYT+z5x
-3UkDF9qQboQ8RNb/BEz/cN3on/LTEnZ7YSraRL11M6cEB8mvmJxddCEquwqccRbs
-Usp8WUB7uRv1w6Anley7N9F/LE1iLPwJasZypRnzWb3aYsJy0cMFOYy+OXVdpktn
-qYqlNIjnt84u4Nil6UXnBbIJNUVOCY8wOFClNvVpubjPkWK1gtdWy3x/hJU5RpAO
-K9cnHxq4M/I4SUWTWO3r7yweQiHG4Jyoc7sP1jkwjBkSG93sDEycfwOdOoZft3wN
-sQIDAQAB
------END PUBLIC KEY-----
--- a/modules/balrog_scriptworker/manifests/init.pp
+++ b/modules/balrog_scriptworker/manifests/init.pp
@@ -1,124 +1,116 @@
class balrog_scriptworker {
- include ::config
include balrog_scriptworker::services
include balrog_scriptworker::settings
include dirs::builds
include packages::mozilla::python35
include packages::mozilla::python27
- include packages::mozilla::git
include packages::mozilla::py27_mercurial
include users::builder
include tweaks::swap_on_instance_storage
include packages::gcc
include packages::make
include packages::libffi
- $env_config = $config::balrog_scriptworker_env_config[$balrogworker_env]
+ $env_config = $balrog_scriptworker::settings::env_config[$balrogworker_env]
python35::virtualenv {
"${balrog_scriptworker::settings::root}":
python3 => "${packages::mozilla::python35::python3}",
require => Class["packages::mozilla::python35"],
user => "${users::builder::username}",
group => "${users::builder::group}",
mode => 700,
packages => [
- "aiohttp==0.22.5",
+ "aiohttp==1.1.2",
"arrow==0.8.0",
+ "async-timeout==1.1.0",
"chardet==2.3.0",
"defusedxml==0.4.1",
"ecdsa==0.13",
- "frozendict==1.0",
- "future==0.15.2",
+ "frozendict==1.2",
+ "future==0.16.0",
"jsonschema==2.5.1",
"mohawk==0.3.3",
- "multidict==1.2.2",
+ "multidict==2.1.2",
"pefile==2016.7.26",
"pexpect==4.2.1",
"ptyprocess==0.5.1",
"pycrypto==2.6.1",
- "python-dateutil==2.5.3",
- "python-gnupg==0.3.8",
- "python-jose==1.2.0",
+ "python-dateutil==2.6.0",
+ "python-gnupg==0.3.9",
+ "python-jose==1.3.2",
+ "PyYAML==3.12",
"requests==2.11.1",
- "scriptworker==0.7.2",
+ "scriptworker==1.0.0b6",
"signtool==2.0.3",
"six==1.10.0",
"slugid==1.0.7",
"taskcluster==0.3.4",
"virtualenv==15.0.3",
+ "yarl==0.7.0",
];
}
python::virtualenv {
"${balrog_scriptworker::settings::root}/py27venv":
python => "${packages::mozilla::python27::python}",
require => Class["packages::mozilla::python27"],
user => "${users::builder::username}",
group => "${users::builder::group}",
packages => [
"balrogclient==0.0.1",
+ "balrogscript==0.0.4",
"boto==2.41.0",
+ "cffi==1.8.3",
"cryptography==1.2.3",
"enum34==1.1.2",
"idna==2.0",
"ipaddress==1.0.16",
+ "jsonschema==2.4.0",
"mar==1.2",
"pyasn1==0.1.9",
+ "pycparser==2.14",
"requests==2.8.1",
"six==1.10.0",
];
}
- git::repo {
- "balrogscript":
- repo => "${balrog_scriptworker::settings::balrogscript_repo}",
- dst_dir => "${balrog_scriptworker::settings::root}/balrogscript",
- user => "${users::builder::username}",
- require => [
- Class["packages::mozilla::git"],
- Python35::Virtualenv["${balrog_scriptworker::settings::root}"],
- ];
+ scriptworker::instance {
+ "${balrog_scriptworker::settings::root}":
+ basedir => "${balrog_scriptworker::settings::root}",
+ task_script_executable => "${balrog_scriptworker::settings::task_script_executable}",
+ task_script => "${balrog_scriptworker::settings::task_script}",
+ task_script_config => "${balrog_scriptworker::settings::task_script_config}",
+ task_max_timeout => $balrog_scriptworker::settings::task_max_timeout,
+ username => "${users::builder::username}",
+ group => "${users::builder::group}",
+ worker_group => "${balrog_scriptworker::settings::worker_group}",
+ worker_type => "${balrog_scriptworker::settings::worker_type}",
+ cot_job_type => "balrog",
+ verbose_logging => $balrog_scriptworker::settings::verbose_logging,
+ taskcluster_client_id => "${balrog_scriptworker::settings::taskcluster_client_id}",
+ taskcluster_access_token => "${balrog_scriptworker::settings::taskcluster_access_token}",
}
mercurial::repo {
"tools":
hg_repo => "${balrog_scriptworker::settings::tools_repo}",
- dst_dir => "${balrog_scriptworker::settings::root}/balrogscript/tools",
+ dst_dir => "${balrog_scriptworker::settings::root}/tools",
user => "${users::builder::username}",
branch => "${balrog_scriptworker::settings::tools_branch}",
require => [
Class["packages::mozilla::py27_mercurial"],
Python35::Virtualenv["${balrog_scriptworker::settings::root}"],
- Git::Repo["balrogscript"],
];
}
file {
- "${balrog_scriptworker::settings::root}/config.json":
+ "${balrog_scriptworker::settings::root}/script_config.json":
require => Python35::Virtualenv["${balrog_scriptworker::settings::root}"],
mode => 600,
owner => "${users::builder::username}",
group => "${users::builder::group}",
- content => template("${module_name}/config.json.erb"),
- show_diff => false;
- # requirement as part of scriptworker pentest bug 1298199#c23
- '/root/certs.sh':
- ensure => absent;
- "${balrog_scriptworker::settings::root}/balrogscript/keys/dep.pubkey":
- source => "puppet:///modules/balrog_scriptworker/dep.pubkey",
- require => Git::Repo["balrogscript"],
- owner => "${users::builder::username}",
- group => "${users::builder::group}";
- "${balrog_scriptworker::settings::root}/balrogscript/keys/nightly.pubkey":
- source => "puppet:///modules/balrog_scriptworker/nightly.pubkey",
- require => Git::Repo["balrogscript"],
- owner => "${users::builder::username}",
- group => "${users::builder::group}";
- "${balrog_scriptworker::settings::root}/balrogscript/keys/release.pubkey":
- source => "puppet:///modules/balrog_scriptworker/release.pubkey",
- require => Git::Repo["balrogscript"],
- owner => "${users::builder::username}",
- group => "${users::builder::group}";
+ content => template("${module_name}/script_config.json.erb"),
+ show_diff => true;
}
}
--- a/modules/balrog_scriptworker/manifests/services.pp
+++ b/modules/balrog_scriptworker/manifests/services.pp
@@ -1,20 +1,20 @@
class balrog_scriptworker::services {
include ::config
include balrog_scriptworker::settings
include packages::mozilla::supervisor
supervisord::supervise {
"balrog_scriptworker":
- command => "${balrog_scriptworker::settings::root}/bin/scriptworker ${balrog_scriptworker::settings::root}/config.json",
+ command => "${balrog_scriptworker::settings::root}/bin/scriptworker ${balrog_scriptworker::settings::root}/scriptworker.yaml",
user => $::config::builder_username,
- require => [ File["${balrog_scriptworker::settings::root}/config.json"]],
+ require => [ File["${balrog_scriptworker::settings::root}/scriptworker.yaml"]],
extra_config => template("${module_name}/supervisor_config.erb");
}
exec {
"restart-balrogworker":
command => "/usr/bin/supervisorctl restart balrog_scriptworker",
refreshonly => true,
subscribe => [Python35::Virtualenv["${balrog_scriptworker::settings::root}"],
- File["${balrog_scriptworker::settings::root}/config.json"]];
+ File["${balrog_scriptworker::settings::root}/scriptworker.yaml"]];
}
}
--- a/modules/balrog_scriptworker/manifests/settings.pp
+++ b/modules/balrog_scriptworker/manifests/settings.pp
@@ -1,8 +1,22 @@
class balrog_scriptworker::settings {
- include ::config
+ $root = "/builds/balrogworker"
+ $task_script_executable = "${root}/py27venv/bin/python"
+ $task_script = "${root}/py27venv/bin/balrogscript"
+ $task_script_config = "${root}/script_config.json"
+ $task_max_timeout = 1800
+ $tools_repo = 'https://hg.mozilla.org/build/tools'
+ $tools_branch = 'default'
+ $worker_group = "balrogworker-v1"
+ $worker_type = "balrogworker-v1"
+ $taskcluster_client_id = secret("balrogworker_dev_taskcluster_client_id")
+ $taskcluster_access_token = secret("balrogworker_dev_taskcluster_access_token")
+ $verbose_logging = true
- $root = $config::balrog_scriptworker_root
- $balrogscript_repo = $config::balrog_scriptworker_git_balrogscript_repo
- $tools_repo = $config::balrog_scriptworker_hg_tools_repo
- $tools_branch = $config::balrog_scriptworker_hg_tools_branch
+ $env_config = {
+ "dev" => {
+ balrog_username => "stage-ffxbld",
+ balrog_password => secret("stage-ffxbld_ldap_password"),
+ balrog_api_root => "https://balrog-admin.stage.mozaws.net/api"
+ }
+ }
}
deleted file mode 100644
--- a/modules/balrog_scriptworker/templates/config.json.erb
+++ /dev/null
@@ -1,34 +0,0 @@
-{
- "provisioner_id": "<%= @env_config["provisioner_id"] %>",
- "worker_group": "<%= @env_config["worker_group"] %>",
- "worker_type": "<%= @env_config["worker_type"] %>",
- "worker_id": "<%= @hostname %>",
-
- "work_dir": "<%= scope.lookupvar("config::balrog_scriptworker_root") %>/work",
- "log_dir": "<%= scope.lookupvar("config::balrog_scriptworker_root") %>/logs",
- "artifact_dir": "<%= scope.lookupvar("config::balrog_scriptworker_root") %>/artifacts",
- "task_log_dir": "<%= scope.lookupvar("config::balrog_scriptworker_root") %>/artifacts/public/logs",
- "valid_artifact_path_regexes": ["^/v1/task/(?P<taskId>[^/]+)(/runs/\\d+)?/artifacts/(?P<filepath>.*)$"],
- "verify_chain_of_trust": false,
- "sign_chain_of_trust": false,
-
-
- "credentials": {
- "clientId": "<%= @env_config["taskcluster_client_id"] %>",
- "accessToken": "<%= @env_config["taskcluster_access_token"] %>"
- },
-
- "artifact_expiration_hours": <%= scope.lookupvar("config::balrog_scriptworker_artifact_expiration_hours") %>,
- "artifact_upload_timeout": <%= scope.lookupvar("config::balrog_scriptworker_artifact_upload_timeout") %>,
- "task_script": ["<%= scope.lookupvar("config::balrog_scriptworker_root") %>/py27venv/bin/python",
- "<%= scope.lookupvar("config::balrog_scriptworker_root") %>/balrogscript/bin/balrogworker.py",
- "--taskdef", "<%= scope.lookupvar("config::balrog_scriptworker_root") %>/work/task.json",
- "--balrog-api-root", "<%= @env_config["balrog_api_root"] %>",
- "--balrog-username", "<%= @env_config["balrog_username"] %>",
- "--balrog-password", "<%= @env_config["balrog_password"] %>",
- "--disable-s3",
- "--verbose"],
-
- "task_max_timeout": <%= scope.lookupvar("config::balrog_scriptworker_task_max_timeout") %>,
- "verbose": <%= scope.lookupvar("config::balrog_scriptworker_verbose_logging") %>
-}
new file mode 100644
--- /dev/null
+++ b/modules/balrog_scriptworker/templates/script_config.json.erb
@@ -0,0 +1,16 @@
+{
+ "work_dir": "<%= scope.lookupvar("balrog_scriptworker::settings::root") %>/work",
+ "artifact_dir": "<%= scope.lookupvar("balrog_scriptworker::settings::root") %>/artifacts",
+
+ "schema_file": "<%= scope.lookupvar("balrog_scriptworker::settings::root") %>/py27venv/lib/python2.7/site-packages/balrogscript/data/balrog_task_schema.json",
+
+ "verbose": <%= scope.lookupvar("balrog_scriptworker::settings::verbose_logging") %>,
+ "dummy": false,
+ "disable_certs": false,
+
+ "api_root": "<%= @env_config["balrog_api_root"] %>",
+ "balrog_username": "<%= @env_config["balrog_username"] %>",
+ "balrog_password": "<%= @env_config["balrog_password"] %>",
+
+ "tools_location": "<%= scope.lookupvar("balrog_scriptworker::settings::root") %>/tools"
+}
--- a/modules/beetmover_scriptworker/manifests/init.pp
+++ b/modules/beetmover_scriptworker/manifests/init.pp
@@ -1,79 +1,86 @@
class beetmover_scriptworker {
- include ::config
include beetmover_scriptworker::services
include beetmover_scriptworker::settings
include dirs::builds
include packages::mozilla::python35
include users::builder
include tweaks::swap_on_instance_storage
include packages::gcc
include packages::make
include packages::libffi
- $env_config = $config::beetmover_scriptworker_env_config[$beetmoverworker_env]
+ $env_config = $beetmover_scriptworker::settings::env_config[$beetmoverworker_env]
python35::virtualenv {
"${beetmover_scriptworker::settings::root}":
python3 => "${packages::mozilla::python35::python3}",
require => Class["packages::mozilla::python35"],
user => "${users::builder::username}",
group => "${users::builder::group}",
mode => 700,
packages => [
- "aiohttp==0.22.5",
+ "aiohttp==1.1.2",
"arrow==0.8.0",
+ "async-timeout==1.1.0",
"chardet==2.3.0",
"defusedxml==0.4.1",
"ecdsa==0.13",
- "frozendict==1.0",
- "future==0.15.2",
+ "frozendict==1.2",
+ "future==0.16.0",
"jsonschema==2.5.1",
"mohawk==0.3.3",
- "multidict==1.2.2",
+ "multidict==2.1.2",
"pefile==2016.7.26",
"pexpect==4.2.1",
"ptyprocess==0.5.1",
"pycrypto==2.6.1",
- "python-dateutil==2.5.3",
- "python-gnupg==0.3.8",
- "python-jose==1.2.0",
+ "python-dateutil==2.6.0",
+ "python-gnupg==0.3.9",
+ "python-jose==1.3.2",
+ "PyYAML==3.12",
"requests==2.11.1",
- "scriptworker==0.7.2",
+ "scriptworker==1.0.0b6",
"signtool==2.0.3",
"six==1.10.0",
"slugid==1.0.7",
"taskcluster==0.3.4",
"virtualenv==15.0.3",
"boto3==1.4.1",
"botocore==1.4.63",
"jmespath==0.9.0",
- "PyYAML==3.12",
"Jinja2==2.8",
- "beetmoverscript==0.0.13",
+ "beetmoverscript==0.1.3",
"MarkupSafe==0.23",
"s3transfer==0.1.8",
"docutils==0.12",
+ "yarl==0.7.0",
+ ];
+ }
- ];
+ scriptworker::instance {
+ "${beetmover_scriptworker::settings::root}":
+ basedir => "${beetmover_scriptworker::settings::root}",
+ task_script_executable => "${beetmover_scriptworker::settings::task_script_executable}",
+ task_script => "${beetmover_scriptworker::settings::task_script}",
+ task_script_config => "${beetmover_scriptworker::settings::task_script_config}",
+ task_max_timeout => $beetmover_scriptworker::settings::task_max_timeout,
+ username => "${users::builder::username}",
+ group => "${users::builder::group}",
+ worker_group => "${beetmover_scriptworker::settings::worker_group}",
+ worker_type => "${beetmover_scriptworker::settings::worker_type}",
+ cot_job_type => "beetmover",
+ verbose_logging => $beetmover_scriptworker::settings::verbose_logging,
+ taskcluster_client_id => "${beetmover_scriptworker::settings::taskcluster_client_id}",
+ taskcluster_access_token => "${beetmover_scriptworker::settings::taskcluster_access_token}",
}
file {
- "${beetmover_scriptworker::settings::root}/config.json":
- require => Python35::Virtualenv["${beetmover_scriptworker::settings::root}"],
- mode => 600,
- owner => "${users::builder::username}",
- group => "${users::builder::group}",
- content => template("${module_name}/config.json.erb"),
- show_diff => false;
"${beetmover_scriptworker::settings::root}/script_config.json":
require => Python35::Virtualenv["${beetmover_scriptworker::settings::root}"],
mode => 600,
owner => "${users::builder::username}",
group => "${users::builder::group}",
content => template("${module_name}/script_config.json.erb"),
- show_diff => false;
- # requirement as part of scriptworker pentest bug 1298199#c23
- '/root/certs.sh':
- ensure => absent;
+ show_diff => true;
}
}
--- a/modules/beetmover_scriptworker/manifests/services.pp
+++ b/modules/beetmover_scriptworker/manifests/services.pp
@@ -1,20 +1,20 @@
class beetmover_scriptworker::services {
include ::config
include beetmover_scriptworker::settings
include packages::mozilla::supervisor
supervisord::supervise {
"beetmover_scriptworker":
- command => "${beetmover_scriptworker::settings::root}/bin/scriptworker ${beetmover_scriptworker::settings::root}/config.json",
+ command => "${beetmover_scriptworker::settings::root}/bin/scriptworker ${beetmover_scriptworker::settings::root}/scriptworker.yaml",
user => $::config::builder_username,
- require => [ File["${beetmover_scriptworker::settings::root}/config.json"]],
+ require => [ File["${beetmover_scriptworker::settings::root}/scriptworker.yaml"]],
extra_config => template("${module_name}/supervisor_config.erb");
}
exec {
"restart-beetmoverworker":
command => "/usr/bin/supervisorctl restart beetmover_scriptworker",
refreshonly => true,
subscribe => [Python35::Virtualenv["${beetmover_scriptworker::settings::root}"],
- File["${beetmover_scriptworker::settings::root}/config.json"]];
+ File["${beetmover_scriptworker::settings::root}/scriptworker.yaml"]];
}
}
--- a/modules/beetmover_scriptworker/manifests/settings.pp
+++ b/modules/beetmover_scriptworker/manifests/settings.pp
@@ -1,5 +1,22 @@
class beetmover_scriptworker::settings {
- include ::config
+ $root = "/builds/beetmoverworker"
+ $task_script_executable = "${root}/bin/python"
+ $task_script = "${root}/bin/beetmoverscript"
+ $task_script_config = "${root}/script_config.json"
+ $task_max_timeout = 1800
- $root = $config::beetmover_scriptworker_root
+ $worker_group = "beetmoverworker-v1"
+ $worker_type = "beetmoverworker-v1"
+ $taskcluster_client_id = secret("beetmoverworker_dev_taskcluster_client_id")
+ $taskcluster_access_token = secret("beetmoverworker_dev_taskcluster_access_token")
+ $verbose_logging = true
+
+ $env_config = {
+ "dev" => {
+ beetmover_aws_access_key_id => secret("nightly-beetmover-aws_access_key_id"),
+ beetmover_aws_secret_access_key => secret("nightly-beetmover-aws_secret_access_key"),
+ beetmover_aws_s3_firefox_bucket => "net-mozaws-prod-delivery-firefox",
+ beetmover_aws_s3_fennec_bucket => "net-mozaws-prod-delivery-archive",
+ }
+ }
}
deleted file mode 100644
--- a/modules/beetmover_scriptworker/templates/config.json.erb
+++ /dev/null
@@ -1,28 +0,0 @@
-{
- "provisioner_id": "<%= @env_config["provisioner_id"] %>",
- "worker_group": "<%= @env_config["worker_group"] %>",
- "worker_type": "<%= @env_config["worker_type"] %>",
- "worker_id": "<%= @hostname %>",
-
- "work_dir": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/work",
- "log_dir": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/logs",
- "artifact_dir": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/artifacts",
- "task_log_dir": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/artifacts/public/logs",
- "valid_artifact_path_regexes": ["^/v1/task/(?P<taskId>[^/]+)(/runs/\\d+)?/artifacts/(?P<filepath>.*)$"],
- "verify_chain_of_trust": false,
- "sign_chain_of_trust": false,
-
-
- "credentials": {
- "clientId": "<%= @env_config["taskcluster_client_id"] %>",
- "accessToken": "<%= @env_config["taskcluster_access_token"] %>"
- },
-
- "artifact_expiration_hours": <%= scope.lookupvar("config::beetmover_scriptworker_artifact_expiration_hours") %>,
- "artifact_upload_timeout": <%= scope.lookupvar("config::beetmover_scriptworker_artifact_upload_timeout") %>,
- "task_script": ["<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/bin/beetmoverscript",
- "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/script_config.json"],
-
- "task_max_timeout": <%= scope.lookupvar("config::beetmover_scriptworker_task_max_timeout") %>,
- "verbose": <%= scope.lookupvar("config::beetmover_scriptworker_verbose_logging") %>
-}
--- a/modules/beetmover_scriptworker/templates/script_config.json.erb
+++ b/modules/beetmover_scriptworker/templates/script_config.json.erb
@@ -1,23 +1,30 @@
{
- "work_dir": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/work",
- "artifact_dir": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/artifacts",
+ "work_dir": "<%= scope.lookupvar("beetmover_scriptworker::settings::root") %>/work",
+ "artifact_dir": "<%= scope.lookupvar("beetmover_scriptworker::settings::root") %>/artifacts",
+ "aiohttp_max_connections": 20,
+
+ "schema_file": "<%= scope.lookupvar("beetmover_scriptworker::settings::root") %>/lib/python3.5/site-packages/beetmoverscript/data/beetmover_task_schema.json",
+
+ "verbose": <%= scope.lookupvar("beetmover_scriptworker::settings::verbose_logging") %>,
+ "dummy": false,
+ "disable_certs": false,
"template_files": {
- "fennec_nightly_unsigned": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/lib/python3.5/site-packages/beetmoverscript/templates/fennec_nightly_en_us_multi_unsigned.yml",
- "fennec_nightly_signed": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/lib/python3.5/site-packages/beetmoverscript/templates/fennec_nightly_en_us_multi_signed.yml",
- "fennec_nightly_repacks_unsigned": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/lib/python3.5/site-packages/beetmoverscript/templates/fennec_nightly_repacks_unsigned.yml",
- "fennec_nightly_repacks_signed": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/lib/python3.5/site-packages/beetmoverscript/templates/fennec_nightly_repacks_signed.yml",
- "firefox_nightly_unsigned": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/lib/python3.5/site-packages/beetmoverscript/templates/firefox_nightly_en_us_unsigned.yml",
- "fennecx86_nightly_unsigned": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/lib/python3.5/site-packages/beetmoverscript/templates/fennecx86_nightly_en_us_multi_unsigned.yml",
- "fennecx86_nightly_signed": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/lib/python3.5/site-packages/beetmoverscript/templates/fennecx86_nightly_en_us_multi_signed.yml",
- "firefox_nightly_signed": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/lib/python3.5/site-packages/beetmoverscript/templates/firefox_nightly_en_us_signed.yml",
- "firefox_nightly_repacks_unsigned": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/lib/python3.5/site-packages/beetmoverscript/templates/firefox_nightly_repacks_unsigned.yml",
- "firefox_nightly_repacks_signed": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/lib/python3.5/site-packages/beetmoverscript/templates/firefox_nightly_repacks_signed.yml"
+ "fennec_nightly_unsigned": "<%= scope.lookupvar("beetmover_scriptworker::settings::root") %>/lib/python3.5/site-packages/beetmoverscript/templates/fennec_nightly_en_us_multi_unsigned.yml",
+ "fennec_nightly_signed": "<%= scope.lookupvar("beetmover_scriptworker::settings::root") %>/lib/python3.5/site-packages/beetmoverscript/templates/fennec_nightly_en_us_multi_signed.yml",
+ "fennec_nightly_repacks_unsigned": "<%= scope.lookupvar("beetmover_scriptworker::settings::root") %>/lib/python3.5/site-packages/beetmoverscript/templates/fennec_nightly_repacks_unsigned.yml",
+ "fennec_nightly_repacks_signed": "<%= scope.lookupvar("beetmover_scriptworker::settings::root") %>/lib/python3.5/site-packages/beetmoverscript/templates/fennec_nightly_repacks_signed.yml",
+ "firefox_nightly_unsigned": "<%= scope.lookupvar("beetmover_scriptworker::settings::root") %>/lib/python3.5/site-packages/beetmoverscript/templates/firefox_nightly_en_us_unsigned.yml",
+ "fennecx86_nightly_unsigned": "<%= scope.lookupvar("beetmover_scriptworker::settings::root") %>/lib/python3.5/site-packages/beetmoverscript/templates/fennecx86_nightly_en_us_multi_unsigned.yml",
+ "fennecx86_nightly_signed": "<%= scope.lookupvar("beetmover_scriptworker::settings::root") %>/lib/python3.5/site-packages/beetmoverscript/templates/fennecx86_nightly_en_us_multi_signed.yml",
+ "firefox_nightly_signed": "<%= scope.lookupvar("beetmover_scriptworker::settings::root") %>/lib/python3.5/site-packages/beetmoverscript/templates/firefox_nightly_en_us_signed.yml",
+ "firefox_nightly_repacks_unsigned": "<%= scope.lookupvar("beetmover_scriptworker::settings::root") %>/lib/python3.5/site-packages/beetmoverscript/templates/firefox_nightly_repacks_unsigned.yml",
+ "firefox_nightly_repacks_signed": "<%= scope.lookupvar("beetmover_scriptworker::settings::root") %>/lib/python3.5/site-packages/beetmoverscript/templates/firefox_nightly_repacks_signed.yml"
},
"s3": {
"firefox": {
"bucket": "<%= @env_config["beetmover_aws_s3_firefox_bucket"] %>",
"credentials": {
"id": "<%= @env_config["beetmover_aws_access_key_id"] %>",
"key": "<%= @env_config["beetmover_aws_secret_access_key"] %>"
@@ -25,13 +32,10 @@
},
"fennec": {
"bucket": "<%= @env_config["beetmover_aws_s3_fennec_bucket"] %>",
"credentials": {
"id": "<%= @env_config["beetmover_aws_access_key_id"] %>",
"key": "<%= @env_config["beetmover_aws_secret_access_key"] %>"
}
}
- },
-
- "schema_file": "<%= scope.lookupvar("config::beetmover_scriptworker_root") %>/lib/python3.5/site-packages/beetmoverscript/data/beetmover_task_schema.json",
- "verbose": <%= scope.lookupvar("config::beetmover_scriptworker_verbose_logging") %>
+ }
}