Bug 1436499 - Spin up a dev treescriptworker. r=aki draft production
authorJustin Wood <Callek@gmail.com>
Fri, 09 Feb 2018 10:37:23 -0500
branchproduction
changeset 6404 39d75d3654a4d3d9ca165a22afd6b2af9effb216
parent 6403 a8ea64cccfec744b85f25650329a3a9e5513717b
push id2709
push userCallek@gmail.com
push dateFri, 09 Feb 2018 17:05:15 +0000
reviewersaki
bugs1436499
Bug 1436499 - Spin up a dev treescriptworker. r=aki MozReview-Commit-ID: 8RHK0M68goW
manifests/moco-nodes.pp
modules/packages/manifests/mozilla/python27.pp
modules/toplevel/manifests/server/treescriptworker.pp
modules/tree_scriptworker/manifests/init.pp
modules/tree_scriptworker/manifests/settings.pp
modules/tree_scriptworker/templates/script_config.json.erb
--- a/manifests/moco-nodes.pp
+++ b/manifests/moco-nodes.pp
@@ -1004,17 +1004,17 @@ node /^shipitworker-.*\.srv\.releng\..*\
 
 # Treescript workers
 
 node /^treescriptworker-dev\d*\.srv\.releng\..*\.mozilla\.com$/ {
     $aspects          = [ 'maximum-security' ]
     $treescriptworker_env = 'dev'
     $timezone         = 'UTC'
     $only_user_ssh    = true
-    include toplevel::server
+    include toplevel::server::treescriptworker
 }
 
 
 ## Loaners
 
 node 'dhouse-1330169.srv.releng.scl3.mozilla.com' {
     $aspects = [ 'low-security' ]
     include toplevel::server
--- a/modules/packages/manifests/mozilla/python27.pp
+++ b/modules/packages/manifests/mozilla/python27.pp
@@ -41,17 +41,18 @@ class packages::mozilla::python27 {
             } -> Anchor['packages::mozilla::python27::end']
 
             case $::operatingsystem {
                 CentOS: {
                     # Bug 1307757 - Deploy python 2.7.12 to releng servers
                     # install Python 2.7.12 for 'buildduty-tools' and 'cruncher-aws' servers in order
                     # to silence the alerts caused by using on older Python version (2.7.3) when
                     # performing HG operation.
-                    if $::hostname in [ 'buildduty-tools', 'cruncher-aws', 'aws-manager1', 'aws-manager2' ] {
+                    if $::hostname in [ 'buildduty-tools', 'cruncher-aws', 'aws-manager1', 'aws-manager2',
+                        'treescriptworker-dev1', 'treescriptworker-1' ] {
                       realize(Packages::Yumrepo['python27-12'])
                       Anchor['packages::mozilla::python27::begin'] ->
                       package {
                           'mozilla-python27':
                               ensure => '2.7.12-1.el6';
                       } -> Anchor['packages::mozilla::python27::end']
                     }
                     else {
new file mode 100644
--- /dev/null
+++ b/modules/toplevel/manifests/server/treescriptworker.pp
@@ -0,0 +1,7 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class toplevel::server::treescriptworker inherits toplevel::server {
+    include ::tree_scriptworker
+}
new file mode 100644
--- /dev/null
+++ b/modules/tree_scriptworker/manifests/init.pp
@@ -0,0 +1,96 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class tree_scriptworker {
+    include tree_scriptworker::settings
+    include dirs::builds
+    include dirs::builds::hg_shared
+    include packages::mozilla::python35
+    include packages::mozilla::python27
+    include packages::mozilla::py27_mercurial
+    include users::builder
+    include tweaks::swap_on_instance_storage
+    include packages::gcc
+    include packages::make
+    include packages::libffi
+    include tweaks::scriptworkerlogrotate
+
+    $env_config = $tree_scriptworker::settings::env_config[$treescriptworker_env]
+
+    python35::virtualenv {
+        $tree_scriptworker::settings::root:
+            python3  => $packages::mozilla::python35::python3,
+            require  => Class['packages::mozilla::python35'],
+            user     => $users::builder::username,
+            group    => $users::builder::group,
+            mode     => '0700',
+            packages => [
+                  'PyYAML==3.12',
+                  'aiohttp==2.3.9',
+                  'arrow==0.12.1',
+                  'async_timeout==1.4.0',
+                  'certifi==2018.1.18',
+                  'chardet==3.0.4',
+                  'defusedxml==0.5.0',
+                  'dictdiffer==0.7.0',
+                  'frozendict==1.2',
+                  'idna==2.6',
+                  'json-e==2.5.0',
+                  'jsonschema==2.6.0',
+                  'mohawk==0.3.4',
+                  'multidict==4.1.0',
+                  'pexpect==4.3.1',
+                  'ptyprocess==0.5.2',
+                  'python-dateutil==2.6.1',
+                  'python-gnupg==0.4.1',
+                  'requests==2.18.4',
+                  'scriptworker==8.0.0',
+                  'treescript==0.1',
+                  'six==1.10.0',
+                  'slugid==1.0.7',
+                  'taskcluster==2.1.3',
+                  'urllib3==1.22',
+                  'virtualenv==15.1.0',
+                  'yarl==1.1.0',
+            ];
+    }
+
+    scriptworker::instance {
+        $tree_scriptworker::settings::root:
+            instance_name            => $module_name,
+            basedir                  => $tree_scriptworker::settings::root,
+
+            task_script_executable   => $tree_scriptworker::settings::task_script_executable,
+            task_script              => $tree_scriptworker::settings::task_script,
+            task_script_config       => $tree_scriptworker::settings::task_script_config,
+
+            username                 => $users::builder::username,
+            group                    => $users::builder::group,
+
+            taskcluster_client_id    => $env_config["taskcluster_client_id"],
+            taskcluster_access_token => $env_config["taskcluster_access_token"],
+            worker_group             => $tree_scriptworker::settings::worker_group,
+            worker_type              => $env_config["worker_type"],
+
+            task_max_timeout         => $tree_scriptworker::settings::task_max_timeout,
+
+            cot_job_type             => 'scriptworker',
+
+            sign_chain_of_trust      => $env_config["sign_chain_of_trust"],
+            verify_chain_of_trust    => $env_config["verify_chain_of_trust"],
+            verify_cot_signature     => $env_config["verify_cot_signature"],
+
+            verbose_logging          => $tree_scriptworker::settings::verbose_logging,
+    }
+
+    file {
+        "${tree_scriptworker::settings::root}/script_config.json":
+            require   => Python35::Virtualenv[$tree_scriptworker::settings::root],
+            mode      => '0600',
+            owner     => $users::builder::username,
+            group     => $users::builder::group,
+            content   => template("${module_name}/script_config.json.erb"),
+            show_diff => false;
+    }
+}
new file mode 100644
--- /dev/null
+++ b/modules/tree_scriptworker/manifests/settings.pp
@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class tree_scriptworker::settings {
+    $root                     = '/builds/scriptworker'
+    $task_script              = "${root}/bin/treescript"
+    $task_script_config       = "${root}/script_config.json"
+    $task_max_timeout         = 1800
+    $upstream_repo            = 'https://hg.mozilla.org/mozilla-unified'
+    $share_base               = '/builds/hg-shared'
+    $worker_group             = 'treescriptworker-v1'
+    $verbose_logging          = true
+
+    $env_config = {
+        'dev' => {
+            taskcluster_client_id => 'project/releng/scriptworker/treescriptworker-dev',
+            taskcluster_access_token => secret('treescriptworker_dev_taskcluster_access_token'),
+            worker_type => 'treescript-dev',
+            sign_chain_of_trust => false,
+            verify_chain_of_trust => false,
+            verify_cot_signature => false,
+        },
+        'prod' => {
+            taskcluster_client_id => 'project/releng/scriptworker/treescriptworker',
+            # No prod secret yet, this line would break puppet if not commented out
+            # taskcluster_access_token => secret('treescriptworker_prod_taskcluster_access_token'),
+            worker_type => 'balrogworker-v1',
+            sign_chain_of_trust => true,
+            verify_chain_of_trust => true,
+            verify_cot_signature => true,
+        }
+    }
+}
new file mode 100644
--- /dev/null
+++ b/modules/tree_scriptworker/templates/script_config.json.erb
@@ -0,0 +1,8 @@
+{
+    "work_dir": "<%= scope.lookupvar("tree_scriptworker::settings::root") %>/work",
+    "artifact_dir": "<%= scope.lookupvar("tree_scriptworker::settings::root") %>/artifacts",
+
+    "hg": "<%= scope.lookupvar("packages::mozilla::py27_mercurial::mercurial") %>",
+    "hg_share_base_dir": "<%= scope.lookupvar("tree_scriptworker::settings::share_base") %>",
+    "upstream_repo": "<%= scope.lookupvar("tree_scriptworker::settings::upstream_repo") %>"
+}