ansible: role to manage version control syncing services (bug 1337173) draft
authorGregory Szorc <gps@mozilla.com>
Tue, 07 Feb 2017 14:59:20 -0800
changeset 10329 07aceeaf2b14c07a4a6647beae67a45d1a89b71d
parent 10328 44ec5eb5430e98556768f0671f3e9f9f92683548
push id1515
push userbmo:gps@mozilla.com
push dateWed, 08 Feb 2017 02:10:07 +0000
bugs1337173
ansible: role to manage version control syncing services (bug 1337173) Introduced is an Ansible role for a machine performing version control "syncing" functionality. The idea is for all our services doing version control sync to be defined in this role. Currently implemented is configuration required for Servo VCS syncing. TODO: * Integrate secrets * Monitoring / Postfix configuration * Replace some hardcoded uses of my personal account info * Docs MozReview-Commit-ID: Ks0T2DvOsOJ
ansible/deploy-vcs-sync.yml
ansible/hosts
ansible/roles/vcs-sync/files/mail-unit-output-override.conf
ansible/roles/vcs-sync/files/servo-linearize.service
ansible/roles/vcs-sync/files/servo-linearize.timer
ansible/roles/vcs-sync/files/servo-pulse-monitor.service
ansible/roles/vcs-sync/files/servo-requirements.txt
ansible/roles/vcs-sync/files/servo-ssh_config
ansible/roles/vcs-sync/files/sudoers-servo-sync
ansible/roles/vcs-sync/meta/main.yml
ansible/roles/vcs-sync/tasks/main.yml
ansible/roles/vcs-sync/tasks/servo-sync.yml
ansible/roles/vcs-sync/templates/servo-linearize.env.j2
ansible/roles/vcs-sync/templates/servo-sync.ini.j2
vcssync/mozvcssync/servo.py
new file mode 100644
--- /dev/null
+++ b/ansible/deploy-vcs-sync.yml
@@ -0,0 +1,11 @@
+---
+- name: deploy vcs syncing service
+  hosts: vcssync
+  user: centos
+  become: true
+
+  # TODO grab secrets from <somewhere> and pass to vcs-sync role.
+
+  roles:
+    - common
+    - vcs-sync
--- a/ansible/hosts
+++ b/ansible/hosts
@@ -17,8 +17,12 @@ hgssh4.dmz.scl3.mozilla.com
 hgssh.stage.dmz.scl3.mozilla.com ansible_ssh_user=root
 
 [hgmo:children]
 hgssh-prod
 hgweb-prod
 
 [reviewbot]
 54.244.13.202 ansible_ssh_user=ubuntu ansible_sudo=yes ansible_python_interpreter=/usr/bin/python2.7
+
+# TODO switch to new, production instance.
+[vcssync]
+54.202.56.34 ansible_ssh_user=centos ansible_sudo=yes
new file mode 100644
--- /dev/null
+++ b/ansible/roles/vcs-sync/files/mail-unit-output-override.conf
@@ -0,0 +1,2 @@
+[Service]
+Environment=TO=developer-services@mozilla.org
new file mode 100644
--- /dev/null
+++ b/ansible/roles/vcs-sync/files/servo-linearize.service
@@ -0,0 +1,36 @@
+# Triggered by Pulse events and a timer to linearize history of the
+# Servo Git repo and convert to Mercurial.
+
+[Unit]
+Description=Convert Servo Git repo to Mercurial
+After=local-fs.target network.target nss-lookup.target
+OnFailure=mail-unit-output@%n.service
+
+[Service]
+Type=oneshot
+EnvironmentFile=/home/servo-sync/servo-linearize.env
+ExecStart=/home/servo-sync/venv/bin/linearize-git-to-hg \
+  --exclude-dir src/test/wpt \
+  --exclude-dir src/test/ref \
+  --exclude-dir tests/ref \
+  --exclude-dir tests/wpt \
+  --summary-prefix servo: \
+  --remove-reviewable \
+  --source-repo-key Source-Repo \
+  --source-revision-key Source-Revision \
+  --normalize-github-merge-message \
+  --committer-action use-author \
+  --use-p2-author \
+  --hg /home/servo-sync/venv/bin/hg \
+  --copy-similarity 75 \
+  --find-copies-harder \
+  --skip-submodules \
+  --git-push-url git@github.com:mozilla/converted-servo.git \
+  --hg-push-url ssh://hg.mozilla.org/projects/converted-servo-linear \
+  https://github.com/servo/servo \
+  master \
+  /home/servo-sync/servo.git \
+  /home/servo-sync/servo-linear
+
+User=servo-sync
+Group=servo-sync
new file mode 100644
--- /dev/null
+++ b/ansible/roles/vcs-sync/files/servo-linearize.timer
@@ -0,0 +1,11 @@
+# Periodically run a process to convert the Servo Git repo to Mercurial
+[Unit]
+Description=Schedules periodic linearization of Servo repo to Mercurial
+After=local-fs.target network.target nss-lookup.target
+
+[Timer]
+OnUnitInactiveSec=600
+OnBootSec=60
+
+[Install]
+WantedBy=multi-user.target
new file mode 100644
--- /dev/null
+++ b/ansible/roles/vcs-sync/files/servo-pulse-monitor.service
@@ -0,0 +1,18 @@
+[Unit]
+Description=Trigger Servo VCS syncing after Pulse events
+After=local-fs.target network.target nss-lookup.target
+
+[Service]
+WorkingDirectory=/
+PrivateTmp=true
+User=servo-sync
+Group=servo-sync
+
+ExecStart=/home/servo-sync/venv/bin/servo-pulse-listen \
+  /home/servo-sync/servo-sync.ini
+
+Restart=always
+TimeoutStopSec=10
+
+[Install]
+WantedBy=multi-user.target
new file mode 100644
--- /dev/null
+++ b/ansible/roles/vcs-sync/files/servo-requirements.txt
@@ -0,0 +1,17 @@
+amqp==1.4.9 \
+    --hash=sha256:e0ed0ce6b8ffe5690a2e856c7908dc557e0e605283d6885dd1361d79f2928908
+
+anyjson==0.3.3 \
+    --hash=sha256:37812d863c9ad3e35c0734c42e0bf0320ce8c3bed82cd20ad54cb34d158157ba
+
+dulwich==0.16.1 \
+    --hash=sha256:470d0feec9d4e7aba091c02f62db7f9cc6549ffe3f623a8039f96f584159da05
+
+kombu==3.0.37 \
+    --hash=sha256:7ceab743e3e974f3e5736082e8cc514c009e254e646d6167342e0e192aee81a6
+
+Mercurial==4.1 \
+    --hash=sha256:7b33c32cdd1d518bc2e2ae223e6ef63c486cf52e9d01a45b99cf8eab7bea5274
+
+python-hglib==1.7 \
+    --hash=sha256:0dc087d15b774cda82d3c8096fb0e514caeb2ddb60eed38e9056b16e279ba3c5
new file mode 100644
--- /dev/null
+++ b/ansible/roles/vcs-sync/files/servo-ssh_config
@@ -0,0 +1,4 @@
+Host hg.mozilla.org
+  # TODO switch to production account
+  User gszorc@mozilla.com
+  IdentityFile ~/.ssh/id_hg
new file mode 100644
--- /dev/null
+++ b/ansible/roles/vcs-sync/files/sudoers-servo-sync
@@ -0,0 +1,2 @@
+# Allow servo-sync user to start various systemd units.
+servo-sync ALL=NOPASSWD: /usr/bin/systemctl start servo-linearize.service
new file mode 100644
--- /dev/null
+++ b/ansible/roles/vcs-sync/meta/main.yml
@@ -0,0 +1,3 @@
+---
+dependencies:
+  - ius-repo
new file mode 100644
--- /dev/null
+++ b/ansible/roles/vcs-sync/tasks/main.yml
@@ -0,0 +1,28 @@
+---
+- name: install system packages
+  yum: name={{ item }} state=present update_cache=yes
+  with_items:
+    - gcc
+    # Version of Git provided by CentOS 7 doesn't have features we need.
+    # So install modern version for IUS.
+    - git2u
+    - openssh-clients
+    - python-devel
+
+- include: ../../../tasks/systemd-mail-unit-output.yml
+
+- name: directory for mail-unit-output override file
+  file: path=/etc/systemd/system/mail-unit-output@.service.d
+        state=directory
+        owner=root
+        group=root
+        mode=0755
+
+- name: install mail-unit-output override
+  copy: src=mail-unit-output-override.conf
+        dest=/etc/systemd/system/mail-unit-output@.service.d/override.conf
+        owner=root
+        group=root
+        mode=0644
+
+- include: servo-sync.yml
new file mode 100644
--- /dev/null
+++ b/ansible/roles/vcs-sync/tasks/servo-sync.yml
@@ -0,0 +1,97 @@
+# This playbook configures a system to perform syncing of Servo version control
+# data.
+---
+- name: create servo-sync user
+  user: name=servo-sync
+        shell=/bin/bash
+        state=present
+
+# Sudoers policy allows user to start systemd services. Ideally we
+# would use systemd user units. But CentOS 7 doesn't have that systemd
+# feature enabled.
+- name: sudoers policy for servo-sync
+  copy: src=sudoers-servo-sync
+        dest=/etc/sudoers.d/servo-sync
+        owner=root
+        group=root
+        mode=0440
+
+- name: configure ssh host keys for github.com
+  known_hosts:
+    path: /home/servo-sync/.ssh/known_hosts
+    host: "{{ item.host }}"
+    key: "{{ item.key }}"
+  with_items:
+    - { host: github.com, key: "{{github_ssh_host_key}}" }
+    - { host: hg.mozilla.org, key: "{{hgmo_ssh_host_key}}" }
+
+- name: ssh config for servo-sync user
+  copy: src=servo-ssh_config
+        dest=/home/servo-sync/.ssh/config
+        mode=0600
+  become_user: servo-sync
+
+- name: create virtualenv for servo syncing
+  include: ../../../tasks/virtualenv.yml
+           venv=/home/servo-sync/venv
+           requirements=../roles/vcs-sync/files/servo-requirements.txt
+  become_user: servo-sync
+
+- name: clone v-c-t
+  command: /home/servo-sync/venv/bin/hg clone {{ vct_url }} /home/servo-sync/version-control-tools creates=/home/servo-sync/version-control-tools
+  become_user: servo-sync
+
+- name: pull v-c-t
+  command: /home/servo-sync/venv/bin/hg -R /home/servo-sync/version-control-tools pull {{ vct_url }}
+  become_user: servo-sync
+
+- name: update v-c-t
+  command: /home/servo-sync/venv/bin/hg -R /home/servo-sync/version-control-tools up -r {{ vct_rev }}
+  become_user: servo-sync
+
+- name: purge v-c-t
+  command: /home/servo-sync/venv/bin/hg -R /home/servo-sync/version-control-tools --config extensions.purge= purge --all
+  become_user: servo-sync
+
+- name: install vcssync in virtualenv
+  command: /home/servo-sync/venv/bin/pip install --upgrade --no-deps --force-reinstall /home/servo-sync/version-control-tools/vcssync
+  become_user: servo-sync
+
+- name: seed Servo Git clone with existing conversion
+  command: /usr/bin/git clone --mirror git@github.com:mozilla/converted-servo.git /home/servo-sync/servo.git creates=/home/servo-sync/servo.git
+  become_user: servo-sync
+
+- name: seed Servo Mercurial repo with existing conversion
+  command: /home/servo-sync/venv/bin/hg clone -U https://hg.mozilla.org/projects/converted-servo-linear /home/servo-sync/servo-linear creates=/home/servo-sync/servo-linear
+  become_user: servo-sync
+
+- name: install servo-linearize config files
+  template: src={{ item }}.j2
+            dest=/home/servo-sync/{{ item }}
+            mode=0644
+  become_user: servo-sync
+  with_items:
+    - servo-linearize.env
+    - servo-sync.ini
+
+- name: install systemd unit files
+  copy: src={{ item }}
+        dest=/etc/systemd/system/{{ item }}
+        owner=root
+        group=root
+        mode=0644
+  with_items:
+    - servo-linearize.service
+    - servo-linearize.timer
+    - servo-pulse-monitor.service
+
+- name: reload systemd units
+  command: /usr/bin/systemctl daemon-reload
+
+- name: systemd services are enabled
+  service: name={{ item }}
+           enabled=yes
+           state=restarted
+  with_items:
+    - servo-linearize.timer
+    - servo-pulse-monitor.service
new file mode 100644
--- /dev/null
+++ b/ansible/roles/vcs-sync/templates/servo-linearize.env.j2
@@ -0,0 +1,2 @@
+GITHUB_USERNAME={{ github_username | mandatory }}
+GITHUB_TOKEN={{ github_token | mandatory }}
new file mode 100644
--- /dev/null
+++ b/ansible/roles/vcs-sync/templates/servo-sync.ini.j2
@@ -0,0 +1,14 @@
+[servo]
+
+pulse_host = pulse.mozilla.org
+pulse_port = 5671
+pulse_ssl = true
+pulse_userid = {{ pulse_userid | mandatory }}
+pulse_password = {{ pulse_password | mandatory }}
+
+pulse_github_exchange = exchange/github-webhooks/v1
+pulse_github_queue = github
+pulse_github_routing_key = #
+
+servo_github_name = servo/servo
+servo_fetch_ref = refs/heads/master
--- a/vcssync/mozvcssync/servo.py
+++ b/vcssync/mozvcssync/servo.py
@@ -3,16 +3,17 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 """Functionality to support VCS syncing for Servo."""
 
 import logging
 import os
+import subprocess
 import sys
 
 from ConfigParser import (
     RawConfigParser,
 )
 
 from . import (
     pulse,
@@ -59,17 +60,19 @@ def run_pulse_listener(c):
         #
         # `systemctl start` will block. This is fine. We want to wait
         # for the conversion to finish in case there are multiple remote
         # pushes queued up. Otherwise, there is a race condition between
         # the initial run finishing and subsequent Pulse events arriving.
         # If a subsequent notification is handled when the service is
         # running, it will no-op and we may not see its push.
         logger.warn('triggering linearization and conversion...')
-        # TODO actually do this.
+        subprocess.check_call([b'/usr/bin/systemctl', b'start',
+                               b'servo-linearize.service'],
+                              cwd='/', bufsize=0)
         message.ack()
 
     # Overlay Servo changesets from the pristine, converted repo onto
     # a Firefox repo in response to new hg changesets.
     def on_hgmo_message(body, message):
         if body['payload']['type'] != 'changegroup.1':
             message.ack()
             return