Bug 1425571: Replace pulse-notify with taskcluster-notify draft
authorChris AtLee <catlee@mozilla.com>
Fri, 23 Mar 2018 09:04:18 -0400
changeset 773966 6968eed4113392187e279184c5d6a9924264cdc9
parent 772574 7b9da7139d94951431a148dcaf8a388640c91b27
push id104364
push usercatlee@mozilla.com
push dateWed, 28 Mar 2018 18:52:55 +0000
bugs1425571
milestone61.0a1
Bug 1425571: Replace pulse-notify with taskcluster-notify MozReview-Commit-ID: FgfNz0fyY3z
taskcluster/ci/release-notify-promote/kind.yml
taskcluster/ci/release-notify-push/kind.yml
taskcluster/ci/release-notify-ship/kind.yml
taskcluster/ci/release-secondary-notify-ship/kind.yml
taskcluster/taskgraph/transforms/job/__init__.py
taskcluster/taskgraph/transforms/release_notifications.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/ci/release-notify-promote/kind.yml
+++ b/taskcluster/ci/release-notify-promote/kind.yml
@@ -24,44 +24,26 @@ job-defaults:
       os: linux
       docker-image: "ubuntu:16.10"
       max-run-time: 600
       command:
          - /bin/bash
          - -c
          - echo "Dummy task"
    notifications:
-      completed:
-         subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} is in the candidates directory"
-         message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} is in the candidates directory"
-         plugins: ["ses"]
-         emails:
-            by-project:
-               mozilla-beta: ["release-signoff@mozilla.org"]
-               mozilla-release: ["release-signoff@mozilla.org"]
-               try: ["{task_def[metadata][owner]}"]
-               maple: ["release+tcstaging@mozilla.com"]
-               default: []
-   routes:
-      - index.releases.v1.{branch}.{revision}.{product}.{underscore_version}.build{build_number}.email-{channel}
-      - index.releases.v1.{branch}.latest.{product}.latest.email-{channel}
-   index:
-      type: release
-      channel:
+      subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} is in the candidates directory"
+      message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} is in the candidates directory"
+      emails:
          by-project:
-            mozilla-beta: beta-candidates
-            mozilla-release: release-candidates
-            maple: maple-candidates
-            default: unknown
+            mozilla-beta: ["release-signoff@mozilla.org"]
+            mozilla-release: ["release-signoff@mozilla.org"]
+            try: ["{task_def[metadata][owner]}"]
+            maple: ["release+tcstaging@mozilla.com"]
+            birch: ["release+tcstaging@mozilla.com"]
+            default: []
 
 jobs:
    fennec:
       shipping-product: fennec
-      index:
-         product: fennec
    firefox:
       shipping-product: firefox
-      index:
-         product: firefox
    devedition:
       shipping-product: devedition
-      index:
-         product: devedition
--- a/taskcluster/ci/release-notify-push/kind.yml
+++ b/taskcluster/ci/release-notify-push/kind.yml
@@ -23,40 +23,24 @@ job-defaults:
       os: linux
       docker-image: "ubuntu:16.10"
       max-run-time: 600
       command:
          - /bin/bash
          - -c
          - echo "Dummy task"
    notifications:
-      completed:
-         subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has been pushed to releases"
-         message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has been pushed to releases"
-         plugins: ["ses"]
-         emails:
-            by-project:
-               mozilla-beta: ["release-signoff@mozilla.org"]
-               mozilla-release: ["release-signoff@mozilla.org"]
-               try: ["{task_def[metadata][owner]}"]
-               maple: ["release+tcstaging@mozilla.com"]
-               default: []
-   routes:
-      - index.releases.v1.{branch}.{revision}.{product}.{underscore_version}.build{build_number}.email-{channel}
-      - index.releases.v1.{branch}.latest.{product}.latest.email-{channel}
-   index:
-      type: release
-      channel:
+      subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has been pushed to cdntest"
+      message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has been pushed to cdntest"
+      emails:
          by-project:
-            mozilla-beta: beta-cdns
-            mozilla-release: release-cdns
-            maple: maple-cdns
-            default: unknown
+            mozilla-beta: ["release-signoff@mozilla.org"]
+            mozilla-release: ["release-signoff@mozilla.org"]
+            try: ["{task_def[metadata][owner]}"]
+            maple: ["release+tcstaging@mozilla.com"]
+            birch: ["release+tcstaging@mozilla.com"]
+            default: []
 
 jobs:
    firefox:
       shipping-product: firefox
-      index:
-         product: firefox
    devedition:
       shipping-product: devedition
-      index:
-         product: devedition
--- a/taskcluster/ci/release-notify-ship/kind.yml
+++ b/taskcluster/ci/release-notify-ship/kind.yml
@@ -27,54 +27,33 @@ job-defaults:
       os: linux
       docker-image: "ubuntu:16.10"
       max-run-time: 600
       command:
          - /bin/bash
          - -c
          - echo "Dummy task"
    notifications:
-      completed:
-         plugins: ["ses"]
-         emails:
-            by-project:
-               mozilla-beta: ["release-signoff@mozilla.org"]
-               mozilla-release: ["release-signoff@mozilla.org"]
-               try: ["{task_def[metadata][owner]}"]
-               maple: ["release+tcstaging@mozilla.com"]
-               default: []
-   routes:
-      - index.releases.v1.{branch}.{revision}.{product}.{underscore_version}.build{build_number}.email-{channel}
-      - index.releases.v1.{branch}.latest.{product}.latest.email-{channel}
-   index:
-      type: release
-      channel:
+      emails:
          by-project:
-            mozilla-beta: beta
-            mozilla-release: release
-            maple: maple
-            default: unknown
+            mozilla-beta: ["release-signoff@mozilla.org"]
+            mozilla-release: ["release-signoff@mozilla.org"]
+            try: ["{task_def[metadata][owner]}"]
+            maple: ["release+tcstaging@mozilla.com"]
+            birch: ["release+tcstaging@mozilla.com"]
+            default: []
 
 jobs:
    fennec:
       shipping-product: fennec
-      index:
-         product: fennec
       notifications:
-         completed:
-            subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has shipped!"
-            message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has shipped!"
+         subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has shipped!"
+         message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has shipped!"
    firefox:
       shipping-product: firefox
-      index:
-         product: firefox
       notifications:
-         completed:
-            subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!"
-            message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!"
+         subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!"
+         message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!"
    devedition:
       shipping-product: devedition
-      index:
-         product: devedition
       notifications:
-         completed:
-            subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!"
-            message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!"
+         subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!"
+         message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!"
--- a/taskcluster/ci/release-secondary-notify-ship/kind.yml
+++ b/taskcluster/ci/release-secondary-notify-ship/kind.yml
@@ -24,45 +24,28 @@ job-defaults:
       os: linux
       docker-image: "ubuntu:16.10"
       max-run-time: 600
       command:
          - /bin/bash
          - -c
          - echo "Dummy task"
    notifications:
-      completed:
-         plugins: ["ses"]
-         emails:
-            by-project:
-               mozilla-beta: ["release-signoff@mozilla.org"]
-               mozilla-release: ["release-signoff@mozilla.org"]
-               try: ["{task_def[metadata][owner]}"]
-               maple: ["release+tcstaging@mozilla.com"]
-               default: []
-   routes:
-      - index.releases.v1.{branch}.{revision}.{product}-rc.{underscore_version}.build{build_number}.email-{channel}
-      - index.releases.v1.{branch}.latest.{product}-rc.latest.email-{channel}
-   index:
-      type: release
-      channel:
+      emails:
          by-project:
-            mozilla-release: beta
-            birch: birch
-            default: unknown
+            mozilla-beta: ["release-signoff@mozilla.org"]
+            mozilla-release: ["release-signoff@mozilla.org"]
+            try: ["{task_def[metadata][owner]}"]
+            maple: ["release+tcstaging@mozilla.com"]
+            birch: ["release+tcstaging@mozilla.com"]
+            default: []
 
 jobs:
    firefox-rc:
       shipping-product: firefox
-      index:
-         product: firefox
       notifications:
-         completed:
-            subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!"
-            message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!"
+         subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!"
+         message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} updates are ready for signoff in Balrog!"
    fennec-rc:
       shipping-product: fennec
-      index:
-         product: fennec
       notifications:
-         completed:
-            subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has shipped!"
-            message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has shipped!"
+         subject: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has shipped!"
+         message: "{task[shipping-product]} {release_config[version]} build{release_config[build_number]}/{config[params][project]} has shipped!"
--- a/taskcluster/taskgraph/transforms/job/__init__.py
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -51,23 +51,21 @@ job_description_schema = Schema({
     Optional('attributes'): task_description_schema['attributes'],
     Optional('job-from'): task_description_schema['job-from'],
     Optional('dependencies'): task_description_schema['dependencies'],
     Optional('expires-after'): task_description_schema['expires-after'],
     Optional('routes'): task_description_schema['routes'],
     Optional('scopes'): task_description_schema['scopes'],
     Optional('tags'): task_description_schema['tags'],
     Optional('extra'): task_description_schema['extra'],
-    Optional('notifications'): task_description_schema['notifications'],
     Optional('treeherder'): task_description_schema['treeherder'],
     Optional('index'): task_description_schema['index'],
     Optional('run-on-projects'): task_description_schema['run-on-projects'],
     Optional('shipping-phase'): task_description_schema['shipping-phase'],
     Optional('shipping-product'): task_description_schema['shipping-product'],
-    Optional('notifications'): task_description_schema['notifications'],
     Optional('coalesce'): task_description_schema['coalesce'],
     Optional('always-target'): task_description_schema['always-target'],
     Exclusive('optimization', 'optimization'): task_description_schema['optimization'],
     Optional('needs-sccache'): task_description_schema['needs-sccache'],
 
     # The "when" section contains descriptions of the circumstances under which
     # this task should be included in the task graph.  This will be converted
     # into an optimization, so it cannot be specified in a job description that
--- a/taskcluster/taskgraph/transforms/release_notifications.py
+++ b/taskcluster/taskgraph/transforms/release_notifications.py
@@ -4,25 +4,27 @@
 """
 Add notifications via taskcluster-notify for release tasks
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.scriptworker import get_release_config, RELEASE_NOTIFICATION_PHASES
+from taskgraph.util.schema import resolve_keyed_by
 
 
 transforms = TransformSequence()
 
 EMAIL_DESTINATIONS = {
     'mozilla-beta': ["release-automation-notifications@mozilla.com"],
     'mozilla-release': ["release-automation-notifications@mozilla.com"],
     'mozilla-esr60': ["release-automation-notifications@mozilla.com"],
     'maple': ["release+tcstaging@mozilla.com"],
+    'birch': ["release+tcstaging@mozilla.com"],
     'jamun': ["release+tcstaging@mozilla.com"],
     # otherwise []
 }
 
 # Only notify on tasks that have issues
 DEFAULT_ROUTES = [
     'notify.email.{email_dest}.on-failed',
     'notify.email.{email_dest}.on-exception',
@@ -40,37 +42,59 @@ def add_notifications(config, jobs):
     for job in jobs:
         # Frankly, my dear, you're all over the place
         shipping_phase = job.get('attributes', {}).get('shipping_phase') or \
             job.get('shipping-phase')
         shipping_product = job.get('attributes', {}).get('shipping_product') or \
             job.get('shipping-product')
         label = job.get('label') or '{}-{}'.format(config.kind, job['name'])
 
-        # We only modify release jobs, or nightly & release being run in the context of a release
-        if shipping_phase in RELEASE_NOTIFICATION_PHASES and \
-                config.params['target_tasks_method'].startswith(RELEASE_NOTIFICATION_PHASES):
-
+        # Handle notification overrides
+        notifications = job.get('notifications')
+        if notifications:
+            resolve_keyed_by(notifications, 'emails', label, project=config.params['project'])
+            emails = notifications['emails']
+            format_kwargs = dict(
+                task=job,
+                config=config.__dict__,
+                release_config=release_config,
+            )
+            subject = notifications['subject'].format(**format_kwargs)
+            message = notifications['message'].format(**format_kwargs)
+            routes = ['notify.email.{email_dest}.on-any']
+            # Don't need this any more
+            del job['notifications']
+        else:
+            emails = email_dest
             format_kwargs = dict(
                 label=label,
                 shipping_product=shipping_product,
                 config=config.__dict__,
                 release_config=release_config,
             )
+            subject = SUBJECT_TEMPLATE.format(**format_kwargs)
+            message = None
+            routes = DEFAULT_ROUTES
+
+        # We only modify release jobs, or nightly & release being run in the context of a release
+        if shipping_phase in RELEASE_NOTIFICATION_PHASES and \
+                config.params['target_tasks_method'].startswith(RELEASE_NOTIFICATION_PHASES):
 
             # Add routes to trigger notifications via tc-notify
-            for dest in email_dest:
+            for dest in emails:
                 job.setdefault('routes', []).extend(
-                    [r.format(email_dest=dest) for r in DEFAULT_ROUTES]
+                    [r.format(email_dest=dest) for r in routes]
                 )
 
             # Customize the email subject to include release name and build number
             job.setdefault('extra', {}).update(
                 {
                    'notify': {
                        'email': {
-                            'subject': SUBJECT_TEMPLATE.format(**format_kwargs)
+                            'subject': subject,
                         }
                     }
                 }
             )
+            if message:
+                job['extra']['notify']['email']['message'] = message
 
         yield job
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -48,46 +48,16 @@ def _run_task_suffix():
 
 
 # shortcut for a string where task references are allowed
 taskref_or_string = Any(
     basestring,
     {Required('task-reference'): basestring},
 )
 
-# For more details look at https://github.com/mozilla-releng/pulse-notify#task-definition
-#
-# Notification fields are keyed by project, which lets you use
-# `by-project` and define different messages or recepients for each
-# project.
-notification_schema = Schema({
-    # notification routes for this task status
-    # https://github.com/mozilla-releng/pulse-notify/tree/master/pulsenotify/plugins
-    Optional('plugins'): optionally_keyed_by('project', [basestring]),
-
-    # notification subject
-    Optional('subject'): optionally_keyed_by('project', basestring),
-
-    # notification message
-    Optional('message'): optionally_keyed_by('project', basestring),
-
-    # emails to be notified (for ses and smtp plugins only)
-    Optional('emails'): optionally_keyed_by('project', [basestring]),
-
-    # IRC nicknames to notify (for irc plugin only)
-    Optional('nicks'): optionally_keyed_by('project', [basestring]),
-
-    # IRC channels to send a notification to (for irc plugin only)
-    Optional('channels'): optionally_keyed_by('project', [basestring]),
-
-    # notify a 'name' based on a configuration in the service
-    # https://github.com/mozilla-releng/pulse-notify/blob/production/pulsenotify/id_configs/prod.yml
-    Optional('ids'): optionally_keyed_by('project', [basestring]),
-})
-
 # A task description is a general description of a TaskCluster task
 task_description_schema = Schema({
     # the label for this task
     Required('label'): basestring,
 
     # description of the task (for metadata)
     Required('description'): basestring,
 
@@ -241,32 +211,16 @@ task_description_schema = Schema({
     # the provisioner-id/worker-type for the task.  The following parameters will
     # be substituted in this string:
     #  {level} -- the scm level of this push
     'worker-type': basestring,
 
     # Whether the job should use sccache compiler caching.
     Required('needs-sccache'): bool,
 
-    # Send notifications using pulse-notifier[1] service:
-    #
-    #     https://github.com/mozilla-releng/pulse-notify
-    #
-    # Notifications are send uppon task completion, failure or when exception
-    # is raised.
-    Optional('notifications'): {
-        Optional('defined'): notification_schema,
-        Optional('pending'): notification_schema,
-        Optional('running'): notification_schema,
-        Optional('artifact-created'): notification_schema,
-        Optional('completed'): notification_schema,
-        Optional('failed'): notification_schema,
-        Optional('exception'): notification_schema,
-    },
-
     # information specific to the worker implementation that will run this task
     'worker': Any({
         Required('implementation'): Any('docker-worker', 'docker-engine'),
         Required('os'): 'linux',
 
         # For tasks that will run in docker-worker or docker-engine, this is the
         # name of the docker image or in-tree docker image to run the task in.  If
         # in-tree, then a dependency will be created automatically.  This is
@@ -1664,55 +1618,16 @@ def build_task(config, tasks):
             'native-engine',
             'docker-worker',
         ):
             payload = task_def.get('payload')
             if payload:
                 env = payload.setdefault('env', {})
                 env['MOZ_AUTOMATION'] = '1'
 
-        notifications = task.get('notifications')
-        if notifications:
-            release_config = get_release_config(config)
-            task_def['extra'].setdefault('notifications', {})
-            for notification_event, notification in notifications.items():
-
-                for notification_option, notification_option_value in notification.items():
-
-                    # resolve by-project
-                    resolve_keyed_by(
-                        notification,
-                        notification_option,
-                        'notifications',
-                        project=config.params['project'],
-                    )
-
-                    # resolve formatting for each of the fields
-                    format_kwargs = dict(
-                            task=task,
-                            task_def=task_def,
-                            config=config.__dict__,
-                            release_config=release_config,
-                    )
-                    if isinstance(notification_option_value, basestring):
-                        notification[notification_option] = notification_option_value.format(
-                            **format_kwargs
-                        )
-                    elif isinstance(notification_option_value, list):
-                        notification[notification_option] = [
-                            i.format(**format_kwargs) for i in notification_option_value
-                        ]
-
-                # change event to correct event
-                if notification_event != 'artifact-created':
-                    notification_event = 'task-' + notification_event
-
-                # update notifications
-                task_def['extra']['notifications'][notification_event] = notification
-
         yield {
             'label': task['label'],
             'task': task_def,
             'dependencies': task.get('dependencies', {}),
             'attributes': attributes,
             'optimization': task.get('optimization', None),
         }