Bug 1268862: expire try jobs in a shorter time-period than default; r=nthomas draft
authorDustin J. Mitchell <dustin@mozilla.com>
Thu, 05 May 2016 21:19:42 +0000
changeset 365308 47111d4d381f43748d40e24373ec884c3cb2d248
parent 357902 01067412db32dd8eee38f64e6d66942b437fa924
child 520512 a398ba393b9b9fff89386a08cdfdf8713d3eb361
push id17696
push userdmitchell@mozilla.com
push dateTue, 10 May 2016 12:45:35 +0000
reviewersnthomas
bugs1268862
milestone49.0a1
Bug 1268862: expire try jobs in a shorter time-period than default; r=nthomas MozReview-Commit-ID: KUe6Sk0122w
testing/taskcluster/mach_commands.py
testing/taskcluster/tasks/decision/try.yml
--- a/testing/taskcluster/mach_commands.py
+++ b/testing/taskcluster/mach_commands.py
@@ -30,16 +30,19 @@ ARTIFACT_URL = 'https://queue.taskcluste
 
 DEFINE_TASK = 'queue:define-task:aws-provisioner-v1/{}'
 
 DEFAULT_TRY = 'try: -b do -p all -u all'
 DEFAULT_JOB_PATH = os.path.join(
     ROOT, 'tasks', 'branches', 'base_jobs.yml'
 )
 
+# time after which a try build's results will expire
+TRY_EXPIRATION = "14 days"
+
 def merge_dicts(*dicts):
     merged_dict = {}
     for dictionary in dicts:
         merged_dict.update(dictionary)
     return merged_dict
 
 def gaia_info():
     '''
@@ -139,16 +142,30 @@ def remove_caches_from_task(task):
     try:
         caches = task["task"]["payload"]["cache"]
         for cache in caches.keys():
             if not any(pat.match(cache) for pat in whitelist):
                 caches.pop(cache)
     except KeyError:
         pass
 
+def set_expiration(task, timestamp):
+    task_def = task['task']
+    task_def['expires'] = timestamp
+    if task_def.get('deadline', timestamp) > timestamp:
+        task_def['deadline'] = timestamp
+
+    try:
+        artifacts = task_def['payload']['artifacts']
+    except KeyError:
+        return
+
+    for artifact in artifacts.values():
+        artifact['expires'] = timestamp
+
 def query_vcs_info(repository, revision):
     """Query the pushdate and pushid of a repository/revision.
     This is intended to be used on hg.mozilla.org/mozilla-central and
     similar. It may or may not work for other hg repositories.
     """
     if not repository or not revision:
         sys.stderr.write('cannot query vcs info because vcs info not provided\n')
         return None
@@ -482,16 +499,17 @@ class Graph(object):
                                     seen_images,
                                     build_parameters,
                                     os.environ.get('TASK_ID', None))
             set_interactive_task(build_task, interactive)
 
             # try builds don't use cache
             if project == "try":
                 remove_caches_from_task(build_task)
+                set_expiration(build_task, json_time_from_now(TRY_EXPIRATION))
 
             if params['revision_hash']:
                 treeherder_transform.add_treeherder_revision_info(build_task['task'],
                                                                   params['head_rev'],
                                                                   params['revision_hash'])
                 routes_transform.decorate_task_treeherder_routes(build_task['task'],
                                                                  treeherder_route)
                 routes_transform.decorate_task_json_routes(build_task['task'],
@@ -562,16 +580,18 @@ class Graph(object):
                                         post_task,
                                         seen_images,
                                         build_parameters,
                                         os.environ.get('TASK_ID', None))
                 set_interactive_task(post_task, interactive)
                 treeherder_transform.add_treeherder_revision_info(post_task['task'],
                                                                   params['head_rev'],
                                                                   params['revision_hash'])
+                if project == "try":
+                    set_expiration(post_task, json_time_from_now(TRY_EXPIRATION))
                 graph['tasks'].append(post_task)
 
             for test in build['dependents']:
                 test = test['allowed_build_tasks'][build['task']]
                 # TODO additional-parameters is currently not an option, only
                 # enabled for build tasks
                 test_parameters = merge_dicts(build_parameters,
                                               test.get('additional-parameters', {}))
@@ -616,16 +636,19 @@ class Graph(object):
                         treeherder_transform.add_treeherder_revision_info(test_task['task'],
                                                                           params['head_rev'],
                                                                           params['revision_hash'])
                         routes_transform.decorate_task_treeherder_routes(
                             test_task['task'],
                             treeherder_route
                         )
 
+                    if project == "try":
+                        set_expiration(test_task, json_time_from_now(TRY_EXPIRATION))
+
                     # This will schedule test jobs N times
                     for i in range(0, trigger_tests):
                         graph['tasks'].append(test_task)
                         # If we're scheduling more tasks each have to be unique
                         test_task = copy.deepcopy(test_task)
                         test_task['taskId'] = slugid()
 
                     define_task = DEFINE_TASK.format(
--- a/testing/taskcluster/tasks/decision/try.yml
+++ b/testing/taskcluster/tasks/decision/try.yml
@@ -14,16 +14,17 @@ scopes:
   - scheduler:*
 
 tasks:
   - taskId: '{{#as_slugid}}decision task{{/as_slugid}}'
     reruns: 3
     task:
       created: '{{now}}'
       deadline: '{{#from_now}}1 day{{/from_now}}'
+      expires: '{{#from_now}}14 day{{/from_now}}'
       metadata:
         owner: mozilla-taskcluster-maintenance@mozilla.com
         source: {{{source}}}
         name: "[tc] Initial decision task for try"
         description: |
           This is the single most important task as it decides how all other tasks
           get built.