Bug 1335651 - Move index_paths from DockerImageTask to the base Task class. r?dustin draft
authorMike Hommey <mh+mozilla@glandium.org>
Tue, 31 Jan 2017 16:33:27 +0900
changeset 484246 a26f4c56589eac66e27ed6367c85c6c96b6c9d28
parent 484245 4899a861d2869b33b0401a19b67ed6f0ed63ebfd
child 484247 882ceab274597aeb29c03e38f63a990071bfd7e5
push id45428
push userbmo:mh+mozilla@glandium.org
push dateWed, 15 Feb 2017 01:40:50 +0000
reviewersdustin
bugs1335651
milestone54.0a1
Bug 1335651 - Move index_paths from DockerImageTask to the base Task class. r?dustin This does slightly change the behavior when artifacts expire, in that if for some reason the artifact for the task that was found expired, we don't try to get the artifact from a lower level task. In practice, that shouldn't be a concern.
taskcluster/taskgraph/task/base.py
taskcluster/taskgraph/task/docker_image.py
--- a/taskcluster/taskgraph/task/base.py
+++ b/taskcluster/taskgraph/task/base.py
@@ -1,15 +1,26 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import abc
+import json
+import os
+import urllib2
+
+
+# if running in a task, prefer to use the taskcluster proxy (http://taskcluster/),
+# otherwise hit the services directly
+if os.environ.get('TASK_ID'):
+    INDEX_URL = 'http://taskcluster/index/v1/task/{}'
+else:
+    INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
 
 
 class Task(object):
     """
     Representation of a task in a TaskGraph.  Each Task has, at creation:
 
     - kind: the name of the task kind
     - label; the label for this task
@@ -23,33 +34,36 @@ class Task(object):
 
     A kind represents a collection of tasks that share common characteristics.
     For example, all build jobs.  Each instance of a kind is intialized with a
     path from which it draws its task configuration.  The instance is free to
     store as much local state as it needs.
     """
     __metaclass__ = abc.ABCMeta
 
-    def __init__(self, kind, label, attributes, task):
+    def __init__(self, kind, label, attributes, task, index_paths=None):
         self.kind = kind
         self.label = label
         self.attributes = attributes
         self.task = task
 
         self.task_id = None
         self.optimized = False
 
         self.attributes['kind'] = kind
 
+        self.index_paths = index_paths or ()
+
     def __eq__(self, other):
         return self.kind == other.kind and \
             self.label == other.label and \
             self.attributes == other.attributes and \
             self.task == other.task and \
-            self.task_id == other.task_id
+            self.task_id == other.task_id and \
+            self.index_paths == other.index_paths
 
     @classmethod
     @abc.abstractmethod
     def load_tasks(cls, kind, path, config, parameters, loaded_tasks):
         """
         Load the tasks for a given kind.
 
         The `kind` is the name of the kind; the configuration for that kind
@@ -85,18 +99,28 @@ class Task(object):
         it should be replaced with.
 
         The return value is a tuple `(optimized, taskId)`.  If `optimized` is
         true, then the task will be optimized (in other words, not included in
         the task graph).  If the second argument is a taskid, then any
         dependencies on this task will isntead depend on that taskId.  It is an
         error to return no taskId for a task on which other tasks depend.
 
-        The default never optimizes.
+        The default optimizes when a taskId can be found for one of the index
+        paths attached to the task.
         """
+        for index_path in self.index_paths:
+            try:
+                url = INDEX_URL.format(index_path)
+                existing_task = json.load(urllib2.urlopen(url))
+
+                return True, existing_task['taskId']
+            except urllib2.HTTPError:
+                pass
+
         return False, None
 
     @classmethod
     def from_json(cls, task_dict):
         """
         Given a data structure as produced by taskgraph.to_json, re-construct
         the original Task object.  This is used to "resume" the task-graph
         generation process, for example in Action tasks.
--- a/taskcluster/taskgraph/task/docker_image.py
+++ b/taskcluster/taskgraph/task/docker_image.py
@@ -1,16 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import logging
-import json
 import os
 import urllib2
 
 from . import base
 from taskgraph.util.docker import (
     docker_image,
     generate_context_hash,
     INDEX_PREFIX,
@@ -19,32 +18,22 @@ from taskgraph.util.templates import Tem
 
 logger = logging.getLogger(__name__)
 GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
 
 # if running in a task, prefer to use the taskcluster proxy (http://taskcluster/),
 # otherwise hit the services directly
 if os.environ.get('TASK_ID'):
     ARTIFACT_URL = 'http://taskcluster/queue/v1/task/{}/artifacts/{}'
-    INDEX_URL = 'http://taskcluster/index/v1/task/{}'
 else:
     ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
-    INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
 
 
 class DockerImageTask(base.Task):
 
-    def __init__(self, *args, **kwargs):
-        self.index_paths = kwargs.pop('index_paths')
-        super(DockerImageTask, self).__init__(*args, **kwargs)
-
-    def __eq__(self, other):
-        return super(DockerImageTask, self).__eq__(other) and \
-               self.index_paths == other.index_paths
-
     @classmethod
     def load_tasks(cls, kind, path, config, params, loaded_tasks):
         parameters = {
             'pushlog_id': params.get('pushlog_id', 0),
             'pushdate': params['moz_build_date'],
             'pushtime': params['moz_build_date'][8:],
             'year': params['moz_build_date'][0:4],
             'month': params['moz_build_date'][4:6],
@@ -94,32 +83,28 @@ class DockerImageTask(base.Task):
                              index_paths=index_paths))
 
         return tasks
 
     def get_dependencies(self, taskgraph):
         return []
 
     def optimize(self, params):
-        for index_path in self.index_paths:
+        optimized, taskId = super(DockerImageTask, self).optimize(params)
+        if optimized and taskId:
             try:
-                url = INDEX_URL.format(index_path)
-                existing_task = json.load(urllib2.urlopen(url))
                 # Only return the task ID if the artifact exists for the indexed
-                # task.  Otherwise, continue on looking at each of the branches.  Method
-                # continues trying other branches in case mozilla-central has an expired
-                # artifact, but 'project' might not. Only return no task ID if all
-                # branches have been tried
+                # task.
                 request = urllib2.Request(
-                    ARTIFACT_URL.format(existing_task['taskId'], 'public/image.tar.zst'))
+                    ARTIFACT_URL.format(taskId, 'public/image.tar.zst'))
                 request.get_method = lambda: 'HEAD'
                 urllib2.urlopen(request)
 
                 # HEAD success on the artifact is enough
-                return True, existing_task['taskId']
+                return True, taskId
             except urllib2.HTTPError:
                 pass
 
         return False, None
 
     @classmethod
     def from_json(cls, task_dict):
         # Generating index_paths for optimization