Bug 1335651 - Move index_paths from DockerImageTask to the base Task class. r?dustin
This does slightly change the behavior when artifacts expire, in that
if for some reason the artifact for the task that was found expired,
we don't try to get the artifact from a lower level task. In practice,
that shouldn't be a concern.
--- a/taskcluster/taskgraph/task/base.py
+++ b/taskcluster/taskgraph/task/base.py
@@ -1,15 +1,26 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import abc
+import json
+import os
+import urllib2
+
+
+# if running in a task, prefer to use the taskcluster proxy (http://taskcluster/),
+# otherwise hit the services directly
+if os.environ.get('TASK_ID'):
+ INDEX_URL = 'http://taskcluster/index/v1/task/{}'
+else:
+ INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
class Task(object):
"""
Representation of a task in a TaskGraph. Each Task has, at creation:
- kind: the name of the task kind
- label; the label for this task
@@ -23,33 +34,36 @@ class Task(object):
A kind represents a collection of tasks that share common characteristics.
For example, all build jobs. Each instance of a kind is intialized with a
path from which it draws its task configuration. The instance is free to
store as much local state as it needs.
"""
__metaclass__ = abc.ABCMeta
- def __init__(self, kind, label, attributes, task):
+ def __init__(self, kind, label, attributes, task, index_paths=None):
self.kind = kind
self.label = label
self.attributes = attributes
self.task = task
self.task_id = None
self.optimized = False
self.attributes['kind'] = kind
+ self.index_paths = index_paths or ()
+
def __eq__(self, other):
return self.kind == other.kind and \
self.label == other.label and \
self.attributes == other.attributes and \
self.task == other.task and \
- self.task_id == other.task_id
+ self.task_id == other.task_id and \
+ self.index_paths == other.index_paths
@classmethod
@abc.abstractmethod
def load_tasks(cls, kind, path, config, parameters, loaded_tasks):
"""
Load the tasks for a given kind.
The `kind` is the name of the kind; the configuration for that kind
@@ -85,18 +99,28 @@ class Task(object):
it should be replaced with.
The return value is a tuple `(optimized, taskId)`. If `optimized` is
true, then the task will be optimized (in other words, not included in
the task graph). If the second argument is a taskid, then any
dependencies on this task will isntead depend on that taskId. It is an
error to return no taskId for a task on which other tasks depend.
- The default never optimizes.
+ The default optimizes when a taskId can be found for one of the index
+ paths attached to the task.
"""
+ for index_path in self.index_paths:
+ try:
+ url = INDEX_URL.format(index_path)
+ existing_task = json.load(urllib2.urlopen(url))
+
+ return True, existing_task['taskId']
+ except urllib2.HTTPError:
+ pass
+
return False, None
@classmethod
def from_json(cls, task_dict):
"""
Given a data structure as produced by taskgraph.to_json, re-construct
the original Task object. This is used to "resume" the task-graph
generation process, for example in Action tasks.
--- a/taskcluster/taskgraph/task/docker_image.py
+++ b/taskcluster/taskgraph/task/docker_image.py
@@ -1,16 +1,15 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import logging
-import json
import os
import urllib2
from . import base
from taskgraph.util.docker import (
docker_image,
generate_context_hash,
INDEX_PREFIX,
@@ -19,32 +18,22 @@ from taskgraph.util.templates import Tem
logger = logging.getLogger(__name__)
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
# if running in a task, prefer to use the taskcluster proxy (http://taskcluster/),
# otherwise hit the services directly
if os.environ.get('TASK_ID'):
ARTIFACT_URL = 'http://taskcluster/queue/v1/task/{}/artifacts/{}'
- INDEX_URL = 'http://taskcluster/index/v1/task/{}'
else:
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
- INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
class DockerImageTask(base.Task):
- def __init__(self, *args, **kwargs):
- self.index_paths = kwargs.pop('index_paths')
- super(DockerImageTask, self).__init__(*args, **kwargs)
-
- def __eq__(self, other):
- return super(DockerImageTask, self).__eq__(other) and \
- self.index_paths == other.index_paths
-
@classmethod
def load_tasks(cls, kind, path, config, params, loaded_tasks):
parameters = {
'pushlog_id': params.get('pushlog_id', 0),
'pushdate': params['moz_build_date'],
'pushtime': params['moz_build_date'][8:],
'year': params['moz_build_date'][0:4],
'month': params['moz_build_date'][4:6],
@@ -94,32 +83,28 @@ class DockerImageTask(base.Task):
index_paths=index_paths))
return tasks
def get_dependencies(self, taskgraph):
return []
def optimize(self, params):
- for index_path in self.index_paths:
+ optimized, taskId = super(DockerImageTask, self).optimize(params)
+ if optimized and taskId:
try:
- url = INDEX_URL.format(index_path)
- existing_task = json.load(urllib2.urlopen(url))
# Only return the task ID if the artifact exists for the indexed
- # task. Otherwise, continue on looking at each of the branches. Method
- # continues trying other branches in case mozilla-central has an expired
- # artifact, but 'project' might not. Only return no task ID if all
- # branches have been tried
+ # task.
request = urllib2.Request(
- ARTIFACT_URL.format(existing_task['taskId'], 'public/image.tar.zst'))
+ ARTIFACT_URL.format(taskId, 'public/image.tar.zst'))
request.get_method = lambda: 'HEAD'
urllib2.urlopen(request)
# HEAD success on the artifact is enough
- return True, existing_task['taskId']
+ return True, taskId
except urllib2.HTTPError:
pass
return False, None
@classmethod
def from_json(cls, task_dict):
# Generating index_paths for optimization