Bug 1305795 - Set mozinfo_json_url property; r?ted
We set the attribute from buildbot properties, iterating TaskCluster
artifacts, inferring it from the path to test_packages.json, or a
command line argument.
Honestly, I'm not sure when these code paths are all used. The fact
we do this N ways is pure madness. The code in testbase.py that
iterates over the buildbot properties file's
['sourcestamp']['changes']['files'] is particularly hacky. Currently
the list only contains the package bz2 file and test_packages.json.
But I can't figure out what actually populates the list. I suspect
the code lives out of tree. While we should be adding the mozinfo.json
URL to this list, it seems just as effective to infer the URL from
the URL of test_packages.json. So I did.
The TaskCluster transform has been updated to pass the argument to
mozharness test tasks.
MozReview-Commit-ID: Hepdq268ZDd
--- a/taskcluster/taskgraph/transforms/tests/make_task_description.py
+++ b/taskcluster/taskgraph/transforms/tests/make_task_description.py
@@ -122,16 +122,18 @@ def worker_setup_function(name):
@worker_setup_function("docker-engine")
@worker_setup_function("docker-worker")
def docker_worker_setup(config, test, taskdesc):
mozharness = test['mozharness']
installer_url = ARTIFACT_URL.format('<build>', mozharness['build-artifact-name'])
test_packages_url = ARTIFACT_URL.format('<build>',
'public/build/target.test_packages.json')
+ mozinfo_json_url = ARTIFACT_URL.format('<build>',
+ 'public/build/target.mozinfo.json')
mozharness_url = ARTIFACT_URL.format('<build>',
'public/build/mozharness.zip')
taskdesc['worker-type'] = {
'default': 'aws-provisioner-v1/desktop-test-large',
'large': 'aws-provisioner-v1/desktop-test-large',
'xlarge': 'aws-provisioner-v1/desktop-test-xlarge',
'legacy': 'aws-provisioner-v1/desktop-test',
@@ -215,16 +217,17 @@ def docker_worker_setup(config, test, ta
'/home/worker/bin/test-linux.sh',
])
if mozharness.get('no-read-buildbot-config'):
command.append("--no-read-buildbot-config")
command.extend([
{"task-reference": "--installer-url=" + installer_url},
{"task-reference": "--test-packages-url=" + test_packages_url},
+ {"task-reference": "--mozinfo-json-url=" + mozinfo_json_url},
])
command.extend(mozharness.get('extra-options', []))
# TODO: remove the need for run['chunked']
if mozharness.get('chunked') or test['chunks'] > 1:
# Implement mozharness['chunking-args'], modifying command in place
if mozharness['chunking-args'] == 'this-chunk':
command.append('--total-chunk={}'.format(test['chunks']))
--- a/testing/mozharness/mozharness/mozilla/taskcluster_helper.py
+++ b/testing/mozharness/mozharness/mozilla/taskcluster_helper.py
@@ -197,27 +197,32 @@ class TaskClusterArtifactFinderMixin(obj
def set_bbb_artifacts(self, task_id, properties_file_path):
""" Find BBB artifacts through properties_file_path and set them. """
p = self.load_json_url(
self.url_to_artifact(task_id, properties_file_path))['properties']
# Set importants artifacts for test jobs
self.set_artifacts(
- p['packageUrl'] if p.get('packageUrl') else None,
- p['testPackagesUrl'] if p.get('testPackagesUrl') else None,
- p['symbolsUrl'] if p.get('symbolsUrl') else None
+ p.get('packageUrl') or None,
+ p.get('testPackagesUrl') or None,
+ p.get('symbolsUrl') or None,
+ p.get('mozinfoJsonUrl') or None,
)
- def set_artifacts(self, installer, tests, symbols):
+ def set_artifacts(self, installer, tests, symbols, mozinfo_json):
""" Sets installer, test and symbols URLs from the artifacts of BBB based task."""
- self.installer_url, self.test_url, self.symbols_url = installer, tests, symbols
+ self.installer_url = installer
+ self.test_url = tests
+ self.symbols_url = symbols
+ self.mozinfo_json_url = mozinfo_json
self.info('Set installer_url: %s' % self.installer_url)
self.info('Set test_url: %s' % self.test_url)
self.info('Set symbols_url: %s' % self.symbols_url)
+ self.info('Set mozinfo_json_url: %s' % self.mozinfo_json_url)
def set_parent_artifacts(self, child_task_id):
""" Find and set installer_url, test_url and symbols_url by querying TaskCluster.
In Buildbot Bridge's normal behaviour we can find the artifacts by inspecting
a child's taskId, determine the task in which it depends on and find the uploaded
artifacts.
@@ -246,17 +251,18 @@ class TaskClusterArtifactFinderMixin(obj
# Case 1: The parent task is a pure TC task
if parent_task['extra'].get('locations'):
# Build tasks generated under TC specify where they upload their builds
installer_path = parent_task['extra']['locations']['build']
self.set_artifacts(
self.url_to_artifact(parent_id, installer_path),
self.url_to_artifact(parent_id, 'public/build/test_packages.json'),
- self.url_to_artifact(parent_id, 'public/build/target.crashreporter-symbols.zip')
+ self.url_to_artifact(parent_id, 'public/build/target.crashreporter-symbols.zip',
+ self.url_to_artifact(parent_id, 'public/build/target.mozinfo.json'))
)
else:
# Case 2: The parent task has an associated BBB task
# graph_props.json is uploaded in buildbase.py
self.set_bbb_artifacts(
task_id=parent_id,
properties_file_path='public/build/buildbot_properties.json'
)
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@@ -81,16 +81,22 @@ testing_config_options = [
"help": "URL to the zip file containing the actual tests",
}],
[["--test-packages-url"],
{"action": "store",
"dest": "test_packages_url",
"default": None,
"help": "URL to a json file describing which tests archives to download",
}],
+ [["--mozinfo-json-url"],
+ {"action": "store",
+ "dest": "mozinfo_json_url",
+ "default": None,
+ "help": "URL to mozinfo.json file",
+ }],
[["--jsshell-url"],
{"action": "store",
"dest": "jsshell_url",
"default": None,
"help": "URL to the jsshell to install",
}],
[["--download-symbols"],
{"action": "store",
@@ -111,16 +117,17 @@ class TestingMixin(VirtualenvMixin, Buil
tests and Talos.
"""
installer_url = None
installer_path = None
binary_path = None
test_url = None
test_packages_url = None
+ mozinfo_json_url = None
symbols_url = None
symbols_path = None
jsshell_url = None
minidump_stackwalk_path = None
default_tools_repo = 'https://hg.mozilla.org/build/tools'
proxxy = None
def query_abs_dirs(self):
@@ -300,16 +307,18 @@ class TestingMixin(VirtualenvMixin, Buil
if not c.get('test_url') and not c.get('test_packages_url'):
self.exception("You must use --test-url or --test-packages-url with developer_config.py")
c["installer_url"] = _replace_url(c["installer_url"], c["replace_urls"])
if c.get("test_url"):
c["test_url"] = _replace_url(c["test_url"], c["replace_urls"])
if c.get("test_packages_url"):
c["test_packages_url"] = _replace_url(c["test_packages_url"], c["replace_urls"])
+ if c.get("mozinfo_json_url"):
+ c["mozinfo_json_url"] = _replace_url(c["mozinfo_json_url"], c["replace_urls"])
for key, value in self.config.iteritems():
if type(value) == str and value.startswith("http"):
self.config[key] = _replace_url(value, c["replace_urls"])
# Any changes to c means that we need credentials
if not c == orig_config:
get_credentials()
@@ -370,16 +379,22 @@ class TestingMixin(VirtualenvMixin, Buil
self.test_url = str(f['name'])
self.info("Found test url %s." % self.test_url)
elif f['name'].endswith('crashreporter-symbols.zip'): # yuk
self.symbols_url = str(f['name'])
self.info("Found symbols url %s." % self.symbols_url)
elif f['name'].endswith('test_packages.json'):
self.test_packages_url = str(f['name'])
self.info("Found a test packages url %s." % self.test_packages_url)
+
+ # This is a bit hacky. But it is easier than teaching buildbot to
+ # expose more URLs.
+ self.mozinfo_json_url = self.test_packages_url.replace('.test_packages.json',
+ '.mozinfo.json')
+ self.info('Inferred mozinfo.json url %s.' % self.mozinfo_json_url)
elif not any(f['name'].endswith(s) for s in ('code-coverage-gcno.zip',)):
if not self.installer_url:
self.installer_url = str(f['name'])
self.info("Found installer url %s." % self.installer_url)
except IndexError, e:
self.error(str(e))
def find_artifacts_from_taskcluster(self):
@@ -396,26 +411,30 @@ class TestingMixin(VirtualenvMixin, Buil
c = self.config
message = "Unable to set %s from the buildbot config"
if c.get("installer_url"):
self.installer_url = c['installer_url']
if c.get("test_url"):
self.test_url = c['test_url']
if c.get("test_packages_url"):
self.test_packages_url = c['test_packages_url']
+ if c.get('mozinfo_json_url'):
+ self.mozinfo_json_url = c['mozinfo_json_url']
# This supports original Buildbot to Buildbot mode
if self.buildbot_config['sourcestamp']['changes']:
self.find_artifacts_from_buildbot_changes()
# This supports TaskCluster/BBB task to Buildbot job
elif 'testPackagesUrl' in self.buildbot_config['properties'] and \
- 'packageUrl' in self.buildbot_config['properties']:
+ 'packageUrl' in self.buildbot_config['properties'] and \
+ 'mozinfoJsonUrl' in self.buildbot_config['properties']:
self.installer_url = self.buildbot_config['properties']['packageUrl']
self.test_packages_url = self.buildbot_config['properties']['testPackagesUrl']
+ self.mozinfo_json_url = self.buildbot_config['properties']['mozinfoJsonUrl']
# This supports TaskCluster/BBB task to TaskCluster/BBB task
elif 'taskId' in self.buildbot_config['properties']:
self.find_artifacts_from_taskcluster()
missing = []
if not self.installer_url:
missing.append("installer_url")
@@ -572,17 +591,17 @@ 2. running via buildbot and running the
def download_and_extract(self, extract_dirs=None, suite_categories=None):
"""
download and extract test zip / download installer
"""
# Swap plain http for https when we're downloading from ftp
# See bug 957502 and friends
from_ = "http://ftp.mozilla.org"
to_ = "https://ftp-ssl.mozilla.org"
- for attr in 'symbols_url', 'installer_url', 'test_packages_url', 'test_url':
+ for attr in 'symbols_url', 'installer_url', 'test_packages_url', 'test_url', 'mozinfo_json_url':
url = getattr(self, attr)
if url and url.startswith(from_):
new_url = url.replace(from_, to_)
self.info("Replacing url %s -> %s" % (url, new_url))
setattr(self, attr, new_url)
if 'test_url' in self.config:
# A user has specified a test_url directly, any test_packages_url will