Bug 1305795 - Set mozinfo_json_url property; r?ted draft
authorGregory Szorc <gps@mozilla.com>
Tue, 27 Sep 2016 12:49:37 -0700
changeset 418563 c62c9fb642e1e365e4951b39ecb76d48b15f1164
parent 418562 5054f1cfa74e13d23ff1de4a26a9d644a918277e
child 418564 08a554e6d9a605d99bb04959c4041f8ca9c4f1ea
push id30707
push userbmo:gps@mozilla.com
push dateWed, 28 Sep 2016 18:02:51 +0000
reviewersted
bugs1305795
milestone52.0a1
Bug 1305795 - Set mozinfo_json_url property; r?ted We set the attribute from buildbot properties, iterating TaskCluster artifacts, inferring it from the path to test_packages.json, or a command line argument. Honestly, I'm not sure when these code paths are all used. The fact we do this N ways is pure madness. The code in testbase.py that iterates over the buildbot properties file's ['sourcestamp']['changes']['files'] is particularly hacky. Currently the list only contains the package bz2 file and test_packages.json. But I can't figure out what actually populates the list. I suspect the code lives out of tree. While we should be adding the mozinfo.json URL to this list, it seems just as effective to infer the URL from the URL of test_packages.json. So I did. The TaskCluster transform has been updated to pass the argument to mozharness test tasks. MozReview-Commit-ID: Hepdq268ZDd
taskcluster/taskgraph/transforms/tests/make_task_description.py
testing/mozharness/mozharness/mozilla/taskcluster_helper.py
testing/mozharness/mozharness/mozilla/testing/testbase.py
--- a/taskcluster/taskgraph/transforms/tests/make_task_description.py
+++ b/taskcluster/taskgraph/transforms/tests/make_task_description.py
@@ -122,16 +122,18 @@ def worker_setup_function(name):
 @worker_setup_function("docker-engine")
 @worker_setup_function("docker-worker")
 def docker_worker_setup(config, test, taskdesc):
     mozharness = test['mozharness']
 
     installer_url = ARTIFACT_URL.format('<build>', mozharness['build-artifact-name'])
     test_packages_url = ARTIFACT_URL.format('<build>',
                                             'public/build/target.test_packages.json')
+    mozinfo_json_url = ARTIFACT_URL.format('<build>',
+                                           'public/build/target.mozinfo.json')
     mozharness_url = ARTIFACT_URL.format('<build>',
                                          'public/build/mozharness.zip')
 
     taskdesc['worker-type'] = {
         'default': 'aws-provisioner-v1/desktop-test-large',
         'large': 'aws-provisioner-v1/desktop-test-large',
         'xlarge': 'aws-provisioner-v1/desktop-test-xlarge',
         'legacy': 'aws-provisioner-v1/desktop-test',
@@ -215,16 +217,17 @@ def docker_worker_setup(config, test, ta
         '/home/worker/bin/test-linux.sh',
     ])
 
     if mozharness.get('no-read-buildbot-config'):
         command.append("--no-read-buildbot-config")
     command.extend([
         {"task-reference": "--installer-url=" + installer_url},
         {"task-reference": "--test-packages-url=" + test_packages_url},
+        {"task-reference": "--mozinfo-json-url=" + mozinfo_json_url},
     ])
     command.extend(mozharness.get('extra-options', []))
 
     # TODO: remove the need for run['chunked']
     if mozharness.get('chunked') or test['chunks'] > 1:
         # Implement mozharness['chunking-args'], modifying command in place
         if mozharness['chunking-args'] == 'this-chunk':
             command.append('--total-chunk={}'.format(test['chunks']))
--- a/testing/mozharness/mozharness/mozilla/taskcluster_helper.py
+++ b/testing/mozharness/mozharness/mozilla/taskcluster_helper.py
@@ -197,27 +197,32 @@ class TaskClusterArtifactFinderMixin(obj
 
     def set_bbb_artifacts(self, task_id, properties_file_path):
         """ Find BBB artifacts through properties_file_path and set them. """
         p = self.load_json_url(
             self.url_to_artifact(task_id, properties_file_path))['properties']
 
         # Set importants artifacts for test jobs
         self.set_artifacts(
-            p['packageUrl'] if p.get('packageUrl') else None,
-            p['testPackagesUrl'] if p.get('testPackagesUrl') else None,
-            p['symbolsUrl'] if p.get('symbolsUrl') else None
+            p.get('packageUrl') or None,
+            p.get('testPackagesUrl') or None,
+            p.get('symbolsUrl') or None,
+            p.get('mozinfoJsonUrl') or None,
         )
 
-    def set_artifacts(self, installer, tests, symbols):
+    def set_artifacts(self, installer, tests, symbols, mozinfo_json):
         """ Sets installer, test and symbols URLs from the artifacts of BBB based task."""
-        self.installer_url, self.test_url, self.symbols_url = installer, tests, symbols
+        self.installer_url = installer
+        self.test_url = tests
+        self.symbols_url = symbols
+        self.mozinfo_json_url = mozinfo_json
         self.info('Set installer_url: %s' % self.installer_url)
         self.info('Set test_url: %s' % self.test_url)
         self.info('Set symbols_url: %s' % self.symbols_url)
+        self.info('Set mozinfo_json_url: %s' % self.mozinfo_json_url)
 
     def set_parent_artifacts(self, child_task_id):
         """ Find and set installer_url, test_url and symbols_url by querying TaskCluster.
 
         In Buildbot Bridge's normal behaviour we can find the artifacts by inspecting
         a child's taskId, determine the task in which it depends on and find the uploaded
         artifacts.
 
@@ -246,17 +251,18 @@ class TaskClusterArtifactFinderMixin(obj
             # Case 1: The parent task is a pure TC task
             if parent_task['extra'].get('locations'):
                 # Build tasks generated under TC specify where they upload their builds
                 installer_path = parent_task['extra']['locations']['build']
 
                 self.set_artifacts(
                     self.url_to_artifact(parent_id, installer_path),
                     self.url_to_artifact(parent_id, 'public/build/test_packages.json'),
-                    self.url_to_artifact(parent_id, 'public/build/target.crashreporter-symbols.zip')
+                    self.url_to_artifact(parent_id, 'public/build/target.crashreporter-symbols.zip',
+                    self.url_to_artifact(parent_id, 'public/build/target.mozinfo.json'))
                 )
             else:
                 # Case 2: The parent task has an associated BBB task
                 # graph_props.json is uploaded in buildbase.py
                 self.set_bbb_artifacts(
                     task_id=parent_id,
                     properties_file_path='public/build/buildbot_properties.json'
                 )
--- a/testing/mozharness/mozharness/mozilla/testing/testbase.py
+++ b/testing/mozharness/mozharness/mozilla/testing/testbase.py
@@ -81,16 +81,22 @@ testing_config_options = [
      "help": "URL to the zip file containing the actual tests",
       }],
     [["--test-packages-url"],
      {"action": "store",
      "dest": "test_packages_url",
      "default": None,
      "help": "URL to a json file describing which tests archives to download",
       }],
+    [["--mozinfo-json-url"],
+     {"action": "store",
+      "dest": "mozinfo_json_url",
+      "default": None,
+      "help": "URL to mozinfo.json file",
+      }],
     [["--jsshell-url"],
      {"action": "store",
      "dest": "jsshell_url",
      "default": None,
      "help": "URL to the jsshell to install",
       }],
     [["--download-symbols"],
      {"action": "store",
@@ -111,16 +117,17 @@ class TestingMixin(VirtualenvMixin, Buil
     tests and Talos.
     """
 
     installer_url = None
     installer_path = None
     binary_path = None
     test_url = None
     test_packages_url = None
+    mozinfo_json_url = None
     symbols_url = None
     symbols_path = None
     jsshell_url = None
     minidump_stackwalk_path = None
     default_tools_repo = 'https://hg.mozilla.org/build/tools'
     proxxy = None
 
     def query_abs_dirs(self):
@@ -300,16 +307,18 @@ class TestingMixin(VirtualenvMixin, Buil
             if not c.get('test_url') and not c.get('test_packages_url'):
                 self.exception("You must use --test-url or --test-packages-url with developer_config.py")
 
         c["installer_url"] = _replace_url(c["installer_url"], c["replace_urls"])
         if c.get("test_url"):
             c["test_url"] = _replace_url(c["test_url"], c["replace_urls"])
         if c.get("test_packages_url"):
             c["test_packages_url"] = _replace_url(c["test_packages_url"], c["replace_urls"])
+        if c.get("mozinfo_json_url"):
+            c["mozinfo_json_url"] = _replace_url(c["mozinfo_json_url"], c["replace_urls"])
 
         for key, value in self.config.iteritems():
             if type(value) == str and value.startswith("http"):
                 self.config[key] = _replace_url(value, c["replace_urls"])
 
         # Any changes to c means that we need credentials
         if not c == orig_config:
             get_credentials()
@@ -370,16 +379,22 @@ class TestingMixin(VirtualenvMixin, Buil
                         self.test_url = str(f['name'])
                         self.info("Found test url %s." % self.test_url)
                 elif f['name'].endswith('crashreporter-symbols.zip'):  # yuk
                     self.symbols_url = str(f['name'])
                     self.info("Found symbols url %s." % self.symbols_url)
                 elif f['name'].endswith('test_packages.json'):
                     self.test_packages_url = str(f['name'])
                     self.info("Found a test packages url %s." % self.test_packages_url)
+
+                    # This is a bit hacky. But it is easier than teaching buildbot to
+                    # expose more URLs.
+                    self.mozinfo_json_url = self.test_packages_url.replace('.test_packages.json',
+                                                                           '.mozinfo.json')
+                    self.info('Inferred mozinfo.json url %s.' % self.mozinfo_json_url)
                 elif not any(f['name'].endswith(s) for s in ('code-coverage-gcno.zip',)):
                     if not self.installer_url:
                         self.installer_url = str(f['name'])
                         self.info("Found installer url %s." % self.installer_url)
         except IndexError, e:
             self.error(str(e))
 
     def find_artifacts_from_taskcluster(self):
@@ -396,26 +411,30 @@ class TestingMixin(VirtualenvMixin, Buil
             c = self.config
             message = "Unable to set %s from the buildbot config"
             if c.get("installer_url"):
                 self.installer_url = c['installer_url']
             if c.get("test_url"):
                 self.test_url = c['test_url']
             if c.get("test_packages_url"):
                 self.test_packages_url = c['test_packages_url']
+            if c.get('mozinfo_json_url'):
+                self.mozinfo_json_url = c['mozinfo_json_url']
 
             # This supports original Buildbot to Buildbot mode
             if self.buildbot_config['sourcestamp']['changes']:
                 self.find_artifacts_from_buildbot_changes()
 
             # This supports TaskCluster/BBB task to Buildbot job
             elif 'testPackagesUrl' in self.buildbot_config['properties'] and \
-                 'packageUrl' in self.buildbot_config['properties']:
+                 'packageUrl' in self.buildbot_config['properties'] and \
+                 'mozinfoJsonUrl' in self.buildbot_config['properties']:
                 self.installer_url = self.buildbot_config['properties']['packageUrl']
                 self.test_packages_url = self.buildbot_config['properties']['testPackagesUrl']
+                self.mozinfo_json_url = self.buildbot_config['properties']['mozinfoJsonUrl']
 
             # This supports TaskCluster/BBB task to TaskCluster/BBB task
             elif 'taskId' in self.buildbot_config['properties']:
                 self.find_artifacts_from_taskcluster()
 
             missing = []
             if not self.installer_url:
                 missing.append("installer_url")
@@ -572,17 +591,17 @@ 2. running via buildbot and running the 
     def download_and_extract(self, extract_dirs=None, suite_categories=None):
         """
         download and extract test zip / download installer
         """
         # Swap plain http for https when we're downloading from ftp
         # See bug 957502 and friends
         from_ = "http://ftp.mozilla.org"
         to_ = "https://ftp-ssl.mozilla.org"
-        for attr in 'symbols_url', 'installer_url', 'test_packages_url', 'test_url':
+        for attr in 'symbols_url', 'installer_url', 'test_packages_url', 'test_url', 'mozinfo_json_url':
             url = getattr(self, attr)
             if url and url.startswith(from_):
                 new_url = url.replace(from_, to_)
                 self.info("Replacing url %s -> %s" % (url, new_url))
                 setattr(self, attr, new_url)
 
         if 'test_url' in self.config:
             # A user has specified a test_url directly, any test_packages_url will